diff --git a/.editorconfig b/.editorconfig index 87bc35d..0026f40 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,52 +1,57 @@ -# EditorConfig — https://editorconfig.org -# Top-most file; stops the search here. root = true -########################################## -# All files -########################################## [*] charset = utf-8 +# lf, not crlf: every tracked file is committed LF. Demanding crlf here would only hold on Windows, +# where core.autocrlf converts on checkout and hides the mismatch. See .gitattributes, which pins +# eol=lf so no contributor's autocrlf setting can change what lands in a blob. end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true indent_style = space indent_size = 4 -[*.{yml,yaml,json,md}] +[*.{yml,yaml,json,md,csproj,props,targets,slnx}] indent_size = 2 +# Two trailing spaces are a hard line break in Markdown. [*.md] -trim_trailing_whitespace = false # trailing spaces are significant (hard line breaks) +trim_trailing_whitespace = false -[*.{csproj,props,targets,slnx}] -indent_size = 2 - -########################################## -# C# files -########################################## [*.cs] -indent_size = 4 max_line_length = 120 -# --- Language conventions --------------------------------------------------- +# The code is block-scoped throughout; switching styles file by file would make every diff a reindent. csharp_style_namespace_declarations = block_scoped:suggestion csharp_using_directive_placement = outside_namespace:suggestion dotnet_sort_system_directives_first = true dotnet_separate_import_directive_groups = false -# var usage — prefer explicit types except when obvious from the right-hand side. csharp_style_var_for_built_in_types = false:suggestion csharp_style_var_when_type_is_apparent = true:suggestion csharp_style_var_elsewhere = false:suggestion -# Expression-bodied members. -csharp_style_expression_bodied_methods = false:suggestion +# STYLEGUIDE.md: "Every method ... must use a block body. Expression bodies (=>) are forbidden on +# methods, no matter how short." Properties are exempted here (project preference overrides the +# guide for the common one-line forwarding getter). +csharp_style_expression_bodied_methods = false:warning +csharp_style_expression_bodied_constructors = false:warning +csharp_style_expression_bodied_operators = false:warning +csharp_style_expression_bodied_local_functions = false:warning csharp_style_expression_bodied_properties = true:suggestion csharp_style_expression_bodied_accessors = false:suggestion +# Accessors (get/set/init) are part of the property exemption: either body form is fine, so IDE0027 +# reports nothing. +dotnet_diagnostic.IDE0027.severity = none + +# STYLEGUIDE.md: "Always use braces on `if`, even single-line bodies. The linter enforces this." +csharp_prefer_braces = true:warning + +# STYLEGUIDE.md ternary rule ("nested ternary" / "branches contain logic" forbidden) has no direct +# analyzer id; IDE0045/IDE0046 (convert if to/from ternary) would fight that rule, so both are off. +dotnet_diagnostic.IDE0045.severity = none +dotnet_diagnostic.IDE0046.severity = none -# Modern C# preferences. -csharp_prefer_braces = true:suggestion csharp_style_prefer_pattern_matching = true:suggestion csharp_style_prefer_switch_expression = true:suggestion csharp_style_throw_expression = true:suggestion @@ -54,19 +59,35 @@ csharp_prefer_simple_using_statement = true:suggestion csharp_style_prefer_range_operator = true:suggestion csharp_style_prefer_index_operator = true:suggestion -# --- Naming: private fields use _camelCase -------------------------------- -dotnet_naming_rule.private_fields_underscore.severity = suggestion -dotnet_naming_rule.private_fields_underscore.symbols = private_fields -dotnet_naming_rule.private_fields_underscore.style = underscore_prefix +dotnet_style_qualification_for_field = false:suggestion +dotnet_style_qualification_for_property = false:suggestion +dotnet_style_qualification_for_method = false:suggestion +dotnet_style_qualification_for_event = false:suggestion + +# Naming: private fields _camelCase, private consts PascalCase, everything else PascalCase/camelCase +# per STYLEGUIDE.md. +dotnet_naming_style.underscore_camel_case.capitalization = camel_case +dotnet_naming_style.underscore_camel_case.required_prefix = _ +dotnet_naming_style.pascal_case.capitalization = pascal_case + +# Consts first — more specific than the general private-field rule below, so it must win. Rule order +# matters: the first matching rule applies. +dotnet_naming_symbols.private_consts.applicable_kinds = field +dotnet_naming_symbols.private_consts.applicable_accessibilities = private +dotnet_naming_symbols.private_consts.required_modifiers = const + +dotnet_naming_rule.private_consts_pascal_case.symbols = private_consts +dotnet_naming_rule.private_consts_pascal_case.style = pascal_case +dotnet_naming_rule.private_consts_pascal_case.severity = warning dotnet_naming_symbols.private_fields.applicable_kinds = field dotnet_naming_symbols.private_fields.applicable_accessibilities = private +dotnet_naming_symbols.private_fields.required_modifiers = -dotnet_naming_style.underscore_prefix.required_prefix = _ -dotnet_naming_style.underscore_prefix.capitalization = camel_case +dotnet_naming_rule.private_fields_underscore_camel_case.symbols = private_fields +dotnet_naming_rule.private_fields_underscore_camel_case.style = underscore_camel_case +dotnet_naming_rule.private_fields_underscore_camel_case.severity = warning -# --- 'this.' qualification -------------------------------------------------- -dotnet_style_qualification_for_field = false:suggestion -dotnet_style_qualification_for_property = false:suggestion -dotnet_style_qualification_for_method = false:suggestion -dotnet_style_qualification_for_event = false:suggestion +dotnet_diagnostic.MA0051.severity = warning +MA0051.maximum_lines_per_method = 90 +MA0051.maximum_statements_per_method = 50 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d8175cc --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Line endings are pinned here rather than left to each contributor's core.autocrlf, so what lands in +# a blob never depends on who committed it. Every tracked text file is already LF, and +# .editorconfig's end_of_line matches. +* text=auto eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 08ad484..c80d8d1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,8 +21,15 @@ updates: test: patterns: - "xunit*" - - "Microsoft.NET.Test.Sdk" - - "coverlet.*" + - "Microsoft.Testing.*" + - "CsCheck" + # The libraries the comparison benchmarks measure against. + benchmarks: + patterns: + - "BenchmarkDotNet" + - "FileHelpers" + - "FlatFiles" + - "RecordParser" labels: - dependencies diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 5d8f219..bfecb03 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -15,7 +15,9 @@ - [ ] Tests added/updated for the change - [ ] `dotnet build --configuration Release` is warning-clean - [ ] `dotnet test --configuration Release` passes -- [ ] Public API changes are documented (XML docs + README if relevant) +- [ ] Public API changes are documented (XML docs, plus `docs/guide/` if relevant) and recorded in `PublicAPI.Unshipped.txt` +- [ ] User-visible changes are in `CHANGELOG.md` +- [ ] Native, Python or C++ changes: the Python, C and C++ tests pass (see CONTRIBUTING.md) - [ ] For performance-sensitive changes: benchmark numbers included below ## Benchmark impact (if applicable) diff --git a/.github/scripts/promote_public_api.py b/.github/scripts/promote_public_api.py new file mode 100644 index 0000000..5744ffd --- /dev/null +++ b/.github/scripts/promote_public_api.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Moves every entry out of each PublicAPI.Unshipped.txt into its PublicAPI.Shipped.txt. + +Run after a release ships: everything that was "unshipped" at release time is now part of +the public API contract. Operates on every src/*/PublicAPI.Unshipped.txt. + +Exit codes: 0 = promoted something, 2 = nothing needed promoting (not an error — the caller +skips the rest of the job), 1 = an actual failure (an uncaught exception, same as Python's +default for any unhandled error). 2 is deliberately not 1 - an uncaught exception used +to exit 1 exactly like the intentional "nothing to promote" case, so release.yml's `if +python3 ...; then changed=true; else changed=false; fi` silently treated a crash (missing +file, permission error) the same as a no-op success, disarming the next release's +breaking-change detection instead of failing the job. +""" + +import glob +import os +import sys + +HEADER = "#nullable enable" + + +def read_entries(path): + with open(path, encoding="utf-8") as f: + return [line.strip() for line in f if line.strip() and line.strip() != HEADER] + + +def write_entries(path, entries): + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(HEADER + "\n") + f.writelines(entry + "\n" for entry in sorted(entries)) + + +REMOVED_PREFIX = "*REMOVED*" + + +def promote_one(unshipped_path, shipped_path): + """Applies unshipped_path's entries onto shipped_path and clears unshipped_path. + + A plain entry is added to Shipped. A `*REMOVED*Foo` entry deletes the plain `Foo` entry + from Shipped (an API removal is meaningful only for one release cycle) and is itself + discarded rather than copied - RS0024 forbids a *REMOVED* line inside Shipped.txt, since + Shipped is the current API surface, not a removal history. + + Returns the number of entries read from unshipped_path (0 if there was nothing to do). + """ + new_entries = read_entries(unshipped_path) + if not new_entries: + return 0 + + shipped_entries = set(read_entries(shipped_path)) if os.path.exists(shipped_path) else set() + for entry in new_entries: + if entry.startswith(REMOVED_PREFIX): + shipped_entries.discard(entry[len(REMOVED_PREFIX):]) + else: + shipped_entries.add(entry) + + write_entries(shipped_path, shipped_entries) + write_entries(unshipped_path, []) + return len(new_entries) + + +def main(): + unshipped_files = sorted(glob.glob("src/*/PublicAPI.Unshipped.txt")) + if not unshipped_files: + print("No src/*/PublicAPI.Unshipped.txt files found.") + return 2 + + promoted_any = False + for unshipped_path in unshipped_files: + shipped_path = unshipped_path.replace("Unshipped.txt", "Shipped.txt") + count = promote_one(unshipped_path, shipped_path) + if count == 0: + print(f"{unshipped_path}: nothing to promote.") + continue + print(f"{unshipped_path}: promoted {count} entries to {shipped_path}.") + promoted_any = True + + return 0 if promoted_any else 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_promote_public_api.py b/.github/scripts/test_promote_public_api.py new file mode 100644 index 0000000..8cd0ae0 --- /dev/null +++ b/.github/scripts/test_promote_public_api.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Unit tests for promote_public_api.py. + +Run: python3 .github/scripts/test_promote_public_api.py +""" + +import os +import tempfile +import unittest + +import promote_public_api as promote + + +class PromoteTests(unittest.TestCase): + def _write(self, path, lines): + with open(path, "w", encoding="utf-8", newline="\n") as f: + f.write(promote.HEADER + "\n") + f.writelines(line + "\n" for line in lines) + + def _read(self, path): + with open(path, encoding="utf-8") as f: + return {line.strip() for line in f if line.strip() and line.strip() != promote.HEADER} + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.shipped = os.path.join(self._tmp.name, "PublicAPI.Shipped.txt") + self.unshipped = os.path.join(self._tmp.name, "PublicAPI.Unshipped.txt") + + def tearDown(self): + self._tmp.cleanup() + + def test_plain_entry_is_added_to_shipped(self): + self._write(self.shipped, ["Foo.Bar() -> void"]) + self._write(self.unshipped, ["Foo.Baz() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.Bar() -> void", "Foo.Baz() -> void"}) + self.assertEqual(self._read(self.unshipped), set()) + + def test_removed_entry_deletes_the_plain_entry_from_shipped(self): + self._write(self.shipped, ["Foo.Bar() -> void", "Foo.Baz() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.Bar() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.Baz() -> void"}) + self.assertEqual(self._read(self.unshipped), set()) + + def test_removed_entry_is_never_written_to_shipped(self): + self._write(self.shipped, ["Foo.Bar() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.Bar() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + for entry in self._read(self.shipped): + self.assertFalse(entry.startswith("*REMOVED*"), entry) + + def test_removed_entry_with_no_matching_shipped_entry_is_a_noop_delete(self): + self._write(self.shipped, ["Foo.Bar() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.NeverShipped() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.Bar() -> void"}) + + def test_mixed_add_and_remove_in_the_same_promotion(self): + self._write(self.shipped, ["Foo.Old() -> void"]) + self._write(self.unshipped, ["*REMOVED*Foo.Old() -> void", "Foo.New() -> void"]) + + promote.promote_one(self.unshipped, self.shipped) + + self.assertEqual(self._read(self.shipped), {"Foo.New() -> void"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml new file mode 100644 index 0000000..a63aa24 --- /dev/null +++ b/.github/workflows/audit.yml @@ -0,0 +1,48 @@ +name: Dependency audit + +# A newly published advisory flags a package without any change here, so the audit runs weekly as well as when +# the dependencies change. +on: + pull_request: + branches: [ master, develop ] + paths: + - '**/*.csproj' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/audit.yml' + push: + branches: [ master ] + paths: + - '**/*.csproj' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/audit.yml' + schedule: + - cron: '0 6 * * 1' + workflow_dispatch: + +permissions: + contents: read + +jobs: + audit: + name: Dependency audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + - name: Restore + run: dotnet restore FixedWidthParser.slnx + + - name: Check for vulnerable packages + run: | + output=$(dotnet list FixedWidthParser.slnx package --vulnerable --include-transitive) + echo "$output" + if echo "$output" | grep -q -E '\b(High|Critical)\b'; then + echo "::error::High or Critical severity vulnerability found in dependencies." + exit 1 + fi diff --git a/.github/workflows/benchmark-pages.yml b/.github/workflows/benchmark-pages.yml index a04099e..451f6bf 100644 --- a/.github/workflows/benchmark-pages.yml +++ b/.github/workflows/benchmark-pages.yml @@ -1,7 +1,8 @@ name: Benchmarks → GitHub Pages -# Runs on every push to master. Benchmark classes are split across parallel -# runners and the results are merged and published to GitHub Pages. +# Runs on each push to master that can move a benchmark. The groups in tests/Benchmarks/benchmark-groups.json +# run on parallel runners, and the results are merged and published to GitHub Pages, where benchmark-pr.yml +# reads them as its baseline. # # Prerequisites (one-time setup in GitHub repo settings): # Settings → Pages → Source: "Deploy from a branch" → Branch: gh-pages / root @@ -11,77 +12,64 @@ name: Benchmarks → GitHub Pages on: push: branches: [master] + paths: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Generator/**' + - 'tests/Benchmarks/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/benchmark-pages.yml' concurrency: group: benchmark-pages cancel-in-progress: true permissions: - contents: write # needed to push data to the gh-pages branch + contents: read jobs: + groups: + name: Load benchmark groups + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load.outputs.matrix }} + steps: + - uses: actions/checkout@v7 + - id: load + run: echo "matrix=$(jq -c . tests/Benchmarks/benchmark-groups.json)" >> "$GITHUB_OUTPUT" + benchmark: name: Run (${{ matrix.group.name }}) + needs: groups runs-on: ubuntu-latest strategy: fail-fast: false matrix: - group: - # Space-separated fully-qualified class patterns. - # Each pattern becomes a separate --filter flag (BenchmarkDotNet ORs them). - - name: parser - filter: >- - Benchmarks.Perf.ParserBenchmarks.* - Benchmarks.Perf.SourceGenParserBenchmarks.* - Benchmarks.Perf.IntHeavyParserBenchmarks.* - - name: reader - filter: >- - Benchmarks.Perf.ReaderBenchmarks.* - Benchmarks.Perf.ReaderStartupBenchmarks.* - Benchmarks.Perf.AsyncReaderBenchmarks.* - - name: byte - filter: >- - Benchmarks.Perf.ByteReaderBenchmarks.* - Benchmarks.Perf.ByteReaderPoolingBenchmarks.* - Benchmarks.Perf.ByteReaderStreamBenchmarks.* - Benchmarks.Perf.PipeReaderBenchmarks.* - - name: writer - filter: Benchmarks.Perf.WriterBenchmarks.* - - name: comparison - filter: >- - Benchmarks.Perf.ComparisonWriterBenchmarks.* - Benchmarks.Perf.ComparisonBenchmarks.* + group: ${{ fromJSON(needs.groups.outputs.matrix) }} steps: - uses: actions/checkout@v7 - uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Restore run: dotnet restore tests/Benchmarks/Benchmarks.csproj - name: Build (Release) - run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore + run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore -p:RunAnalyzers=false - name: Run benchmarks run: | - # --filter accepts multiple space-separated patterns as values of a - # single flag: `--filter Pat1 Pat2 Pat3`. - # Single-quoting the substitution prevents bash glob-expanding the '*'. + # --filter takes several space-separated patterns as values of one flag; single-quoting the + # substitution keeps bash from glob-expanding the '*'. IFS=' ' read -ra patterns <<< '${{ matrix.group.filter }}' dotnet run --project tests/Benchmarks/Benchmarks.csproj \ --configuration Release --no-build \ - -- --filter "${patterns[@]}" --warmupCount 3 --iterationCount 10 + -- --filter "${patterns[@]}" ${{ matrix.group.pagesArgs }} - name: Upload JSON results uses: actions/upload-artifact@v7 @@ -96,6 +84,8 @@ jobs: # Run even when some matrix shards fail so partial results are still published. if: always() && needs.benchmark.result != 'cancelled' runs-on: ubuntu-latest + permissions: + contents: write # pushes to the gh-pages branch steps: - uses: actions/checkout@v7 diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 04835fe..236429f 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -1,16 +1,17 @@ name: Benchmarks → PR Comment -# Runs on every PR push targeting master and develop. -# Results are posted (and updated) as a sticky comment on the PR only — -# no data is written to the gh-pages branch or anywhere else shared. +# Opt-in: runs only on a pull request labelled run-benchmarks whose changes can move a benchmark. The suite +# is several runner-minutes per group, too much to repeat on each push. Results go to a sticky comment on the +# PR, as a delta against the master series benchmark-pages.yml publishes; nothing shared is written. on: pull_request: branches: [master, develop] - types: [opened, synchronize, reopened] + types: [opened, synchronize, reopened, ready_for_review, labeled] concurrency: - group: benchmark-pr-${{ github.event.pull_request.number }} + # Adding any other label must not cancel a run in progress. + group: benchmark-pr-${{ github.event.pull_request.number }}${{ github.event.action == 'labeled' && github.event.label.name != 'run-benchmarks' && format('-{0}', github.run_id) || '' }} cancel-in-progress: true permissions: @@ -18,68 +19,98 @@ permissions: pull-requests: write # needed to post/update the PR comment jobs: + changes: + name: Detect changed paths + if: >- + github.event.pull_request.draft != true && + (github.event.action != 'labeled' || github.event.label.name == 'run-benchmarks') + runs-on: ubuntu-latest + outputs: + benchmarks: ${{ steps.filter.outputs.benchmarks }} + run: ${{ steps.filter.outputs.benchmarks == 'true' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Filter + id: filter + uses: dorny/paths-filter@v4 + with: + filters: | + benchmarks: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Generator/**' + - 'tests/Benchmarks/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/benchmark-pr.yml' + + groups: + name: Load benchmark groups + needs: changes + if: needs.changes.outputs.run == 'true' + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load.outputs.matrix }} + steps: + - uses: actions/checkout@v7 + - id: load + run: echo "matrix=$(jq -c . tests/Benchmarks/benchmark-groups.json)" >> "$GITHUB_OUTPUT" + + # Cheap, so it runs on every PR that touches the benchmarks, labelled or not. + guard: + name: Every benchmark belongs to a group + needs: changes + if: needs.changes.outputs.benchmarks == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' + + - name: Build (Release) + run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release -p:RunAnalyzers=false + + - name: Check group coverage + run: bash tests/Benchmarks/check-benchmark-groups.sh + benchmark: name: Run (${{ matrix.group.name }}) + needs: [changes, groups] + if: needs.changes.outputs.run == 'true' runs-on: ubuntu-latest strategy: fail-fast: false matrix: - group: - # Space-separated fully-qualified class patterns. - # Each pattern becomes a separate --filter flag (BenchmarkDotNet ORs them). - - name: parser - filter: >- - Benchmarks.Perf.ParserBenchmarks.* - Benchmarks.Perf.SourceGenParserBenchmarks.* - Benchmarks.Perf.IntHeavyParserBenchmarks.* - - name: reader - filter: >- - Benchmarks.Perf.ReaderBenchmarks.* - Benchmarks.Perf.ReaderStartupBenchmarks.* - Benchmarks.Perf.AsyncReaderBenchmarks.* - - name: byte - filter: >- - Benchmarks.Perf.ByteReaderBenchmarks.* - Benchmarks.Perf.ByteReaderPoolingBenchmarks.* - Benchmarks.Perf.ByteReaderStreamBenchmarks.* - Benchmarks.Perf.PipeReaderBenchmarks.* - - name: writer - filter: Benchmarks.Perf.WriterBenchmarks.* - - name: comparison - filter: >- - Benchmarks.Perf.ComparisonWriterBenchmarks.* - Benchmarks.Perf.ComparisonBenchmarks.* + group: ${{ fromJSON(needs.groups.outputs.matrix) }} steps: - uses: actions/checkout@v7 - uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Restore run: dotnet restore tests/Benchmarks/Benchmarks.csproj - name: Build (Release) - run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore + run: dotnet build tests/Benchmarks/Benchmarks.csproj --configuration Release --no-restore -p:RunAnalyzers=false - name: Run benchmarks run: | - # --filter accepts multiple space-separated patterns as values of a - # single flag: `--filter Pat1 Pat2 Pat3`. - # Single-quoting the substitution prevents bash glob-expanding the '*'. + # --filter takes several space-separated patterns as values of one flag; single-quoting the + # substitution keeps bash from glob-expanding the '*'. IFS=' ' read -ra patterns <<< '${{ matrix.group.filter }}' dotnet run --project tests/Benchmarks/Benchmarks.csproj \ --configuration Release --no-build \ - -- --filter "${patterns[@]}" --warmupCount 1 --iterationCount 5 + -- --filter "${patterns[@]}" ${{ matrix.group.prArgs }} - name: Upload markdown results uses: actions/upload-artifact@v7 @@ -88,14 +119,32 @@ jobs: path: BenchmarkDotNet.Artifacts/results/*-report-github.md retention-days: 1 + - name: Upload JSON results + uses: actions/upload-artifact@v7 + with: + name: bench-json-${{ matrix.group.name }} + path: BenchmarkDotNet.Artifacts/results/*-report-full.json + retention-days: 1 + comment: name: Post PR Comment - needs: benchmark + needs: [changes, benchmark] # Post partial results even when some shards fail. - if: always() && needs.benchmark.result != 'cancelled' + if: always() && needs.changes.outputs.run == 'true' && needs.benchmark.result != 'cancelled' runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Checkout the published benchmark series + continue-on-error: true + uses: actions/checkout@v7 + with: + ref: gh-pages + path: gh-pages + fetch-depth: 1 + - name: Download all markdown results uses: actions/download-artifact@v8 with: @@ -103,14 +152,25 @@ jobs: merge-multiple: true path: results/ + - name: Download all JSON results + uses: actions/download-artifact@v8 + with: + pattern: bench-json-* + merge-multiple: true + path: results-json/ + - name: Build comment body id: build run: | + delta=(--results results-json) + if [ -f gh-pages/dev/bench/data.js ]; then + delta+=(--baseline gh-pages/dev/bench/data.js) + fi { echo 'body<Full results per group' echo '' for f in results/*-report-github.md; do [ -f "$f" ] || continue @@ -120,6 +180,7 @@ jobs: cat "$f" echo '' done + echo '' echo 'BENCHMARK_EOF' } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9128d26..20d1149 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,14 +1,13 @@ name: CI -# Roda em PRs direcionados à master (o status check que bloqueia o merge) e em -# pushes na master (mantém o branch padrão verde e registra o check no histórico). on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. pull_request: branches: [ master, develop ] + types: [ opened, synchronize, reopened, ready_for_review ] push: branches: [ master ] -# Cancela runs antigas do mesmo PR/branch quando um novo push chega. concurrency: group: ci-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -17,95 +16,128 @@ permissions: contents: read jobs: + changes: + name: Detect changed paths + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + outputs: + dotnet: ${{ github.event_name != 'pull_request' || steps.filter.outputs.dotnet == 'true' }} + aot: ${{ github.event_name != 'pull_request' || steps.filter.outputs.aot == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Filter + id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@v4 + with: + filters: | + dotnet: + - 'src/**' + - 'tests/**' + - 'Directory.Build.props' + - 'global.json' + - 'FixedWidthParser.slnx' + - '.editorconfig' + - 'coverage.runsettings' + - '.github/workflows/ci.yml' + aot: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Generator/**' + - 'tests/AotSmoke/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/ci.yml' + + # "Build & Test ()" are required checks on master, and a check whose job never starts stays pending, + # so the job always runs and skips its steps when no .NET source changed. test: name: Build & Test (${{ matrix.os }}) + needs: changes runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] + env: + RUN: ${{ needs.changes.outputs.dotnet }} + LINUX: ${{ matrix.os == 'ubuntu-latest' }} steps: + - name: Nothing to build + if: env.RUN != 'true' + run: echo "No .NET sources changed; skipping build and tests." + - name: Checkout + if: env.RUN == 'true' uses: actions/checkout@v7 - name: Setup .NET + if: env.RUN == 'true' uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - # Cache restored NuGet packages keyed on the project/props files. - # (setup-dotnet's built-in cache needs a packages.lock.json, which this repo - # does not use, so we cache the global packages folder directly.) - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- - - - name: Restore - run: dotnet restore FixedWidthParser.slnx - - # DeterministicSourcePaths is disabled here so the Microsoft code-coverage - # collector can map IL back to source. With it on (it is enabled on CI via - # ContinuousIntegrationBuild in FixedWidthParser.csproj), source paths get - # remapped and coverage for the library comes out empty. This build is only - # used for test/coverage; the pack/publish pipeline lives in release.yml. - - name: Build (Release) - run: dotnet build FixedWidthParser.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false - - - name: Test (Release) + collect coverage - run: >- - dotnet test --solution FixedWidthParser.slnx - --results-directory ./TestResults - --configuration Release - --verbosity normal - --no-build - -- - --report-xunit-trx - --coverage - --coverage-output-format cobertura + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' + + # Linux builds the whole solution with the analyzers, which are about two thirds of the compile and report + # the same on every OS, packs it and runs the generator tests, which are pure Roslyn. Windows and macOS + # build and run only the library tests, without the analyzers; source generators still run. + - name: Build and test the solution (Linux) + if: env.RUN == 'true' && env.LINUX == 'true' + run: | + dotnet restore FixedWidthParser.slnx + dotnet build FixedWidthParser.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false + dotnet format whitespace FixedWidthParser.slnx --verify-no-changes + # Packing validates the package against the last release when package validation is on. + dotnet pack src/FixedWidthParser/FixedWidthParser.csproj --configuration Release --no-restore --output ./package-validation + dotnet test --solution FixedWidthParser.slnx --results-directory ./TestResults --configuration Release --no-build \ + -- --report-xunit-trx --coverage --coverage-settings coverage.runsettings --coverage-output-format cobertura + + - name: Build and test the library (Windows, macOS) + if: env.RUN == 'true' && env.LINUX != 'true' + shell: bash + run: | + dotnet build tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj --configuration Release -p:RunAnalyzers=false + dotnet test --project tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj --results-directory ./TestResults \ + --configuration Release --no-build -- --report-xunit-trx - name: Upload test results - if: always() + if: always() && env.RUN == 'true' uses: actions/upload-artifact@v7 with: name: test-results-${{ matrix.os }} path: ./TestResults/**/*.trx if-no-files-found: ignore - # Coverage is uploaded once, from the Linux leg only, to avoid duplicate reports. - name: Upload coverage to Codecov - if: matrix.os == 'ubuntu-latest' + if: env.RUN == 'true' && env.LINUX == 'true' uses: codecov/codecov-action@v7 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./TestResults/*.cobertura.xml + flags: csharp fail_ci_if_error: false - # Surfaces vulnerable transitive/direct packages regardless of NoWarn suppressions - # in Directory.Build.props (NU190x warnings are silenced there). - audit: - name: Dependency audit - runs-on: ubuntu-latest + aot-smoke: + name: NativeAOT smoke (win-x64) + needs: changes + if: needs.changes.outputs.aot == 'true' + runs-on: windows-latest steps: - - uses: actions/checkout@v7 + - name: Checkout + uses: actions/checkout@v7 - - uses: actions/setup-dotnet@v6 + - name: Setup .NET + uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - - name: Restore - run: dotnet restore FixedWidthParser.slnx + - name: Publish AotSmoke (PublishAot) + run: dotnet publish tests/AotSmoke/AotSmoke.csproj --configuration Release --runtime win-x64 -p:IlcTreatWarningsAsErrors=true -p:RunAnalyzers=false - - name: Check for vulnerable packages - run: | - echo "Scanning for known-vulnerable packages…" - output=$(dotnet list FixedWidthParser.slnx package --vulnerable --include-transitive) - echo "$output" - if echo "$output" | grep -q -E '\b(High|Critical)\b'; then - echo "::error::High or Critical severity vulnerability found in dependencies." - exit 1 - fi + - name: Run the published binary + run: ./tests/AotSmoke/bin/Release/net10.0/win-x64/publish/AotSmoke.exe diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index af31a41..d108ca3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,10 +1,12 @@ name: CodeQL on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. push: branches: [ master ] pull_request: branches: [ master, develop ] + types: [ opened, synchronize, reopened, ready_for_review ] schedule: # Weekly scan (Mondays 06:00 UTC) to catch newly published query updates. - cron: '0 6 * * 1' @@ -18,30 +20,82 @@ permissions: security-events: write # needed to upload results to the Security tab jobs: + changes: + name: Detect changed languages + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + outputs: + csharp: ${{ github.event_name != 'pull_request' || steps.filter.outputs.csharp == 'true' }} + python: ${{ github.event_name != 'pull_request' || steps.filter.outputs.python == 'true' }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Filter + id: filter + if: github.event_name == 'pull_request' + uses: dorny/paths-filter@v4 + with: + filters: | + csharp: + - 'src/**' + - 'Directory.Build.props' + - 'global.json' + - '.github/workflows/codeql.yml' + python: + - 'python/**' + - '.github/workflows/codeql.yml' + + # Every language always gets a job: "Analyze (C#)" is a required check, and a check whose job never starts + # stays pending. An unchanged language's job skips its steps instead. analyze: - name: Analyze (C#) + name: Analyze (${{ matrix.name }}) + needs: changes runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { language: csharp, name: 'C#' } + - { language: python, name: Python } + env: + RUN: ${{ needs.changes.outputs[matrix.language] }} steps: + - name: Skip (language unchanged) + if: env.RUN != 'true' + run: echo "No ${{ matrix.name }} changes; skipping the analysis." + - name: Checkout + if: env.RUN == 'true' uses: actions/checkout@v7 - name: Setup .NET + if: env.RUN == 'true' && matrix.language == 'csharp' uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Initialize CodeQL + if: env.RUN == 'true' uses: github/codeql-action/init@v4 with: - languages: csharp + languages: ${{ matrix.language }} queries: security-extended - # Manual build so CodeQL sees the source generator + analyzers correctly. + # Only the shipped projects: tests and benchmarks double the build for code no one runs against untrusted + # input. Building them by hand lets CodeQL see the source generator's output. - name: Build (Release) - run: dotnet build FixedWidthParser.slnx --configuration Release + if: env.RUN == 'true' && matrix.language == 'csharp' + run: | + for project in src/*/*.csproj; do + dotnet build "$project" --configuration Release -p:RunAnalyzers=false + done - name: Perform CodeQL Analysis + if: env.RUN == 'true' uses: github/codeql-action/analyze@v4 with: - category: "/language:csharp" + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml new file mode 100644 index 0000000..2ee8c7e --- /dev/null +++ b/.github/workflows/native-bindings.yml @@ -0,0 +1,229 @@ +name: Native bindings + +on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. + push: + branches: [master] + pull_request: + branches: [master, develop] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +concurrency: + group: native-bindings-${{ github.ref }} + cancel-in-progress: true + +jobs: + changes: + name: Detect changed paths + if: github.event.pull_request.draft != true + runs-on: ubuntu-latest + outputs: + native: ${{ steps.filter.outputs.native }} + python: ${{ steps.filter.outputs.python }} + cpp: ${{ steps.filter.outputs.cpp }} + steps: + - name: Checkout + uses: actions/checkout@v7 + + # A change to the native library tests every binding; a change to one binding tests only that one. + - name: Filter + id: filter + uses: dorny/paths-filter@v4 + with: + filters: | + native: + - 'src/FixedWidthParser/**' + - 'src/FixedWidthParser.Native/**' + - 'Directory.Build.props' + - 'global.json' + - 'python/scripts/build_native.py' + - '.github/workflows/native-bindings.yml' + python: + - 'python/**' + - 'tests/FixedWidthParser.NativeSmoke/**' + cpp: + - 'cpp/**' + + # Every platform a release ships, so a release never builds one for the first time. + build-native: + name: Build native library (${{ matrix.os }}) + needs: changes + if: >- + needs.changes.outputs.native == 'true' || + needs.changes.outputs.python == 'true' || + needs.changes.outputs.cpp == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' + + - name: Publish the native library into the Python package + shell: bash + env: + RunAnalyzers: 'false' + run: python python/scripts/build_native.py + + - name: Upload native library + uses: actions/upload-artifact@v7 + with: + name: native-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib/* + if-no-files-found: error + + test-python: + name: Python + C smoke (${{ matrix.os }}) + needs: [changes, build-native] + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.python == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.12' + cache: pip + cache-dependency-path: python/pyproject.toml + + - name: Download native library + uses: actions/download-artifact@v8 + with: + name: native-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib + + - name: Restore executable bit (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: chmod +x python/src/fixedwidth/_lib/* + + - name: Install the Python package + run: pip install -e "python[dev,pandas,polars]" ${{ matrix.os == 'ubuntu-latest' && 'pytest-cov' || '' }} + + - name: Run the Python tests + run: >- + pytest python/tests -v + ${{ matrix.os == 'ubuntu-latest' && '--cov=fixedwidth --cov-report=xml:coverage-python.xml' || '' }} + + - name: Upload Python coverage to Codecov + if: matrix.os == 'ubuntu-latest' + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage-python.xml + flags: python + fail_ci_if_error: false + + - name: Build the wheel and check it installs here + run: | + pip install build + python -m build --wheel python/ --outdir dist/ + python python/scripts/check_wheel_tag.py dist + + - name: Run the Python benchmark (smoke) + run: python python/benchmarks/bench_read.py --lines 2000 --runs 1 + + # lib.exe, which turns fixedwidth.def into an import library, is only on PATH in a developer prompt. + - name: MSVC developer environment (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Build and run the C smoke test + run: | + cmake -S tests/FixedWidthParser.NativeSmoke -B build/nativesmoke -DCMAKE_BUILD_TYPE=Release + cmake --build build/nativesmoke --config Release + ctest --test-dir build/nativesmoke --output-on-failure -C Release + + test-musl: + name: musl library + Python (${{ matrix.runner }}) + needs: changes + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.python == 'true' + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Build, test and wheel in Alpine + env: + RunAnalyzers: 'false' + run: sh python/scripts/build_musl_wheel.sh + + test-cpp: + name: C++ wrapper (${{ matrix.os }}) + needs: [changes, build-native] + if: needs.changes.outputs.native == 'true' || needs.changes.outputs.cpp == 'true' + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Download native library + uses: actions/download-artifact@v8 + with: + name: native-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib + + - name: Restore executable bit (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: chmod +x python/src/fixedwidth/_lib/* + + # lib.exe, which turns fixedwidth.def into an import library, is only on PATH in a developer prompt. + - name: MSVC developer environment (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + + - name: Build and run the C++ tests + run: | + cmake -S cpp -B build/cpp -DCMAKE_BUILD_TYPE=Release + cmake --build build/cpp --config Release + ctest --test-dir build/cpp --output-on-failure -C Release + + - name: Install the package and build a find_package() consumer against it + shell: bash + run: | + cmake --install build/cpp --config Release --prefix "${{ github.workspace }}/build/prefix" + cmake -S cpp/tests/package -B build/consumer -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH="${{ github.workspace }}/build/prefix" + cmake --build build/consumer --config Release + ctest --test-dir build/consumer --output-on-failure -C Release + + # The one check branch protection requires from this workflow. A matrix job skipped by its `if` never reports + # its expanded name, so requiring those names would leave the check pending whenever no binding changed. + result: + name: Bindings result + needs: [changes, build-native, test-python, test-musl, test-cpp] + if: always() && github.event.pull_request.draft != true + runs-on: ubuntu-latest + steps: + - name: Fail if any job failed or was cancelled + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "A bindings job failed or was cancelled." + exit 1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 12fcb4a..8b62ab6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,41 +1,38 @@ name: Release -# Publica no NuGet ao empurrar uma tag de versão (ex.: v1.0.0 -> versão 1.0.0). -# Usa Trusted Publishing (OIDC): nenhuma API key fica armazenada como secret — o GitHub -# troca um token OIDC de curta duração por uma chave temporária via NuGet/login. -# Pré-requisitos no nuget.org: uma policy de Trusted Publishing apontando para este repo + -# workflow, e a variável de repositório NUGET_USER com o usuário do nuget.org. +# Pushing a version tag (v2.0.0, v2.0.0-rc.1) publishes that version everywhere: +# - FixedWidthParser.NET to NuGet, through Trusted Publishing (OIDC): nuget.org needs a policy for this +# repository and workflow, and the repository variable NUGET_USER its user name. +# - fixedwidthparser wheels to PyPI, through Trusted Publishing: PyPI needs a trusted publisher for +# this repository, workflow release.yml. +# - The native library for each platform, attached to the GitHub Release. +# A tag with a "-" is published as a prerelease. on: push: tags: [ 'v*' ] permissions: contents: read - id-token: write # necessário para o OIDC do Trusted Publishing jobs: - publish: - name: Pack & Publish to NuGet + build: + name: Build, test & pack (.NET) runs-on: ubuntu-latest + outputs: + version: ${{ steps.version.outputs.value }} steps: - name: Checkout uses: actions/checkout@v7 with: - fetch-depth: 0 # histórico completo para o SourceLink + fetch-depth: 0 # full history for SourceLink - name: Setup .NET uses: actions/setup-dotnet@v6 with: - dotnet-version: '10.0.x' - - - name: Cache NuGet packages - uses: actions/cache@v6 - with: - path: ~/.nuget/packages - key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj', '**/Directory.Build.props') }} - restore-keys: | - ${{ runner.os }}-nuget- + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' - name: Derive version from tag id: version @@ -60,6 +57,189 @@ jobs: -p:Version=${{ steps.version.outputs.value }} --output ./artifacts + - name: Generate SBOM + uses: anchore/sbom-action@v0 + with: + path: ./artifacts + format: spdx-json + output-file: ./artifacts/fixedwidthparser.spdx.json + upload-release-assets: false + + - name: Upload package artifacts + uses: actions/upload-artifact@v7 + with: + name: nuget-packages + path: ./artifacts/* + if-no-files-found: error + + native: + name: Build native library (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + cache: true + cache-dependency-path: '**/*.csproj' + + - name: Build the native library + shell: bash + env: + RunAnalyzers: 'false' + run: python python/scripts/build_native.py + + - name: Rename asset for this platform + shell: bash + run: | + case "${{ runner.os }}" in + Windows) src=FixedWidthParser.Native.dll; os=win; ext=dll ;; + macOS) src=FixedWidthParser.Native.dylib; os=osx; ext=dylib ;; + *) src=FixedWidthParser.Native.so; os=linux; ext=so ;; + esac + case "$(uname -m)" in arm64|aarch64) arch=arm64 ;; *) arch=x64 ;; esac + mkdir -p release-assets + cp "python/src/fixedwidth/_lib/${src}" "release-assets/fixedwidth-native-${os}-${arch}.${ext}" + + - name: Upload release asset + uses: actions/upload-artifact@v7 + with: + name: release-native-${{ matrix.os }} + path: release-assets/* + if-no-files-found: error + + - name: Upload library for the Python wheel + uses: actions/upload-artifact@v7 + with: + name: python-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib/* + if-no-files-found: error + + python: + name: Build Python wheel (${{ matrix.os }}) + needs: [build, native] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest, macos-15-intel] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: '3.12' + + - name: Download the native library + uses: actions/download-artifact@v8 + with: + name: python-lib-${{ matrix.os }} + path: python/src/fixedwidth/_lib + + - name: Restore executable bit (Linux/macOS) + if: runner.os != 'Windows' + shell: bash + run: chmod +x python/src/fixedwidth/_lib/* + + - name: Set package version + shell: bash + run: | + sed -i.bak "s/^version = \".*\"/version = \"${{ needs.build.outputs.version }}\"/" python/pyproject.toml + rm -f python/pyproject.toml.bak + + - name: Install test dependencies + run: pip install -e "python[dev,pandas,polars]" + + - name: Test (Python) + run: pytest python/tests + + - name: Build the wheel + run: | + pip install build + python -m build --wheel python/ --outdir dist/ + + - name: Check the wheel installs here + run: python python/scripts/check_wheel_tag.py dist + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: wheel-${{ matrix.os }} + path: dist/*.whl + if-no-files-found: error + + musl: + name: Build musl library and wheel (${{ matrix.arch }}) + needs: build + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - { arch: x64, runner: ubuntu-latest } + - { arch: arm64, runner: ubuntu-24.04-arm } + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set package version + run: sed -i "s/^version = \".*\"/version = \"${{ needs.build.outputs.version }}\"/" python/pyproject.toml + + - name: Build, test and wheel in Alpine + env: + RunAnalyzers: 'false' + run: sh python/scripts/build_musl_wheel.sh + + - name: Rename asset for this platform + run: | + mkdir -p release-assets + cp python/src/fixedwidth/_lib/FixedWidthParser.Native.so "release-assets/fixedwidth-native-linux-musl-${{ matrix.arch }}.so" + + - name: Upload release asset + uses: actions/upload-artifact@v7 + with: + name: release-native-musl-${{ matrix.arch }} + path: release-assets/* + if-no-files-found: error + + - name: Upload wheel artifact + uses: actions/upload-artifact@v7 + with: + name: wheel-musl-${{ matrix.arch }} + path: dist/*.whl + if-no-files-found: error + + publish: + name: Publish to NuGet & GitHub Release + needs: [build, native, python, musl] + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + + steps: + - name: Download NuGet packages + uses: actions/download-artifact@v8 + with: + name: nuget-packages + path: artifacts + + - name: Download release assets + uses: actions/download-artifact@v8 + with: + pattern: release-* + path: release-assets + merge-multiple: true + - name: NuGet login (Trusted Publishing / OIDC) uses: NuGet/login@v1 id: nuget-login @@ -73,10 +253,86 @@ jobs: --source https://api.nuget.org/v3/index.json --skip-duplicate - - name: Upload package artifacts - if: always() - uses: actions/upload-artifact@v7 + - name: Upload to the GitHub Release + uses: softprops/action-gh-release@v3 with: - name: nuget-packages - path: ./artifacts/* - if-no-files-found: warn + files: | + release-assets/* + artifacts/*.spdx.json + fail_on_unmatched_files: true + prerelease: ${{ contains(github.ref_name, '-') }} + + publish-pypi: + name: Publish to PyPI + needs: publish + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + steps: + - name: Download wheel artifacts + uses: actions/download-artifact@v8 + with: + pattern: wheel-* + path: dist/ + merge-multiple: true + + - name: Publish to PyPI (Trusted Publishing) + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + skip-existing: true + + promote-public-api: + name: Promote PublicAPI.Unshipped -> Shipped + needs: publish + if: ${{ !contains(github.ref_name, '-') }} + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout develop + uses: actions/checkout@v7 + with: + ref: develop + + - name: Test the promotion script + working-directory: .github/scripts + run: python3 test_promote_public_api.py -v + + - name: Promote unshipped API entries + id: promote + run: | + set +e + python3 .github/scripts/promote_public_api.py + code=$? + set -e + # 2 means nothing to promote, a real no-op. Any other nonzero code is a crash (a missing file, a + # permission error) and must fail the job rather than pass as changed=false. + if [ "$code" -eq 0 ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + elif [ "$code" -eq 2 ]; then + echo "changed=false" >> "$GITHUB_OUTPUT" + else + exit "$code" + fi + + - name: Open PR + if: steps.promote.outputs.changed == 'true' + uses: peter-evans/create-pull-request@v8 + with: + commit-message: "chore: promote public API for ${{ github.ref_name }}" + title: "chore: promote public API for ${{ github.ref_name }}" + body: >- + Everything that was in `PublicAPI.Unshipped.txt` when `${{ github.ref_name }}` was + published is now part of the shipped contract. Moves it into `PublicAPI.Shipped.txt` + so future removals and signature changes get flagged as breaking. + branch: chore/promote-public-api-${{ github.ref_name }} + base: develop + delete-branch: true + add-paths: | + src/*/PublicAPI.Shipped.txt + src/*/PublicAPI.Unshipped.txt diff --git a/.gitignore b/.gitignore index 0808c4a..8409f8c 100644 --- a/.gitignore +++ b/.gitignore @@ -480,3 +480,13 @@ $RECYCLE.BIN/ # Vim temporary swap files *.swp + +# CMake build trees (native smoke and C++ tests) +build/ + +# Python package +python/.venv/ +python/src/fixedwidth/_lib/* +!python/src/fixedwidth/_lib/.gitkeep +__pycache__/ +.pytest_cache/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..5b620f5 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,247 @@ +# Architecture + +A map of how the codebase fits together, not a manual. Start here, then follow the file/type names +into the source: the code is expected to carry its own meaning through naming and flow, so read it +directly rather than looking for a commentary alongside it. + +What the code deliberately does *not* carry is the evidence behind a decision — benchmark numbers, +allocation counts, approaches that were tried and rejected. That lives in [`docs/`](docs/README.md), +and [`STYLEGUIDE.md`](STYLEGUIDE.md#comments) says why. + +## Two paths over one set of helpers + +Every model is parsed and written by one of two paths, both behind the static `FixedWidth` class +(`src/FixedWidthParser/FixedWidth.cs`), where each method has one overload per path: + +| | Layout | Generated | +|---|---|---| +| Model | any type, through a `FixedWidthLayout` from `[FixedColumn]` attributes or from `FixedWidthLayout.Build` | a `partial` type implementing `IFixedWidthModel` | +| Call | `FixedWidth.X(…, layout, …)` | `FixedWidth.X(…)`, calling the generated statics | +| Built | when the layout is built: by reflection and compiled expression trees for attributes, from delegates for `Build` | at compile time by `FixedWidthModelGenerator` | + +Both paths call the same helpers in `FixedWidthRuntime` (`src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs`) +for every column: trim, decode a string, parse a number, format a value, pad and align. That class is +public only because generated code lives in the consumer's assembly. Keeping the semantics in one place +is what makes the two paths agree; `SourceGenParityTests` checks that they do. + +Column positions are bytes throughout. Lines are never transcoded to UTF-16: a column is a +`ReadOnlySpan` slice of the line, and only a `string` column, or a type that parses from +characters alone, is decoded with the configured encoding. + +## Settings: options and the context + +Reads take a `FixedWidthReaderOptions` and writes a `FixedWidthWriterOptions`, validated on each call. Both reduce to a +`FixedWidthContext` (`src/FixedWidthParser/FixedWidthContext.cs`), a `readonly record struct` passed +by `in` to every parse and format call, generated or not, and to every `IFixedWidthConverter`. + +The context is the extension point. `IFixedWidthModel` and `IFixedWidthConverter` are +implemented by users, so a minor version cannot add a member to them; a new setting arrives as a new +property on the context instead. The context also caches what the hot path would otherwise recompute +per column, such as whether the encoding is UTF-8 and the provider's decimal separator. + +## The layout object + +A `FixedWidthLayout` (`src/FixedWidthParser/Layouts/`) holds, per column, a `ColumnParserInfo` +(start, length, name and a `ColumnParser` closure that parses and assigns) and an +`IColumnFormatter` (`Formatters/ColumnFormatters.cs`), plus the line length and the model factory. +It is immutable, so one instance serves every read, write and worker thread. A layout that lacks the +parsers or the formatters reads or writes nothing; `EnsureCanRead`/`EnsureCanWrite` throw before the +first line. + +Two sources fill it, and both reach the same parse and format closures: + +- **`FromAttributes()`.** `ModelColumns.ForEachColumn` (`Attributes/`) walks the `[FixedColumn]` + properties, then fields; `ColumnLayoutValidator` rejects a negative start, a non-positive length and + overlapping columns. `ParserBuilder` compiles a model factory and a setter per column, and + `ColumnParserFactory` picks the parse for the member's runtime `Type` through `MakeGenericMethod`. + `FormatterBuilder` compiles a getter per column and picks its formatter the same way. The result is + cached once per `T` in a static constructor. The read side and the write side build independently: + a model only one side can handle (a column type that formats but does not parse, say) still gets a + layout, and the other side's error is rethrown on its first use. Only an error both sides hit, such + as an overlap, throws from `FromAttributes`. +- **`Build(configure)`.** `FixedWidthLayoutBuilder` receives a `ColumnGetter`/`ColumnSetter` + pair per column, already typed, so it selects the parse and the formatter by generic constraints + (`Column` for UTF-8 parsable types, `DecodedColumn`, `NullableColumn`, `ConvertedColumn`) + and `typeof(T) ==` checks the JIT folds. No reflection: this is the trimming- and AOT-safe source, + which `tests/AotSmoke` publishes with NativeAOT to prove. + +`ColumnParserFactory`'s resolution order — converter, date/time with a `Format`, `string`, +`double`/`float`, `IUtf8SpanParsable`, then `ISpanParsable` over decoded text — is shared by both +sources (`TryBuildSpecial`) and mirrored by the generator's `Classify`; the formatter choice (converter, +string, `IUtf8SpanFormattable`, `ISpanFormattable`, a nullable wrapper) is mirrored by `ClassifyWrite`. +Column options are validated by `FormatterBuilder.ResolveOptions` for both. + +A layout accepts `ref struct` models (`allows ref struct`) for `TryParse`, `TryFormat`, the readers and +the aggregates; an attribute layout over a `ref struct` does not write. An enumerator cannot hold a +`ref struct` in a field, so `MoveNext` only finds the next line and `Current` +parses it on access; `foreach` reads `Current` once, so each line is still parsed once. With +`OnParseError` set, `MoveNext` parses the line too, to skip a bad one before yielding it. + +## Reading + +``` +source -> line splitter -> IRecordLineParser -> model + (bytes in a (struct strategy: + pooled buffer) runtime or generated) +``` + +Two enumerators share one shape, each specialized by a `struct` parse strategy +(`IRecordLineParser`: `LayoutLineParser` or `GeneratedLineParser`, in +`Infrastructure/RecordLineParsers.cs`). Because the strategy is a `struct` type argument, the JIT compiles one +enumerator per `(TModel, TParser)` pair and the parse call devirtualizes. + +| Source | Enumerator | +|---|---| +| `Stream` or file, sync | `RecordEnumerator` (a `struct`, so `foreach` allocates nothing) | +| `Stream` or file, async | `AsyncRecordEnumerator` | + +Every reader, the parallel ones included, splits lines with `LineBufferState` +(`Readers/LineBufferState.cs`). It owns a pooled buffer: it finds the next `\n`, drops a preceding +`\r`, skips a UTF-8 byte order mark once, skips empty lines while counting them, and compacts or grows +the buffer when a line does not fit. Growth stops at `MaxLineLength`, which is what bounds memory on a +source with no line breaks. Keeping one splitter is deliberate: a `PipeReader` reader with its own +splitter over `ReadOnlySequence` was removed because it duplicated these rules and read slower +than `PipeReader.AsStream()` ([pipe-reader.md](docs/pipe-reader.md)). + +The async enumerator keeps the line scan synchronous: `MoveNextAsync` returns a completed +`ValueTask` while the buffer still holds a line, and only `MoveNextSlowAsync` awaits a refill. + +A file path is reopened on each enumeration (`FixedWidthRecordEnumerable` and its async sibling); a +stream is read once and disposed unless the caller passed `leaveOpen`. + +### Parallel aggregation + +`ParallelAggregator` (`Readers/ParallelAggregator.cs`) backs `AggregateFileAsync` on both paths, and the +native library's reads. It reads an `IRandomAccessSource` (a file handle, or native memory), cuts +`[dataStart, length)` into byte ranges (four per worker, 1 MB to 64 MB each), and lets workers pull ranges +from a shared counter. Each worker reads its range through its own `LineBufferState` into its own +accumulator — or, for the native library, its own `ILineSink` — and the caller merges them in range order +and throws the first failure in file order. The accumulator and sink loops are twins on purpose: folding +the accumulator into a generic sink cost one-call reads about 40% under dynamic PGO. + +A line belongs to the range its first byte falls in, the byte after the previous line break, so a +byte order mark belongs to line 1. A worker starts one byte before its range, skips the line it lands +inside, and stops once the next line would start past its range. Fixed-width lines have no quoting, +so unlike ExcelReader's CSV engine there is no guessed start to confirm or re-read: every line is +parsed exactly once. A single range is the sequential read, which is how `DegreeOfParallelism = 1` and +small files run, so there is no separate sequential twin. + +A worker does not know how many lines precede its range. It counts them only when a line fails — +line breaks from the start of the file up to the failing line, continued from there for later failures +in the same range — so errors carry the sequential line number and the happy path pays nothing. + +A `Stream` cannot be read at an offset, so `ParallelStreamAggregator` follows ExcelReader's XLSX +pipeline instead: + +``` +source -> producer (1 thread) -> bounded queue -> N workers -> ordered merge + 256 KB pooled chunks, chunk, index, parse + Add by chunk index + cut after the last '\n' lines before +``` + +The producer carries the bytes after the last line break into the next chunk and counts the line +breaks it hands over, so each chunk knows the number of its first line and workers report sequential +line numbers directly. The queue holds about two chunks per worker, and results are merged in chunk +order as soon as every earlier chunk is merged. Each chunk holds one of four slots per worker from the +time it is read until it is merged, so a slow early chunk stalls the producer instead of letting later +results pile up: chunks and unmerged accumulators stay bounded whatever the source's size. A line longer than a chunk grows the chunk, up to `MaxLineLength`. A failure in the +producer lies past every chunk it emitted, so a worker's failure is thrown first. With +`DegreeOfParallelism = 1` the stream is read by the ordinary async enumerator. + +Measurements are in [parallel-aggregate.md](docs/parallel-aggregate.md). + +### Errors + +A line that fails to parse costs nothing extra on the lines that succeed. On failure, `ParseErrors` +(`Readers/ParseFailure.cs`) parses the line a second time with a `ParseFailure` sink in the context; +the parser reports the rejecting column, or the short line, through +`FixedWidthContext.ReportFailedColumn`/`ReportShortLine`, which do nothing outside this diagnosis. The +result is a `FixedWidthParseException` naming the line, column, position and raw text, handed to +`OnParseError` to skip the line or thrown. + +## Writing + +`RecordWriter` (`Writers/RecordWriter.cs`) writes for both paths, specialized like the readers by a +`struct` strategy (`IRecordFormatter`: `LayoutRecordFormatter` or `GeneratedRecordFormatter`, +in `Writers/RecordFormatters.cs`). Each record is formatted, followed by the encoded `NewLine`, into one +pooled 64 KB batch. A generated model does not declare its line length, so the batch does not reserve +a slot: a record that does not fit the rest of the batch flushes it, and one that does not fit an empty +batch grows it. A write allocates nothing per record. A layout line is filled with spaces first, then +every column formats into its own slice. + +Per column, `FixedWidthRuntime` picks the cheapest route that keeps the bytes the encoding would +produce: + +- `IUtf8SpanFormattable` values (every BCL number and date) format straight into the column's slice + and are aligned in place. The UTF-8 bytes are kept when the encoding is UTF-8 or they are ASCII; + otherwise the value is formatted again as characters and encoded. +- A `double` with no `Format` is written as a scaled integer when it has at most four decimals, the + same bytes as the BCL's shortest round-trip form. Measurements are in + [writer-performance.md](docs/writer-performance.md). +- A left-aligned `string` under UTF-8 is transcoded straight into the slice; truncation stops at a whole + character. Other strings are encoded and placed by `ColumnFormatOptions.WriteInto`. +- Everything else formats as characters, is encoded, and is placed by `WriteInto`, which applies + alignment, padding (keeping a sign ahead of zero padding) and the overflow policy. + +## The source generator + +`src/FixedWidthParser.Generator/` targets `netstandard2.0` so the .NET Framework compiler host in Visual +Studio can load it. It is an incremental generator: `Extract` turns each candidate type into a +`ModelInfo` of plain, equatable values (`ColumnInfo`, `EquatableArray`), so an edit that does not +change a model's columns does not regenerate it. + +`Emit` validates the model — partial, top-level, a supported type for each column, layout, converter +type, overflow, ASCII padding — and reports diagnostics `FWP001`–`FWP012` (`FWP007` and `FWP009` are +retired) instead of generating code +for an invalid model. Otherwise it writes `TryParse` and `TryFormat` as straight-line code: one +`FixedWidthRuntime` call per column. Each column's settings become two `static readonly` fields, a +`ColumnParseOptions` and a `ColumnFormatOptions`, passed to the helpers by `in`. + +## The native library + +`src/FixedWidthParser.Native` is published with NativeAOT as a shared library with a C ABI +(`include/fixedwidth.h`), following ExcelReader.Native: `[UnmanagedCallersOnly]` wrappers in `Exports*.cs` +that only check pointers, over a span-based `*Api` layer the tests drive; opaque handles in a +`NativeHandleTable` that never reissues an id; a per-thread last error; every optional input in a +`struct_size`-versioned options struct. It uses the core through `InternalsVisibleTo` and adds nothing to +its public API. + +``` +fw_column_spec[] -> NativeLayout -> ParallelAggregator -> TableSink per partition -> fw_table + (validated, (byte ranges, (a ColumnBuilder per or Arrow + immutable) header skipped) column, native buffers) +``` + +There is no model type: a `NativeLayout` holds each column's `ColumnParseOptions` and +`ColumnFormatOptions`, and a `ColumnBuilder` parses its slice of each line with the same +`FixedWidthRuntime` helpers generated code calls, appending to `GrowableNativeBuffer`s (`int64`, +`double`, `int32` days or string offsets, UTF-8 bytes, a validity bitmap created on the first null). A line +rejected midway truncates the columns it already filled. A partitioned read merges by adopting the +following partition's builders; publishing concatenates each column once. The Arrow export moves those +buffers into `ArrowArray`s without a copy, except DECIMAL, widened to 128 bits. + +Writing formats a caller's `fw_table` through the layout. Every line has the same length, so the output +size and each batch's offset are known up front: batches format in parallel straight into the output +buffer, or into a pooled buffer written at its offset with `RandomAccess.Write`, and the bytes equal a +serial write. + +`python/` binds the C ABI with `ctypes`; `fixedwidth.hpp` wraps it for C++23, and maps structs through +`FixedWidthMapper`, as ExcelReader's C++ header does: the bindings build the layout, a read converts +the columnar table to `T` row by row, and a write builds columns from the structs. The NativeSmoke C program +pins every struct's layout with static asserts. + +## Public surface + +User-facing types live in the root `FixedWidthParser` namespace. `FixedWidthParser.Infrastructure` +holds what is public only because generated code in the consumer's assembly calls it, or because a +`FixedWidth` method returns it so `foreach` allocates nothing; it is marked +`[EditorBrowsable(Never)]`. Compiled consumer code binds to those signatures, so they evolve only by +addition: a new per-column setting is a new constructor overload and property on `ColumnParseOptions` +or `ColumnFormatOptions`, never a change to an existing helper. Internal types keep their folder +namespaces (`Readers`, `Writers`, `Parsers`, `Processors`, `Formatters`, `Attributes`). + +## Further reading + +- [`STYLEGUIDE.md`](STYLEGUIDE.md) — code style, and the rules for untrusted input, pooled buffers and twins. +- [`CONTRIBUTING.md`](CONTRIBUTING.md) — build expectations and how to submit a change. +- [`SECURITY.md`](SECURITY.md) — supported versions and how to report a vulnerability. diff --git a/CHANGELOG.md b/CHANGELOG.md index bdfb095..4175656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,125 +5,227 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [2.0.0] + +Not released yet. + +2.0 reads and writes **bytes only**, where 1.x had a `char` and a UTF-8 version of each reader, +parser and writer, and goes through **one entry point**: every `FixedWidth` method comes in an +overload for generated models and one that takes a `FixedWidthLayout`, built from the attributes or +fluently in code. The 1.x runtime classes are gone. Column positions are bytes; text columns are decoded with an ASCII-compatible encoding, UTF-8 by default. Settings travel +in options objects and a `FixedWidthContext`, so later versions can add to them without another break. +2.0 targets `net10.0` only. + +### Migrating from 1.x + +| 1.x | 2.0 | +|---|---| +| `FixedWidthReader` over a `TextReader`, `FixedWidthByteReader` | `FixedWidth.Read(stream, FixedWidthLayout.FromAttributes(), …)` or `FixedWidth.ReadFile(path, FixedWidthLayout.FromAttributes(), …)` | +| `FixedWidthUtf8.Read*`, `FixedWidth.Read*` over a `TextReader` | `FixedWidth.Read*` over a `Stream` or a file | +| `new FixedWidthReader(provider, pool, bufferSize).Read(…)` | `FixedWidth.Read(…, FixedWidthLayout.FromAttributes(), options: new FixedWidthReaderOptions { FormatProvider = …, StringPool = …, BufferSize = … })` | +| `FixedWidth.Read(stream, encoding, leaveOpen, provider, pool, bufferSize)` | `FixedWidth.Read(stream, leaveOpen, options)`; the encoding is `FixedWidthReaderOptions.Encoding` | +| `FixedWidthParser`, `Utf8FixedWidthParser` `.TryParse(line, provider, pool, out m)` | `FixedWidth.TryParse(ReadOnlySpan line, FixedWidthLayout.FromAttributes(), in FixedWidthContext context, out m)` | +| `FixedWidth.TryParse(chars, provider, pool, out m)`, `FixedWidthUtf8.TryParse(bytes, …)` | `FixedWidth.TryParse(bytes, in FixedWidthContext context, out m)` | +| `FixedWidth.TryFormat(in m, Span, provider, out n)` | `FixedWidth.TryFormat(in m, Span, in FixedWidthContext context, out n)` | +| `IFixedWidthModel` + `IUtf8FixedWidthModel` | `IFixedWidthModel`, whose generated members take bytes and a context | +| `IFixedWidthConverter` + `IUtf8FixedWidthConverter` with `IFormatProvider?` | `IFixedWidthConverter` over bytes, with `in FixedWidthContext` | +| `FixedWidthWriter.Write*(StreamWriter or Stream, …, provider)` | `FixedWidth.Write*(Stream, …, FixedWidthLayout.FromAttributes(), new FixedWidthWriterOptions { FormatProvider = …, Encoding = …, NewLine = … })` | +| `catch (FormatException)` on a bad line | still works; the exception is now `FixedWidthParseException` | +| `using FixedWidthParser.Attributes;`, `using FixedWidthParser.Processors;` | `using FixedWidthParser;` — `FixedColumnAttribute`, `Alignment`, `TrimMode`, `OverflowBehavior` and `IFixedWidthConverter` moved to the root namespace | +| `IFixedWidthConverter.TryParse(field, …)`, `TryFormat(…, out int written)` | the same, with the parameters named `column` and `bytesWritten` | + +A UTF-8 file with multi-byte text whose layout counts characters no longer lines up: positions are +bytes. Read files written in a single-byte encoding with that encoding (`Encoding.Latin1`, or +Windows-1252 after registering `CodePagesEncodingProvider`). ### Added -- `System.IO.Pipelines` integration on the UTF-8 byte path: `FixedWidthByteReader.ReadAsync(PipeReader)` - (reflection) and `FixedWidthUtf8.ReadAsync(PipeReader)` (source-generated) stream records straight - off a `PipeReader` via `await foreach`. Lines are sliced from the `ReadOnlySequence` and parsed in - place when contiguous, copied into a pooled scratch buffer only when they span segments. Intended for - sources that are already pipes (Kestrel request bodies, sockets, upstream pipeline stages); for plain - files/streams the existing `Stream` overloads remain the faster default (see `PipeReaderBenchmarks`). -- `FixedColumnAttribute.Converter`: a per-property custom converter (`IFixedWidthConverter` for the - `char` path, `IUtf8FixedWidthConverter` for the UTF-8 path — one type can implement both) that - takes priority over the built-in `ISpanParsable`/`IUtf8SpanParsable` fallback, for both parsing and - writing. Wired into reflection (parse + write, both element types) and the source generator (parse, - both element types), with build-time diagnostics `FWP008`/`FWP009` when the converter doesn't - implement the interface the column's type requires. -- Nullable value-type columns (`int?`, `decimal?`, `DateTime?`, …): a blank (trimmed-empty) column - parses to `null` without invoking the underlying parser/converter, and `null` writes as a blank - (padding-filled) column. Works across reflection and generated, char and UTF-8, and composes with - `FixedColumnAttribute.Converter` (the converter always targets the non-nullable `T`). -- **Source-generated writer**: `IFixedWidthModel` gained a static-abstract `TryFormat(in TSelf, - Span, IFormatProvider?, out int)`, implemented by the generator for every model declaring the - `char` marker — reflection-free, AOT-safe writing at parity with the existing generated `TryParse`. - Mirrors `FixedWidthWriter`'s semantics exactly: per-column alignment/padding/format/overflow - (resolved at compile time into a `ColumnFormatOptions` per column), nullable columns write blank when - `null`, and `FixedColumnAttribute.Converter` columns format through the same converter instance used - for parsing. Returns `false` only when the destination span is shorter than the line length; a column - that doesn't fit throws or truncates per its `Overflow`, same as reflection. New facade - `FixedWidth.TryFormat`. New diagnostic `FWP010` when a column's type is neither `string` nor - `ISpanFormattable` and has no converter. (UTF-8 byte writing — reflection or generated — remains out of - scope: no byte writer of either kind exists yet in this library.) -- `FixedColumnAttribute.TrimChar`: the character trimmed from the end of a column when **parsing** - (previously hardcoded `' '` everywhere — `Padding` was write-only). Threaded through reflection - (char/byte) and the source generator (char/byte), including the nullable "blank column is null" check - and the `double`/`float` fast path. On the UTF-8 byte path a non-ASCII `TrimChar` throws - `NotSupportedException` (reflection: at parser construction; generated: from the column's static - `__trim{i}` field, surfacing as `TypeInitializationException` on first use) rather than silently - trimming the wrong byte — mirrors the existing decimal-separator ASCII guard. Unconfigured columns - (the overwhelming majority) still emit/execute the exact space-trim call they did before; the new - argument is only added when `TrimChar` differs from the default. +- `FixedWidthLayout`: how a model maps to the columns of a line, for reading and writing, passed to + the `FixedWidth` methods. `FixedWidthLayout.FromAttributes()` builds it from `[FixedColumn]` + attributes; `FixedWidthLayout.Build(b => …)` describes it in code, binding each column with a + `ColumnGetter`/`ColumnSetter` delegate and `FixedWidthColumnOptions`, with no reflection, so it works + under trimming and NativeAOT. A layout can read only or write only. +- `FixedWidth.Write`, `WriteMany`, `WriteAsync` and `WriteManyAsync` for generated models, which had no + stream writer. +- Parallel aggregation: `FixedWidth.AggregateFileAsync`, for generated models and layouts alike, + reads a file in byte-range partitions on several threads and fold each into an + `IRecordAccumulator`, merged in file order. Every line is parsed exactly once, and + errors carry the same line number as a sequential read. `FixedWidthReaderOptions.DegreeOfParallelism` + sets the thread count. `AggregateAsync` does the same over a `Stream`: one thread + reads chunks, the others parse them. +- `FixedWidthReaderOptions`: `FormatProvider`, `Encoding`, `StringPool`, `BufferSize`, + `MaxLineLength` and `OnParseError`, taken by every read. `FixedWidthWriterOptions`: `FormatProvider`, + `Encoding` and `NewLine`. +- `FixedWidthContext`, passed by `in` reference to generated code and converters: the format provider, + the encoding and the string pool for one line. New settings arrive as new properties. +- `FixedWidthParseException`, a `FormatException` carrying `LineNumber`, `ColumnName`, `ColumnStart`, + `ColumnLength` and `RawValue`. A short line says how long it is and how long the layout needs it to + be. A line that parses pays nothing for this: a failed line is parsed again to find the column. +- `FixedWidthReaderOptions.OnParseError`: return `true` to skip a line that fails to parse. +- `FixedWidthParser.Native`: the library published with NativeAOT as a shared library with a C ABI + (`fixedwidth.h`), for languages without a .NET runtime. Layouts are described at run time + (`fw_layout_create`), including implied decimals (`scale`) for vendor formats such as B3's COTAHIST. + Reads fill a columnar `fw_table` or Arrow C Data Interface arrays and streams, in parallel by byte range, + with `header_lines` and `skip_invalid`; writes take a columnar table and are byte-identical whatever the + thread count. Not on NuGet. +- The `fixedwidth` Python package over it (NumPy views, pyarrow, pandas, polars): `read_pandas` is 96 times + faster than `pandas.read_fwf` on a 1,000,000-line COTAHIST-shaped file. And a header-only C++23 wrapper + (`fixedwidth.hpp`) that also maps structs to columns through `FixedWidthMapper`. It installs as a CMake + package (`find_package(fixedwidth)`) or through FetchContent, and downloads the release's native library for + the platform, so it needs no .NET SDK. +- Writing to a sink: `fw_write_stream` pushes the lines, in order, to an `fw_sink`'s `write` callback while + batches format in parallel, so memory stays at a batch per thread. Python's `fw.write` takes any binary stream + (an open file, `gzip.open(..., "wb")`), as fast as writing to a path; C++ has `write_stream(std::ostream&, ...)`. +- Dictionary-encoded STRING columns for Arrow (`fw_column_spec.dictionary`, Python `fw.Column(dictionary=True)`, + C++ `ColumnSpec::dictionary`): int32 indices into the column's distinct values, which pandas reads as + `category` and polars as `Categorical`. `fw_table` and `fw.read` keep strings; categorical columns write like + strings. +- Batched and Arrow-stream reads from a forward-only source, as ExcelReader's `xl_stream`: an `fw_stream` of + `read` and `release` callbacks (`fw_reader_open_stream`, `fw_read_arrow_stream_stream`, and + `fw_set_source_error` for a callback's failure). Header lines are skipped without seeking, and the batches + stay in constant memory. Python's readers take any object with `readinto()` or `read()` (an open file, + `gzip.open`, `zipfile.ZipFile.open`), and an `io.BytesIO` is read in place; the C++ wrapper takes a + `std::istream&` (`Reader::open_stream`, `TypedReader::open_stream`, `read_arrow_stream`). +- Releases publish the Python package to PyPI as `fixedwidthparser`, with wheels for Windows x64, + macOS arm64 and x64, and Linux glibc and musl on x64 and arm64, and attach each platform's native library + to the GitHub Release. +- `ref struct` models on `Read`, `ReadFile`, `ReadAsync`, `ReadFileAsync`, `AggregateFileAsync` and + `AggregateAsync`, generated and through a layout. The enumerators parse a line when `Current` is + read, so reading it twice parses twice; with `OnParseError` set, each line is parsed once more to + decide whether to skip it. +- `FixedWidthReaderOptions.MaxLineLength` (default 1 MiB): a longer line throws + `InvalidDataException`. Before, a source without line breaks doubled the read buffer until memory ran + out, which a request body made reachable from the network. +- Encodings other than UTF-8 for text columns, dates parsed with a format and types that parse only + from characters. A UTF-8 byte order mark is skipped only when the encoding is UTF-8. +- Columns of any `ISpanParsable` type without a UTF-8 parser, decoded first (they used to be + refused on the byte path). +- `FixedWidth.TryFormat(in model, layout, Span, in FixedWidthContext, out int)`. +- `FixedColumnAttribute.Converter`: a per-property `IFixedWidthConverter` that takes priority over the + built-in parsing and formatting, with diagnostic `FWP008` when it targets the wrong type. +- Enum columns, as ExcelReader reads them: a member's name in any case or its number, and nothing the enum + does not define. They write the name, or the number with `Format = "D"`. Attribute layouts and generated + models take an enum property as it is; fluent layouts add `EnumColumn` and `NullableEnumColumn`. +- Nullable value-type columns (`int?`, `decimal?`, `DateTime?`, …): a blank column parses to `null` + without invoking the parser or converter, and `null` writes as a blank column. +- The source-generated writer: `IFixedWidthModel.TryFormat`, with the same alignment, padding, + format, overflow, nullable and converter handling as the model's attribute layout, and diagnostic `FWP010` + when a column's type cannot be written. +- `FixedColumnAttribute.TrimChar` and `TrimMode`: what to strip from a column when parsing, and from + which side. +- `FixedColumnAttribute.Format` parses `DateTime`, `DateOnly`, `TimeOnly` and `DateTimeOffset` with + that exact format. +- Diagnostic `FWP012`: `Padding` or `TrimChar` is not ASCII. Layouts throw + `InvalidOperationException` for the same models. +- Public API tracking (`PublicAPI.*.txt`) and package validation against the last release, run by CI on + every pull request. ### Changed -- File-backed readers (`FixedWidthReader`, `FixedWidthByteReader`, and their UTF-8/generated - counterparts) now open the underlying `FileStream` with `FileOptions.SequentialScan` and an internal - buffer of 1 byte instead of the configured `bufferSize`: the enumerator core already does its own - block buffering into a pooled buffer, so a matching `FileStream` buffer was pure double-buffering - (an extra memcpy of most of the file) and dropped the OS's sequential-read prefetch hint on Windows. - Async file paths keep `FileOptions.Asynchronous` alongside the new flag. The default `bufferSize` for - `FixedWidthReader` and `FixedWidthByteReader` also increases from 4096 to 65536 bytes, - which is a more appropriate default block size for a throughput-oriented parser. -- **Breaking:** the internal column formatters `StringColumnFormatter` and - `SpanFormattableColumnFormatter` are now `internal` (they were unintentionally - `public`). They are implementation details resolved by the writer; the public extension point is the - `IColumnFormatter` interface. The reader/enumerator strategy types remain `public` because - they appear in `GetAsyncEnumerator()` return types (a deliberate allocation-free design choice). +- **Breaking:** bytes only; see the table above. The column positions of a model are bytes. +- **Breaking:** `FixedColumnAttribute`, `Alignment`, `TrimMode`, `OverflowBehavior` and + `IFixedWidthConverter` moved to the `FixedWidthParser` namespace. The types public only for + generated code or allocation-free enumeration (`FixedWidthRuntime`, `ColumnFormatOptions`, the new + `ColumnParseOptions`, the record enumerables and enumerators, `IRecordLineParser` and the + line-parser structs) moved to `FixedWidthParser.Infrastructure` and are hidden from IntelliSense. +- **Breaking:** without `OnParseError`, a line that fails to parse throws when the enumerator's + `Current` is read, not from `MoveNext`. A `foreach` sees no difference. The non-generic + `IEnumerator.Current` throws `NotSupportedException`. +- **Breaking:** targets `net10.0` only. .NET 8 reaches end of support on 10 November 2026. +- **Breaking:** a `null` format provider means the invariant culture for every column type, on parse and + write. 1.x used it for `double`/`float` only and fell back to `CurrentCulture` for the rest. +- A culture whose decimal separator is not one byte (U+066B, for example) now parses, where 1.1 threw + `NotSupportedException` on the byte path. +- File-backed readers open the `FileStream` with `FileOptions.SequentialScan` and no internal buffer of + their own, since the reader already buffers. The default buffer is 65,536 bytes. +- The writer batches records into one pooled buffer and writes bytes straight to the stream, instead of + going through a `StreamWriter`. +- The writer formats `IUtf8SpanFormattable` columns (every BCL number and date) straight into the line, + and transcodes left-aligned string columns straight into it when the encoding is UTF-8. A formatted + value that is not ASCII under another encoding still goes through that encoding. +- A `double` column with no `Format`, at most four decimals and a magnitude under 1e9 is written as a + scaled integer, about 5x faster, with the same bytes. A line of a string, an `int` and a `double` + writes about 2x faster. +- The generator targets `netstandard2.0`, so Visual Studio's .NET Framework compiler host can load it. +- `double`, `float` and `decimal` columns parse through a built-in fast path (ExcelReader's `FastDouble` + and `FastDecimal`) under any culture whose decimal separator is one ASCII character and whose + negative sign is `-`, not only `'.'`: a pt-BR read is about 40% faster + ([docs/reader-performance.md](docs/reader-performance.md)). Anything the fast path refuses goes to + the BCL parser, which decides the result. +- **Breaking:** the csFastFloat dependency is gone. +- Dates and times with a format made only of fixed-width digit fields (`yyyyMMdd`, `yyyy-MM-dd`, `HHmmss`, …) + and `int`/`long` columns of plain digits parse straight from the bytes, about 5x and 1.5x faster; any + other text still goes to the BCL. A single-threaded B3 COTAHIST read through Python is 26% faster + ([docs/python-benchmarks.md](docs/python-benchmarks.md)). +- Native and Python whole-file reads size their column buffers once per partition, skip bad lines under + `skip_invalid` without numbering them, and concatenate the partitions' columns in parallel: a parallel + COTAHIST read is about 32% faster. +- Native and Python whole-file reads parse every partition straight into the final columns, at the row its + byte offset gives when all lines have the first line's length, instead of copying the partitions together + afterwards. Numeric, date and time values and STRING bytes and offsets skip the copy, and a parallel COTAHIST read is + another 17% faster. A file whose lines vary in length still reads correctly; its columns are copied as before. +- Writing: right-aligned `int`/`long` columns without a format put their digits straight into the line, + digit-only date and time formats reuse the reader's fast path, and `fw_write_file` opens its file for + asynchronous I/O so parallel batches write at once. A COTAHIST write through Python is about 37% faster. +- Reading with a `StringPool` checks a small per-thread cache before the pool, about halving the cost of + a pooled read. +- The library, generator and native library build with warnings as errors. ### Removed -- **Breaking:** `ColumnParserRegistry` and `Utf8ColumnParserRegistry` (and the `ColumnValueParser`/ - `Utf8ColumnValueParser` delegates they used) are gone. They were process-wide mutable state with a - documented footgun (registration only took effect if it happened before a model's first parse) and no - write-side counterpart; `FixedColumnAttribute.Converter` replaces them with a per-column, type-checked - alternative that also covers writing. The built-in `double`/`float` fast path (csFastFloat) no longer - goes through a registry lookup — it's now a direct type check in the parser factories, with no - behavior change for consumers. +- **Breaking:** `FixedWidthParser`, `FixedWidthReader` and `FixedWidthWriter`. Pass + `FixedWidthLayout.FromAttributes()` to the matching `FixedWidth` method instead (see the table above); the layout is + built once per type and shared. +- **Breaking:** reading from a `PipeReader` (`ReadAsync(PipeReader)` and its enumerator types). Pass + `pipeReader.AsStream()` to the `Stream` overloads instead; it reads twice as fast as the dedicated + path did ([docs/pipe-reader.md](docs/pipe-reader.md)). +- **Breaking:** `FixedWidthUtf8`, `FixedWidthByteReader`, `Utf8FixedWidthParser`, + `IUtf8FixedWidthModel`, `IUtf8FixedWidthConverter`, `Utf8FixedWidthRuntime` and the `TextReader` + overloads. Diagnostics `FWP007` and `FWP009`, which only applied to the second path, are retired. +- **Breaking:** `ColumnParserRegistry` and `Utf8ColumnParserRegistry` (and their value-parser delegates): + process-wide mutable state that only took effect before a model's first parse. + `FixedColumnAttribute.Converter` replaces them. +- **Breaking:** types that were public without a public use: `ColumnParser`, `Utf8ColumnParser`, + `RefAction`, `RefGetter`, `IColumnFormatter`, `ColumnLayoutValidator`, `ISource`, `TextReaderSource`, `StreamSource`, + `ILineFormat` and the line-format structs. The enumerator and strategy types that stay public + (they appear in return types, so `foreach` allocates nothing) lost their element-type and source + type parameters. + +### Fixed +- Right-aligned `double`/`float` columns could fail to parse under the invariant fast path: csFastFloat + did not consistently count skipped leading whitespace. +- `double`/`float` columns with a thousands separator under a `'.'` culture (`"1,234.56"`) failed + instead of parsing. +- `double`/`float` columns under a `','` culture could silently return a truncated value: `"1.234,50"` + became `1.0`. The fast path is now trusted only when it consumes the whole field. +- A generated model skipped `[FixedColumn]` members declared on a base class: it left them at their + default when reading and wrote filler over their columns. Inherited columns are now generated as + reflection maps them, with an override or a `new` member replacing the one it hides. +- The decimal-separator cache thrashed when two format providers alternated. + +## [1.1.1] + +### Added +- Targets `net8.0` alongside `net10.0`. `ref struct` models need .NET 9 or later, so they are + `net10.0`-only. + +## [1.1.0] + +### Added +- `System.IO.Pipelines` integration on the UTF-8 byte path: `FixedWidthByteReader.ReadAsync(PipeReader)` + and `FixedWidthUtf8.ReadAsync(PipeReader)` stream records off a `PipeReader` via `await foreach`. + +### Changed +- **Breaking:** the column formatters `StringColumnFormatter` and + `SpanFormattableColumnFormatter` became `internal`; they were unintentionally + `public`. ### Fixed -- **Right-aligned (leading-space-padded) `double`/`float` columns could fail to parse under the - invariant `'.'` fast path** — the single most common numeric layout in fixed-width files. csFastFloat's - `characters_consumed` did not consistently include skipped leading whitespace across its `double`/`float` - overloads (verified empirically: `FastDoubleParser` counted it, `FastFloatParser` didn't, and vice versa - on the UTF-8 overloads), so the full-consumption check used to reject silent truncation could itself - reject a perfectly valid right-aligned value depending on the element type. Leading whitespace is now - trimmed before the fast path runs, on all four `char`/`byte` × `double`/`float` combinations. -- **`double`/`float` columns using a thousands separator under the invariant `'.'` decimal separator - failed to parse instead of accepting the value** (e.g. `"1,234.56"`) — csFastFloat's single - `decimal_separator` override can't express thousands separators, so the fast path's full-consumption - check failed even though the value is valid. A failed fast-path attempt now retries via the real - `NumberFormatInfo`-aware parser (which still rejects genuine garbage) instead of returning `false` - outright. -- **Breaking:** a `null` `IFormatProvider` now means `CultureInfo.InvariantCulture` for **every** column - type, on both parse and write. Previously `double`/`float` columns treated `null` as invariant while - every other `ISpanParsable`/`ISpanFormattable` column (`decimal`, `int`, `DateTime`, …) fell through to - the BCL's own default of `CurrentCulture` — so on a non-invariant machine, the same line could parse a - `double` column and a `decimal` column to different results under one `null`-provider call, and writing - with `null` (`CurrentCulture`) could produce output the same `null`-provider parse then rejected. Pass - an explicit `IFormatProvider` (e.g. `CultureInfo.CurrentCulture`) if culture-dependent parsing/formatting - is what you want; `null` is now deterministic regardless of the running thread's ambient culture. -- **`double`/`float` columns under a non-'.' decimal separator could silently return a truncated, - wrong value instead of failing.** csFastFloat's `decimal_separator` override does not fail on - trailing content it doesn't recognize — it just stops at the first unrecognized character and - reports success with whatever it parsed up to that point. Under `de-DE`/`pt-BR`-style cultures - (`.` groups thousands, `,` is the decimal separator), a field like `"1.234,50"` silently became - `1.0` instead of `1234.50` (or failing outright). The fast path is now only trusted for the - invariant `'.'` separator, and even then only after confirming the whole (trimmed) field was - consumed; any other separator falls back to real `NumberFormatInfo`-aware parsing - (`double.TryParse`/`float.TryParse` with the actual `IFormatProvider`, which validates the whole - input and understands thousands separators correctly). Applies to reflection and generated, char - and UTF-8 (the byte path transcodes the — always short — numeric field to a small char buffer - when it needs the non-fast-path parse). -- The decimal-separator cache was a single-entry memo that thrashed (recomputed on every call) when - two `IFormatProvider`s were used alternately in the same process (e.g. parsing files in different - locales). Replaced with a `ConditionalWeakTable` keyed by provider identity — no thrash, and no - unbounded growth if callers pass many distinct providers (entries are reclaimed with their provider). -- UTF-8 byte parser: a culture whose decimal separator is not a single ASCII character (e.g. the - Arabic decimal separator U+066B) previously had its separator silently truncated to the wrong - byte, mis-parsing `double`/`float` columns. Such cultures now throw a clear `NotSupportedException` - on the byte path; use the `char`-based parser for them. ASCII separators (`.`/`,`) are unaffected. -- Writer: a value whose `ISpanFormattable.TryFormat` never succeeded could overflow the formatter's - buffer-growth counter into a negative/huge `ArrayPool` rent. The grow loop is now bounded (1M chars) - and throws a clear `InvalidOperationException` naming the column instead. +- UTF-8 byte parser: a culture whose decimal separator is not a single ASCII character had its separator + silently truncated to the wrong byte; it now throws `NotSupportedException`. +- Writer: a value whose `ISpanFormattable.TryFormat` never succeeded could overflow the buffer-growth + counter into a negative rent; the loop is bounded and throws a clear error naming the column. ### Documentation -- Corrected the UTF-8 byte path docs: `string` columns **are** interned through a supplied - `StringPool` (identical to the `char` path); the previous "no pooling" note was inaccurate. -- Documented that empty lines are skipped (counted but not yielded) while a non-empty line shorter - than the declared layout is treated as malformed and throws. -- Reconciled the README's stated requirements with the actual multi-target: the package builds for - `net8.0` and `net10.0`, not ".NET 10" only. Added an explicit note (in Requirements and in the - `ref struct` Models section) that `ref struct` model support needs .NET 9+, since the `allows ref - struct` generic constraint it depends on doesn't exist on `net8.0`. +- `string` columns on the UTF-8 path are interned through a supplied `StringPool`, like the char path. +- Empty lines are skipped (counted but not yielded); a non-empty line shorter than the layout throws. ## [1.0.0] @@ -145,8 +247,9 @@ Initial release. - `ref struct` model support on the parser. - A bundled Roslyn source generator: models implementing `IFixedWidthModel` get a reflection-free `TryParse` generated at compile time, with diagnostics (FWP001-FWP007) for invalid - layouts and unsupported column types. Shipped inside the package as an analyzer — no extra package - required. + layouts and unsupported column types. -[Unreleased]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.0.0...HEAD +[2.0.0]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.1.1...HEAD +[1.1.1]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.1.0...v1.1.1 +[1.1.0]: https://github.com/GabrielMarquezMatte/FixedWidthParser/compare/v1.0.0...v1.1.0 [1.0.0]: https://github.com/GabrielMarquezMatte/FixedWidthParser/releases/tag/v1.0.0 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..aa928b4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,48 @@ +# Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our community a +harassment-free experience for everyone, regardless of age, body size, visible or invisible +disability, ethnicity, sex characteristics, gender identity and expression, level of experience, +education, socio-economic status, nationality, personal appearance, race, religion, or sexual +identity and orientation. + +## Our Standards + +Examples of behavior that contributes to a positive environment: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing for mistakes, and learning from the experience + +Examples of unacceptable behavior: + +- The use of sexualized language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information without explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement Responsibilities + +Maintainers are responsible for clarifying and enforcing these standards, and will take appropriate +and fair corrective action in response to any behavior deemed inappropriate, threatening, offensive, +or harmful. + +## Scope + +This Code of Conduct applies within all project spaces (issues, pull requests, discussions) and when +an individual is officially representing the project in public spaces. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the +maintainers via GitHub's private reporting channel on this repository. All complaints will be +reviewed and investigated promptly and fairly. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), +version 2.1. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index eca683f..97c2380 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,64 +1,119 @@ -# Contributing to FixedWidthParser - -Thanks for taking the time to contribute! This document covers how to build, test, -and submit changes. - -## Prerequisites - -- [.NET 10 SDK](https://dotnet.microsoft.com/download) (`net10.0`) -- Git - -## Building - -```bash -dotnet restore FixedWidthParser.slnx -dotnet build FixedWidthParser.slnx --configuration Release -``` - -## Running tests - -```bash -dotnet test FixedWidthParser.slnx --configuration Release -``` - -To collect coverage locally (same as CI): - -```bash -dotnet test FixedWidthParser.slnx --configuration Release --collect:"XPlat Code Coverage" -``` - -The Cobertura report is written under each test project's `TestResults/` folder. - -## Running benchmarks - -```bash -dotnet run --project tests/Benchmarks/Benchmarks.csproj --configuration Release -- --filter '*' -``` - -Filter to a subset with fully-qualified patterns, e.g. -`--filter 'Benchmarks.Perf.ParserBenchmarks.*'`. Results for `master` are published to -[GitHub Pages](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/); every PR -gets a benchmark comparison posted as a comment automatically. - -## Coding standards - -- The repo enforces a large analyzer suite (SonarAnalyzer, Roslynator, Meziantou, - NetAnalyzers, and more) configured in `Directory.Build.props`, plus formatting rules in - `.editorconfig`. Please make sure the build is warning-clean before opening a PR. -- Match the surrounding style: file-scoped namespaces, `_camelCase` private fields, - span-based hot paths, and XML docs on public members. -- This is a **performance-sensitive** library. Avoid introducing allocations on parse/write - hot paths; if a change could affect performance, include benchmark numbers in the PR. - -## Pull request flow - -1. Branch off `develop` (or `master` for hotfixes). -2. Make your change with tests covering it. New behavior without tests will be asked to add them. -3. Ensure `dotnet build` and `dotnet test` pass locally on Release. -4. Open the PR against `master`/`develop`; fill in the PR template. -5. CI (build/test on Linux, Windows, macOS), CodeQL, and the benchmark comparison must be green. +# Contributing + +Thanks for considering a contribution. This project takes small, focused pull requests over large +rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebase before diving in, and +[STYLEGUIDE.md](STYLEGUIDE.md) for the code style, which the analyzers only partly enforce. + +## Build expectations + +- **Warnings are errors in `src/`.** The library, generator and native projects set `TreatWarningsAsErrors`, under a + curated `AnalysisMode=All` analyzer set (NetAnalyzers, Meziantou, SonarAnalyzer, Roslynator and + others, in `Directory.Build.props`). A PR that doesn't build clean locally won't build clean in CI + either — run a full build before pushing: + + ```bash + dotnet build FixedWidthParser.slnx --configuration Release + ``` + +- **Public API changes require a `PublicAPI.Unshipped.txt` entry.** `Microsoft.CodeAnalysis.PublicApiAnalyzers` + is active and fails the build on any unrecorded public member. If you add, change, or remove anything + public, update `src/FixedWidthParser/PublicAPI.Unshipped.txt`; the IDE's code fix writes the line for + you. At a release, the unshipped entries move to `PublicAPI.Shipped.txt`. + +- **Minor versions don't break callers.** The policy is in + [docs/guide/versioning.md](docs/guide/versioning.md); in practice: + - `IFixedWidthModel` and `IFixedWidthConverter` are implemented by users, so a minor + version never adds a member to them. New information reaches them as a new property on + `FixedWidthContext`. + - New inputs go on the options objects or a new overload, never as a new optional parameter on an + existing method. + - `dotnet pack` runs package validation against `PackageValidationBaselineVersion` in + `src/FixedWidthParser/FixedWidthParser.csproj`, and CI runs it on every pull request. It is off + while a major version is in development, since that version breaks the last one on purpose; list + those breaks in the changelog. After each release, turn validation back on with the baseline set + to the version just published. + +- **Tests are required for behavior changes.** Run the suite before opening a PR: + + ```bash + dotnet test --solution FixedWidthParser.slnx --configuration Release + ``` + + CI also collects coverage (`-- --coverage --coverage-output-format cobertura`) and uploads it to + Codecov from the Linux leg. + + A change to one path needs the same change, and a parity test, on its twin: the runtime and + generated paths, the sync and async readers and writers, the sequential and parallel reads, and the + aggregator's model and sink loops ([STYLEGUIDE.md § Twins](STYLEGUIDE.md#twins)). + +- **The native library, its bindings and NativeAOT have their own checks.** A change under + `src/FixedWidthParser.Native`, `python/` or `cpp/` needs the Python, C and C++ tests; a change to the + fluent layout or anything it reaches needs the AOT smoke app to publish without an `IL2xxx`/`IL3xxx` + warning and to run: + + ```bash + python python/scripts/build_native.py # publishes the library into python/src/fixedwidth/_lib + pip install -e "python[dev,pandas,polars]" && pytest python/tests + cmake -S cpp -B build/cpp && cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure + dotnet publish tests/AotSmoke/AotSmoke.csproj -c Release -r win-x64 -p:IlcTreatWarningsAsErrors=true + ``` + + On Windows, run the CMake steps from a Visual Studio developer prompt, which puts `lib.exe` on the path. Line-splitting code reads untrusted input; read + [STYLEGUIDE.md § Untrusted Input](STYLEGUIDE.md#untrusted-input) before touching it. + +- **Performance claims come with numbers.** This is a performance-sensitive library: avoid + allocations on the parse and write hot paths, and if a change could affect speed, run the benchmarks + and put the result in the PR. Findings that justify a design go in [`docs/`](docs/README.md), not in + code comments. + + ```bash + dotnet run --project tests/Benchmarks/Benchmarks.csproj --configuration Release -- --filter '*' + ``` + + Filter with fully-qualified patterns, e.g. `--filter 'Benchmarks.Perf.ParserBenchmarks.*'`. Results + for `master` are published to [GitHub Pages](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/), + and every PR gets a benchmark comparison posted as a comment. + +## Commit messages + +Use [Conventional Commits](https://www.conventionalcommits.org/): `type(scope): summary`, imperative +mood, lowercase summary, no trailing period. Common types in this repo: `feat`, `fix`, `refactor`, +`perf`, `test`, `docs`, `chore`, `ci`, `build`. The scope is usually the area touched (`reader`, +`writer`, `parser`, `generator`). Example: `fix(reader): bound an unterminated line by MaxLineLength`. + +## Pull requests + +- Branch off `develop` (or `master` for a hotfix), and open the PR against the same branch. +- One focused change per PR — don't batch unrelated fixes into one commit or one PR. +- If a change is user-visible (new API, behavior change, performance claim), add it to + [CHANGELOG.md](CHANGELOG.md) and mention it in the PR description. +- CI builds and tests on Linux, Windows, and macOS: the .NET suite, and the native library with its + Python, C and C++ bindings on every platform a release ships. It also checks whitespace formatting, + validates the package against the last release, publishes and runs the NativeAOT smoke app, and runs + CodeQL. Each job runs only when the paths it covers change, and draft PRs run nothing. A change that only + builds on one OS isn't ready to merge. +- Benchmarks are opt-in: label the PR `run-benchmarks` to get a comment comparing it with the latest master + run. A new benchmark class must be added to a group in `tests/Benchmarks/benchmark-groups.json`. + +## Releasing + +Push a `v*` tag (`v2.0.0`, or `v2.0.0-rc.1` for a prerelease). [release.yml](.github/workflows/release.yml) +takes the version from the tag, tests and packs the NuGet package, builds the native library on Windows x64, +macOS arm64 and x64, Linux glibc and musl x64 and arm64, tests one Python wheel per platform, then publishes +the package to NuGet, the wheels to PyPI as `fixedwidthparser`, and the native libraries and an SBOM +to the GitHub Release. Both registries publish through Trusted Publishing, so no API keys are stored: + +- nuget.org: a trusted publishing policy for this repository and `release.yml`, and the repository variable + `NUGET_USER` set to the nuget.org user. +- PyPI: a trusted publisher for this repository and `release.yml`. Before the first release, add it as a + pending publisher for the project name `fixedwidthparser`. ## Reporting bugs / requesting features Use the [issue templates](https://github.com/GabrielMarquezMatte/FixedWidthParser/issues/new/choose). -For security issues, **do not** open a public issue — see [SECURITY.md](SECURITY.md). +For suspected security vulnerabilities, do **not** open a public issue — see [SECURITY.md](SECURITY.md) +for the private reporting channel. + +## Code of conduct + +This project follows the [Code of Conduct](CODE_OF_CONDUCT.md). diff --git a/Directory.Build.props b/Directory.Build.props index 8397e4a..677892e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -2,14 +2,14 @@ latest All - - + + 9999 $(NoWarn);CA1062;CA1008;1591;RS0016;RS0037;RS0026;S1244;SS002;SS008;SS019;MA0016;CA1819;S6964;NU1902;NU1901;IDE0008;IDE0046;IDE0058;CA1515;CA1002 - $(NoWarn);CA1848;CA3003;AD0001;S6960;MA0051;CA1873;MA0049;CA1724;SS017;MA0048 + $(NoWarn);CA1848;CA3003;AD0001;S6960;CA1873;MA0049;CA1724;SS017;MA0048 true - HLQ005;HLQ012 + HLQ005;HLQ012;IDE1006 true @@ -19,19 +19,19 @@ $(DefineConstants.Replace("TRACE;", "")) - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all @@ -39,15 +39,15 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all @@ -63,11 +63,11 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/FixedWidthParser.slnx b/FixedWidthParser.slnx index 3d9b6e9..b510740 100644 --- a/FixedWidthParser.slnx +++ b/FixedWidthParser.slnx @@ -2,8 +2,10 @@ + + diff --git a/README.md b/README.md index e1b339a..3fe3eaf 100644 --- a/README.md +++ b/README.md @@ -8,35 +8,35 @@ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Benchmarks](https://img.shields.io/badge/benchmarks-GitHub%20Pages-informational)](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/) -A high-performance, low-allocation library for **parsing and writing fixed-width (flat) files** in .NET. Columns are declared with attributes, layouts are validated up front, and the hot paths work over spans so fixed-width records can be parsed, streamed and written without the usual per-line churn. +A high-performance, low-allocation library for **parsing and writing fixed-width (flat) files** in .NET. Columns are declared with attributes or described in code, layouts are validated up front, and lines are parsed straight from the file's bytes, without a string per line. -The package includes both: +Everything goes through one static class, `FixedWidth`, whose every method comes in two overloads: -- a runtime parser/writer API for regular attribute-mapped models; and -- a bundled Roslyn source generator for reflection-free `TryParse` implementations, including a UTF-8 byte path. +- one for models the bundled Roslyn source generator implements (`IFixedWidthModel`): reflection-free and statically dispatched; and +- one that takes a `FixedWidthLayout`, built from `[FixedColumn]` attributes or fluently in code. + +Upgrading from 1.x? See [the 2.0 entry in the changelog](CHANGELOG.md#200). ## Features -- **Attribute-driven** column mapping (`[FixedColumn(start, length)]`) on properties and public fields. -- **Runtime parsing** for single lines and lazy batch reading from a `TextReader`, `Stream` or file. -- **Source-generated parsing** for `partial` models implementing `IFixedWidthModel`. -- **UTF-8 byte parsing** via `Utf8FixedWidthParser`, `FixedWidthByteReader` and generated `IUtf8FixedWidthModel` models, avoiding `StreamReader` and UTF-16 transcoding for ASCII-style flat files. -- **Synchronous and asynchronous readers** (`IEnumerable` / `IAsyncEnumerable`) with struct enumerators on the synchronous path. -- **`System.IO.Pipelines` support** on the byte path: read records straight from a `PipeReader` (`FixedWidthByteReader.ReadAsync(PipeReader)` / `FixedWidthUtf8.ReadAsync(PipeReader)`). -- **Writing** for single records and batches, synchronous and asynchronous, with `StreamWriter` reuse and `ReadOnlySpan` overloads for zero-allocation output. +- **Attribute-driven** column mapping (`[FixedColumn(start, length)]`) on properties and public fields, or a **fluent layout** bound with delegates, for types you cannot annotate. Fluent layouts are trimming- and Native AOT-safe. +- **Byte positions.** Lines are read and written as bytes, with no transcoding to UTF-16. Text columns are decoded with the file's encoding: UTF-8 by default, or any ASCII-compatible encoding such as Latin-1 or Windows-1252. +- **Lazy reading** from a `Stream` or a file, synchronous (`IEnumerable` with a struct enumerator) and asynchronous (`IAsyncEnumerable`). +- **Source-generated parsing and writing** for `partial` models implementing `IFixedWidthModel`. +- **Errors that name the column**: a `FixedWidthParseException` with the line number, column, position and raw text, and an `OnParseError` callback to skip bad lines. +- **Bounded memory on untrusted input**: `MaxLineLength` stops a source without line breaks from growing the buffer. +- **Writing** of single records and batches, synchronous and asynchronous, batched into one pooled buffer. - **Configurable formatting** per column: alignment, padding character, format string and explicit overflow policy. -- **Culture-aware** numeric parsing/formatting, including `double`/`float` via csFastFloat and generic `ISpanParsable` / `ISpanFormattable` support. -- **Layout validation** at construction or generation time: negative `Start`, non-positive `Length` and overlapping columns fail clearly. -- **`ref struct` model support** on parser/source-generated single-line parsing (.NET 9+ only, see Requirements). +- **Culture-aware** numbers and dates, with a fast path for `double`, `float` and `decimal`, enums by name or number, and any `IUtf8SpanParsable`/`ISpanParsable` type. +- **Layout validation** when a layout is built or generated: negative `Start`, non-positive `Length`, overlapping columns and non-ASCII padding fail clearly. +- **`ref struct` model support** on single-line parsing and formatting, on every reader and on parallel aggregation. +- **Python and C/C++** through a NativeAOT library with a C ABI: columnar and Arrow reads, parallel reads and writes. See [Python and C/C++](#python-and-cc). ## Requirements -- `net8.0` or `net10.0` (the package multi-targets both). -- `ref struct` models (e.g. `RefPersonModel`-style spans-only types) require .NET 9 or later — the - `allows ref struct` generic constraint that makes them possible doesn't exist on `net8.0`, so that - one feature is compiled out there; everything else behaves identically on both targets. +- `net10.0`. -Dependencies: [CommunityToolkit.HighPerformance](https://www.nuget.org/packages/CommunityToolkit.HighPerformance) (`StringPool`) and [csFastFloat](https://www.nuget.org/packages/csFastFloat) (fast `double`/`float` parsing). +Dependency: [CommunityToolkit.HighPerformance](https://www.nuget.org/packages/CommunityToolkit.HighPerformance) (`StringPool`). ## Installation @@ -44,370 +44,100 @@ Dependencies: [CommunityToolkit.HighPerformance](https://www.nuget.org/packages/ dotnet add package FixedWidthParser.NET ``` -Or as a ``: - -```xml - -``` +The package ships the Roslyn source generator bundled as an analyzer. Models that implement `IFixedWidthModel` get a generated parser and writer automatically; no extra package or setup is required. -The package ships the Roslyn source generator bundled as an analyzer. Models that implement `IFixedWidthModel` or `IUtf8FixedWidthModel` get generated parsers automatically; no extra package or setup is required. +## Quick Start -## Defining a Model - -Runtime/reflection models only need a public parameterless constructor and mapped fields or properties: +Describe a line with `[FixedColumn(start, length)]`, both in bytes, then read and write through the +model's layout: ```csharp -using FixedWidthParser.Attributes; +using FixedWidthParser; public readonly record struct Person { public Person() { Name = string.Empty; - Age = 0; - Salary = 0.0; } [FixedColumn(0, 10)] public string Name { get; init; } [FixedColumn(10, 5)] public int Age { get; init; } [FixedColumn(15, 10)] public double Salary { get; init; } } -``` - -`start` is the 0-based offset and `length` is the column width. - -For source generation, make the model `partial` and implement one or both marker interfaces: - -```csharp -using FixedWidthParser; -using FixedWidthParser.Attributes; - -public readonly partial record struct GeneratedPerson : - IFixedWidthModel, - IUtf8FixedWidthModel -{ - [FixedColumn(0, 10)] public string Name { get; init; } - [FixedColumn(10, 5)] public int Age { get; init; } - [FixedColumn(15, 10)] public double Salary { get; init; } -} -``` - -The generator emits distinct `TryParse` overloads for `ReadOnlySpan` and `ReadOnlySpan` when both interfaces are present. - -## Parsing - -### Runtime Single-Line Parsing - -```csharp -using System.Globalization; -using FixedWidthParser.Parsers; - -var parser = new FixedWidthParser(); - -if (parser.TryParse("John Doe 30 60000.00 ", CultureInfo.InvariantCulture, stringPool: null, out var person)) -{ - // person.Name == "John Doe" - // person.Age == 30 - // person.Salary == 60000.0 -} -``` - -### Source-Generated Single-Line Parsing - -```csharp -using System.Globalization; -using FixedWidthParser; - -if (FixedWidth.TryParse( - "John Doe 30 60000.00 ", - CultureInfo.InvariantCulture, - stringPool: null, - out var person)) -{ - // reflection-free generated parser -} -``` - -### Reading Text Files - -```csharp -using System.Globalization; -using FixedWidthParser.Readers; - -var reader = new FixedWidthReader(CultureInfo.InvariantCulture); - -foreach (var person in reader.ReadFile("people.txt")) -{ - // ... -} -``` - -`Read(TextReader)` and `Read(Stream, encoding, leaveOpen)` are also available. Reading is lazy and reuses a pooled buffer; lines are sliced directly from the buffer, so the reader does not allocate a string per line. A malformed line throws a `FormatException` carrying the line number. - -Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded as records. A line that is non-empty but shorter than the declared layout is treated as malformed and throws. - -The source-generated facade has matching overloads: - -```csharp -foreach (var person in FixedWidth.ReadFile("people.txt", formatProvider: CultureInfo.InvariantCulture)) -{ - // generated TryParse for each line -} -``` - -### Async Reading - -```csharp -await foreach (var person in reader.ReadFileAsync("people.txt")) -{ - // ... -} -``` - -`ReadAsync(TextReader)` and `ReadAsync(Stream, encoding, leaveOpen)` mirror the synchronous overloads; `ReadFileAsync` uses true async file I/O. Cancellation is honored via `WithCancellation`. - -## UTF-8 Byte Parsing - -For ASCII/single-byte fixed-width files, the UTF-8 APIs parse directly from bytes. This avoids `StreamReader`, avoids UTF-8 to UTF-16 transcoding, and keeps offsets measured in bytes. - -```csharp -using System.Globalization; -using FixedWidthParser.Readers; - -var reader = new FixedWidthByteReader(CultureInfo.InvariantCulture); - -foreach (var person in reader.ReadFile("people.txt")) -{ - // parsed from raw UTF-8 bytes -} -``` - -Generated UTF-8 models use `FixedWidthUtf8`: - -```csharp -using System.Globalization; -using FixedWidthParser; - -if (FixedWidthUtf8.TryParse( - "John Doe 30 60000.00 "u8, - CultureInfo.InvariantCulture, - stringPool: null, - out var person)) -{ - // generated byte parser -} - -await using var stream = File.OpenRead("people.txt"); -await foreach (var person in FixedWidthUtf8.ReadAsync(stream, formatProvider: CultureInfo.InvariantCulture)) -{ - // async raw-byte streaming -} -``` - -Column offsets on the UTF-8 path are byte offsets. That is ideal for the ASCII-style payloads common in flat files; with multi-byte UTF-8 characters, byte offsets and character offsets are not the same. - -The byte path supports the same `StringPool` interning as the char path: pass a pool to `FixedWidthByteReader` / the `FixedWidthUtf8` methods (the `stringPool` argument above) and string columns are interned through it; pass `null` to decode a fresh string per value. -### Reading from a PipeReader - -When the source is already a [`System.IO.Pipelines.PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket, a named pipe, or an upstream pipeline stage — you can parse straight off it, letting the pipe own buffering and read-ahead. Both the reflection reader and the generated facade expose a `PipeReader` overload of `ReadAsync`: - -```csharp -using System.IO.Pipelines; -using System.Globalization; -using FixedWidthParser; -using FixedWidthParser.Readers; +var layout = FixedWidthLayout.FromAttributes(); -// Reflection reader: -var reader = new FixedWidthByteReader(CultureInfo.InvariantCulture); -await foreach (var person in reader.ReadAsync(pipeReader)) -{ - // parsed from the pipe -} - -// Source-generated facade: -await foreach (var person in FixedWidthUtf8.ReadAsync(pipeReader, formatProvider: CultureInfo.InvariantCulture)) +foreach (var person in FixedWidth.ReadFile("people.txt", layout)) { // ... } -``` - -Lines are sliced from the pipe's `ReadOnlySequence` and parsed in place when contiguous, copying into a pooled buffer only when a line spans segment boundaries. The same line semantics apply (BOM skipped once, `\n`/`\r\n`, empty lines skipped, trailing line without a newline yielded). By default the reader is completed when iteration ends; pass `leaveOpen: true` to leave it open. - -> **When to use it.** Prefer the `PipeReader` overload when you already hold a pipe. For plain files and streams the `Stream`/file overloads remain the faster default — a `PipeReader` adds per-read overhead that only pays off when there is real I/O to overlap (network, slow disk), not for in-memory or local-file sources. - -## Writing - -```csharp -using System.Globalization; -using FixedWidthParser.Writers; - -var writer = new FixedWidthWriter(); -var people = new[] -{ - new Person { Name = "John Doe", Age = 30, Salary = 60000 }, - new Person { Name = "Jane", Age = 28, Salary = 55000 }, -}; +var people = new[] { new Person { Name = "John Doe", Age = 30, Salary = 60000 } }; using var stream = File.Create("out.txt"); -writer.WriteMany(stream, people.AsSpan(), CultureInfo.InvariantCulture); +FixedWidth.WriteMany(stream, people, layout); ``` -Overloads cover `Stream`/`StreamWriter` with `IEnumerable`/`ReadOnlySpan`, plus `WriteAsync` and `WriteManyAsync`. Reusing a `StreamWriter`, or passing a span, keeps writing allocation-free per line. - -## Formatting Options - -Each column can be tuned through named attribute arguments: +Make the model `partial` and implement `IFixedWidthModel` to have the generator write its +parser and writer instead; the generated overloads take no layout: `FixedWidth.ReadFile("people.txt")`. -```csharp -[FixedColumn(0, 8, Alignment = Alignment.Right, Padding = '0')] public int Id { get; init; } // "00000042" -[FixedColumn(8, 10, Format = "F2")] public double Amount { get; init; } // "1234.50 " -[FixedColumn(18, 5, Overflow = OverflowBehavior.Truncate)] public string Code { get; init; } -``` +## Guide -- **`Alignment`**: `Left` (default) or `Right`. -- **`Padding`**: fill character (default space; for example `'0'` for zero-padding). -- **`Format`**: format string passed to `ISpanFormattable` (for example `"F2"` or `"N0"`); ignored for `string`. -- **`Overflow`**: `Default`, `Truncate` or `Throw`. `Default` resolves per type: strings truncate, numeric types throw. +- [Models and layouts](docs/guide/models.md) — attribute and generated models, fluent layouts for types you + cannot annotate, formatting options, column types, encodings and validation. +- [Reading](docs/guide/reading.md) — files and streams, async, parallel aggregation, options, errors, + untrusted input and single lines. +- [Writing](docs/guide/writing.md) — records and batches, synchronous and asynchronous, and formatting into + your own buffer. +- [Versioning](docs/guide/versioning.md) — what a minor version may change, and the `Infrastructure` namespace. +- [Benchmarks](docs/performance/benchmarks.md) — reading and writing against FileHelpers, FlatFiles and + RecordParser. [Live results](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/). -## Culture Handling +## Python and C/C++ -Pass an `IFormatProvider` to `TryParse`, the reader constructor, the source-generated facade methods or the write methods. The generic path (`ISpanParsable`/`ISpanFormattable`) and the `double`/`float` processors honor it. When the provider is `null`, `'.'` is used as the decimal separator. +`src/FixedWidthParser.Native` publishes the library as a NativeAOT shared library with a C ABI +(`include/fixedwidth.h`), so Python and C/C++ read and write fixed-width files without a .NET runtime. The +caller describes the columns at run time; a read fills one native buffer per column, or Arrow arrays, in +parallel by byte range. -## StringPool +```python +import fixedwidth as fw -Pass a `CommunityToolkit.HighPerformance.Buffers.StringPool` to intern repeated string-column values: - -```csharp -using System.Globalization; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; - -var pool = new StringPool(); -var reader = new FixedWidthReader(CultureInfo.InvariantCulture, stringPool: pool); -``` - -This is a time-vs-memory trade-off: pooling removes repeated string allocations but costs extra CPU for hashing and lookup. Prefer it for GC-sensitive or high-concurrency workloads; skip it for raw throughput. - -Pooling applies to both the `char` and UTF-8 byte paths (`FixedWidthReader`/`FixedWidthByteReader` and the `FixedWidth`/`FixedWidthUtf8` facades), and to `ref struct` models. When no pool is supplied, each string column is decoded into a fresh string. - -## Validation - -Invalid layouts fail fast with an `InvalidOperationException` on the runtime parser/writer paths, or generator diagnostics on generated models. Negative `Start`, `Length < 1`, and overlapping columns are rejected. Adjacent columns are valid. - -## `ref struct` Models - -> Requires `net10.0` (or any .NET 9+ target) — the `allows ref struct` constraint doesn't exist on `net8.0`, so this feature is compiled out there. - -The parser accepts `ref struct` models (`where TModel : new(), allows ref struct`), useful for stack-only row processing: - -```csharp -using FixedWidthParser.Attributes; -using FixedWidthParser.Parsers; - -public ref struct Row -{ - public Row() - { - Name = string.Empty; - Age = 0; - } - - [FixedColumn(0, 10)] public string Name { get; set; } - [FixedColumn(10, 5)] public int Age { get; set; } -} - -var parser = new FixedWidthParser(); -parser.TryParse(line, CultureInfo.InvariantCulture, null, out var row); +layout = fw.Layout([ + fw.Column("codneg", 12, 12, fw.String), + fw.Column("preabe", 56, 13, fw.Float64, scale=2), # B3 N(11)V99: two implied decimals +]) +df = fw.read_pandas("COTAHIST_A2023.TXT", layout, fw.ReadOptions(header_lines=1, skip_invalid=True)) ``` -Batch readers and the writer use regular generic constraints because `IEnumerable` cannot carry a `ref struct`. - -## Performance - -Measured with BenchmarkDotNet (`MemoryDiagnoser`) on .NET 10. Highlights: - -- Parsing a line is allocation-light on the runtime path and reflection-free on the generated path. -- Text readers avoid allocating a string per line by slicing a reusable buffer. -- UTF-8 byte readers avoid `StreamReader` and transcoding for ASCII-style flat files. -- Writing with `StreamWriter` reuse or `ReadOnlySpan` is zero-alloc per line. - -### Comparison with other libraries - -#### Reading - -Batched read of N fixed-width records (10-char string + 5-char int + 10-char double) from an in-memory string, compared against [FileHelpers](https://www.filehelpers.net/), [FlatFiles](https://github.com/jehugaleahsa/FlatFiles), and [RecordParser](https://github.com/leandromoh/RecordParser). - -``` -BenchmarkDotNet v0.15.8, Windows 11 (10.0.26200.8655/25H2/2025Update/HudsonValley2) -13th Gen Intel Core i7-1355U 1.70GHz, 1 CPU, 12 logical and 10 physical cores -.NET SDK 10.0.301 - [Host] : .NET 10.0.9 (10.0.9, 10.0.926.27113), X64 RyuJIT x86-64-v3 - DefaultJob : .NET 10.0.9 (10.0.9, 10.0.926.27113), X64 RyuJIT x86-64-v3 -``` - -| Method | Count | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | -|--------------------------- |------:|-----------:|---------:|----------:|------:|--------:|---------:|--------:|----------:|------------:| -| FixedWidthParser_Generated | 100 | 2.661 μs | 0.052 μs | 0.062 μs | 0.10 | 0.02 | 0.6294 | - | 3.86 KB | 0.09 | -| FixedWidthParser_Read | 100 | 4.460 μs | 0.074 μs | 0.065 μs | 0.16 | 0.04 | 0.6256 | - | 3.86 KB | 0.09 | -| RecordParser_Read | 100 | 7.047 μs | 0.091 μs | 0.080 μs | 0.26 | 0.06 | 0.6866 | - | 4.23 KB | 0.10 | -| FileHelpers_Read | 100 | 28.860 μs | 2.630 μs | 7.671 μs | 1.07 | 0.39 | 6.7444 | 0.2136 | 41.49 KB | 1.00 | -| FlatFiles_Read | 100 | 40.173 μs | 0.747 μs | 0.699 μs | 1.49 | 0.36 | 16.7847 | 0.5493 | 103.13 KB | 2.49 | -| | | | | | | | | | | | -| FixedWidthParser_Generated | 1000 | 26.273 μs | 0.347 μs | 0.325 μs | 0.13 | 0.00 | 6.2256 | - | 38.31 KB | 0.09 | -| FixedWidthParser_Read | 1000 | 43.935 μs | 0.467 μs | 0.390 μs | 0.21 | 0.00 | 6.2256 | - | 38.31 KB | 0.09 | -| RecordParser_Read | 1000 | 72.391 μs | 1.392 μs | 1.429 μs | 0.35 | 0.01 | 6.2256 | - | 38.69 KB | 0.10 | -| FileHelpers_Read | 1000 | 205.683 μs | 3.662 μs | 3.058 μs | 1.00 | 0.02 | 66.1621 | 17.0898 | 406.42 KB | 1.00 | -| FlatFiles_Read | 1000 | 402.237 μs | 6.329 μs | 5.610 μs | 1.96 | 0.04 | 155.2734 | 4.8828 | 953.2 KB | 2.35 | - -The reflection-based reader is **~5× faster** and allocates **11× less** than FileHelpers. The source-generated reader reaches **~8× faster** at the same allocation footprint. - -#### Writing - -Write of N fixed-width records to `Stream.Null` (formatting cost only, no I/O). - -| Method | Count | Mean | Error | StdDev | Median | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | -|----------------------- |------:|--------------:|-------------:|-------------:|--------------:|------:|--------:|----------:|----------:|----------:|------------:| -| FixedWidthParser_Write | 1 | 121.4 ns | 2.01 ns | 1.57 ns | 120.9 ns | 0.71 | 0.01 | - | - | - | 0.00 | -| FileHelpers_Write | 1 | 170.2 ns | 1.40 ns | 1.17 ns | 170.3 ns | 1.00 | 0.01 | 0.0572 | - | 360 B | 1.00 | -| FlatFiles_Write | 1 | 357.2 ns | 5.71 ns | 5.07 ns | 355.9 ns | 2.10 | 0.03 | 0.1135 | - | 712 B | 1.98 | -| RecordParser_Write | 1 | 29,258.0 ns | 542.81 ns | 507.75 ns | 29,232.1 ns | 171.96 | 3.11 | 7.9956 | 1.0376 | 49,640 B | 137.89 | -| | | | | | | | | | | | | -| FixedWidthParser_Write | 100 | 8,665.8 ns | 169.30 ns | 424.73 ns | 8,519.2 ns | 0.77 | 0.04 | - | - | - | 0.00 | -| FileHelpers_Write | 100 | 11,283.8 ns | 187.38 ns | 166.11 ns | 11,255.9 ns | 1.00 | 0.02 | 5.2338 | - | 32,832 B | 1.00 | -| FlatFiles_Write | 100 | 25,250.0 ns | 493.33 ns | 411.95 ns | 25,367.7 ns | 2.24 | 0.05 | 11.3220 | - | 71,200 B | 2.17 | -| RecordParser_Write | 100 | 87,556.0 ns | 1,820.98 ns | 5,224.72 ns | 86,343.9 ns | 7.76 | 0.47 | 8.3008 | 0.9766 | 51,953 B | 1.58 | -| | | | | | | | | | | | | -| FixedWidthParser_Write | 1000 | 86,454.8 ns | 1,695.66 ns | 1,952.72 ns | 86,407.9 ns | 0.71 | 0.03 | - | - | - | 0.00 | -| FileHelpers_Write | 1000 | 122,427.3 ns | 2,439.24 ns | 5,797.11 ns | 120,769.0 ns | 1.00 | 0.06 | 52.2461 | - | 328,032 B | 1.00 | -| FlatFiles_Write | 1000 | 258,895.1 ns | 5,051.42 ns | 4,725.10 ns | 258,890.0 ns | 2.12 | 0.10 | 113.2813 | - | 712,000 B | 2.17 | -| RecordParser_Write | 1000 | 754,218.4 ns | 14,792.11 ns | 29,880.79 ns | 749,527.8 ns | 6.17 | 0.37 | 173.8281 | 171.8750 | 1,076,624 B | 3.28 | - -The writer is **~30% faster** than FileHelpers and allocates **nothing** per batch, regardless of record count. - -Run benchmarks: - -```bash -dotnet run -c Release --project tests/Benchmarks/Benchmarks.csproj -- --filter "*Comparison*" -dotnet run -c Release --project tests/Benchmarks/Benchmarks.csproj -- --filter "*ReaderBenchmarks*" -``` - -Benchmark reports are written to `tests/Benchmarks/BenchmarkDotNet.Artifacts/results`. +On a 1,000,000-line COTAHIST-shaped file, `fw.read_pandas` is 96 times faster than `pandas.read_fwf`, and +`fw.read_polars` 114 times +([measurements](docs/python-benchmarks.md)). The [Python README](python/README.md), the +[C++ README](cpp/README.md) and the [native library README](src/FixedWidthParser.Native/README.md) cover +the rest. ## Project Layout ```text src/FixedWidthParser/ The library src/FixedWidthParser.Generator/ Roslyn source generator -tests/FixedWidthParser.Tests/ Runtime, reader, writer and parity tests +src/FixedWidthParser.Native/ NativeAOT C ABI, its C and C++ headers +python/ The fixedwidth Python package +cpp/ CMake target and tests for the C++ wrapper +tests/FixedWidthParser.Tests/ Runtime, reader, writer, parity and native-layer tests tests/FixedWidthParser.Generator.Tests/Source generator tests +tests/FixedWidthParser.NativeSmoke/ C consumer of the native library tests/Benchmarks/ BenchmarkDotNet benchmarks +tests/AotSmoke/ Fluent layout published with NativeAOT +docs/ Usage guide and the measurements behind design decisions ``` +[ARCHITECTURE.md](ARCHITECTURE.md) maps the code; [CONTRIBUTING.md](CONTRIBUTING.md) and +[STYLEGUIDE.md](STYLEGUIDE.md) cover how to change it. + ## Building and Testing ```bash dotnet build FixedWidthParser.slnx -c Release -dotnet test tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj -dotnet test tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +dotnet test --solution FixedWidthParser.slnx -c Release ``` diff --git a/SECURITY.md b/SECURITY.md index 4be3525..64882f9 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,31 +1,26 @@ # Security Policy -## Supported versions +## Supported Versions -Security fixes are applied to the latest released `1.x` version published on NuGet as -[`FixedWidthParser.NET`](https://www.nuget.org/packages/FixedWidthParser.NET). +Only the latest published release of [`FixedWidthParser.NET`](https://www.nuget.org/packages/FixedWidthParser.NET) +receives security fixes. Older releases are not patched retroactively; upgrade to the latest version +before reporting an issue. The native library (`FixedWidthParser.Native`), the `fixedwidth` Python +package and the C++ header are built from the same source and reported the same way. -| Version | Supported | -| ------- | ------------------ | -| 1.x | :white_check_mark: | -| < 1.0 | :x: | +## Reporting a Vulnerability -## Reporting a vulnerability +Please **do not** open a public GitHub issue for security vulnerabilities. -**Please do not report security vulnerabilities through public GitHub issues.** +Report vulnerabilities privately via +[GitHub Private Vulnerability Reporting](https://github.com/GabrielMarquezMatte/FixedWidthParser/security/advisories/new), +or by email to **gabrielandremarquez.matte@gmail.com**. -Instead, use one of the following private channels: +Include, where possible: +- A minimal reproduction: the model and the input that triggers it +- The reading or writing path involved (file or stream, sync, async or parallel, generated or runtime, + or the native library through C, C++ or Python) +- The impact you observed (crash, excessive memory/CPU, incorrect data, etc.) +- The affected version(s) -1. **Preferred:** GitHub's private vulnerability reporting — - [open a report](https://github.com/GabrielMarquezMatte/FixedWidthParser/security/advisories/new). -2. Email the maintainer at **gabrielandremarquez.matte@gmail.com** with the details. - -Please include: - -- a description of the vulnerability and its impact; -- steps to reproduce (a minimal model + input is ideal); -- affected version(s). - -You can expect an initial acknowledgement within **5 business days**. Once the issue is -confirmed, a fix and coordinated disclosure timeline will be agreed with you before any -public announcement. +You should receive an initial response within 5 business days. If the report is +confirmed, a fix will be prepared and released before any public disclosure. diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md new file mode 100644 index 0000000..3896934 --- /dev/null +++ b/STYLEGUIDE.md @@ -0,0 +1,376 @@ +# FixedWidthParser Style Guide + +Code style rules. For build/PR/test process, see [CONTRIBUTING.md](CONTRIBUTING.md); for the shape of +the codebase, see [ARCHITECTURE.md](ARCHITECTURE.md). + +## Priorities + +1. **Correctness** — the code must be right. No shortcuts that sacrifice safety or accuracy. +2. **Performance** — optimize where it matters, but never at the cost of readability. +3. **Readability** — code is read far more than it is written. + +--- + +## Ternaries + +Use a ternary only when the entire expression fits on one line and both branches are single values — no function calls, no method chains, no nesting. + +```csharp +// OK: two literal values +int resolvedOverflow = writeKind == WriteKind.String ? 1 : 2; + +// OK: one local each side +int byteCount = ascii ? text.Length : encodedLength; + +// NOT OK: nested ternary +string result = a ? b ? "x" : "y" : "z"; + +// NOT OK: branches contain logic +int value = condition ? ComputeA() : ComputeB(); + +// NOT OK: a long condition, even with plain values +byte sign = Padding != ' ' && length > 0 && (first == (byte)'-' || first == (byte)'+') ? first : padding; + +// Use an if that returns instead (see "Avoid else" below): +private static int Value(bool condition) +{ + if (condition) + { + return ComputeA(); + } + return ComputeB(); +} +``` + +When in doubt, use `if`. + +--- + +## Avoid `else` + +Whenever possible, do not write `else`. A branch that ends the method — `return`, `continue`, +`break`, `throw` — makes the `else` redundant, and the code after it reads as the main path. + +```csharp +// NOT OK: two branches at the same depth +if (Alignment != Alignment.Right) +{ + content.CopyTo(slice); + slice[length..].Fill(padding); +} +else +{ + slice[..pad].Fill(padding); + content.CopyTo(slice[pad..]); +} + +// OK: the special case returns, the other case follows unindented +if (Alignment != Alignment.Right) +{ + content.CopyTo(slice); + slice[length..].Fill(padding); + return; +} +slice[..pad].Fill(padding); +content.CopyTo(slice[pad..]); +``` + +When neither branch can end the method, extract the `if` into its own method that returns. Keep +`else` only when both branches must fall through to shared code that follows and extracting them +would split one idea across two methods. + +--- + +## Control Flow — Prefer Early Return + +Avoid wrapping the happy path in a condition. Guard and exit early; keep the main logic at the lowest indentation level. + +```csharp +// NOT OK: body is nested under the guard +internal void Compact() +{ + if (Start > 0) + { + int len = End - Start; + ... + } +} + +// OK: guard exits early +internal void Compact() +{ + if (Start <= 0) + { + return; + } + int len = End - Start; + ... +} +``` + +Always use braces on `if`, even single-line bodies. The linter enforces this. + +--- + +## Scope Nesting — Maximum 3 Levels + +Count every block that introduces indentation: `if`, `else`, `for`, `foreach`, `while`, `using`, `try`, `switch`. A function body is level 0. Nesting beyond 3 is a signal to extract a method. + +``` +level 0: function body +level 1: while (true) +level 2: if (ScanFrom < End) +level 3: if (rel >= 0) ← maximum allowed +level 4: if (...) ← extract this into a private method +``` + +When you hit level 4, stop and ask: what is this inner block doing? Name it and move it. + +--- + +## Method Bodies — Always Block Body + +Every method, constructor, operator and local function must use a block body with braces. Expression +bodies (`=>`) are forbidden on them, no matter how short. + +```csharp +// NOT OK +private static string Foo() => Bar(); + +// OK +private static string Foo() +{ + return Bar(); +} +``` + +Properties and their accessors are exempt: `public TModel Current => _current;` and +`get => field ?? Encoding.UTF8;` are both fine, as is a block body. `.editorconfig` encodes the same +split and turns IDE0027 off. + +This rule exists to keep diffs readable and the line count honest. A one-liner expression body hides complexity behind a single line and makes the 70-line rule easy to game. + +--- + +## Function Length — Maximum 70 Lines + +Count every non-blank, non-comment line inside the function body (opening and closing brace included). If a function exceeds 70 lines, split it. + +70 is the review target. The build enforces a looser backstop through MA0051 (90 lines / 50 statements, see `.editorconfig`), so a method between 71 and 90 lines compiles but still fails review. + +Do not game the limit by collapsing logic onto single lines. The limit exists to keep methods focused, not to reward line-golf. A method that packs three operations on one line to stay under 70 is worse than one that splits them across 71 readable lines. + +Common splits: + +- Read one column's attribute vs. assemble the model from its columns +- Find a line boundary vs. parse the line within it +- The rare growth or error path extracted from the caller + +--- + +## Naming + +- Private fields: `_camelCase` +- Private static methods: `PascalCase` +- Local variables: `camelCase` +- Prefer full words over abbreviations (`index` over `idx`, `source` over `src`) unless the abbreviation is idiomatic in the domain (`buf`, `len`, `pos`) + +### The runtime and generated paths are peers + +A model is parsed and written either through a layout (`FixedWidth.X(…, FixedWidthLayout layout, …)`) +or by the generated path (`FixedWidth.X(…)` over `IFixedWidthModel`). Every `FixedWidth` +operation has both overloads, side by side, with the same parameters in the same order apart from the +layout, and the same options object configures both. A caller who learns one path should be able to +guess the other. When you add an overload for one, add its sibling in the same change. + +--- + +## Comments + +**The default is no comment.** Code is expected to explain itself through naming and execution flow. +Reaching for a comment is the signal to look at the code again first: if a reader cannot follow the +logic, that is a defect in the code, not a gap in its documentation. Rename the variable, extract the +method, flatten the branch. Fix it there, and the comment becomes unnecessary. + +A comment is justified only once that has been tried and the code still cannot carry the meaning — +because the reason lives outside the code and no amount of naming can bring it in: + +- A constraint imposed by a platform, a dependency, or the bytes of a file. +- An invariant a reader could silently break, where breaking it stays compiling and passes review. + +```csharp +// OK: a dependency's behavior the code cannot express +// The BCL's UTF-8 parser does not take a space for fr-FR's no-break-space group separator. + +// NOT OK: describes what the code does +// Loop through each column and parse its value. + +// NOT OK: the code should have said this itself +// i is the index of the '\n' that ends the line +int i = ...; // -> name it lineFeedIndex and delete the comment +``` + +Never reference the task, the ticket, or the caller. + +### Findings do not belong in code + +Measurements, benchmark results, allocation counts, and the reasoning that picked one approach over +another are **not comments**. They go in [`docs/`](docs/README.md). + +This includes the whole class of "why it is this way" content: a throughput figure that justified a +buffer size, an allocation count that motivated pooling, a design that was prototyped and measured +and then rejected. Put it in a document, with the numbers, the machine, and the input. Then the +next person can re-run it and disagree with the evidence — which a comment asserting `// 5x faster` +never lets them do. + +What may stay in the code is the constraint itself, stated without its evidence: + +```csharp +// OK: the ceiling, not the benchmark that found it +// ponytail: at most four decimals; anything finer takes the BCL formatter. + +// NOT OK: a finding, and it rots the moment anything around it changes +// Measured 8 ns vs 46 ns for double.TryFormat, so the scaled path stays. +``` + +### A comment must stay true, or it is worse than no comment + +Comments carry maintenance cost. Two failure modes are specifically banned: + +- **Do not describe a state the code is no longer in.** When you refactor, reread the comments on what + you touched. +- **Do not cite a file that can disappear.** Never point a comment at a plan, a spec, or a section + marker as the authoritative explanation. Those are working artifacts: they get renamed, merged and + deleted, and the comment then sends a reader to nothing, which is worse than silence. A committed + document under [`docs/`](docs/README.md) is different in kind — it is permanent, and pointing at one is + how a finding stays out of the code. The test is whether the target is expected to still exist in a + year, not whether a link is involved. + +--- + +## Error Handling + +Throw at trust boundaries: constructor arguments, public API parameters, malformed external data. Use `ArgumentOutOfRangeException.ThrowIfNegative` and similar .NET 6+ throw helpers. Do not add defensive checks for conditions that are impossible given the internal invariants. + +Match the exception to whose fault it is: + +| Fault | Exception | +|---|---| +| A line that does not fit the layout | `FixedWidthParseException`, naming the line, column and raw text | +| A source that breaks a configured limit (`MaxLineLength`) | `InvalidDataException` | +| A model whose layout is invalid | `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`, or a generator diagnostic | +| A value too wide for its column when writing | `InvalidOperationException` naming the column | +| A bad argument from the caller | `ArgumentException` and its subtypes | + +Never answer a malformed file with `ArgumentException`/`ArgumentOutOfRangeException`: that tells the +caller they made a mistake when the input is what is broken. If file bytes can reach a .NET throw +helper, you are missing a validation step upstream. + +A layout error throws `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`. +`FromAttributes` builds once per type in a static constructor; an error there is captured and rethrown +from every call, never left to surface as a `TypeInitializationException`. An error only one side of an +attribute layout hits (it can read but not write, or the reverse) is rethrown when that side is first +used, before any line is touched. + +--- + +## Untrusted Input + +Files and request bodies are attacker-controlled. These rules are not optional in reader code. + +- **Bound every buffer a source can grow.** A source without line breaks must hit `MaxLineLength` + before it drives an allocation, on every reader: sync, async and both parallel aggregators. +- **Split lines in one place.** Every reader goes through `LineBufferState`. The parallel stream + producer also grows a chunk for an unterminated line, so a guard added to `LineBufferState.GrowIfFull` + belongs there too; a limit enforced in one and not the other is a hole. +- **Validate before you allocate, not before you use.** Renting a bigger buffer first and discovering the + line is too long afterwards still hands the attacker the allocation. +- **Never let a malformed value silently change semantics.** A column that does not parse fails the + line; it never becomes a default value. +- New or changed line-splitting code needs a test that feeds the malformed shape (no line break, a + line one byte over the limit, a `\r` at a buffer boundary). Assert the exception type and its line + number, not merely that something threw. + +--- + +## Pooled Buffers + +`ArrayPool.Shared` is used throughout the readers and the writer. Misuse here is a correctness bug, +not a performance nit. + +- Every `Rent` needs a matching `Return` on **every** exit path, including thrown exceptions — a + `try`/`finally` around the use. A dropped buffer silently shrinks the pool for the whole process. +- Never `Return` a buffer that anything still references, and never touch a buffer after returning it. +- Pooled arrays come back oversized and dirty. Bound reads by the logical length you tracked, never by + `array.Length`. +- Buffers that live beyond one call are returned in `Dispose`/`DisposeAsync`; guard against double-return, + since `Dispose` may be called twice. + +--- + +## Twins + +Four sets of code paths in this library must stay behaviorally identical, and nothing in the compiler +keeps them honest: + +- **Layout and generated.** `FixedWidthLayout` (from attributes or `Build`) and the generator's + `TryParse`/`TryFormat` call the same `FixedWidthRuntime` helpers. Put shared behavior there, not in + either side. +- **Sequential and parallel reads.** `AggregateFileAsync` and `AggregateAsync` must return what a + sequential read folded into one accumulator would, with the same line numbers in errors. Their + per-line work shares `ParallelAggregator.TryAdd`. +- **Sync and async.** The readers (`RecordEnumerator` and `AsyncRecordEnumerator`) split and parse lines + the same way, and the writers (`WriteMany` and `WriteManyAsync`) batch the same way. +- **Model and sink loops.** `ParallelAggregator.AggregateChunk` folds models into an accumulator and + `SinkChunk` feeds the native layer's sinks; they are copies on purpose (the comment above them says + why). + +So: + +- A behavior fix to one twin **must** be applied to the others in the same change. Read all of them + before you edit any. +- Behavior shared by twins belongs in a helper none of them duplicates — the split should cover only + the awaiting or the code generation, not the logic around it. +- Changes to a twinned path need a parity test asserting identical output, including at buffer + boundaries (a line straddling a refill or a partition) — that is exactly where implementations drift. +- A synchronous method does not take a `CancellationToken`. + +--- + +## Tests and Benchmarks + +Harness code is held to the same standard as library code, because a broken harness reports success. + +- **Fail loudly, never silently.** A setup that never ran, an input that parsed to nothing — assert it. + The dangerous case is not the harness that crashes; it is the one that measures zero work and + publishes a number. +- **Benchmarks against another library must compare matched work, or say plainly that they do not.** + If our side writes bytes straight to the stream while the competitor goes through a `StreamWriter`, + say so next to the numbers. An unlabelled comparison that flatters us is a defect. +- **State the machine for every published number**, and never derive a ratio across results measured on + different hardware. Allocation figures are deterministic and comparable across machines; timings are + not. +- Assert real behavior — exact bytes, exact values, exception types and their line and column. A test + that calls a method and asserts no throw documents nothing. + +--- + +## Performance Notes + +These rules apply only when a function is on a measured hot path (splitting lines, parsing or +formatting a column): + +- Work on `ReadOnlySpan`; decode to `string` only for a `string` column. +- Use `ArrayPool.Shared` for buffers that live beyond one stack frame, `stackalloc` for small ones. +- Avoid LINQ in hot paths; use `IndexOf` on spans, which benefits from SIMD. +- Specialize generic code over `struct` strategies (`IRecordLineParser`) so the JIT devirtualizes, + rather than over interfaces or delegates. +- Split cold paths (growth, error diagnosis, throw helpers) into `[MethodImpl(MethodImplOptions.NoInlining)]` + methods so they do not bloat the hot caller's IL and cost it inlining. +- Optimize against a measurement, and record what it showed in [`docs/`](docs/README.md) — never in a comment. + An optimization with no number behind it is a guess that costs readability. + +Outside the hot path (building a layout), ordinary allocations are fine, and reflection is fine in +`FromAttributes`. `FixedWidthLayout.Build` and the generated overloads must stay free of reflection, +`MakeGenericMethod` and `Expression.Compile`, so they keep working under trimming and NativeAOT; +`tests/AotSmoke` checks it. diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..1d2eab8 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,22 @@ +coverage: + status: + project: + csharp: + flags: + - csharp + python: + flags: + - python + +# Each language uploads under its own flag from a path-filtered job, so a flag a commit did not run keeps its +# last report instead of dropping to nothing. +flag_management: + default_rules: + carryforward: true + individual_flags: + - name: csharp + paths: + - src/ + - name: python + paths: + - python/ diff --git a/coverage.runsettings b/coverage.runsettings new file mode 100644 index 0000000..291aac2 --- /dev/null +++ b/coverage.runsettings @@ -0,0 +1,30 @@ + + + + + + + + + + + .*[\/]obj[\/].* + + .*[\/]FixedWidthParser\.Native[\/]Exports(\.\w+)?\.cs + + + + + *<.*>.* + + + + + + + + diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt new file mode 100644 index 0000000..10d9ab8 --- /dev/null +++ b/cpp/CMakeLists.txt @@ -0,0 +1,102 @@ +# The C++ wrapper: a header-only INTERFACE target, fw::fixedwidth, linked against the published +# FixedWidthParser.Native library. See cpp/README.md for using it from another project. +# +# cmake -S cpp -B build/cpp +# cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure +# cmake --install build/cpp --prefix # then find_package(fixedwidth) +cmake_minimum_required(VERSION 3.16) +project(FixedWidthParser LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(FIXEDWIDTH_NATIVE_LIB "" CACHE FILEPATH + "FixedWidthParser.Native shared library to link; empty uses the repo's published copy or downloads the release's") +set(FIXEDWIDTH_VERSION "" CACHE STRING + "Release tag whose native library is downloaded (v2.0.0); empty takes the tag checked out, from git describe") +if(NOT FIXEDWIDTH_VERSION) + find_package(Git QUIET) + if(Git_FOUND) + execute_process(COMMAND ${GIT_EXECUTABLE} describe --tags --exact-match + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE _git_tag OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET RESULT_VARIABLE _git_result) + endif() + if(_git_result EQUAL 0 AND _git_tag) + set(FIXEDWIDTH_VERSION "${_git_tag}") + else() + set(FIXEDWIDTH_VERSION "v0.0.0") + endif() +endif() + +set(_include_dir "${CMAKE_CURRENT_SOURCE_DIR}/../src/FixedWidthParser.Native/include") + +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/NativeLib.cmake") +include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/CopyNativeLibrary.cmake") +fixedwidth_resolve_native_lib(_native_lib) +fixedwidth_import_native_lib("${_native_lib}" "${_include_dir}/fixedwidth.def") + +add_library(fw_fixedwidth INTERFACE) +target_include_directories(fw_fixedwidth INTERFACE "${_include_dir}") +target_compile_features(fw_fixedwidth INTERFACE cxx_std_23) +target_link_libraries(fw_fixedwidth INTERFACE fw::native) +add_library(fw::fixedwidth ALIAS fw_fixedwidth) + +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + set(_top_level ON) +else() + set(_top_level OFF) +endif() +option(FIXEDWIDTH_INSTALL "Generate install rules and a find_package(fixedwidth) config package" ${_top_level}) +option(FIXEDWIDTH_BUILD_TESTS "Build the C++ wrapper's tests" ${_top_level}) + +if(FIXEDWIDTH_INSTALL) + include(GNUInstallDirs) + include(CMakePackageConfigHelpers) + + get_filename_component(FIXEDWIDTH_INSTALLED_NATIVE_NAME "${_native_lib}" NAME) + if(WIN32) + set(FIXEDWIDTH_INSTALLED_NATIVE_DIR "${CMAKE_INSTALL_BINDIR}") + else() + set(FIXEDWIDTH_INSTALLED_NATIVE_DIR "${CMAKE_INSTALL_LIBDIR}") + endif() + set(FIXEDWIDTH_INSTALLED_IMPLIB_DIR "${CMAKE_INSTALL_LIBDIR}") + set(FIXEDWIDTH_INSTALLED_INCLUDE_DIR "${CMAKE_INSTALL_INCLUDEDIR}") + + install(DIRECTORY "${_include_dir}/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + FILES_MATCHING PATTERN "*.h" PATTERN "*.hpp") + install(FILES "${_native_lib}" DESTINATION "${FIXEDWIDTH_INSTALLED_NATIVE_DIR}") + if(WIN32) + get_target_property(_native_implib fixedwidth_native IMPORTED_IMPLIB) + get_filename_component(FIXEDWIDTH_INSTALLED_IMPLIB_NAME "${_native_implib}" NAME) + install(FILES "${_native_implib}" DESTINATION "${FIXEDWIDTH_INSTALLED_IMPLIB_DIR}") + endif() + + set(_config_dir "${CMAKE_INSTALL_LIBDIR}/cmake/fixedwidth") + configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/fixedwidthConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfig.cmake" + INSTALL_DESTINATION "${_config_dir}" + PATH_VARS FIXEDWIDTH_INSTALLED_NATIVE_DIR FIXEDWIDTH_INSTALLED_IMPLIB_DIR FIXEDWIDTH_INSTALLED_INCLUDE_DIR) + + # The package version takes only the tag's digits: v2.0.0-rc.1 installs as 2.0.0. + if(FIXEDWIDTH_VERSION MATCHES "^v?([0-9]+(\\.[0-9]+)*)") + set(_pkg_version "${CMAKE_MATCH_1}") + else() + set(_pkg_version "0.0.0") + endif() + write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfigVersion.cmake" + VERSION "${_pkg_version}" + COMPATIBILITY SameMajorVersion) + + install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/fixedwidthConfigVersion.cmake" + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/CopyNativeLibrary.cmake" + DESTINATION "${_config_dir}") +endif() + +if(FIXEDWIDTH_BUILD_TESTS) + enable_testing() + add_subdirectory(tests) +endif() diff --git a/cpp/README.md b/cpp/README.md new file mode 100644 index 0000000..1384ae7 --- /dev/null +++ b/cpp/README.md @@ -0,0 +1,137 @@ +# FixedWidthParser for C++ + +`fixedwidth.hpp` (and `fixedwidth_arrow.hpp`) wrap the C ABI in C++23: RAII `Layout`, `Table`, `Reader`, +`ArrowResult` and `ArrowStream`, options structs with defaults, and `std::expected` results. + +```cpp +#include + +std::vector columns{ + {.name = "codneg", .start = 12, .length = 12, .type = fw::Type::String}, + {.name = "preabe", .start = 56, .length = 13, .type = fw::Type::Float64, .scale = 2}, +}; +auto layout = fw::Layout::create(columns).value(); +auto table = fw::read_file("COTAHIST_A2023.TXT", layout, {.header_lines = 1, .skip_invalid = true}); +if (!table) { std::puts(table.error().message.c_str()); return 1; } +std::span prices = table->values(1); +``` + +## Structs + +Specialize `fw::FixedWidthMapper` once, as with ExcelReader's `ExcelMapper`, and the layout comes from the +bindings: `read_file`, `read_memory` and `TypedReader` read rows as `T`, and `write_file`/`write_memory` +write a range of them. + +```cpp +struct Quote +{ + std::string ticker; + double open = 0; + std::optional volume; +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() + { + return std::tuple{ + fw::make_field("codneg", 12, 12, &Quote::ticker), + fw::make_field("preabe", 56, 13, &Quote::open, {.scale = 2}), + fw::make_field("quatot", 152, 18, &Quote::volume), + }; + } +}; + +auto quotes = fw::read_file("COTAHIST_A2023.TXT", {.header_lines = 1, .skip_invalid = true}); +for (const Quote& quote : *quotes) { /* ... */ } +auto bytes = fw::write_memory(std::vector{/* ... */}); +``` + +A field's type picks the column type: `std::string` or `std::string_view` (String), an integer (Int64, or +Decimal holding the unscaled value when the field sets a `scale`), `float` or `double` (Float64), +`std::chrono::sys_days` or `year_month_day` (Date), `std::chrono::microseconds` or `hh_mm_ss` +(Time), `std::chrono::system_clock::time_point` (Timestamp). `std::optional` makes the column nullable; a +null read leaves a plain field at its default. `make_field`'s last argument, `fw::FieldOptions`, takes the +rest of `ColumnSpec`: `scale`, `format`, `trim_char`, `trim_mode`, `alignment`, `padding`, `overflow`. + +`TableView` keeps the library's columns and converts a row on access; a `std::string_view` field points +into it. `view[i]` is unchecked, `view.at(i)` returns an empty optional out of range, and `view.table()` is +the `fw::Table` underneath, with `lines_read()` and `lines_skipped()`. `fw::TypedReader::open_file`, +`open_memory` or `open_stream` reads in batches: `next()` returns the next `TableView`, or an empty optional at +the end. `fw::layout_for()` returns the `fw::Layout` the bindings describe, for the untyped API. + +`fw::Reader::open_stream`, `TypedReader::open_stream` and `fw::read_arrow_stream` take a `std::istream&` and +read it forward a batch at a time, in constant memory, so a decompressing or network stream works. The stream +must outlive the reader; a stream that turns `bad()` fails the read. `fw::write_stream(std::ostream&, table, +layout)` writes the same way, in order, a batch at a time. + +`ColumnSpec::dictionary` hands a String column to Arrow (`read_arrow_*`) dictionary-encoded: int32 indices into +its distinct values. `fw::Table` keeps the strings. + +```cpp +std::ifstream file("COTAHIST_A2023.TXT", std::ios::binary); +auto reader = fw::TypedReader::open_stream(file, {.read = {.header_lines = 1, .skip_invalid = true}}); +while (auto batch = reader->next(); batch && *batch) { /* (*batch)->size() rows */ } +``` + +An integer field narrower than `int64_t` that a value does not fit fails the read with +`FW_INVALID_ARGUMENT` instead of wrapping. + +## Install + +Needs a C++23 compiler and CMake 3.16 or later; on Windows, `lib.exe` (run CMake from a Visual Studio developer +prompt) or MinGW's `dlltool`. No .NET SDK: CMake downloads the prebuilt native library for your platform from +the GitHub Release of the tag you build (Windows x64, macOS arm64/x64, Linux glibc x64/arm64). + +**FetchContent**, the simplest: + +```cmake +include(FetchContent) +FetchContent_Declare(fixedwidth + GIT_REPOSITORY https://github.com/GabrielMarquezMatte/FixedWidthParser.git + GIT_TAG v2.0.0 + SOURCE_SUBDIR cpp) +FetchContent_MakeAvailable(fixedwidth) + +target_link_libraries(app PRIVATE fw::fixedwidth) +fixedwidth_copy_native_library(app) # Windows: puts the DLL next to app.exe; a no-op elsewhere +``` + +**Installed package**, for `find_package`: + + git clone --branch v2.0.0 https://github.com/GabrielMarquezMatte/FixedWidthParser.git + cmake -S FixedWidthParser/cpp -B build -DCMAKE_BUILD_TYPE=Release + cmake --install build --prefix + +```cmake +find_package(fixedwidth 2 REQUIRED) # with in CMAKE_PREFIX_PATH +target_link_libraries(app PRIVATE fw::fixedwidth) +fixedwidth_copy_native_library(app) +``` + +The install puts the headers in `include/`, the library in `bin/` (Windows) or `lib/`, the Windows import +library in `lib/`, and the package in `lib/cmake/fixedwidth`. `find_package` never downloads anything. + +Settings: + +- `FIXEDWIDTH_VERSION`: the release tag whose library is downloaded. Empty takes the tag checked out, which + `GIT_TAG` and `--branch` above set. +- `FIXEDWIDTH_NATIVE_LIB` (CMake or environment variable): a library you built or downloaded yourself, such as + musl's `fixedwidth-native-linux-musl-x64.so` from the release, saved as `FixedWidthParser.Native.so`. Nothing + is downloaded then. + +On Windows an import library is generated from `include/fixedwidth.def`. GCC's `-Wextra` warns about designated +initializers that leave defaulted options out; that is the intended use, so build with +`-Wno-missing-field-initializers`. + +## Developing + + python python/scripts/build_native.py + cmake -S cpp -B build/cpp + cmake --build build/cpp && ctest --test-dir build/cpp --output-on-failure + +In a checkout, the library `build_native.py` publishes into `python/src/fixedwidth/_lib` takes precedence over +the download. `FIXEDWIDTH_BUILD_TESTS` and `FIXEDWIDTH_INSTALL` default to on when `cpp/` is the top-level +project and off under FetchContent or `add_subdirectory`. `cpp/tests/package` builds against an installed prefix, +as CI does. diff --git a/cpp/cmake/CopyNativeLibrary.cmake b/cpp/cmake/CopyNativeLibrary.cmake new file mode 100644 index 0000000..1bb7e51 --- /dev/null +++ b/cpp/cmake/CopyNativeLibrary.cmake @@ -0,0 +1,15 @@ +# fixedwidth_copy_native_library() copies the native library next to an executable after it links. +# +# On Windows a DLL beside neither the executable nor PATH is not found, and the process exits with 0xc0000135 +# (STATUS_DLL_NOT_FOUND) before main runs. A no-op on Linux and macOS, where the rpath resolves it. +function(fixedwidth_copy_native_library target) + if(NOT WIN32) + return() + endif() + if(NOT TARGET ${target}) + message(FATAL_ERROR "fixedwidth_copy_native_library: '${target}' is not a target") + endif() + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different "$" "$" + COMMENT "Copying FixedWidthParser.Native next to ${target}") +endfunction() diff --git a/cpp/cmake/NativeLib.cmake b/cpp/cmake/NativeLib.cmake new file mode 100644 index 0000000..14282d9 --- /dev/null +++ b/cpp/cmake/NativeLib.cmake @@ -0,0 +1,93 @@ +# fixedwidth_resolve_native_lib() finds the FixedWidthParser.Native shared library for this platform: +# +# 1. FIXEDWIDTH_NATIVE_LIB, as a CMake or environment variable: a library you built or downloaded. +# 2. The copy python/scripts/build_native.py publishes into python/src/fixedwidth/_lib, in a checkout of the repo. +# 3. Otherwise the release asset for FIXEDWIDTH_VERSION, downloaded once into the build tree. +set(_fixedwidth_repo_root "${CMAKE_CURRENT_LIST_DIR}/../..") + +function(fixedwidth_resolve_native_lib out_var) + if(WIN32) + set(_os "win") + set(_ext "dll") + elseif(APPLE) + set(_os "osx") + set(_ext "dylib") + else() + set(_os "linux") + set(_ext "so") + endif() + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm64|aarch64|ARM64)$") + set(_arch "arm64") + else() + set(_arch "x64") + endif() + set(_lib_name "FixedWidthParser.Native.${_ext}") + + if(FIXEDWIDTH_NATIVE_LIB) + set(_lib "${FIXEDWIDTH_NATIVE_LIB}") + elseif(DEFINED ENV{FIXEDWIDTH_NATIVE_LIB}) + set(_lib "$ENV{FIXEDWIDTH_NATIVE_LIB}") + elseif(EXISTS "${_fixedwidth_repo_root}/python/src/fixedwidth/_lib/${_lib_name}") + set(_lib "${_fixedwidth_repo_root}/python/src/fixedwidth/_lib/${_lib_name}") + else() + # Saved under the library's own name, not the asset's: the import library on Windows and the SONAME on + # Linux both record the name the loader looks for. + set(_lib "${CMAKE_BINARY_DIR}/fixedwidth-native/${FIXEDWIDTH_VERSION}/${_os}-${_arch}/${_lib_name}") + if(NOT EXISTS "${_lib}") + set(_url "https://github.com/GabrielMarquezMatte/FixedWidthParser/releases/download/${FIXEDWIDTH_VERSION}/fixedwidth-native-${_os}-${_arch}.${_ext}") + message(STATUS "Downloading FixedWidthParser.Native: ${_url}") + file(DOWNLOAD "${_url}" "${_lib}" STATUS _status) + list(GET _status 0 _code) + if(NOT _code EQUAL 0) + file(REMOVE "${_lib}") + list(GET _status 1 _message) + message(FATAL_ERROR "Downloading ${_url} failed: ${_message}. Set FIXEDWIDTH_VERSION to a release " + "tag, or FIXEDWIDTH_NATIVE_LIB to a library you have.") + endif() + endif() + endif() + + # A relative path would be re-resolved against the build directory once baked into the imported target. + get_filename_component(_lib "${_lib}" ABSOLUTE BASE_DIR "${CMAKE_BINARY_DIR}") + if(NOT EXISTS "${_lib}") + message(FATAL_ERROR "FixedWidthParser.Native not found at ${_lib}") + endif() + set(${out_var} "${_lib}" PARENT_SCOPE) +endfunction() + +# fixedwidth_import_native_lib( ) creates the IMPORTED target fw::native. +# +# NativeAOT ships no import library on Windows, so one is generated from the checked-in .def (lib.exe +# under MSVC, dlltool under MinGW), with a LIBRARY line naming the actual DLL so the loader finds it. +function(fixedwidth_import_native_lib library def_file) + add_library(fixedwidth_native SHARED IMPORTED GLOBAL) + set_target_properties(fixedwidth_native PROPERTIES IMPORTED_LOCATION "${library}") + + if(WIN32) + get_filename_component(_dll_name "${library}" NAME) + set(_generated_def "${CMAKE_BINARY_DIR}/fixedwidth.generated.def") + set(_implib "${CMAKE_BINARY_DIR}/fixedwidth_native.lib") + file(READ "${def_file}" _exports) + file(WRITE "${_generated_def}" "LIBRARY ${_dll_name}\n${_exports}") + if(MSVC) + # The target's architecture, as the compiler reports it (x64, ARM64, X86), not the host's: + # an x64 machine building for ARM64 (-A ARM64, or an ARM64 developer prompt) needs an ARM64 import library. + set(_machine "${CMAKE_CXX_COMPILER_ARCHITECTURE_ID}") + if(NOT _machine) + message(FATAL_ERROR "cannot tell the MSVC target architecture for the import library") + endif() + find_program(_lib_exe NAMES lib REQUIRED) + execute_process(COMMAND "${_lib_exe}" "/def:${_generated_def}" "/out:${_implib}" "/machine:${_machine}" + RESULT_VARIABLE _result) + else() + find_program(_dlltool NAMES dlltool REQUIRED) + execute_process(COMMAND "${_dlltool}" -d "${_generated_def}" -l "${_implib}" RESULT_VARIABLE _result) + endif() + if(NOT _result EQUAL 0) + message(FATAL_ERROR "generating the import library for ${_dll_name} failed") + endif() + set_target_properties(fixedwidth_native PROPERTIES IMPORTED_IMPLIB "${_implib}") + endif() + + add_library(fw::native ALIAS fixedwidth_native) +endfunction() diff --git a/cpp/cmake/fixedwidthConfig.cmake.in b/cpp/cmake/fixedwidthConfig.cmake.in new file mode 100644 index 0000000..deddbcf --- /dev/null +++ b/cpp/cmake/fixedwidthConfig.cmake.in @@ -0,0 +1,26 @@ +@PACKAGE_INIT@ + +# The same two targets cpp/CMakeLists.txt defines for an in-tree build, pointed at the installed copies. The +# native library was resolved once, at install time, so find_package() never downloads anything. + +if(NOT TARGET fw::native) + add_library(fw::native SHARED IMPORTED) + set_target_properties(fw::native PROPERTIES + IMPORTED_LOCATION "@PACKAGE_FIXEDWIDTH_INSTALLED_NATIVE_DIR@/@FIXEDWIDTH_INSTALLED_NATIVE_NAME@") + if(WIN32) + set_target_properties(fw::native PROPERTIES + IMPORTED_IMPLIB "@PACKAGE_FIXEDWIDTH_INSTALLED_IMPLIB_DIR@/@FIXEDWIDTH_INSTALLED_IMPLIB_NAME@") + endif() +endif() + +if(NOT TARGET fw::fixedwidth) + add_library(fw::fixedwidth INTERFACE IMPORTED) + set_target_properties(fw::fixedwidth PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "@PACKAGE_FIXEDWIDTH_INSTALLED_INCLUDE_DIR@" + INTERFACE_COMPILE_FEATURES cxx_std_23 + INTERFACE_LINK_LIBRARIES fw::native) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/CopyNativeLibrary.cmake") + +check_required_components(fixedwidth) diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt new file mode 100644 index 0000000..9d6bedf --- /dev/null +++ b/cpp/tests/CMakeLists.txt @@ -0,0 +1,15 @@ +foreach(test read write arrow mapped) + add_executable(fw_cpp_${test} ${test}.cpp) + target_link_libraries(fw_cpp_${test} PRIVATE fw::fixedwidth) + if(MSVC) + target_compile_options(fw_cpp_${test} PRIVATE /W4 /WX) + # The tests use standard std::fopen, which the CRT deprecates (C4996) in favour of fopen_s. + target_compile_definitions(fw_cpp_${test} PRIVATE _CRT_SECURE_NO_WARNINGS) + else() + # Designated initializers that leave defaulted options out are the intended use. + target_compile_options(fw_cpp_${test} PRIVATE -Wall -Wextra -Werror -Wno-missing-field-initializers) + endif() + add_test(NAME fw_cpp_${test} COMMAND fw_cpp_${test} WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") + set_tests_properties(fw_cpp_${test} PROPERTIES PASS_REGULAR_EXPRESSION "${test} ok") + fixedwidth_copy_native_library(fw_cpp_${test}) +endforeach() diff --git a/cpp/tests/arrow.cpp b/cpp/tests/arrow.cpp new file mode 100644 index 0000000..2627256 --- /dev/null +++ b/cpp/tests/arrow.cpp @@ -0,0 +1,58 @@ +#include "common.hpp" + +#include + +#include +#include +#include + +int main() +{ + auto layout = cotahist_layout(); + + auto result = fw::read_arrow_memory(content, layout, skip_header_and_invalid(4)); + CHECK(result.has_value()); + const ArrowSchema& schema = result->schema(); + const ArrowArray& array = result->array(); + CHECK(std::strcmp(schema.format, "+s") == 0 && schema.n_children == 4); + CHECK(std::strcmp(schema.children[2]->name, "preabe") == 0); + CHECK(array.length == 3 && array.children[3]->null_count == 1); + CHECK(static_cast(array.children[2]->buffers[1])[1] == 678.9); + + auto stream = fw::read_arrow_stream_memory(content, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(stream.has_value()); + ArrowSchema stream_schema; + CHECK(stream->get()->get_schema(stream->get(), &stream_schema) == 0); + stream_schema.release(&stream_schema); + int64_t rows = 0; + ArrowArray batch; + while (stream->get()->get_next(stream->get(), &batch) == 0 && batch.release != nullptr) { + rows += batch.length; + batch.release(&batch); + } + CHECK(rows == 3); + + std::istringstream in{std::string(content)}; + auto from_stream = fw::read_arrow_stream(in, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(from_stream.has_value()); + int64_t streamed_rows = 0; + while (from_stream->get()->get_next(from_stream->get(), &batch) == 0 && batch.release != nullptr) { + streamed_rows += batch.length; + batch.release(&batch); + } + CHECK(streamed_rows == 3); + + // A dictionary column: int32 indices whose dictionary holds the distinct strings. + std::vector coded{{.name = "codneg", .start = 2, .length = 12, .type = fw::Type::String, .dictionary = true}}; + auto coded_layout = fw::Layout::create(coded); + CHECK(coded_layout.has_value()); + auto encoded = fw::read_arrow_memory(content, *coded_layout, skip_header_and_invalid()); + CHECK(encoded.has_value()); + const ArrowSchema& codes_schema = *encoded->schema().children[0]; + const ArrowArray& codes = *encoded->array().children[0]; + CHECK(std::strcmp(codes_schema.format, "i") == 0 && std::strcmp(codes_schema.dictionary->format, "u") == 0); + CHECK(codes.length == 4 && codes.dictionary->length == 4); + + std::puts("arrow ok"); + return 0; +} diff --git a/cpp/tests/common.hpp b/cpp/tests/common.hpp new file mode 100644 index 0000000..8ba45d9 --- /dev/null +++ b/cpp/tests/common.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include +#include +#include + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #cond); \ + std::exit(1); \ + } \ + } while (0) + +// A COTAHIST-like file: a header, three quotes (one with a blank quantity), an invalid line, a trailer. +inline constexpr std::string_view content = + "00COTAHIST.2023\n" + "01PETR4 0000000123456000000000000001000\n" + "01VALE3 0000000067890 \n" + "01ITUB4 0000000004321000000000000000050\n" + "01XXXXX 00000000ABCDE000000000000000001\n" + "99TRAILER\n"; + +inline fw::Layout cotahist_layout() +{ + std::vector columns{ + {.name = "tipreg", .start = 0, .length = 2, .type = fw::Type::String}, + {.name = "codneg", .start = 2, .length = 12, .type = fw::Type::String}, + {.name = "preabe", .start = 14, .length = 13, .type = fw::Type::Float64, .scale = 2, .padding = '0'}, + {.name = "quatot", .start = 27, .length = 18, .type = fw::Type::Int64, .nullable = true, .padding = '0'}, + }; + auto layout = fw::Layout::create(columns); + CHECK(layout.has_value()); + return std::move(*layout); +} + +inline fw::ReadOptions skip_header_and_invalid(int32_t degree = 1) +{ + return {.header_lines = 1, .skip_invalid = true, .degree_of_parallelism = degree}; +} diff --git a/cpp/tests/mapped.cpp b/cpp/tests/mapped.cpp new file mode 100644 index 0000000..f7d8d18 --- /dev/null +++ b/cpp/tests/mapped.cpp @@ -0,0 +1,155 @@ +#include "common.hpp" + +#include +#include +#include +#include +#include +#include +#include + +using namespace std::chrono; + +// Maps part of the COTAHIST-like content in common.hpp: the record type and gaps stay unmapped. +struct Quote +{ + std::string ticker; + double open = 0; + std::optional volume; +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() + { + return std::tuple{ + fw::make_field("codneg", 2, 12, &Quote::ticker), + fw::make_field("preabe", 14, 13, &Quote::open, {.scale = 2, .padding = '0'}), + fw::make_field("quatot", 27, 18, &Quote::volume, {.padding = '0'}), + }; + } +}; + +// Every other column type. +struct Trade +{ + std::string code; + int32_t quantity = 0; + int64_t cents = 0; + sys_days day; + std::optional> time; + system_clock::time_point at; + + // hh_mm_ss has no operator==. + bool operator==(const Trade& other) const + { + auto duration = [](const std::optional>& t) { return t.transform([](auto v) { return v.to_duration(); }); }; + return code == other.code && quantity == other.quantity && cents == other.cents && day == other.day && + duration(time) == duration(other.time) && at == other.at; + } +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() + { + return std::tuple{ + fw::make_field("code", 0, 6, &Trade::code), + fw::make_field("quantity", 6, 5, &Trade::quantity), + fw::make_field("cents", 11, 9, &Trade::cents, {.scale = 2}), + fw::make_field("day", 20, 8, &Trade::day, {.format = "yyyyMMdd"}), + fw::make_field("time", 28, 6, &Trade::time, {.format = "HHmmss"}), + fw::make_field("at", 34, 14, &Trade::at, {.format = "yyyyMMddHHmmss"}), + }; + } +}; + +struct Small +{ + int8_t value = 0; +}; + +template <> +struct fw::FixedWidthMapper +{ + static constexpr auto get_bindings() { return std::tuple{fw::make_field("value", 0, 4, &Small::value)}; } +}; + +static_assert(fw::mapped_range>); +static_assert(!fw::mapped_range>); +static_assert(std::ranges::random_access_range>); + +static void check_quotes(const fw::TableView& quotes) +{ + CHECK(quotes.size() == 3); + const Quote first = quotes[0]; + CHECK(first.ticker == "PETR4" && first.open == 1234.56 && first.volume == 1000); + CHECK(quotes[1].ticker == "VALE3" && !quotes[1].volume.has_value()); + CHECK(quotes.at(2)->open == 43.21 && quotes.at(2)->volume == 50); + CHECK(!quotes.at(3).has_value()); + std::vector tickers; + for (const Quote& quote : quotes) { + tickers.push_back(quote.ticker); + } + CHECK((tickers == std::vector{"PETR4", "VALE3", "ITUB4"})); + CHECK(std::ranges::distance(quotes) == 3 && (*(quotes.end() - 1)).ticker == "ITUB4"); +} + +int main() +{ + auto quotes = fw::read_memory(content, skip_header_and_invalid()); + CHECK(quotes.has_value()); + check_quotes(*quotes); + CHECK(quotes->table().lines_skipped() == 3); + + auto parallel = fw::read_memory(content, skip_header_and_invalid(4)); + CHECK(parallel.has_value()); + check_quotes(*parallel); + + auto reader = fw::TypedReader::open_memory(content, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(reader.has_value()); + std::vector batched; + while (true) { + auto batch = reader->next(); + CHECK(batch.has_value()); + if (!batch->has_value()) { + break; + } + for (const Quote& quote : **batch) { + batched.push_back(quote.ticker); + } + } + CHECK((batched == std::vector{"PETR4", "VALE3", "ITUB4"})); + + auto failed = fw::read_memory(content, {.header_lines = 1}); + CHECK(!failed.has_value() && failed.error().status == FW_PARSE_ERROR && failed.error().line_number == 5); + + const std::vector trades{ + {"PETR4", 100, 123456, sys_days(2023y / October / 24), hh_mm_ss(10h + 30min + 5s), + sys_days(2023y / October / 24) + 10h + 30min + 5s}, + {"VALE3", -7, -5, sys_days(1999y / December / 31), std::nullopt, sys_days(2000y / January / 1)}, + }; + auto written = fw::write_memory(trades); + CHECK(written.has_value()); + const std::string text(written->begin(), written->end()); + // The decimal's two decimals are implied; the empty time is blank. + CHECK(text == "PETR4 100 1234562023102410300520231024103005\n" + "VALE3 -7 -519991231 20000101000000\n"); + + auto path = std::string("fw_cpp_mapped.txt"); + CHECK(fw::write_file(path, trades).has_value()); + auto from_file = fw::read_file(path, {.degree_of_parallelism = 1}); + std::remove(path.c_str()); + CHECK(from_file.has_value()); + CHECK(from_file->size() == 2); + CHECK((*from_file)[0] == trades[0] && (*from_file)[1] == trades[1]); + + auto too_big = fw::read_memory(" 12\n 300\n"); + CHECK(!too_big.has_value() && too_big.error().status == FW_INVALID_ARGUMENT); + CHECK(too_big.error().message.find("'value' holds 300 at row 1") != std::string::npos); + + std::puts("mapped ok"); + return 0; +} diff --git a/cpp/tests/package/CMakeLists.txt b/cpp/tests/package/CMakeLists.txt new file mode 100644 index 0000000..786be19 --- /dev/null +++ b/cpp/tests/package/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.16) + +# Consumes the installed package the way a downstream project does - find_package() against an install prefix, +# not add_subdirectory() against this repo. Its own project, so a config file that only works inside the build +# tree fails here. +project(fixedwidth_package_consumer LANGUAGES CXX) + +find_package(fixedwidth REQUIRED) + +add_executable(package_consumer main.cpp) +target_link_libraries(package_consumer PRIVATE fw::fixedwidth) +fixedwidth_copy_native_library(package_consumer) + +enable_testing() +add_test(NAME package_consumer COMMAND package_consumer) +set_tests_properties(package_consumer PROPERTIES PASS_REGULAR_EXPRESSION "package ok") diff --git a/cpp/tests/package/main.cpp b/cpp/tests/package/main.cpp new file mode 100644 index 0000000..09514e0 --- /dev/null +++ b/cpp/tests/package/main.cpp @@ -0,0 +1,32 @@ +#include + +#include +#include +#include + +int main() +{ + std::vector columns{ + {.name = "code", .start = 0, .length = 5, .type = fw::Type::String}, + {.name = "price", .start = 5, .length = 8, .type = fw::Type::Float64, .scale = 2}, + }; + auto layout = fw::Layout::create(columns); + if (!layout) + { + std::puts(layout.error().message.c_str()); + return 1; + } + auto table = fw::read_memory(std::string_view{"PETR400001234\nVALE300005678\n"}, *layout); + if (!table) + { + std::puts(table.error().message.c_str()); + return 1; + } + if (table->row_count() != 2 || table->string(0, 1) != "VALE3" || table->values(1)[0] != 12.34) + { + std::puts("unexpected values"); + return 1; + } + std::puts("package ok"); + return 0; +} diff --git a/cpp/tests/read.cpp b/cpp/tests/read.cpp new file mode 100644 index 0000000..e0848a6 --- /dev/null +++ b/cpp/tests/read.cpp @@ -0,0 +1,91 @@ +#include "common.hpp" + +#include +#include +#include +#include + +static void check_table(const fw::Table& table) +{ + CHECK(table.row_count() == 3); + CHECK(table.column_count() == 4); + CHECK(table.string(1, 0) == "PETR4"); + CHECK(table.string(1, 2) == "ITUB4"); + auto prices = table.values(2); + CHECK(prices.size() == 3 && prices[0] == 1234.56 && prices[2] == 43.21); + auto quantities = table.values(3); + CHECK(quantities[0] == 1000 && quantities[2] == 50); + CHECK(table.is_valid(3, 0) && !table.is_valid(3, 1)); + CHECK(table.is_valid(2, 1)); +} + +int main() +{ + auto layout = cotahist_layout(); + + auto memory = fw::read_memory(content, layout, skip_header_and_invalid()); + CHECK(memory.has_value()); + check_table(*memory); + CHECK(memory->lines_read() == 3 && memory->lines_skipped() == 3); + + auto parallel = fw::read_memory(content, layout, skip_header_and_invalid(4)); + CHECK(parallel.has_value()); + check_table(*parallel); + + const std::string path = "fw_cpp_read.txt"; + std::FILE* file = std::fopen(path.c_str(), "wb"); + CHECK(file != nullptr); + std::fwrite(content.data(), 1, content.size(), file); + std::fclose(file); + auto from_file = fw::read_file(path, layout, skip_header_and_invalid()); + CHECK(from_file.has_value()); + check_table(*from_file); + std::remove(path.c_str()); + + auto failed = fw::read_memory(content, layout, {.header_lines = 1}); + CHECK(!failed.has_value()); + CHECK(failed.error().status == FW_PARSE_ERROR); + CHECK(failed.error().line_number == 5 && failed.error().column == 2); + CHECK(failed.error().message.find("ABCDE") != std::string::npos); + + auto reader = fw::Reader::open_memory(content, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(reader.has_value()); + int64_t rows = 0; + while (true) { + auto batch = reader->next(); + CHECK(batch.has_value()); + if (!batch->has_value()) { + break; + } + rows += (*batch)->row_count(); + } + CHECK(rows == 3); + + // The same batches from a std::istream, read forward without seeking. + std::istringstream in{std::string(content)}; + auto streamed = fw::Reader::open_stream(in, layout, {.read = skip_header_and_invalid(), .batch_rows = 2}); + CHECK(streamed.has_value()); + std::vector codes; + while (true) { + auto batch = streamed->next(); + CHECK(batch.has_value()); + if (!batch->has_value()) { + break; + } + for (int64_t row = 0; row < (*batch)->row_count(); ++row) { + codes.emplace_back((*batch)->string(1, row)); + } + } + CHECK((codes == std::vector{"PETR4", "VALE3", "ITUB4"})); + + std::vector overlap{ + {.name = "a", .start = 0, .length = 4, .type = fw::Type::String}, + {.name = "b", .start = 2, .length = 4, .type = fw::Type::String}, + }; + auto bad = fw::Layout::create(overlap); + CHECK(!bad.has_value() && bad.error().status == FW_INVALID_ARGUMENT); + CHECK(bad.error().message.find("Overlapping") != std::string::npos); + + std::puts("read ok"); + return 0; +} diff --git a/cpp/tests/write.cpp b/cpp/tests/write.cpp new file mode 100644 index 0000000..a776564 --- /dev/null +++ b/cpp/tests/write.cpp @@ -0,0 +1,46 @@ +#include "common.hpp" + +#include +#include +#include + +int main() +{ + auto layout = cotahist_layout(); + auto table = fw::read_memory(content, layout, skip_header_and_invalid()); + CHECK(table.has_value()); + + auto serial = fw::write_memory(*table, layout, {.degree_of_parallelism = 1}); + auto parallel = fw::write_memory(*table, layout, {.degree_of_parallelism = 4, .batch_rows = 1}); + CHECK(serial.has_value() && parallel.has_value()); + CHECK(*serial == *parallel); + const std::string expected = + "01PETR4 0000000123456000000000000001000\n" + "01VALE3 0000000067890 \n" + "01ITUB4 0000000004321000000000000000050\n"; + CHECK(std::string(serial->begin(), serial->end()) == expected); + + auto crlf = fw::write_memory(*table, layout, {.newline = "\r\n"}); + CHECK(crlf.has_value() && crlf->size() == expected.size() + 3); + + std::ostringstream out; + CHECK(fw::write_stream(out, *table, layout, {.degree_of_parallelism = 4, .batch_rows = 1}).has_value()); + CHECK(out.str() == std::string(serial->begin(), serial->end())); + + const std::string path = "fw_cpp_write.txt"; + CHECK(fw::write_file(path, *table, layout).has_value()); + std::FILE* file = std::fopen(path.c_str(), "rb"); + CHECK(file != nullptr); + std::string written(expected.size(), '\0'); + CHECK(std::fread(written.data(), 1, written.size(), file) == written.size()); + std::fclose(file); + std::remove(path.c_str()); + CHECK(written == expected); + + std::vector narrow{{.name = "tipreg", .start = 0, .length = 2, .type = fw::Type::String}}; + auto mismatch = fw::write_memory(*table, *fw::Layout::create(narrow)); + CHECK(!mismatch.has_value() && mismatch.error().status == FW_INVALID_ARGUMENT); + + std::puts("write ok"); + return 0; +} diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..f95643c --- /dev/null +++ b/docs/README.md @@ -0,0 +1,26 @@ +# Documentation + +The evidence behind the code: measurements, and the designs they picked or rejected. Code comments state +a constraint; the numbers that justified it live here, so they can be re-run and disputed +([STYLEGUIDE.md § Comments](../STYLEGUIDE.md#findings-do-not-belong-in-code)). + +- [writer-performance.md](writer-performance.md) — where a written line spends its time, and the + formatting shortcuts that came out of it. +- [reader-performance.md](reader-performance.md) — `double` parsing in comma cultures, the cache in + front of `StringPool`, and the read size the file engine kept. +- [parallel-aggregate.md](parallel-aggregate.md) — how `AggregateFileAsync` and `AggregateAsync` + scale with the number of workers. +- [pipe-reader.md](pipe-reader.md) — why a `PipeReader` is read through `AsStream()` rather than + its own enumerator. +- [fluent-layout.md](fluent-layout.md) — reading and writing through a `FixedWidthLayout` against + the classes it replaced. +- [python-benchmarks.md](python-benchmarks.md) — the Python package reading a COTAHIST-shaped file + against `pandas.read_fwf`. +- [performance/benchmarks.md](performance/benchmarks.md) — reading and writing against FileHelpers, + FlatFiles and RecordParser. + +Usage documentation lives in [guide/](guide/), split out of the [README](../README.md) so the NuGet +landing page stays short: [models and layouts](guide/models.md), [reading](guide/reading.md), +[writing](guide/writing.md) and [versioning](guide/versioning.md). It is prose for a caller deciding how +to use the library; what each member does is in its `///` comments. The shape of the code is +[ARCHITECTURE.md](../ARCHITECTURE.md). diff --git a/docs/fluent-layout.md b/docs/fluent-layout.md new file mode 100644 index 0000000..9f85b70 --- /dev/null +++ b/docs/fluent-layout.md @@ -0,0 +1,36 @@ +# Moving the runtime path onto `FixedWidthLayout` + +2.0 replaced `FixedWidthParser`, `FixedWidthReader` and `FixedWidthWriter` with one layout +object passed to the `FixedWidth` methods. The parsers and formatters moved from `static readonly` +fields of a generic class, built in a static constructor, to fields of a layout instance, and writing +moved into `RecordWriter`, generic over a format strategy. This page checks that neither move cost +anything measurable, and records what generated models gained. + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), 32 GB, Windows 10 Pro 19045, .NET SDK 11.0.100-rc.2, `net10.0`, Release. +A Stopwatch console app (not BenchmarkDotNet) over a model of three columns (`string` 10, `int` 5, +`double` 10, lines of 25 bytes plus a line break): + +- **read**: `ReadFile` over a 1,000,000-line file, summing one column; +- **write**: `WriteMany` of 1,000,000 records to `Stream.Null`. + +Each figure is the best of five runs after one warm-up run. The baseline is `d36b958` (the old +classes); the current build calls `FixedWidth.*(…, FixedWidthLayout.FromAttributes(), …)`. The two +programs ran alternately, twice each; both rounds are shown. + +## Result + +| Operation | `d36b958` | Layout API | +|---|--:|--:| +| read, attribute model | 66.9 / 64.4 ms | 64.6 / 67.3 ms | +| read, generated model | 66.8 / 66.6 ms | 66.6 / 69.7 ms | +| write, attribute model | 60.5 / 58.0 ms | 60.8 / 60.5 ms | +| write, generated model | — (no stream writer) | 47.1 / 45.9 ms | + +Every pair sits within the run-to-run spread of the same build (up to 4.7 %, on the generated read, +whose code did not change). The layout instance costs nothing measurable over the static fields, and +batching through `RecordWriter` matches the old writer. + +Generated models can now write to a stream. They do so about a quarter faster than attribute models, +since the generated `TryFormat` has no per-column delegate or interface call. diff --git a/docs/guide/models.md b/docs/guide/models.md new file mode 100644 index 0000000..ca322f3 --- /dev/null +++ b/docs/guide/models.md @@ -0,0 +1,118 @@ +# Models and layouts + +A layout maps byte ranges of a line to the members of a type. It comes from `[FixedColumn]` attributes, from the source generator, or from a fluent description in code; every read and write goes through one. + +## Defining a Model + +An attribute model needs a public parameterless constructor and mapped fields or properties; `FixedWidthLayout.FromAttributes()` turns it into a layout, built once per type by reflection: + +```csharp +using FixedWidthParser; + +public readonly record struct Person +{ + public Person() + { + Name = string.Empty; + } + + [FixedColumn(0, 10)] public string Name { get; init; } + [FixedColumn(10, 5)] public int Age { get; init; } + [FixedColumn(15, 10)] public double Salary { get; init; } +} +``` + +`start` is the 0-based offset and `length` is the column width, both **in bytes**. In UTF-8 a character such as `é` takes two bytes, so a layout that counts characters only matches a UTF-8 file whose text is ASCII; for accented text, read the file in the single-byte encoding it was written in (see [Encodings](#encodings)). + +For source generation, make the model `partial` and implement the marker interface: + +```csharp +using FixedWidthParser; + +public readonly partial record struct GeneratedPerson : IFixedWidthModel +{ + [FixedColumn(0, 10)] public string Name { get; init; } + [FixedColumn(10, 5)] public int Age { get; init; } + [FixedColumn(15, 10)] public double Salary { get; init; } +} +``` + +Columns can be declared on a base class: its `[FixedColumn]` members are part of the layout, for +attribute and generated models alike. An override keeps the attribute of the member it overrides unless +it declares its own, and a `new` member replaces the one it hides. + +## Layouts without attributes + +For a type you cannot annotate, or a layout known only at run time, describe the columns in code. Each column binds through a getter (used to write) and a setter (used to read), so nothing is reflected over: + +```csharp +using FixedWidthParser; + +var layout = FixedWidthLayout.Build(b => b + .Factory(() => new Order()) + .Column("Id", 0, 6, (in Order o) => o.Id, (ref Order o, int v) => o.Id = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) + .Column("Name", 6, 20, (in Order o) => o.Name, (ref Order o, string v) => o.Name = v) + .NullableColumn("Discount", 26, 10, (in Order o) => o.Discount, (ref Order o, decimal? v) => o.Discount = v) + .DecodedColumn("Date", 36, 8, (in Order o) => o.Date, (ref Order o, DateOnly v) => o.Date = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" }) + .ConvertedColumn("Total", 44, 12, (in Order o) => o.Total, (ref Order o, Money v) => o.Total = v, new MoneyConverter())); +``` + +| Method | Column type | +|---|---| +| `Column` | `string` | +| `Column` | a type that parses from and formats to UTF-8: numbers, `Guid`, … | +| `DecodedColumn` | a type that parses only from characters (decoded first) | +| `NullableColumn` | `T?` of a `Column` type; a blank column is `null` | +| `NullableDecodedColumn` | `T?` of a `DecodedColumn` type, such as `DateOnly?` | +| `ConvertedColumn` | any type, through an `IFixedWidthConverter` instance | +| `NullableConvertedColumn` | `T?` of a value type, through an `IFixedWidthConverter` instance | + +`FixedWidthColumnOptions` carries the same settings as `[FixedColumn]`, with the same defaults (see [Formatting Options](#formatting-options)). A reference-type model needs `Factory` to be read; a value type without one starts from `default`. Pass `null` as the getter or the setter to build a layout that only reads or only writes; using it the other way throws `InvalidOperationException` before any line is touched. The column name appears in error messages. + +## Formatting Options + +Each column can be tuned through named attribute arguments: + +```csharp +[FixedColumn(0, 8, Alignment = Alignment.Right, Padding = '0')] public int Id { get; init; } // "00000042" +[FixedColumn(8, 10, Format = "F2")] public double Amount { get; init; } // "1234.50 " +[FixedColumn(18, 5, Overflow = OverflowBehavior.Truncate)] public string Code { get; init; } +[FixedColumn(23, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public int Count { get; init; } +``` + +- **`Alignment`**: `Left` (default) or `Right`. +- **`Padding`**: fill byte when writing (default space; for example `'0'` for zero-padding). Must be ASCII. +- **`TrimChar`** / **`TrimMode`**: what to strip when parsing, and from which side. Must be ASCII. +- **`Format`**: format string for `ISpanFormattable` when writing, and the exact format for `DateTime`, `DateOnly`, `TimeOnly` and `DateTimeOffset` when parsing. +- **`Overflow`**: `Default`, `Truncate` or `Throw`. `Default` resolves per type: strings truncate, everything else throws. A truncated string keeps whole characters, so a multi-byte character is never split. +- **`Converter`**: a type implementing `IFixedWidthConverter`, used instead of the built-in parsing and formatting. + +## Column Types + +`string`, `double` and `float`, enums, any `IUtf8SpanParsable` (parsed straight from bytes), any other `ISpanParsable` (decoded first), `T?` of any of them (a blank column is `null`), and anything with a `Converter`. Writing needs `string`, `ISpanFormattable` (enums included) or a converter. + +An enum column reads a member's name in any case or its number (`2`, `02`), and rejects a number no member has and a combination of `[Flags]` members. It writes the name; `Format = "D"` writes the number, which `Padding = '0'` and `Alignment.Right` zero-pad. Fluent layouts add one with `EnumColumn` or `NullableEnumColumn`. + +```csharp +public enum Side { Buy = 1, Sell = 2 } + +[FixedColumn(0, 2, Format = "D", Padding = '0', Alignment = Alignment.Right)] +public Side Side { get; set; } // "02" reads and writes as Side.Sell +``` + +## Encodings + +Column positions are bytes, so the encoding only decides how text inside a column is read and written. UTF-8 files with ASCII text and single-byte encodings (Latin-1, Windows-1252, …) keep one byte per character. UTF-16 and EBCDIC are not supported. + +```csharp +var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; +var records = FixedWidth.ReadFile("legacy.txt", options); +``` + +On .NET, Windows-1252 and other code pages need `Encoding.RegisterProvider(CodePagesEncodingProvider.Instance)` first. + +## Validation + +Invalid layouts fail fast with an `InvalidOperationException` from `FixedWidthLayout.Build` or `FromAttributes`, or a generator diagnostic on generated models: negative `Start`, `Length < 1`, overlapping columns, a non-ASCII `Padding` or `TrimChar` (FWP012), an unsupported column type and a converter for the wrong type. Adjacent columns are valid. An attribute model that can be read but not written (or the reverse), such as one with a column type that only formats, gets a layout anyway; the side it cannot do throws on first use. diff --git a/docs/guide/reading.md b/docs/guide/reading.md new file mode 100644 index 0000000..d06dd32 --- /dev/null +++ b/docs/guide/reading.md @@ -0,0 +1,118 @@ +# Reading + +```csharp +using FixedWidthParser; + +var layout = FixedWidthLayout.FromAttributes(); + +foreach (var person in FixedWidth.ReadFile("people.txt", layout)) +{ + // ... +} + +foreach (var person in FixedWidth.ReadFile("people.txt")) +{ + // generated TryParse for each line +} +``` + +`Read(Stream, …, leaveOpen)` reads a stream once; `ReadFile` reopens the file on each enumeration. Reading is lazy and reuses a pooled buffer; each line is parsed in place, without a string per line. A layout is immutable: build it once and share it across reads and threads. + +Empty lines (including a trailing newline at end of file) are skipped: they are counted toward the line number but not yielded. Lines end at `\n`, with a preceding `\r` dropped, and a leading UTF-8 byte order mark is skipped. + +## Async + +```csharp +await foreach (var person in FixedWidth.ReadFileAsync("people.txt", layout)) +{ + // ... +} +``` + +`ReadAsync(Stream, leaveOpen)` mirrors the synchronous overload, and cancellation is honored via `WithCancellation`. A [`PipeReader`](https://learn.microsoft.com/dotnet/standard/io/pipelines) — a Kestrel request body, a socket — is read through `pipeReader.AsStream()`. + +## Parallel aggregation + +To fold a large file into one result — totals, counts, groups — read it on several threads: + +```csharp +public sealed class Payroll : IRecordAccumulator +{ + public double Total { get; private set; } + + public void Add(in Person person) + { + Total += person.Salary; + } + + public void Merge(Payroll following) + { + Total += following.Total; + } +} + +var payroll = await FixedWidth.AggregateFileAsync("people.txt", layout); +// GeneratedPayroll: the same accumulator over GeneratedPerson. +var generated = await FixedWidth.AggregateFileAsync("people.txt"); +``` + +The file is cut into byte ranges, each folded into its own accumulator on a worker thread, and the +accumulators are merged left to right in file order. `Add` must touch only its own instance; `Merge` +runs on one thread. Every line is parsed exactly once, and a failing line reports the same line number +as a sequential read. `FixedWidthReaderOptions.DegreeOfParallelism` caps the threads (`0`, the +default, means one per processor; `1` reads the file in one pass); other reads ignore it. `OnParseError`, when set, runs on +the worker threads. + +`AggregateAsync` does the same over a `Stream`, which is read once: one thread reads it into chunks +and the others parse. + +## Options + +Every read takes an optional `FixedWidthReaderOptions`: + +```csharp +var options = new FixedWidthReaderOptions +{ + FormatProvider = CultureInfo.GetCultureInfo("pt-BR"), + Encoding = Encoding.Latin1, + StringPool = new StringPool(), + BufferSize = 64 * 1024, + MaxLineLength = 4096, + OnParseError = ex => { log.Warning(ex.Message); return true; }, // skip the line +}; +var people = FixedWidth.ReadFile("people.txt", layout, options); +``` + +| Option | Default | Meaning | +|---|---|---| +| `FormatProvider` | invariant culture | Numbers and dates. | +| `Encoding` | UTF-8 | Text columns. Must be ASCII-compatible. | +| `StringPool` | none | Interns repeated string values. | +| `BufferSize` | 65,536 | Bytes read from the source at a time. | +| `MaxLineLength` | 1,048,576 | Longest line accepted; a longer one throws `InvalidDataException`. | +| `OnParseError` | throw | Return `true` to skip a line that fails to parse. | +| `DegreeOfParallelism` | processor count | Threads for `AggregateFileAsync`/`AggregateAsync`; `1` reads in one pass. | + +## Errors + +A line that fails to parse throws `FixedWidthParseException`, a `FormatException` that says where: + +```text +Line 2 could not be parsed into Person: column 'Age' (start 10, length 5) holds "XX ". +``` + +`LineNumber`, `ColumnName`, `ColumnStart`, `ColumnLength` and `RawValue` carry the same information. A line shorter than the layout reports how long it is and how long the layout needs it to be. The parser pays nothing for this on lines that parse: a failed line is parsed a second time to find the column. + +## Untrusted input + +A reader buffers a line until it finds its line break. `MaxLineLength` bounds that buffer, so a request body or file without line breaks throws `InvalidDataException` instead of growing the buffer until memory runs out. Set it close to your layout's line length when the source is untrusted. + +## Single Lines + +```csharp +FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, default, out Person person); + +FixedWidth.TryParse(lineBytes, new FixedWidthContext { FormatProvider = culture }, out var generated); +``` + +`FixedWidthContext` carries the format provider, the encoding and the string pool for one line; `default` means invariant culture, UTF-8 and no pool. diff --git a/docs/guide/versioning.md b/docs/guide/versioning.md new file mode 100644 index 0000000..f93d1f1 --- /dev/null +++ b/docs/guide/versioning.md @@ -0,0 +1,7 @@ +# Versioning + +The package follows [Semantic Versioning](https://semver.org/). A minor version may add types, members, options and enum values, but does not remove or change anything; every pull request checks the package, binary-compatibly, against the last release. + +- **Settings** arrive as new properties on the options objects (`FixedWidthReaderOptions`, `FixedWidthWriterOptions`, `FixedWidthColumnOptions`) or on `FixedWidthContext`, never as a new parameter on an existing method. +- **Interfaces you implement** — `IFixedWidthModel` (normally through the generator), `IFixedWidthConverter` and `IRecordAccumulator` — never gain a required member. Anything added comes with a default implementation (`static virtual` or a default interface method). +- **`FixedWidthParser.Infrastructure`** holds what is public only because generated code calls it (`FixedWidthRuntime`, `ColumnParseOptions`, `ColumnFormatOptions`) or because a method returns it so `foreach` allocates nothing (the `*RecordEnumerable`/`*RecordEnumerator` types and the `*LineParser` strategies). These types are hidden from IntelliSense and not meant for direct use, but they follow the same rules as everything else: code compiled against one 2.x version, generated code included, keeps working with every later 2.x version. A new setting for generated code is a new constructor overload on the options structs. diff --git a/docs/guide/writing.md b/docs/guide/writing.md new file mode 100644 index 0000000..6db549b --- /dev/null +++ b/docs/guide/writing.md @@ -0,0 +1,17 @@ +# Writing + +```csharp +var people = new[] +{ + new Person { Name = "John Doe", Age = 30, Salary = 60000 }, + new Person { Name = "Jane", Age = 28, Salary = 55000 }, +}; + +using var stream = File.Create("out.txt"); +FixedWidth.WriteMany(stream, people.AsSpan(), layout); +FixedWidth.WriteMany(stream, generatedPeople); // generated models need no layout +``` + +`Write`, `WriteMany` (span or enumerable) and their async versions write each record followed by `NewLine`, batched into one pooled buffer. `FixedWidthWriterOptions` sets the `FormatProvider`, the `Encoding` and the `NewLine` (default `Environment.NewLine`). `FixedWidth.TryFormat(in model, layout, Span, in FixedWidthContext, out int)`, or its generated overload without the layout, formats one line into your own buffer. + +Column alignment, padding, format strings and overflow are set per column: see [Formatting Options](models.md#formatting-options). diff --git a/docs/parallel-aggregate.md b/docs/parallel-aggregate.md new file mode 100644 index 0000000..2b693b1 --- /dev/null +++ b/docs/parallel-aggregate.md @@ -0,0 +1,55 @@ +# Parallel aggregation + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, `net10.0`, Release. + +A Stopwatch console app over a 104 MB file of 4,000,000 lines, each a 10-byte string, a 5-byte `int` +and a 10-byte `double`. The accumulator sums the `int` and the `double`. Each figure is the best of +five runs after one warm-up run, with the file in the OS cache; every run's total is checked against +the first. The sequential rows are a `foreach` over `ReadFile` into the same accumulator. + +## Result + +| Path | Sequential `foreach` | dop 1 | dop 2 | dop 4 | dop 8 | dop 16 | +|---|--:|--:|--:|--:|--:|--:| +| runtime | 246.1 ms | 260.9 | 136.1 | 83.2 | 54.2 | 49.7 | +| generated | 210.0 ms | 183.9 | 103.9 | 57.5 | 42.0 | 41.2 | + +Scaling is close to linear up to 4 workers and flattens past the 8 physical cores. Each line still +allocates its `string` column, so the GC and memory bandwidth are the likely ceiling; a model without +string columns, or with a `StringPool`, was not measured. + +`dop 1` is one range through the same worker code as the parallel case, and lands within a few +percent of the sequential enumerator on both paths. + +## Stream + +Same file, generated path, the source opened fresh for each run. The stream engine runs `dop` parsing +workers beside the producer, so `dop 8` is eight parsing threads and one reading flow. The pipe rows +read a `PipeReader.Create(stream, ...)` through `AsStream()`, with the segment size shown. + +| Source | dop 1 | dop 2 | dop 4 | dop 8 | dop 16 | +|---|--:|--:|--:|--:|--:| +| file (`AggregateFileAsync`) | 181.5 | 104.5 | 61.8 | 46.8 | 46.4 | +| `Stream` | 192.2 | 85.1 | 44.9 | 35.6 | 31.8 | +| `PipeReader.AsStream()`, 64 KB segments | 194.8 | 78.1 | 46.8 | 38.0 | 39.6 | +| `PipeReader.AsStream()`, default 4 KB segments | 423.7 | 108.1 | 111.9 | 112.5 | 109.2 | + +A pipe with 4 KB segments caps the run at the speed one thread can copy 4 KB reads into chunks, so +the pipe's own buffer size, not the engine, sets the ceiling. + +These rows predate merging the stream engine's results as they complete, which bounds its memory (each +chunk now holds a slot until it is merged). A Stopwatch check afterwards, on a 2,000,000-line file of the +same model, put the `Stream` engine at `dop 8` at 8.2–8.7 ns a line against 8.9 here; the table was not +re-run. + +The stream engine beats the file engine from `dop 2` up on this cached file. The file engine's ranges +are 3.25 MB at `dop 8` against the stream engine's 256 KB chunks, and each file worker refills a 64 KB +buffer with one `RandomAccess.Read` call at a time. Measured warm, the read size is not it, and the +gap is nearer 10 % than these figures suggest ([reader-performance.md](reader-performance.md)). + +## Not measured + +- A cold file: every worker reads its own range, so on a spinning disk the access pattern is random. +- Files on a network share. diff --git a/docs/performance/benchmarks.md b/docs/performance/benchmarks.md new file mode 100644 index 0000000..bb1a933 --- /dev/null +++ b/docs/performance/benchmarks.md @@ -0,0 +1,81 @@ +# Benchmarks against other libraries + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, .NET SDK 11.0.100-rc.2, `net10.0` (.NET 10.0.13), +Release. BenchmarkDotNet 0.15.8, default job, `MemoryDiagnoser`. Measured on 2.0 (`feat/v2-api`, October 2026). + +The model has three columns: a 10-byte string, a 5-byte `int` and a 10-byte `double`, 25 bytes a line. +The other libraries are [FileHelpers](https://www.filehelpers.net/) 3.5.2, +[FlatFiles](https://github.com/jehugaleahsa/FlatFiles) 5.0.4 and +[RecordParser](https://github.com/leandromoh/RecordParser) 2.3.0, with parallelism off. + +The comparison is not like-for-like, by design: + +- **Reading.** The others read text, so they are handed the already-decoded `string`; FixedWidthParser + reads the file's bytes from a `MemoryStream`, which is what a caller holding the file would give it. + The others are spared the UTF-8 decode. +- **Writing.** Everything targets `Stream.Null`, to measure formatting rather than I/O. The others write + text through a `StreamWriter`; FixedWidthParser writes bytes straight to the stream. + +Run them with: + +```bash +dotnet run -c Release --project tests/Benchmarks/Benchmarks.csproj -- --filter "*Comparison*" +``` + +Reports are written to `tests/Benchmarks/BenchmarkDotNet.Artifacts/results`; the +[live results](https://gabrielmarquezmatte.github.io/FixedWidthParser/dev/bench/) track the library's own suite +across commits. + +## Reading + +N records, summing one column. + +| Method | Count | Mean | Error | StdDev | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | +|--------------------------- |------ |-----------:|----------:|----------:|------:|--------:|--------:|-------:|----------:|------------:| +| FixedWidthParser_Generated | 100 | 3.851 μs | 0.0131 μs | 0.0110 μs | 0.18 | 0.00 | 0.2365 | - | 3.89 KB | 0.09 | +| FixedWidthParser_Read | 100 | 5.268 μs | 0.1010 μs | 0.1994 μs | 0.24 | 0.01 | 0.2365 | - | 3.89 KB | 0.09 | +| RecordParser_Read | 100 | 7.956 μs | 0.0140 μs | 0.0117 μs | 0.37 | 0.00 | 0.2441 | - | 4.23 KB | 0.10 | +| FileHelpers_Read | 100 | 21.708 μs | 0.3205 μs | 0.2503 μs | 1.00 | 0.02 | 2.5330 | 0.0610 | 41.49 KB | 1.00 | +| FlatFiles_Read | 100 | 41.812 μs | 0.8312 μs | 0.6489 μs | 1.93 | 0.04 | 6.2866 | 0.1831 | 103.13 KB | 2.49 | +| | | | | | | | | | | | +| FixedWidthParser_Generated | 1000 | 40.220 μs | 0.7687 μs | 1.4248 μs | 0.18 | 0.01 | 2.3193 | - | 38.34 KB | 0.09 | +| FixedWidthParser_Read | 1000 | 50.214 μs | 0.2093 μs | 0.1748 μs | 0.23 | 0.00 | 2.3193 | - | 38.34 KB | 0.09 | +| RecordParser_Read | 1000 | 73.700 μs | 0.1759 μs | 0.1560 μs | 0.33 | 0.00 | 2.3193 | - | 38.69 KB | 0.10 | +| FileHelpers_Read | 1000 | 220.773 μs | 1.1137 μs | 0.9872 μs | 1.00 | 0.01 | 24.6582 | 5.8594 | 406.42 KB | 1.00 | +| FlatFiles_Read | 1000 | 414.094 μs | 5.4961 μs | 4.8721 μs | 1.88 | 0.02 | 58.1055 | 1.9531 | 953.2 KB | 2.35 | + +The generated model reads about 5.5 times faster than FileHelpers and the attribute layout about 4.2 times, +both with a tenth of its allocation, which is the strings the records hold. RecordParser, the fastest of +the others, takes 1.8 to 2.1 times as long as the generated model. + +## Writing + +N records written to `Stream.Null` through the attribute layout. + +| Method | Count | Mean | Error | StdDev | Median | Ratio | RatioSD | Gen0 | Gen1 | Allocated | Alloc Ratio | +|----------------------- |------ |--------------:|-------------:|-------------:|--------------:|-------:|--------:|--------:|--------:|----------:|------------:| +| FixedWidthParser_Write | 1 | 66.84 ns | 0.273 ns | 0.256 ns | 66.80 ns | 0.52 | 0.02 | - | - | - | 0.00 | +| FileHelpers_Write | 1 | 128.67 ns | 2.606 ns | 6.294 ns | 126.76 ns | 1.00 | 0.07 | 0.0215 | - | 360 B | 1.00 | +| FlatFiles_Write | 1 | 273.24 ns | 3.325 ns | 3.110 ns | 272.65 ns | 2.13 | 0.10 | 0.0424 | - | 712 B | 1.98 | +| RecordParser_Write | 1 | 15,581.73 ns | 550.422 ns | 1,570.385 ns | 14,975.60 ns | 121.37 | 13.44 | 4.0283 | 0.6714 | 65384 B | 181.62 | +| | | | | | | | | | | | | +| FixedWidthParser_Write | 100 | 5,159.48 ns | 8.964 ns | 8.385 ns | 5,156.54 ns | 0.43 | 0.00 | - | - | - | 0.00 | +| FileHelpers_Write | 100 | 11,897.36 ns | 77.800 ns | 64.966 ns | 11,890.04 ns | 1.00 | 0.01 | 1.9531 | - | 32832 B | 1.00 | +| FlatFiles_Write | 100 | 30,719.14 ns | 510.520 ns | 452.562 ns | 30,769.02 ns | 2.58 | 0.04 | 4.2114 | - | 71200 B | 2.17 | +| RecordParser_Write | 100 | 54,366.37 ns | 1,017.927 ns | 952.169 ns | 54,458.00 ns | 4.57 | 0.08 | 4.0283 | 0.6104 | 67696 B | 2.06 | +| | | | | | | | | | | | | +| FixedWidthParser_Write | 1000 | 57,478.75 ns | 65.891 ns | 61.634 ns | 57,493.25 ns | 0.47 | 0.00 | - | - | - | 0.00 | +| FileHelpers_Write | 1000 | 121,174.48 ns | 500.343 ns | 443.541 ns | 121,058.04 ns | 1.00 | 0.00 | 19.5313 | - | 328032 B | 1.00 | +| FlatFiles_Write | 1000 | 262,811.90 ns | 3,274.975 ns | 3,063.413 ns | 262,086.38 ns | 2.17 | 0.03 | 42.4805 | - | 712000 B | 2.17 | +| RecordParser_Write | 1000 | 346,479.69 ns | 5,197.862 ns | 4,862.083 ns | 344,529.69 ns | 2.86 | 0.04 | 53.7109 | 52.7344 | 880427 B | 2.68 | + +The writer takes about half of FileHelpers' time and allocates nothing, whatever the record count. + +## Not measured + +- Files on disk, cold or cached: every figure here is in memory. +- RecordParser and the parallel modes of FixedWidthParser with parallelism on; + [parallel-aggregate.md](../parallel-aggregate.md) covers FixedWidthParser's own. +- Other models: wider lines, `decimal` and date columns, non-ASCII text. diff --git a/docs/pipe-reader.md b/docs/pipe-reader.md new file mode 100644 index 0000000..438a0d6 --- /dev/null +++ b/docs/pipe-reader.md @@ -0,0 +1,28 @@ +# Reading a `PipeReader` + +Through 2.0 development the library read a `PipeReader` with its own enumerator, +`PipeRecordEnumerator`, which split lines over `ReadOnlySequence` and parsed a line in place when +it sat in one segment. It was removed in favour of `pipeReader.AsStream()` into the `Stream` reader. + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, `net10.0`, Release. A Stopwatch console app reads +a 104 MB file of 4,000,000 lines, held in memory as a `MemoryStream`, through the generated path and +sums two columns. Best of seven runs after one warm-up run. + +## Result + +| Read | 4 KB segments | 64 KB segments | +|---|--:|--:| +| `ReadAsync(PipeReader)`, parsed in the segment | 472.0 ms | 474.6 ms | +| `ReadAsync(pipeReader.AsStream())` | 230.4 ms | 187.3 ms | +| `ReadAsync(Stream)` over the `MemoryStream` | 174.4 ms | | + +Avoiding the copy did not pay for itself. The dedicated enumerator's `MoveNextAsync` was `async` on +every line, with no synchronous path for a line already in the buffer, and built a `SequenceReader` +and several `SequencePosition`s per line. The `Stream` path copies a block at a time and then only +slices spans. Which of those costs dominated was not profiled. + +The enumerator also duplicated every line rule — the byte order mark, `\r\n`, empty lines, a last +line without a break, `MaxLineLength` — that `LineBufferState` already implements, so removing it +leaves one line splitter for every reader. diff --git a/docs/python-benchmarks.md b/docs/python-benchmarks.md new file mode 100644 index 0000000..6e96904 --- /dev/null +++ b/docs/python-benchmarks.md @@ -0,0 +1,143 @@ +# Python: fixedwidth against pandas and polars + +`python/benchmarks/bench_read.py` generates a synthetic file shaped like B3's COTAHIST quote history (245-byte +lines, all 26 fields of record type 01, a 00 header and a 99 trailer; layout in `python/benchmarks/cotahist.py`), +checks that every reader returns the same values as `pandas.read_fwf` given the same column specs (fixedwidth +exactly, the polars reference within floating-point rounding), and times each one, best of several runs. +`python/benchmarks/bench_write.py` writes the same table back out (see [Writing](#writing)). + + python python/benchmarks/bench_read.py --lines 1000000 --runs 5 + python python/benchmarks/bench_write.py --lines 1000000 --runs 5 + +`read_fwf` uses its Python engine, which `skipfooter` needs to drop the trailer. The generator caps a day's +quantity at 10^9 shares, so a volume has at most 15 digits and survives a FLOAT64 column exactly; the writers +could not reproduce a 16-digit one from a float64 either. + +## Reading + +1,000,000 lines (235 MiB). AMD Ryzen 7 5700X (8 cores, 16 threads), Windows 10, Python 3.14.7, pandas 3.0.5, +pyarrow 26.0.0, polars 1.44.1. `read_fwf` best of 1 run, the others best of 5. + +| Reader | Seconds | vs read_fwf | +|---|---:|---:| +| pandas.read_fwf | 15.353 | 1.0x | +| polars str.slice (reference) | 0.217 | 71x | +| fw.read_pandas | 0.160 | 96x | +| fw.read_pandas, 1 thread | 0.625 | 25x | +| fw.read_polars | 0.135 | 114x | +| fw.read_arrow | 0.123 | 125x | +| fw.read (NumPy views) | 0.125 | 123x | +| fw.read, 1 thread | 0.581 | 26x | + +A single thread is already 26x faster than `read_fwf`; the default, one thread per processor, adds another 4.6x. +`fw.read_pandas` costs about 40 ms over `fw.read_arrow`, pyarrow's conversion to pandas. The sections below trace +how the read got here; their before and after columns come from runs of their own. + +## Writing + +`bench_write.py` reads the generated file once, before any timing, into pandas, polars and pyarrow tables, and +times each writer turning that table back into the record lines. The expected output is the generated file's +body, so every writer must produce it byte for byte; the script asserts that. Neither pandas nor polars writes +fixed-width files, so the baselines are what their users write instead: + +- `numpy.savetxt` over the pandas columns with one printf field per column (`%-12s`, `%013d`), floats scaled to + their implied decimals and dates turned into `yyyymmdd` integers first. +- polars string expressions: `pad_end` for text, the scaled integers zero-padded with the sign kept leftmost, + `strftime` for dates, joined with `concat_str` into one column written by `write_csv`. + +Same machine and versions; `numpy.savetxt` best of 1 run, the others best of 5. + +| Writer | Seconds | vs savetxt | +|---|---:|---:| +| numpy.savetxt (pandas) | 4.288 | 1.0x | +| polars pad + concat_str | 0.710 | 6.0x | +| fw.write (pandas) | 0.221 | 19x | +| fw.write (polars) | 0.190 | 23x | +| fw.write (pyarrow) | 0.148 | 29x | +| fw.write (pyarrow), 1 thread | 0.559 | 7.7x | +| fw.write_bytes (pyarrow) | 0.337 | 13x | + +`fw.write` is 4.8x faster than the polars pipeline. A pandas frame costs ~70 ms more than a pyarrow table, its +conversion to Arrow; `write_bytes` timing includes copying the 235 MB into a `bytes` and writing it to the file. +Of `fw.write`'s 0.15 s, ~20 ms is the Python side handing the columns over and the rest the native write. + +### Making the native write faster + +The first run measured 0.259 s, ~240 ms of it native. Per value, a zero-padded integer (16 of the 26 columns) +cost 19 ns and a `yyyyMMdd` date 47 ns. At 16 threads, `fw_write_file` took 240 ms where `fw_write_memory` +took 141 ms, with the two equal on one thread. Windows runs one synchronous I/O at a time on a file handle, so +the parallel batches queued for it. + +- The output file is opened for asynchronous I/O. Each batch still waits for its own write, but they run + together: 240 → 170 ms. +- A right-aligned integer with no format writes its digits from the right straight into the column, then the + padding, with "-" kept leftmost: 19 → 9 ns. Formatting at the start and aligning afterwards, as before, cost + a copy, and counting the digits first cost a second pass. +- A date or time whose format is only digit fields and `- . T / :` reuses the reader's compiled `FastDate` + to write its digits: 47 → 23 ns. + +A test checks both against the BCL's char path in four cultures, both alignments, and zero and space padding. + +| Writer | Before | After | +|---|---:|---:| +| fw.write (pyarrow) | 0.259 s | 0.162 s | +| fw.write (pyarrow), 1 thread | 0.838 s | 0.550 s | + +## Fast paths for dates and integers + +The two `yyyyMMdd` date columns cost 85 ns each through `DateOnly.TryParseExact` on the decoded text, and the +sixteen integer and implied-decimal columns 12–14 ns each through `long.TryParse`. A format made only of +fixed-width digit fields (`yyyy MM dd HH mm ss` and `- . T / :`) is now compiled once per column and read straight +from the bytes (18 ns), and a digits-only integer goes through fast_float's eight-digit step (9 ns), each falling +back to the BCL for anything else. Same machine, same run order, before and after: + +| Reader | Before | After | +|---|---:|---:| +| fw.read, 1 thread | 0.920 s | 0.678 s | +| fw.read | 0.261 s | 0.250 s | + +The parallel read barely moved, so parsing is not what bounds it. + +## The parallel read + +Timed inside the native read at 16 threads, the 0.25 s split into ~170 ms of partitioned parsing and ~40 ms of +concatenating the partitions' columns on one thread. Every partition was busy all along, yet the work took ~2.1 s +of thread time against ~0.7 s on one thread, and more threads made each partition slower (1.0x at 2 threads, +1.3x at 4, 1.7x at 8, 3x at 16). An in-memory source changed little, so it was not file I/O. What grows with +the threads is new memory: a read writes ~200 MB of columns, 50,000 demand-zero page faults on one thread and +twice that in parallel, since the concatenation copies everything into fresh memory again, and Windows takes +those faults under a per-process lock. + +Three changes, together: + +- A partition sizes its column buffers once from its byte count (fixed-width lines make that a row count) + instead of doubling them from 256 elements. +- `skip_invalid` skips a bad line without building its exception. Numbering the line meant counting line + breaks from the start of the file, on one thread, for B3's trailer at the very end: ~27 ms. +- The concatenation runs one column per thread. + +| Reader | Before | After | +|---|---:|---:| +| fw.read | 0.250 s | 0.169 s | +| fw.read, 1 thread | 0.678 s | 0.580 s | + +### Parsing into the final columns + +The concatenation still wrote every column into fresh memory a second time. Fixed-width lines fix where each +partition's rows land in the result: with the first line's length as the stride, a partition starting at byte +`b` starts at row `(b - dataStart) / stride`, rounded up. The read now allocates the final columns once and lends +each partition its slice; numeric, date and time values go straight there, and so do STRING bytes, with room +for the column's full width on every row. Publishing moves each partition down to close the gaps that skipped +lines and trimmed text leave, in memory already touched, and hands the buffers over. A partition with more +rows than its slice, as when line lengths vary, copies its column out, and that column is concatenated as +before. STRING offsets are lent too: each slice's index 0 is the previous slice's last offset, so only the first +slice writes the column's leading 0, and publishing rebases the rest in place. That last copy was ~32 MB here and +saved only about 4 ms. + +Same machine, same run order, best of 9: + +| Reader | Before | After | +|---|---:|---:| +| fw.read | 0.184 s | 0.153 s | +| fw.read_polars | 0.198 s | 0.165 s | +| fw.read, 1 thread | 0.588 s | 0.593 s | diff --git a/docs/reader-performance.md b/docs/reader-performance.md new file mode 100644 index 0000000..fd1ded6 --- /dev/null +++ b/docs/reader-performance.md @@ -0,0 +1,52 @@ +# Reader performance + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, .NET SDK 11.0.100-rc.2, `net10.0`, Release. A +Stopwatch console app reads 2,000,000 lines of a generated model (a 10-byte string, a 5-byte `int`, a +10-byte `double`) with `ReadFile`, summing every column. Each figure is the best of five runs after one +warm-up run, file in the OS cache. Before and after were measured in the same position of the same +program: the first configuration of each kind a run measures comes out slower while the JIT is still +tiering up, and comparing across positions misled the file-engine experiment below. + +## `double` columns in a comma culture + +csFastFloat parsed only when the decimal separator was `.`; any other culture decoded the column and +ran the BCL parser. It now runs with any one-ASCII-character separator when the negative sign is `-` +(`FixedWidthContext.DecimalSeparator`, which the writer already used), still falling back unless every +byte was consumed, so thousands separators keep going through the BCL. `CultureTests` checks the +fast path against `double.TryParse` in de-DE, pt-BR and fr-FR. + +| Read | Before | After | +|---|--:|--:| +| `.` separator, invariant | 57.1 ns/line | 58.5 | +| `,` separator, pt-BR | 98.4 ns/line | 58.5 | + +One value: 42.3 ns decoded and BCL-parsed, 7.7 ns through csFastFloat. csFastFloat has since been +replaced by a port of ExcelReader's `FastDouble`, which keeps the same rule: a fast parse counts only +when it takes the whole field. + +## `StringPool` + +CommunityToolkit's `StringPool.GetOrAdd(bytes, encoding)` costs 87 ns per 6-byte value against 19 ns +for `Encoding.UTF8.GetString` (76 ns even decoding on the stack first): hashing, a lock and its +priority map. A per-thread, 4-way set-associative cache of UTF-8 columns now sits in front of it, as +ExcelReader's `Utf8StringCache` does; a miss still goes to the pool. + +| Read | No pool | Pool, before | Pool, after | +|---|--:|--:|--:| +| 997 distinct names | 57.1 ns/line | 137.8 | 69.8 | +| 50 distinct names | 58.6 ns/line | — | 69.7 | + +A 256-set cache (ExcelReader's size) left the 997-name column at 119.5 ns/line: four ways overflowed +in many sets and the cyclic order missed every time. 1,024 sets (48 KB per thread that reads with a +pool) holds it. A pool still costs ~11 ns/line over allocating: short-lived strings are cheap, and +the pool exists to save memory, not time. + +## Not done: larger reads in `AggregateFileAsync` + +The file engine trailed the stream engine by 29–35 % in the +[parallel-aggregate](parallel-aggregate.md) runs, and each file worker refills a 64 KB buffer one +`RandomAccess.Read` at a time. Measured warm and in either order, reading 256 KB or 1 MB at a time +moved dop 8 from 21.2 to 19.5 ms at best, within run-to-run spread, and the gap to the stream engine +was ~10 % (19–21 ms against 17.4–18.3 ms), not 30 %. Not changed. diff --git a/docs/writer-performance.md b/docs/writer-performance.md new file mode 100644 index 0000000..cc9173b --- /dev/null +++ b/docs/writer-performance.md @@ -0,0 +1,69 @@ +# Writer performance + +## Machine and method + +AMD Ryzen 7 5700X (8C/16T), Windows 10 Pro 19045, `net10.0`, Release. + +Two harnesses: + +- BenchmarkDotNet, `--job short`, over 1,000 rows of the benchmark `SampleModel` (a 10-byte string, a + 5-byte `int`, a 10-byte `double`), `OperationsPerInvoke = 1000`, so every figure is per row. +- A Stopwatch console app for before/after: 2,000 warm-up rounds, then the best of five runs of 4,000 + rounds over the same 1,000 rows. Its absolute numbers run higher than BenchmarkDotNet's; only the + ratio between its two columns is meaningful. + +`Salary` is `1000.0 + i`, so every `double` is an integer value. + +## Where a row's time goes + +BenchmarkDotNet, before the shortcuts below: + +| Operation | ns/row | +|---|--:| +| whole line, runtime writer | 82.7 | +| whole line, generated model | 68.8 | +| `double` column (`FormatUtf8Value`) | 54.8 | +| `double.TryFormat` to UTF-8, alone | 46.1 | +| `double` as a scaled integer, alone | 8.1 | +| `int` column | 8.1 | +| `int.TryFormat` to UTF-8, alone | 1.7 | +| `string` column | 4.5 | + +The `double` column is two thirds of the line. Its cost is the shortest round-trip search, which .NET +runs the same way for `char` and `byte` output, so formatting straight to UTF-8 instead of UTF-16 and +transcoding saved nothing measurable on this model: the `WriterBenchmarks` suite moved within its own +noise. + +The runtime writer's extra ~14 ns over the generated model is one interface call and one compiled +getter delegate per column. Compiling the whole line into one delegate would remove it, but the +generated model already does, so it was not built. + +## The shortcuts + +**Scaled-integer `double`.** A value with at most four decimals and a magnitude under 1e9 is written as +`long` digits with the decimal separator inserted: the smallest scale whose quotient reproduces the +value exactly is the shortest round-trip form. It comes from ExcelReader's XLSX writer, which checks it +against `Utf8Formatter`; here `DoubleFormatTests` checks it against `double.ToString` over ~300,000 +values in the invariant culture and pt-BR. It applies only with no `Format` and a provider whose +decimal separator is one ASCII character and whose negative sign is `-`. + +**Formatting into the column.** `IUtf8SpanFormattable` values format straight into the line's slice +and are aligned there, instead of into a stack buffer that `WriteInto` then copies. The `int` column's +6 ns above `int.TryFormat` was that buffer and copy. + +## Result + +Stopwatch console app, per row: + +| Case | Before | After | +|---|--:|--:| +| runtime writer, `TryFormat` | 110.1 | 43.1 | +| generated model, invariant, `double` with two decimals | 101.3 | 42.6 | +| generated model, pt-BR, `double` with two decimals | 101.4 | 49.3 | +| runtime writer, `WriteMany` to `Stream.Null` | 116.0 | 57.5 | + +## Not done + +- `float` and `decimal` still take the BCL formatter. +- Formatting a batch in parallel: records have a fixed length, so record *i* starts at + `i * RecordLength` and batches split cleanly. Worth it only for very large writes; unmeasured. diff --git a/global.json b/global.json index 8f73781..ce2af11 100644 --- a/global.json +++ b/global.json @@ -1,5 +1,10 @@ { + "sdk": { + "version": "10.0.402", + "rollForward": "latestMinor", + "allowPrerelease": false + }, "test": { "runner": "Microsoft.Testing.Platform" } -} \ No newline at end of file +} diff --git a/python/README.md b/python/README.md new file mode 100644 index 0000000..9b78674 --- /dev/null +++ b/python/README.md @@ -0,0 +1,132 @@ +# fixedwidthparser + +Read and write fixed-width files from Python through FixedWidthParser's NativeAOT library: no .NET runtime, +parallel reads, results as NumPy views, Arrow, pandas or polars. + +```bash +pip install "fixedwidthparser[numpy]" # fw.read: NumPy views +pip install "fixedwidthparser[pandas]" # or [pyarrow], [polars] +``` + +Wheels ship for Windows x64, macOS (arm64, x64) and Linux (glibc and musl, x64 and arm64). The package installs +as `fixedwidthparser` and imports as `fixedwidth`. pyarrow has no musllinux wheels, so on Alpine and other +musl systems Arrow, pandas, polars and writing need a pyarrow built for them. + +```python +import fixedwidth as fw + +# B3's COTAHIST quote history: 245-byte lines, a header (00) and a trailer (99) around the quotes (01). +layout = fw.Layout([ + fw.Column("tipreg", 0, 2, fw.String), + fw.Column("data_pregao", 2, 8, fw.Date, format="yyyyMMdd"), + fw.Column("codneg", 12, 12, fw.String), + fw.Column("preabe", 56, 13, fw.Float64, scale=2), # N(11)V99: two implied decimals + fw.Column("quatot", 152, 18, fw.Int64), +]) +options = fw.ReadOptions(header_lines=1, skip_invalid=True) # the trailer fails to parse and is skipped + +df = fw.read_pandas("COTAHIST_A2023.TXT", layout, options) +``` + +## Reading + +| Function | Returns | +|---|---| +| `fw.read(source, layout, options)` | `fw.Table`: NumPy views over the native columns, no copy | +| `fw.read_arrow(source, layout, options)` | `pyarrow.Table` (needs pyarrow) | +| `fw.read_pandas(source, layout, options)` | `pandas.DataFrame` (needs pyarrow and pandas) | +| `fw.read_polars(source, layout, options)` | `polars.DataFrame` | +| `fw.read_batches(source, layout, fw.ReaderOptions(batch_rows=...))` | an iterator of `fw.Table` | +| `fw.read_arrow_stream(source, layout, fw.ReaderOptions(...))` | an Arrow PyCapsule stream for `polars.DataFrame(...)` or `pyarrow.RecordBatchReader.from_stream(...)` | + +`source` is a path, `bytes`/`bytearray`/`memoryview`/`io.BytesIO` to read from memory (a `BytesIO` in place, +from its position), or any stream with `readinto()` or `read()`: an open file, a pipe, `gzip.open(...)`, +`zipfile.ZipFile(...).open(...)`. `read_batches` and `read_arrow_stream` pull from a stream as the batches are +taken, in constant memory; the whole-table reads read it into memory first, since they split it into byte ranges. + +```python +import zipfile + +with zipfile.ZipFile("COTAHIST_A2023.ZIP") as archive, archive.open("COTAHIST_A2023.TXT") as stream: + for batch in fw.read_batches(stream, layout, fw.ReaderOptions(read=options, batch_rows=100_000)): + ... +``` + +Every optional setting lives in +`fw.ReadOptions`: `header_lines`, `skip_invalid`, `degree_of_parallelism` (0 = one thread per processor, +1 = sequential), `encoding` (a code page or `"latin-1"`, `"cp1252"`, ...), `max_line_length` and +`skip_byte_order_mark`. A parallel read gives the same rows, in the same order, as a sequential one. + +A line that fails to parse raises `fw.ParseError` with `line_number` and `column`, unless `skip_invalid` is set. + +`fw.Table` maps a column name to a NumPy array (a list for strings), and has `num_rows`, `column_names`, +`lines_read`, `lines_skipped`, `validity(name)` (a bool array, `True` where the row holds a value, or `None` +when the column has no nulls) and `to_arrow()`, which shares the native buffers. Without pyarrow, +`read_polars` reads sequentially through the Arrow stream. + +## Threads and asyncio + +Every function releases the GIL for its native work, so reads and writes on several threads, or through +`asyncio.to_thread`, run in parallel: + +```python +frames = await asyncio.gather(*(asyncio.to_thread(fw.read_polars, path, layout, options) for path in paths)) +``` + +- A `fw.Layout` is immutable and can be shared by every thread at once. +- A failure is raised on the thread whose call failed, with that call's message and line number. +- A `read_batches` iterator, and the stream it reads, belong to one thread at a time. +- A stream's `read()` is called on the thread doing the read and takes the GIL while it runs; Python's files, + sockets and `zlib` release it again while they wait or decompress, so other threads keep going. +- `degree_of_parallelism=0` gives each call one thread per processor. When many calls already run at once, pass + `1`: the calls are the parallelism, and splitting each one too only makes them queue for the same cores. + +## Columns + +`fw.Column(name, start, length, type, *, nullable, scale, format, trim_char, trim_mode, alignment, padding, overflow, dictionary)`. +Positions are bytes. Types: `fw.String`, `fw.Int64`, `fw.Float64`, `fw.Decimal` (int64 unscaled, Arrow +`decimal128(18, scale)`), `fw.Date`, `fw.Time`, `fw.Timestamp`. `scale` is the number of implied decimals: +`0000000123456` with `scale=2` reads as `1234.56`. `format` uses .NET date syntax (`yyyyMMdd`, `HHmmss`). +`trim_mode`, `alignment` and `overflow` take `fw.TrimMode`, `fw.Alignment` and `fw.Overflow`. + +`dictionary=True` on a `fw.String` column hands it to Arrow dictionary-encoded, so `read_pandas` gives a +`category` column and `read_polars` a `Categorical` one: for a column of few distinct values (a market code, a +currency, a record type) far less memory, and faster group-bys. The encoding costs a pass over the column, about +3 ms per million rows per column, run in parallel across columns. `fw.read`'s Table keeps plain strings, and its +`to_arrow()` encodes the same columns. Categorical columns write like strings. + +`fw.Layout(columns, fw.LayoutOptions(line_length=..., filler=...))` sets how written lines are laid out: +`line_length` past the last column pads each line, and `filler` (default a space) fills the gaps between +columns. + +## Writing + +```python +fw.write(df, "out.txt", layout, fw.WriteOptions(degree_of_parallelism=0)) +data = fw.write_bytes(df, layout) + +with gzip.open("out.txt.gz", "wb") as stream: # or an open file, a socket's makefile("wb"), ... + fw.write(df, stream, layout) +``` + +`fw.write` takes a path or a binary stream with `write()`. A stream gets the lines in order, a batch per call on +the calling thread, while the next batches format in parallel, so memory stays at a batch per thread; writing +to an open file is as fast as writing to its path. For the bytes themselves, `write_bytes` beats an `io.BytesIO`, +which copies itself each time it grows. + +Takes pandas, polars, pyarrow or anything with `__arrow_c_stream__` (needs pyarrow), matching columns by name. +`padding="0"` writes zero-padded numbers with the sign leftmost; a null writes as its column filled with +`trim_char`. Parallel writes are byte-identical to serial ones. `fw.WriteOptions` also takes `encoding`, +`newline` (default `"\n"`) and `batch_rows`. + +A `fw.Decimal` column stores its value times `10**scale` as an int64. Decimals and integers are scaled +exactly, and a decimal with more digits than `scale` raises. Floats round half to even to `scale`. +A value that is NaN, infinite or outside int64 once scaled raises before anything is written. + +## Building from source + + python python/scripts/build_native.py # requires the .NET 10 SDK; publishes the library into src/fixedwidth/_lib + pip install -e "python[dev,pandas,polars]" + pytest python/tests + +To point at a library built elsewhere, set `FIXEDWIDTH_NATIVE_LIB` to its full path. diff --git a/python/benchmarks/bench_read.py b/python/benchmarks/bench_read.py new file mode 100644 index 0000000..1d372b1 --- /dev/null +++ b/python/benchmarks/bench_read.py @@ -0,0 +1,123 @@ +"""Reads a synthetic COTAHIST file with fixedwidth and with pandas.read_fwf, checks they agree, and times them. + + python python/benchmarks/bench_read.py [--lines 1000000] [--runs 3] [--fwf-runs 1] +""" + +from __future__ import annotations + +import argparse +import platform +import sys +import tempfile +import time +from pathlib import Path + +import fixedwidth as fw +import numpy as np +import pandas as pd +import polars as pl + +sys.path.insert(0, str(Path(__file__).parent)) +import cotahist # noqa: E402 + +STRING_FIELDS = [name for name, _, _, kind, _ in cotahist.FIELDS if kind is fw.String] + + +def read_fwf(path: Path) -> pd.DataFrame: + frame = pd.read_fwf( + path, + colspecs=cotahist.colspecs(), + names=[name for name, *_ in cotahist.FIELDS], + dtype={name: str for name in STRING_FIELDS}, + skiprows=1, + skipfooter=1, + engine="python", + ) + for name, _, _, kind, scale in cotahist.FIELDS: + if kind is fw.Float64: + frame[name] = frame[name] / 10**scale + if kind is fw.Date: + frame[name] = pd.to_datetime(frame[name].astype(str), format="%Y%m%d").dt.date + return frame + + +def polars_slices(path: Path) -> pl.DataFrame: + lines = pl.read_csv(path, has_header=False, separator="\x01", new_columns=["line"], quote_char=None, skip_rows=1) + lines = lines.filter(pl.col("line").str.starts_with("01")) + expressions = [] + for name, first, length, kind, scale in cotahist.FIELDS: + text = pl.col("line").str.slice(first - 1, length).str.strip_chars() + if kind is fw.String: + expressions.append(text.alias(name)) + elif kind is fw.Date: + expressions.append(text.str.to_date("%Y%m%d").alias(name)) + elif kind is fw.Float64: + expressions.append((text.cast(pl.Int64) / 10**scale).alias(name)) + else: + expressions.append(text.cast(pl.Int64).alias(name)) + return lines.select(expressions) + + +def check(native: pd.DataFrame, expected: pd.DataFrame, exact: bool = True) -> None: + assert len(native) == len(expected), (len(native), len(expected)) + for name, _, _, kind, _ in cotahist.FIELDS: + left, right = native[name], expected[name] + if kind is fw.String: + assert left.fillna("").tolist() == right.fillna("").tolist(), name + elif kind is fw.Date: + assert [pd.Timestamp(v).date() for v in left] == [pd.Timestamp(v).date() for v in right], name + else: + same = np.array_equal if exact else np.allclose + assert same(left.to_numpy(dtype=float), right.to_numpy(dtype=float)), name + + +def best(runs: int, work) -> float: + times = [] + for _ in range(runs): + start = time.perf_counter() + work() + times.append(time.perf_counter() - start) + return min(times) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lines", type=int, default=1_000_000) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--fwf-runs", type=int, default=1, help="read_fwf is slow; one run is usually enough") + args = parser.parse_args() + + layout = cotahist.layout() + options = fw.ReadOptions(header_lines=1, skip_invalid=True) + sequential = fw.ReadOptions(header_lines=1, skip_invalid=True, degree_of_parallelism=1) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "COTAHIST_SYNTHETIC.TXT" + cotahist.generate(path, args.lines) + size = path.stat().st_size / 2**20 + print(f"{args.lines:,} lines, {size:.0f} MiB, Python {platform.python_version()}, pandas {pd.__version__}, polars {pl.__version__}") + + expected = read_fwf(path) + check(fw.read_pandas(path, layout, options), expected) + check(fw.read_pandas(path, layout, sequential), expected) + check(polars_slices(path).to_pandas(), expected, exact=False) + print("values agree with pandas.read_fwf (fixedwidth exactly; the polars reference within rounding)") + + rows = [ + ("pandas.read_fwf", best(args.fwf_runs, lambda: read_fwf(path))), + ("polars str.slice (reference)", best(args.runs, lambda: polars_slices(path))), + ("fw.read_pandas", best(args.runs, lambda: fw.read_pandas(path, layout, options))), + ("fw.read_pandas, 1 thread", best(args.runs, lambda: fw.read_pandas(path, layout, sequential))), + ("fw.read_polars", best(args.runs, lambda: fw.read_polars(path, layout, options))), + ("fw.read_arrow", best(args.runs, lambda: fw.read_arrow(path, layout, options))), + ("fw.read (NumPy views)", best(args.runs, lambda: fw.read(path, layout, options))), + ("fw.read, 1 thread", best(args.runs, lambda: fw.read(path, layout, sequential))), + ] + baseline = rows[0][1] + print(f"{'reader':<32}{'seconds':>10}{'vs read_fwf':>14}") + for name, seconds in rows: + print(f"{name:<32}{seconds:>10.3f}{baseline / seconds:>13.1f}x") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/benchmarks/bench_write.py b/python/benchmarks/bench_write.py new file mode 100644 index 0000000..9c63500 --- /dev/null +++ b/python/benchmarks/bench_write.py @@ -0,0 +1,129 @@ +"""Writes a synthetic COTAHIST table with fixedwidth and with the ways pandas and polars users write one, checks +every writer produces the same bytes, and times them. + + python python/benchmarks/bench_write.py [--lines 1000000] [--runs 5] [--savetxt-runs 1] + +The table is read once, before any timing, from a generated file; every writer then turns that same table back +into its record lines, so the expected output is the file's body, byte for byte. pandas and polars have no +fixed-width writer: the baselines are numpy.savetxt with one printf field per column, and polars string +expressions padded and concatenated into one column written as CSV. +""" + +from __future__ import annotations + +import argparse +import platform +import sys +import tempfile +import time +from pathlib import Path + +import fixedwidth as fw +import numpy as np +import pandas as pd +import polars as pl +import pyarrow as pa + +sys.path.insert(0, str(Path(__file__).parent)) +import cotahist # noqa: E402 + + +def savetxt(frame: pd.DataFrame, path: Path) -> None: + columns, formats = [], [] + for name, _, length, kind, scale in cotahist.FIELDS: + column = frame[name] + if kind is fw.String: + columns.append(column.to_numpy(dtype=object)) + formats.append(f"%-{length}s") + elif kind is fw.Date: + dates = pd.to_datetime(column) + columns.append((dates.dt.year * 10000 + dates.dt.month * 100 + dates.dt.day).to_numpy()) + formats.append(f"%0{length}d") + elif kind is fw.Float64: + columns.append(np.round(column.to_numpy() * 10**scale).astype(np.int64)) + formats.append(f"%0{length}d") + else: + columns.append(column.to_numpy()) + formats.append(f"%0{length}d") + rows = np.empty((len(frame), len(columns)), dtype=object) + for index, values in enumerate(columns): + rows[:, index] = values + # A binary file, or Windows turns every "\n" into "\r\n". + with path.open("wb") as file: + np.savetxt(file, rows, fmt="".join(formats), newline="\n") + + +def zero_padded(integers: pl.Expr, length: int) -> pl.Expr: + # The sign stays leftmost, ahead of the zeros, as printf's %0Nd and fixedwidth write it. + digits = integers.abs().cast(pl.String) + return pl.when(integers < 0).then(pl.lit("-") + digits.str.pad_start(length - 1, "0")).otherwise(digits.str.pad_start(length, "0")) + + +def polars_strings(frame: pl.DataFrame, path: Path) -> None: + fields = [] + for name, _, length, kind, scale in cotahist.FIELDS: + column = pl.col(name) + if kind is fw.String: + fields.append(column.str.pad_end(length)) + elif kind is fw.Date: + fields.append(column.dt.strftime("%Y%m%d")) + elif kind is fw.Float64: + fields.append(zero_padded((column * 10**scale).round().cast(pl.Int64), length)) + else: + fields.append(zero_padded(column, length)) + frame.select(pl.concat_str(fields).alias("line")).write_csv(path, include_header=False, quote_style="never") + + +def best(runs: int, work, path: Path, expected: bytes) -> float: + times = [] + for _ in range(runs): + path.unlink(missing_ok=True) + start = time.perf_counter() + work() + times.append(time.perf_counter() - start) + assert path.read_bytes() == expected, f"{path.name} differs from the expected record lines" + return min(times) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--lines", type=int, default=1_000_000) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--savetxt-runs", type=int, default=1, help="numpy.savetxt is slow; one run is usually enough") + args = parser.parse_args() + + layout = cotahist.layout() + read = fw.ReadOptions(header_lines=1, skip_invalid=True) + sequential = fw.WriteOptions(degree_of_parallelism=1) + with tempfile.TemporaryDirectory() as directory: + source = Path(directory) / "COTAHIST_SYNTHETIC.TXT" + cotahist.generate(source, args.lines) + lines = source.read_bytes().splitlines(keepends=True) + expected = b"".join(lines[1:-1]) + arrow = fw.read_arrow(source, layout, read) + frame = fw.read_pandas(source, layout, read) + polars = pl.from_arrow(arrow) + size = len(expected) / 2**20 + print(f"{args.lines:,} lines, {size:.0f} MiB, Python {platform.python_version()}, pandas {pd.__version__}, " + f"polars {pl.__version__}, pyarrow {pa.__version__}") + + out = Path(directory) / "out.txt" + rows = [ + ("numpy.savetxt (pandas)", best(args.savetxt_runs, lambda: savetxt(frame, out), out, expected)), + ("polars pad + concat_str", best(args.runs, lambda: polars_strings(polars, out), out, expected)), + ("fw.write (pandas)", best(args.runs, lambda: fw.write(frame, out, layout), out, expected)), + ("fw.write (polars)", best(args.runs, lambda: fw.write(polars, out, layout), out, expected)), + ("fw.write (pyarrow)", best(args.runs, lambda: fw.write(arrow, out, layout), out, expected)), + ("fw.write (pyarrow), 1 thread", best(args.runs, lambda: fw.write(arrow, out, layout, sequential), out, expected)), + ("fw.write_bytes (pyarrow)", best(args.runs, lambda: out.write_bytes(fw.write_bytes(arrow, layout)), out, expected)), + ] + print("every writer produced the generated file's record lines, byte for byte") + baseline = rows[0][1] + print(f"{'writer':<32}{'seconds':>10}{'vs savetxt':>13}") + for name, seconds in rows: + print(f"{name:<32}{seconds:>10.3f}{baseline / seconds:>12.1f}x") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/benchmarks/cotahist.py b/python/benchmarks/cotahist.py new file mode 100644 index 0000000..173bb55 --- /dev/null +++ b/python/benchmarks/cotahist.py @@ -0,0 +1,119 @@ +"""B3's COTAHIST quote history layout (record type 01) and a synthetic file generator. + +Positions follow B3's "Layout do Arquivo - Cotações Históricas": 26 fields that tile bytes 1..245 +exactly (checked in `check_layout`). The real files are B3's and are not committed; `generate` writes a +file of the same shape with random quotes, a 00 header and a 99 trailer. +""" + +from __future__ import annotations + +import random +from pathlib import Path + +import fixedwidth as fw + +LINE_LENGTH = 245 + +# (name, 1-based first byte, length, type, scale) +FIELDS = [ + ("tipreg", 1, 2, fw.String, 0), + ("data_pregao", 3, 8, fw.Date, 0), + ("codbdi", 11, 2, fw.String, 0), + ("codneg", 13, 12, fw.String, 0), + ("tpmerc", 25, 3, fw.Int64, 0), + ("nomres", 28, 12, fw.String, 0), + ("especi", 40, 10, fw.String, 0), + ("prazot", 50, 3, fw.String, 0), + ("modref", 53, 4, fw.String, 0), + ("preabe", 57, 13, fw.Float64, 2), + ("premax", 70, 13, fw.Float64, 2), + ("premin", 83, 13, fw.Float64, 2), + ("premed", 96, 13, fw.Float64, 2), + ("preult", 109, 13, fw.Float64, 2), + ("preofc", 122, 13, fw.Float64, 2), + ("preofv", 135, 13, fw.Float64, 2), + ("totneg", 148, 5, fw.Int64, 0), + ("quatot", 153, 18, fw.Int64, 0), + ("voltot", 171, 18, fw.Float64, 2), + ("preexe", 189, 13, fw.Float64, 2), + ("indopc", 202, 1, fw.Int64, 0), + ("datven", 203, 8, fw.Date, 0), + ("fatcot", 211, 7, fw.Int64, 0), + ("ptoexe", 218, 13, fw.Float64, 6), + ("codisi", 231, 12, fw.String, 0), + ("dismes", 243, 3, fw.Int64, 0), +] + + +def check_layout() -> None: + position = 1 + for name, first, length, _, _ in FIELDS: + assert first == position, f"{name} starts at {first}, expected {position}" + position += length + assert position - 1 == LINE_LENGTH + + +def layout() -> fw.Layout: + check_layout() + return fw.Layout( + [ + fw.Column( + name, first - 1, length, kind, + scale=scale, + format="yyyyMMdd" if kind is fw.Date else None, + padding="0" if kind in (fw.Int64, fw.Float64) else None, + ) + for name, first, length, kind, scale in FIELDS + ] + ) + + +def colspecs() -> list[tuple[int, int]]: + return [(first - 1, first - 1 + length) for _, first, length, _, _ in FIELDS] + + +TICKERS = [f"{a}{b}{c}{d}{n}" for a in "ABCPV" for b in "AEIO" for c in "TRLN" for d in "SAEB" for n in (3, 4, 11)] + + +def _line(rng: random.Random, day: int) -> str: + ticker = rng.choice(TICKERS) + open_ = rng.randint(100, 9_999_999) + prices = [open_ + rng.randint(-500, 500) for _ in range(6)] + trades = rng.randint(1, 99_999) + # At most 10**9 shares keeps the volume (quantity * price) within the 15 digits a float64 round-trips. + quantity = rng.randint(100, 10**9) + return ( + "01" + + f"202310{day:02d}" + + "02" + + f"{ticker:<12}" + + "010" + + f"{ticker[:4] + ' SA':<12}" + + f"{'ON NM':<10}" + + " " + + "R$ " + + "".join(f"{p:013d}" for p in [open_, *prices]) + + f"{trades:05d}" + + f"{quantity:018d}" + + f"{quantity * open_ // 100:018d}" + + f"{0:013d}" + + "0" + + "99991231" + + "0000001" + + f"{0:013d}" + + f"BR{ticker[:4]}ACNOR{rng.randint(0, 9)}" + + f"{rng.randint(0, 999):03d}" + ) + + +def generate(path: Path, lines: int, seed: int = 2023) -> None: + rng = random.Random(seed) + header = f"{'00COTAHIST.2023BOVESPA 20231229':<{LINE_LENGTH}}" + trailer = f"{'99COTAHIST.2023BOVESPA 20231229' + f'{lines + 2:011d}':<{LINE_LENGTH}}" + with path.open("w", encoding="ascii", newline="\n") as file: + file.write(header + "\n") + for i in range(lines): + line = _line(rng, 2 + i * 27 // max(1, lines)) + assert len(line) == LINE_LENGTH, len(line) + file.write(line + "\n") + file.write(trailer + "\n") diff --git a/python/hatch_build.py b/python/hatch_build.py new file mode 100644 index 0000000..1a76fb3 --- /dev/null +++ b/python/hatch_build.py @@ -0,0 +1,60 @@ +"""Forces a platform-specific wheel tag: the wheel bundles a native library per OS +(see _lib/), so it isn't pure Python even though every .py file in it is. The package +has no CPython C-extension (it's ctypes calling a NativeAOT shared library), so it's +ABI/interpreter-independent - only the platform component of the tag should be specific. + +The platform component comes from the bundled library, not from the build machine: a +runner's own tag (macosx_26_0, manylinux_2_39) shuts out every older system the library +actually loads on.""" + +import platform +import re +import struct +from pathlib import Path + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface +from packaging.tags import platform_tags + +LIB_DIR = Path(__file__).parent / "src" / "fixedwidth" / "_lib" +LC_BUILD_VERSION = 0x32 + + +def glibc_floor(library: bytes) -> str: + """The newest GLIBC_2.N symbol version the library needs, as manylinux's `2_N`.""" + minors = [int(minor) for minor in re.findall(rb"GLIBC_2\.(\d+)", library)] + return f"2_{max(minors, default=17)}" + + +def macos_floor(library: bytes) -> str: + """The minimum macOS a thin 64-bit Mach-O declares in LC_BUILD_VERSION, as `MAJOR_0`.""" + commands = struct.unpack_from("> 16}_0" + offset += size + raise ValueError("the native library has no LC_BUILD_VERSION load command") + + +def platform_tag() -> str: + machine = platform.machine().lower() + system = platform.system() + if system == "Linux": + library = (LIB_DIR / "FixedWidthParser.Native.so").read_bytes() + if b"libc.musl-" in library: + return next(tag for tag in platform_tags() if tag.startswith("musllinux_")) + return f"manylinux_{glibc_floor(library)}_{machine}" + if system == "Darwin": + library = (LIB_DIR / "FixedWidthParser.Native.dylib").read_bytes() + return f"macosx_{macos_floor(library)}_{machine}" + return next(iter(platform_tags())) + + +class NativeLibraryBuildHook(BuildHookInterface): + def initialize(self, version, build_data): + build_data["pure_python"] = False + build_data["infer_tag"] = True + tag = next(iter(platform_tags())) if version == "editable" else platform_tag() + build_data["tag"] = f"py3-none-{tag}" diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 0000000..1cbe236 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = ["hatchling", "packaging"] +build-backend = "hatchling.build" + +[project] +name = "fixedwidthparser" +version = "0.1.0" +description = "Read and write fixed-width files through the FixedWidthParser NativeAOT library" +readme = "README.md" +requires-python = ">=3.10" +license = { text = "MIT" } +authors = [{ name = "Gabriel Matte" }] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", +] +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=7.0"] +numpy = ["numpy>=1.21"] +pyarrow = ["pyarrow>=14.0"] +pandas = ["pandas>=2.0", "pyarrow>=14.0"] +polars = ["polars>=1.0"] + +[project.urls] +Homepage = "https://github.com/GabrielMarquezMatte/FixedWidthParser" + +[tool.hatch.build.targets.wheel] +packages = ["src/fixedwidth"] +exclude = ["src/fixedwidth/_lib"] + +[tool.hatch.build.targets.wheel.force-include] +"src/fixedwidth/_lib" = "fixedwidth/_lib" + +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["tests"] diff --git a/python/scripts/build_musl_wheel.sh b/python/scripts/build_musl_wheel.sh new file mode 100644 index 0000000..b6f8358 --- /dev/null +++ b/python/scripts/build_musl_wheel.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Builds the linux-musl native library and its musllinux wheel in Alpine containers, running the +# Python tests against it. Needs a Linux host with Docker; the wheel lands in dist/. pyarrow has no +# musllinux wheels, so the pandas, polars, Arrow and write tests skip here. +# +# Run from the repository root: sh python/scripts/build_musl_wheel.sh +set -eu + +docker run --rm -e RunAnalyzers -v "$PWD:/repo" -w /repo mcr.microsoft.com/dotnet/sdk:10.0-alpine \ + sh -c 'apk add --no-cache clang build-base zlib-dev python3 && python3 python/scripts/build_native.py' + +docker run --rm -v "$PWD:/repo" -w /repo python:3.12-alpine \ + sh -c 'pip install -e "python[dev,numpy]" build && pytest python/tests && python -m build --wheel python/ --outdir dist/ && python python/scripts/check_wheel_tag.py dist' diff --git a/python/scripts/build_native.py b/python/scripts/build_native.py new file mode 100644 index 0000000..c50500f --- /dev/null +++ b/python/scripts/build_native.py @@ -0,0 +1,75 @@ +"""Publish FixedWidthParser.Native for this machine and copy the binary into the Python package. + +Run from anywhere: python python/scripts/build_native.py +""" + +import argparse +import os +import platform +import shutil +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +CSPROJ = REPO_ROOT / "src" / "FixedWidthParser.Native" / "FixedWidthParser.Native.csproj" +PACKAGE_LIB_DIR = REPO_ROOT / "python" / "src" / "fixedwidth" / "_lib" + +LIB_NAMES = { + "Windows": "FixedWidthParser.Native.dll", + "Linux": "FixedWidthParser.Native.so", + "Darwin": "FixedWidthParser.Native.dylib", +} + + +def default_rid() -> str: + system = platform.system() + machine = platform.machine().lower() + arch = "arm64" if machine in {"arm64", "aarch64"} else "x64" + if system == "Windows": + return f"win-{arch}" + if system == "Darwin": + return f"osx-{arch}" + if platform.libc_ver()[0] != "glibc": + return f"linux-musl-{arch}" + return f"linux-{arch}" + + +def publish_environment() -> dict[str, str]: + """The NativeAOT linker step finds MSVC through vswhere, which the installer keeps off PATH.""" + environment = dict(os.environ) + if platform.system() == "Windows" and shutil.which("vswhere") is None: + program_files = environment.get("ProgramFiles(x86)", r"C:\Program Files (x86)") + installer = Path(program_files) / "Microsoft Visual Studio" / "Installer" + if (installer / "vswhere.exe").exists(): + environment["PATH"] = environment.get("PATH", "") + os.pathsep + str(installer) + return environment + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--rid", default=default_rid(), help="dotnet runtime identifier") + parser.add_argument("--framework", default="net10.0") + args = parser.parse_args() + + subprocess.run( + ["dotnet", "publish", str(CSPROJ), "-c", "Release", "-f", args.framework, "-r", args.rid], + check=True, + env=publish_environment(), + ) + + publish_dir = CSPROJ.parent / "bin" / "Release" / args.framework / args.rid / "publish" + name = LIB_NAMES[platform.system()] + source = publish_dir / name + if not source.exists(): + print(f"error: expected native library at {source}", file=sys.stderr) + return 1 + + PACKAGE_LIB_DIR.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, PACKAGE_LIB_DIR / name) + print(f"copied {source} -> {PACKAGE_LIB_DIR / name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/scripts/check_wheel_tag.py b/python/scripts/check_wheel_tag.py new file mode 100644 index 0000000..d5874a0 --- /dev/null +++ b/python/scripts/check_wheel_tag.py @@ -0,0 +1,29 @@ +"""Fails unless every wheel in the given directory installs on this machine. + +Run after building a wheel: python python/scripts/check_wheel_tag.py dist +""" + +import sys +from pathlib import Path + +from packaging.tags import sys_tags +from packaging.utils import parse_wheel_filename + + +def main() -> int: + supported = set(sys_tags()) + wheels = sorted(Path(sys.argv[1]).glob("*.whl")) + if not wheels: + print(f"error: no wheel in {sys.argv[1]}", file=sys.stderr) + return 1 + for wheel in wheels: + tags = parse_wheel_filename(wheel.name)[3] + if tags.isdisjoint(supported): + print(f"error: {wheel.name} does not install on this machine", file=sys.stderr) + return 1 + print(f"ok: {wheel.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/src/fixedwidth/__init__.py b/python/src/fixedwidth/__init__.py new file mode 100644 index 0000000..edbed2c --- /dev/null +++ b/python/src/fixedwidth/__init__.py @@ -0,0 +1,69 @@ +"""Read and write fixed-width files through the FixedWidthParser native library. + + import fixedwidth as fw + + layout = fw.Layout([ + fw.Column("codneg", 12, 12, fw.String), + fw.Column("preabe", 56, 13, fw.Float64, scale=2), + ]) + df = fw.read_pandas("COTAHIST_A2023.TXT", layout, fw.ReadOptions(header_lines=1, skip_invalid=True)) +""" + +from fixedwidth.arrow_stream import ArrowStream +from fixedwidth.layout import Layout +from fixedwidth.reader import read, read_arrow, read_arrow_stream, read_batches, read_pandas, read_polars +from fixedwidth.table import Table +from fixedwidth.types import ( + Alignment, + Column, + ColumnType, + FixedWidthError, + LayoutOptions, + Overflow, + ParseError, + ReaderOptions, + ReadOptions, + TrimMode, + WriteOptions, +) +from fixedwidth.writer import write, write_bytes + +String = ColumnType.STRING +Int64 = ColumnType.INT64 +Float64 = ColumnType.FLOAT64 +Decimal = ColumnType.DECIMAL +Date = ColumnType.DATE +Time = ColumnType.TIME +Timestamp = ColumnType.TIMESTAMP + +__all__ = [ + "Alignment", + "ArrowStream", + "Column", + "ColumnType", + "Date", + "Decimal", + "FixedWidthError", + "Float64", + "Int64", + "Layout", + "LayoutOptions", + "Overflow", + "ParseError", + "ReadOptions", + "ReaderOptions", + "String", + "Table", + "Time", + "Timestamp", + "TrimMode", + "WriteOptions", + "read", + "read_arrow", + "read_arrow_stream", + "read_batches", + "read_pandas", + "read_polars", + "write", + "write_bytes", +] diff --git a/python/src/fixedwidth/_lib/.gitkeep b/python/src/fixedwidth/_lib/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/python/src/fixedwidth/_native.py b/python/src/fixedwidth/_native.py new file mode 100644 index 0000000..99951cc --- /dev/null +++ b/python/src/fixedwidth/_native.py @@ -0,0 +1,273 @@ +"""ctypes binding to the FixedWidthParser NativeAOT shared library. + +Everything here mirrors src/FixedWidthParser.Native/include/fixedwidth.h. If you change one, change both. +""" + +from __future__ import annotations + +import ctypes +import os +import platform +from functools import lru_cache +from pathlib import Path + +FW_OK = 0 +FW_EOF = -1 +FW_BUFFER_TOO_SMALL = -2 +FW_INVALID_HANDLE = -3 +FW_INVALID_ARGUMENT = -4 +FW_ERROR = -5 +FW_PARSE_ERROR = -6 + +FW_ABI_VERSION = 1 + +FW_OPT_DEFAULT = 0 +FW_OPT_FALSE = 1 +FW_OPT_TRUE = 2 + +_u8p = ctypes.POINTER(ctypes.c_uint8) + + +class ColumnSpec(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("name", _u8p), + ("name_len", ctypes.c_int32), + ("start", ctypes.c_int32), + ("length", ctypes.c_int32), + ("type", ctypes.c_int32), + ("nullable", ctypes.c_int32), + ("scale", ctypes.c_int32), + ("format", _u8p), + ("format_len", ctypes.c_int32), + ("trim_char", ctypes.c_int32), + ("trim_mode", ctypes.c_int32), + ("alignment", ctypes.c_int32), + ("padding", ctypes.c_int32), + ("overflow", ctypes.c_int32), + ("dictionary", ctypes.c_int32), + ] + + +class LayoutOptions(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("line_length", ctypes.c_int32), ("filler", ctypes.c_int32)] + + +class ReadOptions(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("header_lines", ctypes.c_int64), + ("skip_invalid", ctypes.c_int32), + ("degree_of_parallelism", ctypes.c_int32), + ("encoding", ctypes.c_int32), + ("max_line_length", ctypes.c_int32), + ("skip_byte_order_mark", ctypes.c_int32), + ] + + +class ReaderOptions(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("read", ReadOptions), ("batch_rows", ctypes.c_int64)] + + +StreamRead = ctypes.CFUNCTYPE(ctypes.c_int64, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64) +StreamRelease = ctypes.CFUNCTYPE(None, ctypes.c_void_p) + + +class NativeStream(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("user_data", ctypes.c_void_p), + ("read", StreamRead), + ("release", StreamRelease), + ] + + +SinkWrite = ctypes.CFUNCTYPE(ctypes.c_int32, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64) + + +class Sink(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("user_data", ctypes.c_void_p), ("write", SinkWrite)] + + +class Column(ctypes.Structure): + _fields_ = [ + ("type", ctypes.c_int32), + ("scale", ctypes.c_int32), + ("length", ctypes.c_int64), + ("values", ctypes.c_void_p), + ("validity", ctypes.c_void_p), + ("data", ctypes.c_void_p), + ("data_len", ctypes.c_int64), + ] + + +class Table(ctypes.Structure): + _fields_ = [("column_count", ctypes.c_int32), ("row_count", ctypes.c_int64), ("columns", ctypes.POINTER(Column))] + + +class ReadResult(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("lines_read", ctypes.c_int64), ("lines_skipped", ctypes.c_int64)] + + +class ParseErrorInfo(ctypes.Structure): + _fields_ = [("struct_size", ctypes.c_int32), ("line_number", ctypes.c_int64), ("column", ctypes.c_int32)] + + +class WriteOptions(ctypes.Structure): + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("encoding", ctypes.c_int32), + ("newline", _u8p), + ("newline_len", ctypes.c_int32), + ("degree_of_parallelism", ctypes.c_int32), + ("batch_rows", ctypes.c_int64), + ] + + +class Buffer(ctypes.Structure): + _fields_ = [("data", ctypes.c_void_p), ("length", ctypes.c_int64)] + + +class ArrowSchema(ctypes.Structure): + pass + + +class ArrowArray(ctypes.Structure): + pass + + +_ArrowSchemaRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowSchema)) +_ArrowArrayRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowArray)) + +ArrowSchema._fields_ = [ + ("format", ctypes.c_char_p), + ("name", ctypes.c_char_p), + ("metadata", ctypes.c_char_p), + ("flags", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("children", ctypes.POINTER(ctypes.POINTER(ArrowSchema))), + ("dictionary", ctypes.POINTER(ArrowSchema)), + ("release", _ArrowSchemaRelease), + ("private_data", ctypes.c_void_p), +] +ArrowArray._fields_ = [ + ("length", ctypes.c_int64), + ("null_count", ctypes.c_int64), + ("offset", ctypes.c_int64), + ("n_buffers", ctypes.c_int64), + ("n_children", ctypes.c_int64), + ("buffers", ctypes.POINTER(ctypes.c_void_p)), + ("children", ctypes.POINTER(ctypes.POINTER(ArrowArray))), + ("dictionary", ctypes.POINTER(ArrowArray)), + ("release", _ArrowArrayRelease), + ("private_data", ctypes.c_void_p), +] + + +class ArrowArrayStream(ctypes.Structure): + pass + + +_ArrowStreamGetSchema = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ArrowArrayStream), ctypes.POINTER(ArrowSchema)) +_ArrowStreamGetNext = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.POINTER(ArrowArrayStream), ctypes.POINTER(ArrowArray)) +_ArrowStreamGetLastError = ctypes.CFUNCTYPE(ctypes.c_char_p, ctypes.POINTER(ArrowArrayStream)) +_ArrowStreamRelease = ctypes.CFUNCTYPE(None, ctypes.POINTER(ArrowArrayStream)) + +ArrowArrayStream._fields_ = [ + ("get_schema", _ArrowStreamGetSchema), + ("get_next", _ArrowStreamGetNext), + ("get_last_error", _ArrowStreamGetLastError), + ("release", _ArrowStreamRelease), + ("private_data", ctypes.c_void_p), +] + +_P = ctypes.POINTER +_SIGNATURES: dict[str, tuple[object, list[object]]] = { + "fw_abi_version": (ctypes.c_int32, []), + "fw_last_error": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(ctypes.c_int32)]), + "fw_last_parse_error": (ctypes.c_int32, [_P(ParseErrorInfo)]), + "fw_layout_create": (ctypes.c_int32, [_P(ColumnSpec), ctypes.c_int32, _P(LayoutOptions), _P(ctypes.c_void_p)]), + "fw_layout_free": (None, [ctypes.c_void_p]), + "fw_read_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReadOptions), _P(Table), _P(ReadResult)]), + "fw_read_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReadOptions), _P(Table), _P(ReadResult)]), + "fw_free_table": (None, [_P(Table)]), + "fw_reader_open_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), + "fw_reader_open_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), + "fw_reader_open_stream": (ctypes.c_int32, [_P(NativeStream), ctypes.c_void_p, _P(ReaderOptions), _P(ctypes.c_void_p)]), + "fw_set_source_error": (None, [ctypes.c_char_p, ctypes.c_int32]), + "fw_reader_next": (ctypes.c_int32, [ctypes.c_void_p, _P(Table)]), + "fw_reader_close": (None, [ctypes.c_void_p]), + "fw_read_arrow_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]), + "fw_read_arrow_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReadOptions), _P(ArrowArray), _P(ArrowSchema), _P(ReadResult)]), + "fw_read_arrow_stream_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), + "fw_read_arrow_stream_memory": (ctypes.c_int32, [ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), + "fw_read_arrow_stream_stream": (ctypes.c_int32, [_P(NativeStream), ctypes.c_void_p, _P(ReaderOptions), _P(ArrowArrayStream)]), + "fw_write_file": (ctypes.c_int32, [_u8p, ctypes.c_int32, _P(Table), ctypes.c_void_p, _P(WriteOptions)]), + "fw_write_memory": (ctypes.c_int32, [_P(Table), ctypes.c_void_p, _P(WriteOptions), _P(Buffer)]), + "fw_write_stream": (ctypes.c_int32, [_P(Table), ctypes.c_void_p, _P(WriteOptions), _P(Sink)]), + "fw_free_buffer": (None, [_P(Buffer)]), +} + + +def library_filename() -> str: + system = platform.system() + if system == "Windows": + return "FixedWidthParser.Native.dll" + if system == "Darwin": + return "FixedWidthParser.Native.dylib" + return "FixedWidthParser.Native.so" + + +def _candidate_paths() -> list[Path]: + paths = [] + override = os.environ.get("FIXEDWIDTH_NATIVE_LIB") + if override: + paths.append(Path(override)) + paths.append(Path(__file__).parent / "_lib" / library_filename()) + return paths + + +@lru_cache(maxsize=1) +def load_library() -> ctypes.CDLL: + """Loads the bundled library (or FIXEDWIDTH_NATIVE_LIB) once and checks its ABI version.""" + tried = [] + for path in _candidate_paths(): + if not path.exists(): + tried.append(str(path)) + continue + lib = ctypes.CDLL(str(path)) + for name, (restype, argtypes) in _SIGNATURES.items(): + function = getattr(lib, name) + function.restype = restype + function.argtypes = argtypes + version = lib.fw_abi_version() + if version != FW_ABI_VERSION: + raise OSError(f"{path} has ABI version {version}; this package needs {FW_ABI_VERSION}") + return lib + raise OSError( + "FixedWidthParser.Native was not found (looked in " + ", ".join(tried) + "). " + "Build it with `python python/scripts/build_native.py` or set FIXEDWIDTH_NATIVE_LIB." + ) + + +def last_error() -> str: + lib = load_library() + buffer = (ctypes.c_uint8 * 4096)() + length = ctypes.c_int32() + status = lib.fw_last_error(buffer, len(buffer), ctypes.byref(length)) + if status == FW_BUFFER_TOO_SMALL: + buffer = (ctypes.c_uint8 * length.value)() + lib.fw_last_error(buffer, len(buffer), ctypes.byref(length)) + return bytes(buffer[: length.value]).decode("utf-8", "replace") + + +def last_parse_error() -> tuple[int, int]: + info = ParseErrorInfo(struct_size=ctypes.sizeof(ParseErrorInfo)) + load_library().fw_last_parse_error(ctypes.byref(info)) + return int(info.line_number), int(info.column) + + +def utf8(text: str) -> tuple[ctypes.Array, int]: + """A UTF-8 buffer and its length, for a pointer + length argument.""" + encoded = text.encode("utf-8") + return (ctypes.c_uint8 * max(1, len(encoded))).from_buffer_copy(encoded.ljust(1, b"\0")), len(encoded) diff --git a/python/src/fixedwidth/arrow_stream.py b/python/src/fixedwidth/arrow_stream.py new file mode 100644 index 0000000..61eb440 --- /dev/null +++ b/python/src/fixedwidth/arrow_stream.py @@ -0,0 +1,127 @@ +"""An Arrow C stream exposed through the Arrow PyCapsule interface, with no pyarrow dependency.""" + +from __future__ import annotations + +import ctypes + +from fixedwidth import _native +from fixedwidth.types import FixedWidthError + +_CAPSULE_NAME = b"arrow_array_stream" + +_capsule_new = ctypes.pythonapi.PyCapsule_New +_capsule_new.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_void_p] +_capsule_new.restype = ctypes.py_object + +_capsule_pointer = ctypes.pythonapi.PyCapsule_GetPointer +_capsule_pointer.argtypes = [ctypes.c_void_p, ctypes.c_char_p] +_capsule_pointer.restype = ctypes.c_void_p + +_ReadFn = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p) +_ErrorFn = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p) +_ReleaseFn = ctypes.CFUNCTYPE(None, ctypes.c_void_p) + + +class _Target: + """The native stream behind a proxy, re-typed so ctypes releases the GIL in its callbacks, and the + caller's bytes a memory read reads lazily.""" + + def __init__(self, stream: _native.ArrowArrayStream, keep: object) -> None: + self.stream = stream + self.keep = keep + self.address = ctypes.addressof(stream) + self.get_schema = _ReadFn(_address_of(stream.get_schema)) + self.get_next = _ReadFn(_address_of(stream.get_next)) + self.get_last_error = _ErrorFn(_address_of(stream.get_last_error)) + + +def _address_of(function: object) -> int: + return ctypes.cast(function, ctypes.c_void_p).value + + +_streams: dict[int, _Target] = {} +_proxies: dict[int, _native.ArrowArrayStream] = {} + + +def _target(stream: int) -> _Target: + return _streams[_native.ArrowArrayStream.from_address(stream).private_data] + + +@_ReadFn +def _get_schema(stream: int, out: int) -> int: + target = _target(stream) + return target.get_schema(target.address, out) + + +@_ReadFn +def _get_next(stream: int, out: int) -> int: + target = _target(stream) + return target.get_next(target.address, out) + + +@_ErrorFn +def _get_last_error(stream: int) -> int | None: + target = _target(stream) + return target.get_last_error(target.address) + + +def _close(proxy: _native.ArrowArrayStream) -> None: + _release(_streams.pop(proxy.private_data).stream) + proxy.release = _native._ArrowStreamRelease() + + +@_ReleaseFn +def _release_proxy(stream: int) -> None: + _close(_native.ArrowArrayStream.from_address(stream)) + + +def _release(stream: _native.ArrowArrayStream) -> None: + if stream.release: + stream.release(ctypes.byref(stream)) + + +def _proxy(stream: _native.ArrowArrayStream, keep: object) -> _native.ArrowArrayStream: + target = _Target(stream, keep) + _streams[id(target)] = target + proxy = _native.ArrowArrayStream() + proxy.get_schema = ctypes.cast(_get_schema, _native._ArrowStreamGetSchema) + proxy.get_next = ctypes.cast(_get_next, _native._ArrowStreamGetNext) + proxy.get_last_error = ctypes.cast(_get_last_error, _native._ArrowStreamGetLastError) + proxy.release = ctypes.cast(_release_proxy, _native._ArrowStreamRelease) + proxy.private_data = id(target) + return proxy + + +def _destroy_capsule(capsule: int) -> None: + proxy = _proxies.pop(_capsule_pointer(capsule, _CAPSULE_NAME)) + if proxy.release: + _close(proxy) + + +_destroy_capsule_callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(_destroy_capsule) + + +class ArrowStream: + """A single-use stream of Arrow record batches implementing `__arrow_c_stream__`. + + Hand it to any consumer of the Arrow PyCapsule interface — `polars.DataFrame(stream)`, + `pyarrow.RecordBatchReader.from_stream(stream)`. Once a consumer takes it, the stream is spent; + dropping it unconsumed releases the native read. + """ + + def __init__(self, stream: _native.ArrowArrayStream, keep: object = None) -> None: + self._stream: _native.ArrowArrayStream | None = stream + self._keep = keep + + def __arrow_c_stream__(self, requested_schema: object = None) -> object: + if self._stream is None: + raise FixedWidthError("this Arrow stream was already consumed") + stream, self._stream = self._stream, None + proxy = _proxy(stream, self._keep) + address = ctypes.addressof(proxy) + _proxies[address] = proxy + return _capsule_new(address, _CAPSULE_NAME, _destroy_capsule_callback) + + def __del__(self) -> None: + if getattr(self, "_stream", None) is not None: + _release(self._stream) diff --git a/python/src/fixedwidth/layout.py b/python/src/fixedwidth/layout.py new file mode 100644 index 0000000..535c3dd --- /dev/null +++ b/python/src/fixedwidth/layout.py @@ -0,0 +1,100 @@ +"""Layout: the columns of a line, validated once by the native library and shared by every call.""" + +from __future__ import annotations + +import ctypes +from collections.abc import Sequence + +from fixedwidth import _native +from fixedwidth.types import Column, ColumnType, FixedWidthError, LayoutOptions, ParseError + + +def check(status: int) -> None: + """Raises the error a native status reports.""" + if status == _native.FW_OK: + return + message = _native.last_error() or f"native status {status}" + if status == _native.FW_PARSE_ERROR: + line_number, column = _native.last_parse_error() + raise ParseError(message, line_number, column) + raise FixedWidthError(message) + + +def ascii_option(value: str | None, what: str) -> int: + if value is None: + return 0 + if len(value) != 1 or ord(value) > 127: + raise ValueError(f"{what} must be one ASCII character, got {value!r}") + return ord(value) + + +class Layout: + """How the columns of a line map to names and types; pass it to every read and write. + + The native layout is immutable and may be shared across threads. + """ + + def __init__(self, columns: Sequence[Column], options: LayoutOptions | None = None) -> None: + self._lib = _native.load_library() + self._handle: int | None = None + self._columns = list(columns) + specs = (_native.ColumnSpec * max(1, len(self._columns)))() + keep = [self._fill(specs[i], column) for i, column in enumerate(self._columns)] + raw = None + if options is not None: + raw = _native.LayoutOptions( + struct_size=ctypes.sizeof(_native.LayoutOptions), + line_length=options.line_length, + filler=ascii_option(options.filler, "filler"), + ) + handle = ctypes.c_void_p() + check(self._lib.fw_layout_create(specs, len(self._columns), ctypes.byref(raw) if raw else None, ctypes.byref(handle))) + self._handle = handle.value + del keep + + @staticmethod + def _fill(spec: _native.ColumnSpec, column: Column) -> tuple[object, ...]: + name, name_len = _native.utf8(column.name) + fmt, fmt_len = _native.utf8(column.format) if column.format is not None else (None, 0) + spec.struct_size = ctypes.sizeof(_native.ColumnSpec) + spec.name = ctypes.cast(name, ctypes.POINTER(ctypes.c_uint8)) + spec.name_len = name_len + spec.start = column.start + spec.length = column.length + spec.type = int(ColumnType(column.type)) + spec.nullable = int(column.nullable) + spec.scale = column.scale + spec.format = ctypes.cast(fmt, ctypes.POINTER(ctypes.c_uint8)) if fmt is not None else None + spec.format_len = fmt_len + spec.trim_char = ascii_option(column.trim_char, "trim_char") + spec.trim_mode = int(column.trim_mode or 0) + spec.alignment = int(column.alignment or 0) + spec.padding = ascii_option(column.padding, "padding") + spec.overflow = int(column.overflow or 0) + spec.dictionary = int(column.dictionary) + return name, fmt + + @property + def columns(self) -> list[Column]: + return list(self._columns) + + @property + def names(self) -> list[str]: + return [column.name for column in self._columns] + + @property + def dictionaries(self) -> frozenset[str]: + """The columns set to reach Arrow dictionary-encoded.""" + return frozenset(column.name for column in self._columns if column.dictionary) + + @property + def handle(self) -> int: + return self._handle or 0 + + def __del__(self) -> None: + if getattr(self, "_handle", None): + self._lib.fw_layout_free(self._handle) + self._handle = None + + def __repr__(self) -> str: + return f"Layout({len(self._columns)} columns: {', '.join(self.names)})" diff --git a/python/src/fixedwidth/py.typed b/python/src/fixedwidth/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/python/src/fixedwidth/reader.py b/python/src/fixedwidth/reader.py new file mode 100644 index 0000000..90a1695 --- /dev/null +++ b/python/src/fixedwidth/reader.py @@ -0,0 +1,226 @@ +"""Reading: whole sources into a Table, Arrow, pandas or polars, and batched reads.""" + +from __future__ import annotations + +import ctypes +import io +import itertools +import operator +import os +from collections.abc import Iterator +from typing import IO, Any, Union + +from fixedwidth import _native +from fixedwidth.arrow_stream import ArrowStream +from fixedwidth.layout import Layout, check +from fixedwidth.table import Table, _Owner +from fixedwidth.types import ReaderOptions, ReadOptions, code_page + +Source = Union[str, "os.PathLike[str]", bytes, bytearray, memoryview, IO[bytes]] + + +def _is_stream(source: object) -> bool: + """An object with readinto() or read() that is not an in-memory buffer: an open file, a pipe, a decompressor.""" + return not isinstance(source, (str, os.PathLike, bytes, bytearray, memoryview, io.BytesIO)) and ( + hasattr(source, "readinto") or hasattr(source, "read") + ) + + +class _Input: + """A file path or a buffer, as the pointer + length arguments the native calls take. A stream is read whole.""" + + def __init__(self, source: Source) -> None: + self.keep: object = None + if isinstance(source, (str, os.PathLike)): + self.path, self.path_len = _native.utf8(os.fspath(source)) + return + self.path = None + if isinstance(source, io.BytesIO): + # Read in place from the current position; the exported buffer stops the BytesIO resizing meanwhile. + source = source.getbuffer()[source.tell():] + elif _is_stream(source): + source = source.read() + if source is None: + raise OSError("the stream has no data ready; reading it needs a blocking stream") + view = memoryview(source) + if view.readonly or not view.contiguous: + data = bytes(view) + self.keep = data + self.address = ctypes.cast(ctypes.c_char_p(data), ctypes.c_void_p).value + else: + buffer = (ctypes.c_char * view.nbytes).from_buffer(view) + self.keep = (view, buffer) + self.address = ctypes.addressof(buffer) + self.length = view.nbytes + + @property + def is_file(self) -> bool: + return self.path is not None + + +def _read_options(options: ReadOptions | None) -> _native.ReadOptions | None: + if options is None: + return None + bom = options.skip_byte_order_mark + return _native.ReadOptions( + struct_size=ctypes.sizeof(_native.ReadOptions), + header_lines=options.header_lines, + skip_invalid=int(options.skip_invalid), + degree_of_parallelism=options.degree_of_parallelism, + encoding=code_page(options.encoding), + max_line_length=options.max_line_length, + skip_byte_order_mark=_native.FW_OPT_DEFAULT if bom is None else (_native.FW_OPT_TRUE if bom else _native.FW_OPT_FALSE), + ) + + +def _reader_options(options: ReaderOptions | None) -> _native.ReaderOptions | None: + if options is None: + return None + raw = _native.ReaderOptions(struct_size=ctypes.sizeof(_native.ReaderOptions), batch_rows=options.batch_rows) + read = _read_options(options.read) + if read is not None: + raw.read = read + return raw + + +def _ref(value: object) -> object: + return ctypes.byref(value) if value is not None else None + + +# Streams the native side is reading, by the key it passes back to the callbacks; release drops them. +_keys = itertools.count(1) +_live: dict[int, object] = {} + + +def _fail(error: BaseException) -> int: + message = (str(error) or type(error).__name__).encode("utf-8", errors="replace") + _native.load_library().fw_set_source_error(message, len(message)) + return -1 + + +@_native.StreamRead +def _read(key: int, buffer: int, length: int) -> int: + try: + stream = _live[key] + readinto = getattr(stream, "readinto", None) + if readinto is None: + data = bytes(stream.read(length)) + if len(data) > length: + raise OSError("the stream returned more bytes than requested") + ctypes.memmove(buffer, data, len(data)) + return len(data) + count = readinto(memoryview((ctypes.c_ubyte * length).from_address(buffer)).cast("B")) + if count is None: + raise OSError("the stream has no data ready; reading it needs a blocking stream") + count = operator.index(count) + if count > length: + raise OSError("the stream returned more bytes than requested") + return count + except BaseException as error: # noqa: BLE001 - every failure, KeyboardInterrupt included, must reach the native side + return _fail(error) + + +@_native.StreamRelease +def _release(key: int) -> None: + _live.pop(key, None) + + +def _stream(source: object) -> _native.NativeStream: + """An fw_stream over `source`, kept alive until the native side releases it.""" + key = next(_keys) + _live[key] = source + return _native.NativeStream(ctypes.sizeof(_native.NativeStream), key, _read, _release) + + +def read(source: Source, layout: Layout, options: ReadOptions | None = None) -> Table: + """Reads a whole file, buffer or stream into a Table (see `ReadOptions` for header lines, skipping and threads). + A stream is read into memory first, as the parallel read needs all of it; `read_batches()` streams it instead.""" + lib = _native.load_library() + data = _Input(source) + raw = _read_options(options) + table = _native.Table() + result = _native.ReadResult(struct_size=ctypes.sizeof(_native.ReadResult)) + if data.is_file: + status = lib.fw_read_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(table), ctypes.byref(result)) + else: + status = lib.fw_read_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(table), ctypes.byref(result)) + check(status) + return Table(_Owner(table), layout.names, int(result.lines_read), int(result.lines_skipped), layout.dictionaries) + + +def read_arrow(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any: + """Reads a whole file or buffer into a `pyarrow.Table`, handed over without a copy. Requires pyarrow.""" + import pyarrow + + lib = _native.load_library() + data = _Input(source) + raw = _read_options(options) + array = _native.ArrowArray() + schema = _native.ArrowSchema() + if data.is_file: + status = lib.fw_read_arrow_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(array), ctypes.byref(schema), None) + else: + status = lib.fw_read_arrow_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(array), ctypes.byref(schema), None) + check(status) + batch = pyarrow.RecordBatch._import_from_c(ctypes.addressof(array), ctypes.addressof(schema)) + return pyarrow.Table.from_batches([batch]) + + +def read_pandas(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any: + """`read_arrow()` as a `pandas.DataFrame`; each Arrow buffer is freed as pandas takes it. Requires pyarrow and pandas.""" + return read_arrow(source, layout, options).to_pandas(self_destruct=True, split_blocks=True) + + +def read_polars(source: Source, layout: Layout, options: ReadOptions | None = None) -> Any: + """`read_arrow()` as a `polars.DataFrame`, zero-copy. Without pyarrow, reads sequentially through an Arrow stream.""" + import polars + + try: + return polars.from_arrow(read_arrow(source, layout, options)) + except ImportError: + return polars.DataFrame(read_arrow_stream(source, layout, ReaderOptions(read=options or ReadOptions()))) + + +def read_batches(source: Source, layout: Layout, options: ReaderOptions | None = None) -> Iterator[Table]: + """Reads sequentially, yielding a Table of at most `options.batch_rows` rows at a time. A stream (an open file, + a pipe, `gzip.open(...)`, `zipfile.ZipFile(...).open(...)`) is read as the batches are taken, in constant memory.""" + lib = _native.load_library() + raw = _reader_options(options) + reader = ctypes.c_void_p() + if _is_stream(source): + data = None + status = lib.fw_reader_open_stream(ctypes.byref(_stream(source)), layout.handle, _ref(raw), ctypes.byref(reader)) + elif (data := _Input(source)).is_file: + status = lib.fw_reader_open_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(reader)) + else: + status = lib.fw_reader_open_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(reader)) + check(status) + try: + while True: + table = _native.Table() + status = lib.fw_reader_next(reader, ctypes.byref(table)) + if status == _native.FW_EOF: + return + check(status) + yield Table(_Owner(table), layout.names, dictionaries=layout.dictionaries) + finally: + lib.fw_reader_close(reader) + del data + + +def read_arrow_stream(source: Source, layout: Layout, options: ReaderOptions | None = None) -> ArrowStream: + """The batches of `read_batches()` as a single-use Arrow PyCapsule stream, for `polars.DataFrame(stream)` + or `pyarrow.RecordBatchReader.from_stream(stream)`. Needs no third-party package to create. A stream source is + read lazily, as in `read_batches()`.""" + lib = _native.load_library() + raw = _reader_options(options) + stream = _native.ArrowArrayStream() + if _is_stream(source): + data = None + status = lib.fw_read_arrow_stream_stream(ctypes.byref(_stream(source)), layout.handle, _ref(raw), ctypes.byref(stream)) + elif (data := _Input(source)).is_file: + status = lib.fw_read_arrow_stream_file(data.path, data.path_len, layout.handle, _ref(raw), ctypes.byref(stream)) + else: + status = lib.fw_read_arrow_stream_memory(data.address, data.length, layout.handle, _ref(raw), ctypes.byref(stream)) + check(status) + return ArrowStream(stream, keep=data) diff --git a/python/src/fixedwidth/table.py b/python/src/fixedwidth/table.py new file mode 100644 index 0000000..fc36c65 --- /dev/null +++ b/python/src/fixedwidth/table.py @@ -0,0 +1,139 @@ +"""Table: a columnar read whose numeric columns are NumPy views over the native buffers.""" + +from __future__ import annotations + +import ctypes +from collections.abc import Callable +from decimal import Decimal +from typing import Any + +from fixedwidth import _native +from fixedwidth.types import ColumnType + +_ELEMENTS = { + ColumnType.INT64: (ctypes.c_int64, "int64"), + ColumnType.FLOAT64: (ctypes.c_double, "float64"), + ColumnType.DECIMAL: (ctypes.c_int64, "int64"), + ColumnType.DATE: (ctypes.c_int32, "int32"), + ColumnType.TIME: (ctypes.c_int64, "int64"), + ColumnType.TIMESTAMP: (ctypes.c_int64, "int64"), +} + + +class _Owner: + """Owns one native fw_table; frees it once nothing (the Table or any view) references it.""" + + def __init__(self, table: _native.Table) -> None: + self.table = table + self._on_free: Callable[[bool], object] | None = None + + def view(self, address: int, element: type, count: int) -> ctypes.Array: + buffer = (element * count).from_address(address) + buffer._owner = self # type: ignore[attr-defined] # a view keeps the native memory alive + return buffer + + def __del__(self) -> None: + _native.load_library().fw_free_table(ctypes.byref(self.table)) + if self._on_free is not None: + self._on_free(True) + + +class Table: + """One read: `table[name]` is a NumPy view for numeric and date/time columns (as stored: int64, + float64, DECIMAL's unscaled int64, DATE's int32 days, TIME/TIMESTAMP's int64 microseconds) and a + list of `str` for strings. Null rows are marked by `validity(name)`. Views stay valid after the + Table is gone; the native memory is freed when the last one is.""" + + def __init__( + self, owner: _Owner, names: list[str], lines_read: int = 0, lines_skipped: int = 0, dictionaries: frozenset[str] = frozenset() + ) -> None: + self._owner = owner + self._names = names + self._dictionaries = dictionaries + self.lines_read = lines_read + self.lines_skipped = lines_skipped + + @property + def num_rows(self) -> int: + return int(self._owner.table.row_count) + + @property + def column_names(self) -> list[str]: + return list(self._names) + + def __len__(self) -> int: + return self.num_rows + + def _column(self, name: str) -> _native.Column: + try: + return self._owner.table.columns[self._names.index(name)] + except ValueError: + raise KeyError(name) from None + + def __getitem__(self, name: str) -> Any: + column = self._column(name) + kind = ColumnType(column.type) + if kind is ColumnType.STRING: + return _strings(column) + import numpy + + element, dtype = _ELEMENTS[kind] + if column.length == 0: + return numpy.empty(0, dtype=dtype) + return numpy.frombuffer(self._owner.view(column.values, element, column.length), dtype=dtype) + + def validity(self, name: str) -> Any: + """A NumPy bool array, True where the row holds a value, or None when the column has no nulls.""" + column = self._column(name) + if not column.validity: + return None + import numpy + + bits = numpy.frombuffer(ctypes.string_at(column.validity, (column.length + 7) // 8), dtype=numpy.uint8) + return numpy.unpackbits(bits, bitorder="little")[: column.length].astype(bool) + + def to_arrow(self) -> Any: + """The same columns as a `pyarrow.Table`, sharing the native buffers. Requires pyarrow.""" + import pyarrow + + arrays = [self._arrow_column(name, pyarrow) for name in self._names] + return pyarrow.Table.from_arrays(arrays, names=self._names) + + def _arrow_column(self, name: str, pa: Any) -> Any: + column = self._column(name) + kind = ColumnType(column.type) + n = int(column.length) + validity = pa.foreign_buffer(column.validity, (n + 7) // 8, base=self._owner) if column.validity else None + if kind is ColumnType.DECIMAL: + # ponytail: decimals go through Python objects; fw.read_arrow() hands them over natively. + mask = self.validity(name) + values = self[name] + decimals = [None if mask is not None and not mask[i] else Decimal(int(v)).scaleb(-column.scale) for i, v in enumerate(values)] + return pa.array(decimals, type=pa.decimal128(18, column.scale)) + if kind is ColumnType.STRING: + offsets = pa.foreign_buffer(column.values, (n + 1) * 4, base=self._owner) + data = pa.foreign_buffer(column.data, column.data_len, base=self._owner) if column.data else pa.py_buffer(b"") + strings = pa.Array.from_buffers(pa.string(), n, [validity, offsets, data]) + return strings.dictionary_encode() if name in self._dictionaries else strings + arrow_type = { + ColumnType.INT64: pa.int64(), + ColumnType.FLOAT64: pa.float64(), + ColumnType.DATE: pa.date32(), + ColumnType.TIME: pa.time64("us"), + ColumnType.TIMESTAMP: pa.timestamp("us"), + }[kind] + width = 4 if kind is ColumnType.DATE else 8 + values = pa.foreign_buffer(column.values, n * width, base=self._owner) if n else pa.py_buffer(b"") + return pa.Array.from_buffers(arrow_type, n, [validity, values]) + + def __repr__(self) -> str: + return f"Table({self.num_rows} rows: {', '.join(self._names)})" + + +def _strings(column: _native.Column) -> list[str]: + n = int(column.length) + if n == 0: + return [] + offsets = (ctypes.c_int32 * (n + 1)).from_address(column.values) + data = ctypes.string_at(column.data, column.data_len) if column.data else b"" + return [data[offsets[i] : offsets[i + 1]].decode("utf-8") for i in range(n)] diff --git a/python/src/fixedwidth/types.py b/python/src/fixedwidth/types.py new file mode 100644 index 0000000..bd4c42a --- /dev/null +++ b/python/src/fixedwidth/types.py @@ -0,0 +1,139 @@ +"""The public value types: column types, column descriptions, options and errors.""" + +from __future__ import annotations + +import re +from dataclasses import KW_ONLY, dataclass, field +from enum import IntEnum + + +class ColumnType(IntEnum): + """FW_T_*: how a column is parsed and what its values are.""" + + STRING = 0 + INT64 = 1 + FLOAT64 = 2 + DECIMAL = 3 + DATE = 4 + TIME = 5 + TIMESTAMP = 6 + + +class TrimMode(IntEnum): + BOTH = 0 + TRAILING = 1 + LEADING = 2 + + +class Alignment(IntEnum): + DEFAULT = 0 + LEFT = 1 + RIGHT = 2 + + +class Overflow(IntEnum): + THROW = 0 + TRUNCATE = 1 + + +@dataclass(frozen=True) +class Column: + """One column of a layout: its name, zero-based byte `start`, byte `length` and `type`. + + - `nullable`: a blank column (only `trim_char` and spaces) reads as null instead of failing. + - `scale`: implied decimals. FLOAT64 divides digits-only text by 10**scale (B3's `N(11)V99`); + DECIMAL keeps the unscaled integer. + - `format`: the exact date/time format (.NET syntax, e.g. `yyyyMMdd`), or a number format on write. + - `trim_char`, `trim_mode`: what is stripped before parsing; a null is written as `trim_char`. + - `alignment`, `padding`, `overflow`: how a value is written. `padding='0'` keeps a sign leftmost. + - `dictionary`: a String column reaches Arrow dictionary-encoded, so pandas reads it as `category` and polars + as `Categorical`: much less memory for a column of few distinct values. `fw.read`'s Table keeps strings. + """ + + name: str + start: int + length: int + type: ColumnType + _: KW_ONLY + nullable: bool = False + scale: int = 0 + format: str | None = None + trim_char: str | None = None + trim_mode: TrimMode | None = None + alignment: Alignment | None = None + padding: str | None = None + overflow: Overflow | None = None + dictionary: bool = False + + +@dataclass(frozen=True) +class LayoutOptions: + """`line_length`: 0 ends lines at the last column; longer pads them with `filler` on write.""" + + line_length: int = 0 + filler: str | None = None + + +@dataclass(frozen=True) +class ReadOptions: + """Mirrors fw_read_options. `encoding` is a code page or a codec name (`"latin-1"`, `"cp1252"`).""" + + header_lines: int = 0 + skip_invalid: bool = False + degree_of_parallelism: int = 0 + encoding: int | str | None = None + max_line_length: int = 0 + skip_byte_order_mark: bool | None = None + + +@dataclass(frozen=True) +class ReaderOptions: + """Mirrors fw_reader_options: a sequential read in tables of `batch_rows` rows (0 = 65,536).""" + + read: ReadOptions = field(default_factory=ReadOptions) + batch_rows: int = 0 + + +@dataclass(frozen=True) +class WriteOptions: + """Mirrors fw_write_options.""" + + encoding: int | str | None = None + newline: str = "\n" + degree_of_parallelism: int = 0 + batch_rows: int = 0 + + +class FixedWidthError(Exception): + """Raised when the native library reports a failure.""" + + +class ParseError(FixedWidthError): + """A line failed to parse. `line_number` is 1-based; `column` is the layout index, -1 for a short line.""" + + def __init__(self, message: str, line_number: int, column: int) -> None: + super().__init__(message) + self.line_number = line_number + self.column = column + + +_CODE_PAGES = { + "utf-8": 0, "utf8": 0, "ascii": 20127, "us-ascii": 20127, + "latin-1": 28591, "latin1": 28591, "iso-8859-1": 28591, "iso8859-1": 28591, + "windows-1252": 1252, "cp1252": 1252, "iso-8859-15": 28605, "latin-9": 28605, +} + + +def code_page(encoding: int | str | None) -> int: + """The code page the native options take for `encoding`.""" + if encoding is None: + return 0 + if isinstance(encoding, int): + return encoding + name = encoding.strip().lower().replace("_", "-") + if name in _CODE_PAGES: + return _CODE_PAGES[name] + match = re.fullmatch(r"(?:cp|windows-)(\d+)", name) + if match: + return int(match.group(1)) + raise ValueError(f"unknown encoding {encoding!r}; pass a code page number") diff --git a/python/src/fixedwidth/writer.py b/python/src/fixedwidth/writer.py new file mode 100644 index 0000000..31678a2 --- /dev/null +++ b/python/src/fixedwidth/writer.py @@ -0,0 +1,224 @@ +"""Writing: a pandas, polars or pyarrow table through a layout, to a file, a stream or bytes. Requires pyarrow.""" + +from __future__ import annotations + +import ctypes +import io +import operator +import os +from typing import IO, Any + +from fixedwidth import _native +from fixedwidth.layout import Layout, check +from fixedwidth.reader import _fail +from fixedwidth.types import ( + Column, + ColumnType, + FixedWidthError, + WriteOptions, + code_page, +) + + +def write(data: Any, destination: str | os.PathLike[str] | IO[bytes], layout: Layout, options: WriteOptions | None = None) -> None: + """Writes every row of `data` (pandas, polars, pyarrow, or anything with `__arrow_c_stream__`) to `destination`: + a path, or a binary stream with `write()` (an open file, `gzip.open(..., "wb")`, a socket's `makefile("wb")`). + + A stream gets the lines in order, a batch per `write()` call on this thread, while batches format in parallel; + memory stays at a batch per thread. Columns are matched to the layout by name. A null writes as its column + filled with `trim_char`.""" + lib = _native.load_library() + table, keep = _native_table(data, layout) + raw, newline = _write_options(options) + raw_ref = ctypes.byref(raw) if raw else None + if isinstance(destination, (str, os.PathLike)): + name, name_len = _native.utf8(os.fspath(destination)) + check(lib.fw_write_file(name, name_len, ctypes.byref(table), layout.handle, raw_ref)) + else: + sink, callback = _sink(destination) + check(lib.fw_write_stream(ctypes.byref(table), layout.handle, raw_ref, ctypes.byref(sink))) + del callback + del keep, newline + + +def _sink(stream: Any) -> tuple[_native.Sink, object]: + """An fw_sink over `stream`'s write(); keep the callback alive until the native call returns.""" + # The standard streams copy or write a buffer before write() returns, so they get the native memory itself; + # anything else gets a copy it may keep. + zero_copy = isinstance(stream, io.IOBase) + + def write(_user_data: int, buffer: int, length: int) -> int: + try: + view = memoryview((ctypes.c_ubyte * length).from_address(buffer)).cast("B") + data = view if zero_copy else bytes(view) + done = 0 + while done < length: + count = stream.write(data[done:]) + if count is None: + if isinstance(stream, io.RawIOBase): + raise OSError("the stream would block; writing needs a blocking stream") + break # a write() that returns nothing took everything + count = operator.index(count) + if count <= 0: + raise OSError("the stream accepted no bytes") + done += count + return 0 + except BaseException as error: # noqa: BLE001 - every failure must reach the native side + return _fail(error) + + callback = _native.SinkWrite(write) + return _native.Sink(ctypes.sizeof(_native.Sink), None, callback), callback + + +def write_bytes(data: Any, layout: Layout, options: WriteOptions | None = None) -> bytes: + """Same as `write()`, returning the bytes.""" + lib = _native.load_library() + table, keep = _native_table(data, layout) + raw, newline = _write_options(options) + buffer = _native.Buffer() + check(lib.fw_write_memory(ctypes.byref(table), layout.handle, ctypes.byref(raw) if raw else None, ctypes.byref(buffer))) + try: + return ctypes.string_at(buffer.data, buffer.length) if buffer.length else b"" + finally: + lib.fw_free_buffer(ctypes.byref(buffer)) + del keep, newline + + +def _write_options(options: WriteOptions | None) -> tuple[_native.WriteOptions | None, object]: + if options is None: + return None, None + newline, newline_len = _native.utf8(options.newline) + raw = _native.WriteOptions( + struct_size=ctypes.sizeof(_native.WriteOptions), + encoding=code_page(options.encoding), + newline=ctypes.cast(newline, ctypes.POINTER(ctypes.c_uint8)), + newline_len=newline_len, + degree_of_parallelism=options.degree_of_parallelism, + batch_rows=options.batch_rows, + ) + return raw, newline + + +def _arrow_table(data: Any) -> Any: + import pyarrow + + if isinstance(data, pyarrow.RecordBatch): + return pyarrow.Table.from_batches([data]) + if isinstance(data, pyarrow.Table): + return data + if type(data).__module__.startswith("pandas"): + return pyarrow.Table.from_pandas(data, preserve_index=False) + if hasattr(data, "__arrow_c_stream__"): + return pyarrow.table(data) + raise TypeError(f"cannot write {type(data).__name__}: pass a pandas, polars or pyarrow table") + + +def _native_table(data: Any, layout: Layout) -> tuple[_native.Table, list[object]]: + import pyarrow + + arrow = _arrow_table(data).combine_chunks() + columns = (_native.Column * max(1, len(layout.columns)))() + keep: list[object] = [columns] + for index, spec in enumerate(layout.columns): + if spec.name not in arrow.column_names: + raise FixedWidthError(f"column '{spec.name}' of the layout is not in the data") + chunked = arrow.column(spec.name) + array = chunked.chunk(0) if chunked.num_chunks else pyarrow.array([], type=chunked.type) + _fill(columns[index], spec, array, keep) + table = _native.Table(column_count=len(layout.columns), row_count=arrow.num_rows, columns=columns) + return table, keep + + +def _fill(target: _native.Column, spec: Column, array: Any, keep: list[object]) -> None: + import numpy + import pyarrow + + kind = ColumnType(spec.type) + target.type = int(kind) + target.scale = spec.scale if kind is ColumnType.DECIMAL else 0 + target.length = len(array) + if array.null_count: + valid = numpy.packbits(~array.is_null().to_numpy(zero_copy_only=False), bitorder="little") + keep.append(valid) + target.validity = valid.ctypes.data + if kind is ColumnType.STRING: + _fill_strings(target, array.cast(pyarrow.string()).fill_null(""), keep) + return + values = numpy.ascontiguousarray(_values(kind, spec, array, pyarrow, numpy)) + keep.append(values) + target.values = values.ctypes.data if len(values) else None + + +def _values(kind: ColumnType, spec: Column, array: Any, pa: Any, np: Any) -> Any: + if kind is ColumnType.INT64: + return array.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + if kind is ColumnType.FLOAT64: + return array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) + if kind is ColumnType.DECIMAL: + return _decimals(spec, array, pa, np) + if kind is ColumnType.DATE: + return array.cast(pa.date32()).cast(pa.int32()).fill_null(0).to_numpy(zero_copy_only=False) + if kind is ColumnType.TIME: + return array.cast(pa.time64("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + if pa.types.is_timestamp(array.type) and array.type.tz is not None: + array = array.cast(pa.timestamp(array.type.unit)) + return array.cast(pa.timestamp("us")).cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) + + +def _decimals(spec: Column, array: Any, pa: Any, np: Any) -> Any: + """The unscaled integers a DECIMAL column stores, each of which must fit an int64. + + Decimals and integers scale exactly, and a decimal with more digits than the scale fails. Floats, binary + approximations to begin with, round half to even to the scale; NaN and infinity fail.""" + if pa.types.is_decimal(array.type): + return np.array([_unscaled(v, spec) for v in array.to_pylist()], dtype=np.int64) + factor = 10**spec.scale + if pa.types.is_integer(array.type): + import pyarrow.compute + + bounds = pyarrow.compute.min_max(array).as_py() + for value in (bounds["min"], bounds["max"]): + if value is not None: + _check_int64(value * factor, value, spec) + return array.cast(pa.int64()).fill_null(0).to_numpy(zero_copy_only=False) * factor + floats = array.cast(pa.float64()).fill_null(0).to_numpy(zero_copy_only=False) + with np.errstate(over="ignore", invalid="ignore"): + scaled = np.round(floats * factor) + # Written so NaN fails too; 2**63 is exact in a float64, the int64 range is not. + outside = ~((scaled >= -(2.0**63)) & (scaled < 2.0**63)) + if outside.any(): + value = floats[int(np.argmax(outside))] + raise FixedWidthError(f"value {value} of column '{spec.name}' is not finite or does not fit an int64 at scale {spec.scale}") + return scaled.astype(np.int64) + + +def _unscaled(value: Any, spec: Column) -> int: + """A decimal as the integer the column stores; more decimals than its scale would be lost, so they fail.""" + if value is None: + return 0 + scaled = value.scaleb(spec.scale) + if scaled != scaled.to_integral_value(): + raise FixedWidthError(f"value {value} of column '{spec.name}' has more decimals than its scale {spec.scale}") + return _check_int64(int(scaled), value, spec) + + +def _check_int64(unscaled: int, value: Any, spec: Column) -> int: + if not -(2**63) <= unscaled < 2**63: + raise FixedWidthError(f"value {value} of column '{spec.name}' does not fit an int64 at scale {spec.scale}") + return unscaled + + +def _fill_strings(target: _native.Column, array: Any, keep: list[object]) -> None: + import numpy + + buffers = array.buffers() + n = len(array) + offsets = numpy.frombuffer(buffers[1], dtype=numpy.int32)[array.offset : array.offset + n + 1] if n else numpy.zeros(1, numpy.int32) + base = int(offsets[0]) + rebased = numpy.ascontiguousarray(offsets - base, dtype=numpy.int32) + keep.append(rebased) + keep.append(array) + target.values = rebased.ctypes.data + data_len = int(rebased[-1]) + target.data_len = data_len + target.data = buffers[2].address + base if data_len and buffers[2] is not None else None diff --git a/python/tests/conftest.py b/python/tests/conftest.py new file mode 100644 index 0000000..94cb3ec --- /dev/null +++ b/python/tests/conftest.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import pytest + +import fixedwidth as fw + +# A COTAHIST-like file: a header, three quotes (one with a blank quantity), an invalid line, a trailer. +CONTENT = ( + "00COTAHIST.2023\n" + "0120231024PETR4 PETROBRAS PN0000000123456000000000000001000\n" + "0120231024VALE3 VALE ON 0000000067890 \n" + "0120231025ITUB4 ITAU UNI PN 0000000004321000000000000000050\n" + "0120231025XXXXX BROKEN 00000000ABCDE000000000000000001\n" + "99TRAILER\n" +) + + +def layout() -> fw.Layout: + return fw.Layout( + [ + fw.Column("tipreg", 0, 2, fw.String), + fw.Column("data", 2, 8, fw.Date, format="yyyyMMdd"), + fw.Column("codneg", 10, 12, fw.String), + fw.Column("nomres", 22, 12, fw.String), + fw.Column("preabe", 34, 13, fw.Float64, scale=2, padding="0"), + fw.Column("quatot", 47, 18, fw.Int64, nullable=True, padding="0"), + ] + ) + + +@pytest.fixture +def cotahist(tmp_path): + path = tmp_path / "cotahist.txt" + path.write_bytes(CONTENT.encode("utf-8")) + return path + + +@pytest.fixture +def options() -> fw.ReadOptions: + return fw.ReadOptions(header_lines=1, skip_invalid=True, degree_of_parallelism=1) diff --git a/python/tests/test_arrow.py b/python/tests/test_arrow.py new file mode 100644 index 0000000..9e711e4 --- /dev/null +++ b/python/tests/test_arrow.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import datetime as dt + +import pytest + +import fixedwidth as fw +from conftest import CONTENT, layout + +# No pyarrow wheels for musllinux; the musl wheel job tests without it. +pl = pytest.importorskip("polars") +pa = pytest.importorskip("pyarrow") + + +def test_read_arrow(cotahist, options): + table = fw.read_arrow(cotahist, layout(), options) + + assert table.schema.field("preabe").type == pa.float64() + assert table.schema.field("data").type == pa.date32() + assert table.schema.field("quatot").nullable + assert table.column("codneg").to_pylist() == ["PETR4", "VALE3", "ITUB4"] + assert table.column("data").to_pylist()[2] == dt.date(2023, 10, 25) + + +def test_read_arrow_decimal(): + columns = fw.Layout([fw.Column("v", 0, 8, fw.Decimal, scale=2)]) + + table = fw.read_arrow(b"00012345\n", columns) + + assert table.schema.field("v").type == pa.decimal128(18, 2) + assert str(table.column("v")[0].as_py()) == "123.45" + + +def test_arrow_stream_to_pyarrow_and_polars(cotahist, options): + reader_options = fw.ReaderOptions(read=options, batch_rows=2) + + batches = list(pa.RecordBatchReader.from_stream(fw.read_arrow_stream(cotahist, layout(), reader_options))) + frame = pl.DataFrame(fw.read_arrow_stream(CONTENT.encode("utf-8"), layout(), reader_options)) + + assert [b.num_rows for b in batches] == [2, 1] + assert frame["codneg"].to_list() == ["PETR4", "VALE3", "ITUB4"] + + +def test_arrow_stream_error_surfaces(options): + columns = fw.Layout([fw.Column("n", 0, 2, fw.Int64)]) + stream = fw.read_arrow_stream(b"01\nXX\n", columns, fw.ReaderOptions(batch_rows=1)) + + reader = pa.RecordBatchReader.from_stream(stream) + reader.read_next_batch() + try: + reader.read_next_batch() + raise AssertionError("expected an error") + except OSError as error: + assert "Line 2" in str(error) + + +def test_table_to_arrow(cotahist, options): + table = fw.read(cotahist, layout(), options) + + assert table.to_arrow().column("preabe").to_pylist() == [1234.56, 678.9, 43.21] + + +def dictionary_layout() -> fw.Layout: + return fw.Layout( + [ + fw.Column(c.name, c.start, c.length, c.type, scale=c.scale, format=c.format, nullable=c.nullable, + padding=c.padding, dictionary=c.type == fw.String) + for c in layout().columns + ] + ) + + +def test_dictionary_columns_reach_pandas_and_polars_as_categories(cotahist, options): + table = fw.read_arrow(cotahist, dictionary_layout(), options) + frame = fw.read_pandas(cotahist, dictionary_layout(), options) + polars_frame = fw.read_polars(cotahist, dictionary_layout(), options) + + assert pa.types.is_dictionary(table.schema.field("codneg").type) + assert table.column("codneg").to_pylist() == ["PETR4", "VALE3", "ITUB4"] + assert str(frame["codneg"].dtype) == "category" + assert list(frame["codneg"]) == ["PETR4", "VALE3", "ITUB4"] + assert polars_frame.schema["codneg"] == pl.Categorical + assert table.column("preabe").to_pylist() == fw.read_arrow(cotahist, layout(), options).column("preabe").to_pylist() + + +def test_dictionary_columns_in_to_arrow_and_streams(cotahist, options): + converted = fw.read(cotahist, dictionary_layout(), options).to_arrow() + streamed = pa.RecordBatchReader.from_stream( + fw.read_arrow_stream(cotahist, dictionary_layout(), fw.ReaderOptions(read=options, batch_rows=2)) + ).read_all() + + assert pa.types.is_dictionary(converted.schema.field("codneg").type) + assert converted.column("codneg").to_pylist() == streamed.column("codneg").to_pylist() == ["PETR4", "VALE3", "ITUB4"] + + +def test_categorical_columns_write_like_strings(cotahist, options): + frame = fw.read_pandas(cotahist, dictionary_layout(), options) + + assert fw.write_bytes(frame, layout()) == fw.write_bytes(fw.read_pandas(cotahist, layout(), options), layout()) + + +def test_dictionary_only_on_string_columns(): + with pytest.raises(fw.FixedWidthError, match="dictionary"): + fw.Layout([fw.Column("n", 0, 4, fw.Int64, dictionary=True)]) diff --git a/python/tests/test_native.py b/python/tests/test_native.py new file mode 100644 index 0000000..e2be040 --- /dev/null +++ b/python/tests/test_native.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest + +import fixedwidth as fw +from fixedwidth import _native + + +def test_abi_version_matches(): + assert _native.load_library().fw_abi_version() == _native.FW_ABI_VERSION + + +def test_overlapping_columns_raise(): + with pytest.raises(fw.FixedWidthError, match="Overlapping"): + fw.Layout([fw.Column("a", 0, 5, fw.String), fw.Column("b", 3, 5, fw.String)]) + + +def test_layout_validates_options(): + with pytest.raises(fw.FixedWidthError, match="scale"): + fw.Layout([fw.Column("a", 0, 5, fw.Int64, scale=2)]) + with pytest.raises(ValueError, match="one ASCII character"): + fw.Layout([fw.Column("a", 0, 5, fw.String, padding="ab")]) + + +def test_layout_lists_its_columns(): + columns = [fw.Column("a", 0, 2, fw.String), fw.Column("b", 2, 3, fw.Int64)] + + assert fw.Layout(columns).columns == columns + + +def test_unknown_encoding_raises(): + with pytest.raises(ValueError, match="unknown encoding"): + fw.read(b"ab\n", fw.Layout([fw.Column("a", 0, 2, fw.String)]), fw.ReadOptions(encoding="klingon")) diff --git a/python/tests/test_read.py b/python/tests/test_read.py new file mode 100644 index 0000000..edf4772 --- /dev/null +++ b/python/tests/test_read.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import datetime as dt +import gc +import io + +import numpy as np +import pytest + +import fixedwidth as fw +from conftest import CONTENT, layout + + +def test_read_every_type(cotahist, options): + table = fw.read(cotahist, layout(), options) + + assert table.num_rows == 3 + assert table.column_names == ["tipreg", "data", "codneg", "nomres", "preabe", "quatot"] + assert table["codneg"] == ["PETR4", "VALE3", "ITUB4"] + assert table["nomres"][2] == "ITAU UNI PN" + assert table["preabe"].tolist() == [1234.56, 678.9, 43.21] + assert table["quatot"].dtype == np.int64 + assert table.validity("quatot").tolist() == [True, False, True] + assert table.validity("preabe") is None + days = (dt.date(2023, 10, 24) - dt.date(1970, 1, 1)).days + assert table["data"].tolist() == [days, days, days + 1] + + +def test_read_options_header_and_skip(cotahist): + table = fw.read(cotahist, layout(), fw.ReadOptions(header_lines=1, skip_invalid=True)) + + assert table.num_rows == 3 + assert table.lines_skipped == 3 + + +def test_parse_error_has_line_and_column(cotahist): + with pytest.raises(fw.ParseError) as error: + fw.read(cotahist, layout(), fw.ReadOptions(header_lines=1)) + + assert error.value.line_number == 5 + assert error.value.column == 4 + assert "ABCDE" in str(error.value) + + +def test_read_memory_equals_file(cotahist, options): + from_file = fw.read(cotahist, layout(), options) + from_bytes = fw.read(CONTENT.encode("utf-8"), layout(), options) + from_view = fw.read(memoryview(bytearray(CONTENT.encode("utf-8"))), layout(), options) + + for name in from_file.column_names: + assert list(from_bytes[name]) == list(from_file[name]) + assert list(from_view[name]) == list(from_file[name]) + + +def test_parallel_equals_sequential(tmp_path): + body = "".join(f"{i:06d}{i * 7 % 1000:04d}\n" for i in range(20000)) + path = tmp_path / "big.txt" + path.write_text(body) + columns = fw.Layout([fw.Column("id", 0, 6, fw.Int64), fw.Column("n", 6, 4, fw.Float64, scale=1)]) + + sequential = fw.read(path, columns, fw.ReadOptions(degree_of_parallelism=1)) + parallel = fw.read(path, columns, fw.ReadOptions(degree_of_parallelism=4)) + + assert np.array_equal(sequential["id"], parallel["id"]) + assert np.array_equal(sequential["n"], parallel["n"]) + assert parallel.num_rows == 20000 + + +def test_views_keep_table_alive(cotahist, options): + table = fw.read(cotahist, layout(), options) + prices = table["preabe"] + collected = [] + table._owner._on_free = collected.append + + del table + gc.collect() + assert prices.tolist() == [1234.56, 678.9, 43.21] + assert collected == [] + + del prices + gc.collect() + assert collected == [True] + + +def test_read_pandas_matches_read_fwf(cotahist, options): + pd = pytest.importorskip("pandas") + pytest.importorskip("pyarrow") + frame = fw.read_pandas(cotahist, layout(), options) + + expected = pd.read_fwf( + io.StringIO(CONTENT), + colspecs=[(0, 2), (2, 10), (10, 22), (22, 34), (34, 47), (47, 65)], + names=["tipreg", "data", "codneg", "nomres", "preabe", "quatot"], + dtype={"tipreg": str}, + skiprows=1, + skipfooter=2, + engine="python", + ) + expected = expected[expected["codneg"] != "XXXXX"].reset_index(drop=True) + assert frame["codneg"].tolist() == expected["codneg"].tolist() + assert frame["nomres"].tolist() == expected["nomres"].tolist() + assert frame["preabe"].tolist() == (expected["preabe"] / 100).tolist() + assert frame["quatot"].tolist()[0::2] == expected["quatot"].tolist()[0::2] + assert pd.isna(frame["quatot"][1]) + assert frame["data"].tolist()[0] == dt.date(2023, 10, 24) + + +def test_read_polars(cotahist, options): + pytest.importorskip("polars") + pytest.importorskip("pyarrow") + frame = fw.read_polars(cotahist, layout(), options) + + assert frame.shape == (3, 6) + assert frame["codneg"].to_list() == ["PETR4", "VALE3", "ITUB4"] + assert frame["quatot"].null_count() == 1 + + +def test_read_batches(cotahist, options): + batches = list(fw.read_batches(cotahist, layout(), fw.ReaderOptions(read=options, batch_rows=2))) + + assert [b.num_rows for b in batches] == [2, 1] + assert batches[0]["codneg"] + batches[1]["codneg"] == ["PETR4", "VALE3", "ITUB4"] + + +def test_read_latin1(tmp_path): + path = tmp_path / "latin1.txt" + path.write_bytes("AÇÃO 01\n".encode("latin-1")) + columns = fw.Layout([fw.Column("name", 0, 6, fw.String), fw.Column("n", 6, 2, fw.Int64)]) + + assert fw.read(path, columns, fw.ReadOptions(encoding="latin-1"))["name"] == ["AÇÃO"] diff --git a/python/tests/test_stream.py b/python/tests/test_stream.py new file mode 100644 index 0000000..64605bf --- /dev/null +++ b/python/tests/test_stream.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import gc +import gzip +import io + +import numpy as np +import pytest + +import fixedwidth as fw +from conftest import CONTENT, layout + +DATA = CONTENT.encode("utf-8") + + +class Trickle: + """A stream with only read(), handing out a few bytes per call, that can fail partway.""" + + def __init__(self, data: bytes, chunk: int = 3, fail_at: int = -1) -> None: + self.data, self.chunk, self.fail_at, self.position = data, chunk, fail_at, 0 + + def read(self, size: int = -1) -> bytes: + if 0 <= self.fail_at <= self.position: + raise ConnectionError("the network went away") + size = len(self.data) if size < 0 else min(size, self.chunk) + out = self.data[self.position : self.position + size] + self.position += len(out) + return out + + +def codes(tables) -> list[str]: + return [code for table in tables for code in table["codneg"]] + + +@pytest.mark.parametrize("make", [lambda: io.BufferedReader(io.BytesIO(DATA)), lambda: Trickle(DATA)]) +def test_read_batches_from_a_stream_equals_memory(options, make): + reader = fw.ReaderOptions(read=options, batch_rows=2) + expected = list(fw.read_batches(DATA, layout(), reader)) + + batches = list(fw.read_batches(make(), layout(), reader)) + + assert [b.num_rows for b in batches] == [b.num_rows for b in expected] + assert codes(batches) == codes(expected) == ["PETR4", "VALE3", "ITUB4"] + assert np.array_equal(np.concatenate([b["quatot"] for b in batches]), np.concatenate([b["quatot"] for b in expected])) + + +def test_read_batches_from_gzip(tmp_path, options): + path = tmp_path / "cotahist.txt.gz" + with gzip.open(path, "wb") as out: + out.write(DATA) + + with gzip.open(path, "rb") as stream: + batches = list(fw.read_batches(stream, layout(), fw.ReaderOptions(read=options))) + + assert codes(batches) == ["PETR4", "VALE3", "ITUB4"] + + +def test_whole_read_from_a_stream_equals_memory(options): + table = fw.read(Trickle(DATA, chunk=1000), layout(), options) + + assert table["codneg"] == fw.read(DATA, layout(), options)["codneg"] + assert table.lines_skipped == 3 # the header, the invalid line and the trailer + + +def test_bytesio_is_read_from_its_position(options): + buffer = io.BytesIO(b"junk" + DATA) + buffer.seek(4) + + assert fw.read(buffer, layout(), options)["codneg"] == ["PETR4", "VALE3", "ITUB4"] + assert codes(fw.read_batches(buffer, layout(), fw.ReaderOptions(read=options))) == ["PETR4", "VALE3", "ITUB4"] + + +def test_a_failing_stream_reports_its_error(options): + with pytest.raises(fw.FixedWidthError, match="the network went away"): + list(fw.read_batches(Trickle(DATA, fail_at=80), layout(), fw.ReaderOptions(read=options, batch_rows=1))) + + +def test_parse_error_counts_the_header_lines(): + strict = fw.ReaderOptions(read=fw.ReadOptions(header_lines=1, degree_of_parallelism=1)) + + with pytest.raises(fw.ParseError) as error: + list(fw.read_batches(Trickle(DATA), layout(), strict)) + + assert error.value.line_number == 5 + + +def test_the_stream_is_released_when_the_reader_closes(options): + from fixedwidth import reader as module + + batches = fw.read_batches(Trickle(DATA), layout(), fw.ReaderOptions(read=options, batch_rows=1)) + next(batches) + assert len(module._live) == 1 + batches.close() + gc.collect() + + assert module._live == {} + + +def test_arrow_stream_from_a_stream(options): + pl = pytest.importorskip("polars") + + frame = pl.DataFrame(fw.read_arrow_stream(Trickle(DATA), layout(), fw.ReaderOptions(read=options, batch_rows=2))) + + assert frame["codneg"].to_list() == ["PETR4", "VALE3", "ITUB4"] diff --git a/python/tests/test_write.py b/python/tests/test_write.py new file mode 100644 index 0000000..27de360 --- /dev/null +++ b/python/tests/test_write.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import datetime as dt + +import pytest + +import fixedwidth as fw +from conftest import CONTENT, layout + +# Writing goes through pyarrow, which has no musllinux wheels; the musl wheel job tests without it. +pd = pytest.importorskip("pandas") +pl = pytest.importorskip("polars") +pa = pytest.importorskip("pyarrow") + +QUOTES = "".join(line + "\n" for line in CONTENT.splitlines()[1:4]) + + +def test_write_roundtrip_pandas_polars_pyarrow(cotahist, options): + frame = fw.read_pandas(cotahist, layout(), options) + + from_pandas = fw.write_bytes(frame, layout()) + from_polars = fw.write_bytes(pl.from_pandas(frame), layout()) + from_arrow = fw.write_bytes(pa.Table.from_pandas(frame, preserve_index=False), layout()) + + assert from_pandas == from_polars == from_arrow + assert from_pandas.decode("utf-8") == QUOTES + + +def test_write_bytes_equals_file(tmp_path, cotahist, options): + frame = fw.read_arrow(cotahist, layout(), options) + path = tmp_path / "out.txt" + + fw.write(frame, path, layout(), fw.WriteOptions(degree_of_parallelism=4, batch_rows=1)) + + assert path.read_bytes() == fw.write_bytes(frame, layout()) + + +def test_write_options(): + columns = fw.Layout([fw.Column("s", 0, 4, fw.String)], fw.LayoutOptions(line_length=6, filler=".")) + data = pa.table({"s": ["AÇÃO", None]}) + + written = fw.write_bytes(data, columns, fw.WriteOptions(encoding="latin-1", newline="\r\n")) + + assert written == "AÇÃO..\r\n".encode("latin-1") + b" ..\r\n" + + +def test_write_values_from_python_types(): + columns = fw.Layout( + [ + fw.Column("d", 0, 8, fw.Date, format="yyyyMMdd"), + fw.Column("p", 8, 8, fw.Float64, scale=2, padding="0"), + fw.Column("q", 16, 4, fw.Int64), + ] + ) + frame = pd.DataFrame({"d": [dt.date(2023, 10, 24)], "p": [-12.5], "q": [7]}) + + assert fw.write_bytes(frame, columns) == b"20231024-0001250 7\n" + + +def test_write_decimal_with_more_digits_than_the_scale_raises(): + import decimal + + columns = fw.Layout([fw.Column("v", 0, 8, fw.Decimal, scale=2)]) + exact = pa.table({"v": pa.array([decimal.Decimal("1.23"), decimal.Decimal("-0.5")], type=pa.decimal128(10, 3))}) + lossy = pa.table({"v": pa.array([decimal.Decimal("1.239")], type=pa.decimal128(10, 3))}) + + assert fw.write_bytes(exact, columns) == b" 123\n -50\n" + with pytest.raises(fw.FixedWidthError, match="1.239"): + fw.write_bytes(lossy, columns) + + +def test_write_decimal_from_integers_is_exact(): + columns = fw.Layout([fw.Column("v", 0, 20, fw.Decimal, scale=1)]) + data = pa.table({"v": pa.array([9007199254740991, None, -3], type=pa.int64())}) + + assert fw.write_bytes(data, columns) == b" 90071992547409910\n \n -30\n" + + +@pytest.mark.parametrize( + "values", + [ + pa.array([1e30]), + pa.array([float("nan")]), + pa.array([float("-inf")]), + pa.array([2**62], type=pa.int64()), + pa.array([2**64 - 1], type=pa.uint64()), + pa.array([10**17], type=pa.decimal128(38, 0)), + ], +) +def test_write_decimal_out_of_range_or_not_finite_raises(values): + columns = fw.Layout([fw.Column("v", 0, 20, fw.Decimal, scale=2)]) + + with pytest.raises(fw.FixedWidthError, match="column 'v'"): + fw.write_bytes(pa.table({"v": values}), columns) + + +def test_write_decimal_from_floats_rounds_half_to_even_to_the_scale(): + columns = fw.Layout([fw.Column("v", 0, 6, fw.Decimal, scale=2)]) + data = pa.table({"v": pa.array([1.234, 0.125, -2.5, None])}) + + assert fw.write_bytes(data, columns) == b" 123\n 12\n -250\n \n" + + +def test_read_invalid_utf8_reaches_polars_and_pandas_as_valid_text(): + columns = fw.Layout([fw.Column("name", 0, 6, fw.String), fw.Column("n", 6, 2, fw.Int64)]) + data = "AÇÃO 01\n".encode("latin-1") + + assert fw.read_arrow(data, columns).validate(full=True) is None + assert fw.read_polars(data, columns)["name"].to_list()[0].startswith("A") + assert fw.read_pandas(data, columns)["name"][0].startswith("A") + + +def test_write_mismatched_columns_raise(): + columns = fw.Layout([fw.Column("a", 0, 2, fw.String), fw.Column("b", 2, 2, fw.Int64)]) + + with pytest.raises(fw.FixedWidthError, match="b"): + fw.write_bytes(pa.table({"a": ["x"]}), columns) + + +def test_write_to_streams_equals_write_bytes(tmp_path, cotahist, options): + import gzip + import io + + table = fw.read_arrow(cotahist, layout(), options) + batched = fw.WriteOptions(degree_of_parallelism=4, batch_rows=1) + expected = fw.write_bytes(table, layout()) + + buffer = io.BytesIO() + fw.write(table, buffer, layout(), batched) + with gzip.open(tmp_path / "out.txt.gz", "wb") as compressed: + fw.write(table, compressed, layout(), batched) + + assert buffer.getvalue() == expected + assert gzip.decompress((tmp_path / "out.txt.gz").read_bytes()) == expected + + +class Chunks: + """A duck-typed sink that keeps every chunk it is given and returns nothing from write().""" + + def __init__(self) -> None: + self.chunks: list[object] = [] + + def write(self, data) -> None: + self.chunks.append(data) + + +def test_write_to_a_duck_typed_sink_gets_copies_it_may_keep(cotahist, options): + table = fw.read_arrow(cotahist, layout(), options) + sink = Chunks() + + fw.write(table, sink, layout(), fw.WriteOptions(degree_of_parallelism=2, batch_rows=1)) + + assert all(isinstance(chunk, bytes) for chunk in sink.chunks) + assert b"".join(sink.chunks) == fw.write_bytes(table, layout()) + + +class Full: + def write(self, data) -> int: + raise OSError("no space left on device") + + +def test_a_failing_sink_reports_its_error(cotahist, options): + with pytest.raises(fw.FixedWidthError, match="no space left on device"): + fw.write(fw.read_arrow(cotahist, layout(), options), Full(), layout()) diff --git a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs index d6c14dd..5876018 100644 --- a/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs +++ b/src/FixedWidthParser.Generator/FixedWidthModelGenerator.cs @@ -1,27 +1,28 @@ -using System; +using System.Collections.Generic; using System.Collections.Immutable; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; +using System; using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; namespace FixedWidthParser.Generator { /// - /// Emits a reflection-free TryParse for every type that declares - /// FixedWidthParser.IFixedWidthModel<TSelf>, mirroring the runtime parser's semantics. + /// Emits a reflection-free TryParse and TryFormat over bytes for every + /// type that declares FixedWidthParser.IFixedWidthModel<TSelf>, + /// mirroring the runtime parser's and writer's semantics. /// [Generator(LanguageNames.CSharp)] public sealed class FixedWidthModelGenerator : IIncrementalGenerator { private const string MarkerMetadataName = "FixedWidthParser.IFixedWidthModel`1"; - private const string Utf8MarkerMetadataName = "FixedWidthParser.IUtf8FixedWidthModel`1"; - private const string ColumnAttributeMetadataName = "FixedWidthParser.Attributes.FixedColumnAttribute"; - private const string ConverterInterfaceMetadataName = "FixedWidthParser.Processors.IFixedWidthConverter`1"; - private const string Utf8ConverterInterfaceMetadataName = "FixedWidthParser.Processors.IUtf8FixedWidthConverter`1"; + private const string ColumnAttributeMetadataName = "FixedWidthParser.FixedColumnAttribute"; + private const string ConverterInterfaceMetadataName = "FixedWidthParser.IFixedWidthConverter`1"; private const string SpanFormattableMetadataName = "System.ISpanFormattable"; + private const string RuntimeFqn = "global::FixedWidthParser.Infrastructure.FixedWidthRuntime"; private static readonly DiagnosticDescriptor MustBePartial = new( "FWP001", "Fixed-width model must be partial", @@ -35,12 +36,7 @@ public sealed class FixedWidthModelGenerator : IIncrementalGenerator private static readonly DiagnosticDescriptor UnsupportedColumnType = new( "FWP003", "Unsupported fixed-width column type", - "Column '{0}' has type '{1}', which is not string, double, float or ISpanParsable; generated parsing cannot handle it", - "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); - - private static readonly DiagnosticDescriptor UnsupportedUtf8ColumnType = new( - "FWP007", "Unsupported UTF-8 fixed-width column type", - "Column '{0}' has type '{1}', which is not string, double, float or IUtf8SpanParsable; generated UTF-8 parsing cannot handle it", + "Column '{0}' has type '{1}', which is not string, double, float, an enum, IUtf8SpanParsable or ISpanParsable; set a Converter for it", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); private static readonly DiagnosticDescriptor NegativeStart = new( @@ -63,14 +59,9 @@ public sealed class FixedWidthModelGenerator : IIncrementalGenerator "Converter '{0}' for column '{1}' does not implement IFixedWidthConverter<{2}>", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); - private static readonly DiagnosticDescriptor UnsupportedUtf8ConverterType = new( - "FWP009", "Converter does not implement IUtf8FixedWidthConverter", - "Converter '{0}' for column '{1}' does not implement IUtf8FixedWidthConverter<{2}>", - "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); - private static readonly DiagnosticDescriptor UnsupportedWriteColumnType = new( "FWP010", "Unsupported fixed-width column type for writing", - "Column '{0}' has type '{1}', which is not string or ISpanFormattable; generated writing cannot handle it", + "Column '{0}' has type '{1}', which is not string or ISpanFormattable; set a Converter for it", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); private static readonly DiagnosticDescriptor InvalidOverflowBehavior = new( @@ -78,6 +69,11 @@ public sealed class FixedWidthModelGenerator : IIncrementalGenerator "Column '{0}' in '{1}' has type '{2}' and specifies OverflowBehavior.Truncate, which is only supported for string columns", "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); + private static readonly DiagnosticDescriptor NonAsciiCharacter = new( + "FWP012", "Padding and TrimChar must be ASCII", + "Column '{0}' in '{1}' sets {2} to '{3}', which is not an ASCII character; column positions are bytes, so it must be one byte in every supported encoding", + "FixedWidthParser", DiagnosticSeverity.Error, isEnabledByDefault: true); + public void Initialize(IncrementalGeneratorInitializationContext context) { var models = context.SyntaxProvider.CreateSyntaxProvider( @@ -87,6 +83,7 @@ public void Initialize(IncrementalGeneratorInitializationContext context) context.RegisterSourceOutput(models, static (spc, model) => Emit(spc, model!)); } + private static ModelInfo? Extract(GeneratorSyntaxContext ctx, System.Threading.CancellationToken ct) { var declaration = (TypeDeclarationSyntax)ctx.Node; @@ -98,97 +95,30 @@ public void Initialize(IncrementalGeneratorInitializationContext context) var compilation = ctx.SemanticModel.Compilation; var marker = compilation.GetTypeByMetadataName(MarkerMetadataName); - var utf8Marker = compilation.GetTypeByMetadataName(Utf8MarkerMetadataName); var columnAttribute = compilation.GetTypeByMetadataName(ColumnAttributeMetadataName); - if (columnAttribute is null || (marker is null && utf8Marker is null)) - { - return null; - } - - // Implements IFixedWidthModel (char) and/or IUtf8FixedWidthModel (byte)? - bool implementsChar = ImplementsMarker(symbol, marker); - bool implementsUtf8 = ImplementsMarker(symbol, utf8Marker); - if (!implementsChar && !implementsUtf8) + if (columnAttribute is null || marker is null || !ImplementsMarker(symbol, marker)) { return null; } - bool isPartial = declaration.Modifiers.Any(m => m.IsKind(SyntaxKind.PartialKeyword)); - bool isNested = symbol.ContainingType is not null; - + // Inherited columns too, as reflection's GetProperties/GetFields see them: walking from the + // model to its bases, the first public member of a name wins, so an override or a 'new' + // member replaces the base member it hides. var columns = ImmutableArray.CreateBuilder(); - foreach (var member in symbol.GetMembers()) + var seen = new HashSet(StringComparer.Ordinal); + for (var type = symbol; type is not null; type = type.BaseType) { - if (member.IsStatic) - { - continue; - } - - ITypeSymbol? memberType = member switch + foreach (var member in type.GetMembers()) { - IPropertySymbol { SetMethod: not null, DeclaredAccessibility: Accessibility.Public } p => p.Type, - IFieldSymbol { IsReadOnly: false, IsConst: false, DeclaredAccessibility: Accessibility.Public } f => f.Type, - _ => null - }; - if (memberType is null) - { - continue; - } - - var attribute = member.GetAttributes().FirstOrDefault(a => - SymbolEqualityComparer.Default.Equals(a.AttributeClass, columnAttribute)); - if (attribute is null || attribute.ConstructorArguments.Length < 2) - { - continue; - } - - int start = attribute.ConstructorArguments[0].Value is int s ? s : 0; - int length = attribute.ConstructorArguments[1].Value is int l ? l : 0; - - ITypeSymbol? converterType = null; - int alignment = 0; // Alignment.Left - char padding = ' '; - string? format = null; - int overflow = 0; // OverflowBehavior.Default - char trimChar = ' '; - int trimMode = 0; // TrimMode.Trailing - foreach (var namedArg in attribute.NamedArguments) - { - switch (namedArg.Key) + if (member is not (IPropertySymbol or IFieldSymbol) || member.DeclaredAccessibility != Accessibility.Public || !seen.Add(member.Name)) { - case "Converter" when namedArg.Value.Value is ITypeSymbol converterSymbol: - converterType = converterSymbol; - break; - case "Alignment" when namedArg.Value.Value is int a: - alignment = a; - break; - case "Padding" when namedArg.Value.Value is char p: - padding = p; - break; - case "Format": - format = namedArg.Value.Value as string; - break; - case "Overflow" when namedArg.Value.Value is int o: - overflow = o; - break; - case "TrimChar" when namedArg.Value.Value is char t: - trimChar = t; - break; - case "TrimMode" when namedArg.Value.Value is int tm: - trimMode = tm; - break; + continue; + } + if (ExtractColumn(member, columnAttribute, compilation) is { } column) + { + columns.Add(column); } } - - var (kind, parsableTypeFqn, converterFqn, isNullable, isCharParsable, isUtf8Parsable) = Classify(memberType, converterType, format, compilation); - var (writeKind, isCharFormattable) = ClassifyWrite(memberType, converterType, compilation); - // Mirrors FixedWidthWriter.DetermineOverflowBehavior: an explicit Overflow wins; otherwise - // string truncates and everything else throws. - int resolvedOverflow = overflow != 0 ? overflow : (writeKind == WriteKind.String ? 1 : 2); - - columns.Add(new ColumnInfo( - member.Name, start, length, kind, parsableTypeFqn, converterFqn, isNullable, isCharParsable, isUtf8Parsable, - writeKind, isCharFormattable, alignment, padding, format, resolvedOverflow, trimChar, trimMode)); } string? ns = symbol.ContainingNamespace.IsGlobalNamespace @@ -201,102 +131,175 @@ public void Initialize(IncrementalGeneratorInitializationContext context) symbol.Name, symbol.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), new EquatableArray(columns.ToImmutable()), - isPartial, - isNested, - implementsChar, - implementsUtf8, + declaration.Modifiers.Any(m => m.IsKind(SyntaxKind.PartialKeyword)), + symbol.ContainingType is not null, declaration.Identifier.GetLocation()); } - // Implements I…Model for the given (open) marker interface? - private static bool ImplementsMarker(INamedTypeSymbol symbol, INamedTypeSymbol? marker) + // A settable public instance property or field carrying [FixedColumn]; null for any other member. + private static ColumnInfo? ExtractColumn(ISymbol member, INamedTypeSymbol columnAttribute, Compilation compilation) + { + if (member.IsStatic) + { + return null; + } + + ITypeSymbol? memberType = member switch + { + IPropertySymbol { SetMethod: not null, DeclaredAccessibility: Accessibility.Public } p => p.Type, + IFieldSymbol { IsReadOnly: false, IsConst: false, DeclaredAccessibility: Accessibility.Public } f => f.Type, + _ => null + }; + if (memberType is null) + { + return null; + } + + var attribute = ColumnAttribute(member, columnAttribute); + if (attribute is null || attribute.ConstructorArguments.Length < 2) + { + return null; + } + + int start = attribute.ConstructorArguments[0].Value is int s ? s : 0; + int length = attribute.ConstructorArguments[1].Value is int l ? l : 0; + + ITypeSymbol? converterType = null; + int alignment = 0; // Alignment.Left + char padding = ' '; + string? format = null; + int overflow = 0; // OverflowBehavior.Default + char trimChar = ' '; + int trimMode = 0; // TrimMode.Trailing + foreach (var namedArg in attribute.NamedArguments) + { + switch (namedArg.Key) + { + case "Converter" when namedArg.Value.Value is ITypeSymbol converterSymbol: + converterType = converterSymbol; + break; + case "Alignment" when namedArg.Value.Value is int a: + alignment = a; + break; + case "Padding" when namedArg.Value.Value is char p: + padding = p; + break; + case "Format": + format = namedArg.Value.Value as string; + break; + case "Overflow" when namedArg.Value.Value is int o: + overflow = o; + break; + case "TrimChar" when namedArg.Value.Value is char t: + trimChar = t; + break; + case "TrimMode" when namedArg.Value.Value is int tm: + trimMode = tm; + break; + } + } + + var underlying = UnwrapNullable(memberType); + bool isNullable = !SymbolEqualityComparer.Default.Equals(underlying, memberType); + string? converterFqn = converterType?.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); + var kind = Classify(underlying, converterType, format, compilation); + var writeKind = ClassifyWrite(underlying, converterType, compilation); + // Mirrors FormatterBuilder.ResolveOptions: an explicit Overflow wins; otherwise string truncates and everything else throws. + int resolvedOverflow = overflow; + if (resolvedOverflow == 0) + { + resolvedOverflow = writeKind == WriteKind.String ? 1 : 2; + } + + return new ColumnInfo( + member.Name, start, length, kind, underlying.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat), converterFqn, + isNullable, writeKind, alignment, padding, format, resolvedOverflow, trimChar, trimMode); + } + + // [FixedColumn] is inherited: an override without one takes it from the property it overrides, as reflection does. + private static AttributeData? ColumnAttribute(ISymbol member, INamedTypeSymbol columnAttribute) { - return marker is not null && symbol.AllInterfaces.Any(i => + for (ISymbol? current = member; current is not null; current = (current as IPropertySymbol)?.OverriddenProperty) + { + var attribute = current.GetAttributes().FirstOrDefault(a => + SymbolEqualityComparer.Default.Equals(a.AttributeClass, columnAttribute)); + if (attribute is not null) + { + return attribute; + } + } + return null; + } + + // Implements IFixedWidthModel? + private static bool ImplementsMarker(INamedTypeSymbol symbol, INamedTypeSymbol marker) + { + return symbol.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, marker) && i.TypeArguments.Length == 1 && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], symbol)); } - private static (ColumnKind Kind, string TypeFqn, string? ConverterFqn, bool IsNullable, bool IsCharParsable, bool IsUtf8Parsable) Classify( - ITypeSymbol type, ITypeSymbol? converterType, string? format, Compilation compilation) + // Mirrors ColumnParserFactory: converter → date/time with Format → string → double/float → enum → + // IUtf8SpanParsable → ISpanParsable over the decoded text. + private static ColumnKind Classify(ITypeSymbol type, ITypeSymbol? converterType, string? format, Compilation compilation) { - // T? (Nullable): classify the underlying T (including any converter, which targets T, - // not T?) and flag IsNullable so Emit/codegen wrap it with the blank-is-null check. - if (type is INamedTypeSymbol { OriginalDefinition.SpecialType: SpecialType.System_Nullable_T } named) + if (converterType is not null) { - var underlying = Classify(named.TypeArguments[0], converterType, format, compilation); - return (underlying.Kind, underlying.TypeFqn, underlying.ConverterFqn, true, underlying.IsCharParsable, underlying.IsUtf8Parsable); + return ImplementsGeneric(converterType, type, compilation, ConverterInterfaceMetadataName) ? ColumnKind.Converter : ColumnKind.BadConverter; } string fqn = type.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); - - // FixedColumnAttribute.Converter takes priority over the built-in primitive/parsable - // handling, for both string and non-string column types. - if (converterType is not null) + if (format is not null && ExactHelper(fqn) is not null) { - string converterFqn = converterType.ToDisplayString(SymbolDisplayFormat.FullyQualifiedFormat); - bool convertsChar = ImplementsConverter(converterType, type, compilation, ConverterInterfaceMetadataName); - bool convertsUtf8 = ImplementsConverter(converterType, type, compilation, Utf8ConverterInterfaceMetadataName); - var convKind = convertsChar || convertsUtf8 ? ColumnKind.Converter : ColumnKind.Unsupported; - return (convKind, fqn, converterFqn, false, convertsChar, convertsUtf8); + return ColumnKind.Exact; } switch (type.SpecialType) { case SpecialType.System_String: - return (ColumnKind.String, "string", null, false, true, true); + return ColumnKind.String; case SpecialType.System_Double: - return (ColumnKind.Double, "double", null, false, true, true); + return ColumnKind.Double; case SpecialType.System_Single: - return (ColumnKind.Float, "float", null, false, true, true); + return ColumnKind.Float; default: break; } - bool isCharParsable = ImplementsParsable(type, compilation, "System.ISpanParsable`1"); - bool isUtf8Parsable = ImplementsParsable(type, compilation, "System.IUtf8SpanParsable`1"); - - bool isDateTimeType = IsDateTimeType(fqn); - if (isDateTimeType && format is not null) + if (type.TypeKind == TypeKind.Enum) { - isCharParsable = true; - isUtf8Parsable = true; + return ColumnKind.Enum; } - - // SpanParsable when at least one path can handle it; otherwise unsupported by every path. - var kind = isCharParsable || isUtf8Parsable ? ColumnKind.SpanParsable : ColumnKind.Unsupported; - return (kind, fqn, null, false, isCharParsable, isUtf8Parsable); - } - - private static bool IsDateTimeType(string fqn) - { - return fqn is "global::System.DateTime" or "System.DateTime" - or "global::System.DateOnly" or "System.DateOnly" - or "global::System.TimeOnly" or "System.TimeOnly" - or "global::System.DateTimeOffset" or "System.DateTimeOffset"; + if (ImplementsGeneric(type, type, compilation, "System.IUtf8SpanParsable`1")) + { + return ColumnKind.Utf8Parsable; + } + return ImplementsGeneric(type, type, compilation, "System.ISpanParsable`1") ? ColumnKind.DecodedParsable : ColumnKind.Unsupported; } - // Classification for the write side: mirrors FixedWidthWriter.CreateFormatter's own resolution - // (converter → string → ISpanFormattable fallback), but write has no "primitive" concept — - // double/float are ISpanFormattable like everything else, no special-casing. - private static (WriteKind Kind, bool IsCharFormattable) ClassifyWrite(ITypeSymbol type, ITypeSymbol? converterType, Compilation compilation) + // Mirrors FormatterBuilder: converter → string → ISpanFormattable. + private static WriteKind ClassifyWrite(ITypeSymbol type, ITypeSymbol? converterType, Compilation compilation) { - var underlying = UnwrapNullable(type); - if (converterType is not null) { - bool convertsChar = ImplementsConverter(converterType, underlying, compilation, ConverterInterfaceMetadataName); - bool convertsUtf8 = ImplementsConverter(converterType, underlying, compilation, Utf8ConverterInterfaceMetadataName); - return (convertsChar || convertsUtf8 ? WriteKind.Converter : WriteKind.Unsupported, false); + return ImplementsGeneric(converterType, type, compilation, ConverterInterfaceMetadataName) ? WriteKind.Converter : WriteKind.Unsupported; } - - if (underlying.SpecialType == SpecialType.System_String) + if (type.SpecialType == SpecialType.System_String) + { + return WriteKind.String; + } + if (!Implements(type, compilation, SpanFormattableMetadataName)) { - return (WriteKind.String, true); + return WriteKind.Unsupported; } + return Implements(type, compilation, "System.IUtf8SpanFormattable") ? WriteKind.Utf8Formattable : WriteKind.Formattable; + } - bool isCharFormattable = ImplementsInterface(underlying, compilation, SpanFormattableMetadataName); - return (isCharFormattable ? WriteKind.Formattable : WriteKind.Unsupported, isCharFormattable); + private static bool Implements(ITypeSymbol type, Compilation compilation, string interfaceMetadataName) + { + var iface = compilation.GetTypeByMetadataName(interfaceMetadataName); + return iface is not null && type.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i, iface)); } private static ITypeSymbol UnwrapNullable(ITypeSymbol type) @@ -306,31 +309,26 @@ private static ITypeSymbol UnwrapNullable(ITypeSymbol type) : type; } - // Implements the given (non-generic) BCL interface metadata name? - private static bool ImplementsInterface(ITypeSymbol type, Compilation compilation, string interfaceMetadataName) + // Does `type` implement the open interface `interfaceMetadataName` closed over `argument`? + private static bool ImplementsGeneric(ITypeSymbol type, ITypeSymbol argument, Compilation compilation, string interfaceMetadataName) { var iface = compilation.GetTypeByMetadataName(interfaceMetadataName); - return iface is not null && type.AllInterfaces.Any(i => SymbolEqualityComparer.Default.Equals(i, iface)); - } - - // Implements I…SpanParsable for the given (open) BCL interface metadata name? - private static bool ImplementsParsable(ITypeSymbol type, Compilation compilation, string interfaceMetadataName) - { - var parsable = compilation.GetTypeByMetadataName(interfaceMetadataName); - return parsable is not null && type.AllInterfaces.Any(i => - SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, parsable) + return iface is not null && type.AllInterfaces.Any(i => + SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, iface) && i.TypeArguments.Length == 1 - && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], type)); + && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], argument)); } - // Implements I…Converter (the given open interface) for the column's converter type? - private static bool ImplementsConverter(ITypeSymbol converterType, ITypeSymbol columnType, Compilation compilation, string interfaceMetadataName) + private static string? ExactHelper(string fqn) { - var iface = compilation.GetTypeByMetadataName(interfaceMetadataName); - return iface is not null && converterType.AllInterfaces.Any(i => - SymbolEqualityComparer.Default.Equals(i.OriginalDefinition, iface) - && i.TypeArguments.Length == 1 - && SymbolEqualityComparer.Default.Equals(i.TypeArguments[0], columnType)); + return fqn switch + { + "global::System.DateTime" => "TryDateTimeExact", + "global::System.DateOnly" => "TryDateOnlyExact", + "global::System.TimeOnly" => "TryTimeOnlyExact", + "global::System.DateTimeOffset" => "TryDateTimeOffsetExact", + _ => null, + }; } private static string BuildTypeKeywords(INamedTypeSymbol symbol) @@ -369,122 +367,103 @@ private static void Emit(SourceProductionContext spc, ModelInfo model) return; } - // Structural errors (geometry) block ALL generated methods (parse char/byte + write). - // Per-column-type errors block only the affected path: FWP003 the char parse method, - // FWP007 the byte parse method, FWP010/FWP008 the write method. - bool structuralError = false; - bool charTypeError = false; - bool utf8TypeError = false; - bool writeTypeError = false; + // The interface needs both methods, so any column error blocks the whole emission. + bool error = false; var columns = model.Columns.AsImmutableArray(); foreach (var column in columns) { - bool isPrimitive = column.Kind is ColumnKind.String or ColumnKind.Double or ColumnKind.Float; - if (!isPrimitive) - { - if (model.ImplementsChar && !column.IsCharParsable) - { - spc.ReportDiagnostic(column.ConverterFqn is not null - ? Diagnostic.Create(UnsupportedConverterType, model.Location, column.ConverterFqn, column.Name, column.TypeFqn) - : Diagnostic.Create(UnsupportedColumnType, model.Location, column.Name, column.TypeFqn)); - charTypeError = true; - } - if (model.ImplementsUtf8 && !column.IsUtf8Parsable) - { - spc.ReportDiagnostic(column.ConverterFqn is not null - ? Diagnostic.Create(UnsupportedUtf8ConverterType, model.Location, column.ConverterFqn, column.Name, column.TypeFqn) - : Diagnostic.Create(UnsupportedUtf8ColumnType, model.Location, column.Name, column.TypeFqn)); - utf8TypeError = true; - } - } - if (model.ImplementsChar) - { - bool writeOk = column.WKind switch - { - WriteKind.String => true, - WriteKind.Converter => column.IsCharParsable, // IFixedWidthConverter is symmetric read/write - WriteKind.Formattable => column.IsCharFormattable, - _ => false - }; - if (!writeOk) - { - spc.ReportDiagnostic(column.ConverterFqn is not null - ? Diagnostic.Create(UnsupportedConverterType, model.Location, column.ConverterFqn, column.Name, column.TypeFqn) - : Diagnostic.Create(UnsupportedWriteColumnType, model.Location, column.Name, column.TypeFqn)); - writeTypeError = true; - } - if (column.Overflow == 1 && column.WKind != WriteKind.String) - { - spc.ReportDiagnostic(Diagnostic.Create(InvalidOverflowBehavior, model.Location, column.Name, model.TypeName, column.TypeFqn)); - writeTypeError = true; - } - } - if (column.Start < 0) - { - spc.ReportDiagnostic(Diagnostic.Create(NegativeStart, model.Location, column.Name, model.TypeName, column.Start)); - structuralError = true; - } - if (column.Length < 1) - { - spc.ReportDiagnostic(Diagnostic.Create(InvalidLength, model.Location, column.Name, model.TypeName, column.Length)); - structuralError = true; - } + error |= ReportColumnErrors(spc, model, column); } - - if (columns.Length > 1) + error |= ReportOverlaps(spc, model, columns); + if (error) { - var sorted = columns.ToArray(); - Array.Sort(sorted, static (a, b) => - a.Start != b.Start ? a.Start.CompareTo(b.Start) : a.Length.CompareTo(b.Length)); + return; + } - var farthest = sorted[0]; - int maxEnd = farthest.Start + farthest.Length; - foreach (ref readonly var current in sorted.AsSpan(1)) - { - if (current.Start < maxEnd) - { - spc.ReportDiagnostic(Diagnostic.Create( - OverlappingColumns, - model.Location, - model.TypeName, - farthest.Name, - farthest.Start, - farthest.Start + farthest.Length, - current.Name, - current.Start, - current.Start + current.Length)); - structuralError = true; - } + spc.AddSource(model.FullyQualifiedName.Replace("global::", string.Empty) + ".FixedWidth.g.cs", BuildSource(model)); + } + + private static bool ReportColumnErrors(SourceProductionContext spc, ModelInfo model, ColumnInfo column) + { + bool error = false; + void Report(DiagnosticDescriptor descriptor, params object?[] args) + { + spc.ReportDiagnostic(Diagnostic.Create(descriptor, model.Location, args)); + error = true; + } - int end = current.Start + current.Length; - if (end > maxEnd) + switch (column.Kind) + { + case ColumnKind.BadConverter: + Report(UnsupportedConverterType, column.ConverterFqn, column.Name, column.TypeFqn); + break; + case ColumnKind.Unsupported: + Report(UnsupportedColumnType, column.Name, column.TypeFqn); + break; + default: + if (column.WKind == WriteKind.Unsupported) { - maxEnd = end; - farthest = current; + Report(UnsupportedWriteColumnType, column.Name, column.TypeFqn); } - } + break; } - if (structuralError) + if (column.Overflow == 1 && column.WKind != WriteKind.String) { - return; + Report(InvalidOverflowBehavior, column.Name, model.TypeName, column.TypeFqn); + } + if (column.Padding > '\x7F') + { + Report(NonAsciiCharacter, column.Name, model.TypeName, "Padding", column.Padding); + } + if (column.TrimChar > '\x7F') + { + Report(NonAsciiCharacter, column.Name, model.TypeName, "TrimChar", column.TrimChar); + } + if (column.Start < 0) + { + Report(NegativeStart, column.Name, model.TypeName, column.Start); + } + if (column.Length < 1) + { + Report(InvalidLength, column.Name, model.TypeName, column.Length); } + return error; + } - bool emitChar = model.ImplementsChar && !charTypeError; - bool emitUtf8 = model.ImplementsUtf8 && !utf8TypeError; - bool emitWrite = model.ImplementsChar && !writeTypeError; - if (!emitChar && !emitUtf8 && !emitWrite) + private static bool ReportOverlaps(SourceProductionContext spc, ModelInfo model, ImmutableArray columns) + { + if (columns.Length < 2) { - return; + return false; } - string hint = model.FullyQualifiedName.Replace("global::", string.Empty) + ".FixedWidth.g.cs"; - spc.AddSource(hint, BuildSource(model, emitChar, emitUtf8, emitWrite)); - } + bool error = false; + var sorted = columns.ToArray(); + Array.Sort(sorted, static (a, b) => a.Start != b.Start ? a.Start.CompareTo(b.Start) : a.Length.CompareTo(b.Length)); + var farthest = sorted[0]; + int maxEnd = farthest.Start + farthest.Length; + foreach (ref readonly var current in sorted.AsSpan(1)) + { + if (current.Start < maxEnd) + { + spc.ReportDiagnostic(Diagnostic.Create( + OverlappingColumns, model.Location, model.TypeName, + farthest.Name, farthest.Start, farthest.Start + farthest.Length, + current.Name, current.Start, current.Start + current.Length)); + error = true; + } - private const string CharRuntimeFqn = "global::FixedWidthParser.FixedWidthRuntime"; - private const string Utf8RuntimeFqn = "global::FixedWidthParser.Utf8FixedWidthRuntime"; + int end = current.Start + current.Length; + if (end > maxEnd) + { + maxEnd = end; + farthest = current; + } + } + return error; + } - private static string BuildSource(ModelInfo model, bool emitChar, bool emitUtf8, bool emitWrite) + private static string BuildSource(ModelInfo model) { var sb = new StringBuilder(); sb.AppendLine("// "); @@ -506,39 +485,11 @@ private static string BuildSource(ModelInfo model, bool emitChar, bool emitUtf8, sb.Append(indent).AppendLine("{"); string body = indent + " "; - bool wroteConverterFields = AppendConverterFields(sb, body, model); - bool wroteOptionsFields = emitWrite && AppendWriteOptionsFields(sb, body, model); - bool wroteTrimFields = emitUtf8 && AppendUtf8TrimFields(sb, body, model); - if (wroteConverterFields || wroteOptionsFields || wroteTrimFields) - { - sb.AppendLine(); - } - - // TryParse (char / byte) and TryFormat (char) coexist in one partial; each static abstract - // interface member binds implicitly to its matching span element type. - bool wroteMethod = false; - if (emitChar) - { - AppendMethod(sb, body, model, "char", CharRuntimeFqn); - wroteMethod = true; - } - if (emitUtf8) - { - if (wroteMethod) - { - sb.AppendLine(); - } - AppendMethod(sb, body, model, "byte", Utf8RuntimeFqn); - wroteMethod = true; - } - if (emitWrite) - { - if (wroteMethod) - { - sb.AppendLine(); - } - AppendWriteMethod(sb, body, model); - } + AppendFields(sb, body, model); + sb.AppendLine(); + AppendParseMethod(sb, body, model); + sb.AppendLine(); + AppendFormatMethod(sb, body, model); sb.Append(indent).AppendLine("}"); if (model.Namespace is not null) @@ -548,408 +499,250 @@ private static string BuildSource(ModelInfo model, bool emitChar, bool emitUtf8, return sb.ToString(); } - // Emits one static readonly field per converter-backed column, so the converter instance is - // created once (not per parse call) and shared across both TryParse overloads. Returns whether - // any field was written. - private static bool AppendConverterFields(StringBuilder sb, string body, ModelInfo model) + // One converter instance per converter column (created once, shared by parse and format), and one + // ColumnParseOptions and one ColumnFormatOptions per column, resolved at compile time. + private static void AppendFields(StringBuilder sb, string body, ModelInfo model) { var columns = model.Columns.AsImmutableArray(); - bool any = false; for (int i = 0; i < columns.Length; i++) { - if (columns[i].Kind != ColumnKind.Converter) + var c = columns[i]; + if (c.Kind == ColumnKind.Converter) { - continue; + sb.Append(body).Append("private static readonly ").Append(c.ConverterFqn) + .Append(" __converter").Append(i).Append(" = new ").Append(c.ConverterFqn).AppendLine("();"); } - sb.Append(body).Append("private static readonly ").Append(columns[i].ConverterFqn) - .Append(" __converter").Append(i).Append(" = new ").Append(columns[i].ConverterFqn).AppendLine("();"); - any = true; + sb.Append(body).Append("private static readonly global::FixedWidthParser.Infrastructure.ColumnParseOptions __parse").Append(i) + .Append(" = new(").Append(SymbolDisplay.FormatLiteral(c.TrimChar, true)).Append(", ").Append(TrimModeLiteral(c.TrimMode)) + .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)).AppendLine(");"); + sb.Append(body).Append("private static readonly global::FixedWidthParser.Infrastructure.ColumnFormatOptions __options").Append(i) + .Append(" = new(").Append(AlignmentLiteral(c.Alignment)).Append(", ").Append(SymbolDisplay.FormatLiteral(c.Padding, true)) + .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)) + .Append(", ").Append(OverflowLiteral(c.Overflow)).AppendLine(");"); } - return any; } - // Emits one static readonly byte field per column whose TrimChar isn't the default space, - // converting the char to its single-byte ASCII representation once (at type-init) instead of - // on every parse call. A non-ASCII TrimChar throws there with a clear message, mirroring how - // CultureHelpers rejects a non-ASCII decimal separator on the byte path. - private static bool AppendUtf8TrimFields(StringBuilder sb, string body, ModelInfo model) + private static int LineLength(ModelInfo model) + { + int length = 0; + foreach (var c in model.Columns.AsImmutableArray()) + { + length = Math.Max(length, c.Start + c.Length); + } + return length; + } + + private static void AppendParseMethod(StringBuilder sb, string body, ModelInfo model) { var columns = model.Columns.AsImmutableArray(); - bool any = false; + sb.Append(body) + .AppendLine("[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]"); + sb.Append(body) + .Append("public static bool TryParse(global::System.ReadOnlySpan line, in global::FixedWidthParser.FixedWidthContext context, out ") + .Append(model.FullyQualifiedName).AppendLine(" model)"); + sb.Append(body).AppendLine("{"); + + string stmt = body + " "; + int lineLength = LineLength(model); + if (lineLength > 0) + { + sb.Append(stmt).Append("if (line.Length < ").Append(lineLength).Append(") { context.ReportShortLine(").Append(lineLength) + .AppendLine("); model = default!; return false; }"); + } + for (int i = 0; i < columns.Length; i++) { - if (columns[i].TrimChar == ' ') + var c = columns[i]; + string col = "line.Slice(" + c.Start + ", " + c.Length + ")"; + if (!c.IsNullable) { + AppendColumnParse(sb, stmt, c, col, i, "__v" + i); continue; } - sb.Append(body).Append("private static readonly byte __trim").Append(i) - .Append(" = global::FixedWidthParser.Utf8FixedWidthRuntime.ToAsciiByte(") - .Append(SymbolDisplay.FormatLiteral(columns[i].TrimChar, true)).Append(", ") - .Append(SymbolDisplay.FormatLiteral(columns[i].Name, true)).AppendLine(");"); - any = true; + + // T?: a blank column assigns null without invoking the underlying parser. + sb.Append(stmt).Append(c.TypeFqn).Append("? __v").Append(i).AppendLine(";"); + sb.Append(stmt).Append("if (").Append(RuntimeFqn).Append(".IsBlank(").Append(col).Append(", in __parse").Append(i).AppendLine("))"); + sb.Append(stmt).AppendLine("{"); + sb.Append(stmt).Append(" __v").Append(i).AppendLine(" = null;"); + sb.Append(stmt).AppendLine("}"); + sb.Append(stmt).AppendLine("else"); + sb.Append(stmt).AppendLine("{"); + AppendColumnParse(sb, stmt + " ", c, col, i, "__u" + i); + sb.Append(stmt).Append(" __v").Append(i).Append(" = __u").Append(i).AppendLine(";"); + sb.Append(stmt).AppendLine("}"); } - return any; - } - // Emits one static readonly ColumnFormatOptions field per column (alignment/padding/format/ - // overflow, resolved at Extract time), shared by the write method's per-column formatting calls. - private static bool AppendWriteOptionsFields(StringBuilder sb, string body, ModelInfo model) - { - var columns = model.Columns.AsImmutableArray(); + sb.Append(stmt).Append("model = new ").Append(model.FullyQualifiedName).AppendLine(); + sb.Append(stmt).AppendLine("{"); for (int i = 0; i < columns.Length; i++) { - var c = columns[i]; - sb.Append(body).Append("private static readonly global::FixedWidthParser.Formatters.ColumnFormatOptions __options").Append(i) - .Append(" = new(").Append(AlignmentLiteral(c.Alignment)).Append(", ").Append(SymbolDisplay.FormatLiteral(c.Padding, true)) - .Append(", ").Append(c.Format is null ? "null" : SymbolDisplay.FormatLiteral(c.Format, true)) - .Append(", ").Append(OverflowLiteral(c.Overflow)).AppendLine(");"); + sb.Append(stmt).Append(" ").Append(columns[i].Name).Append(" = __v").Append(i).AppendLine(","); } - return columns.Length > 0; + sb.Append(stmt).AppendLine("};"); + sb.Append(stmt).AppendLine("return true;"); + sb.Append(body).AppendLine("}"); } - private static string AlignmentLiteral(int value) + // Emits the kind-specific parse statement for one column into a local named localName. + private static void AppendColumnParse(StringBuilder sb, string stmt, ColumnInfo c, string col, int i, string localName) { - return value == 1 - ? "global::FixedWidthParser.Attributes.Alignment.Right" - : "global::FixedWidthParser.Attributes.Alignment.Left"; - } + string parse = ", in __parse" + i; + string fail = ")) { context.ReportFailedColumn(" + SymbolDisplay.FormatLiteral(c.Name, true) + ", " + c.Start + ", " + c.Length + + "); model = default!; return false; }"; - private static string OverflowLiteral(int resolvedValue) - { - return resolvedValue switch + switch (c.Kind) { - 1 => "global::FixedWidthParser.Attributes.OverflowBehavior.Truncate", - 2 => "global::FixedWidthParser.Attributes.OverflowBehavior.Throw", - _ => "global::FixedWidthParser.Attributes.OverflowBehavior.Default", - }; + case ColumnKind.Exact: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append('.').Append(ExactHelper(c.TypeFqn)).Append('(').Append(col) + .Append(", context").Append(parse).Append(", out var ").Append(localName).AppendLine(fail); + break; + case ColumnKind.String: + sb.Append(stmt).Append("string ").Append(localName) + .Append(" = ").Append(RuntimeFqn).Append(".String(").Append(col).Append(", context").Append(parse).AppendLine(");"); + break; + case ColumnKind.Double: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryDouble(").Append(col) + .Append(", context").Append(parse).Append(", out double ").Append(localName).AppendLine(fail); + break; + case ColumnKind.Float: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryFloat(").Append(col) + .Append(", context").Append(parse).Append(", out float ").Append(localName).AppendLine(fail); + break; + case ColumnKind.Utf8Parsable: + case ColumnKind.DecodedParsable: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(c.Kind == ColumnKind.Utf8Parsable ? ".TryParse<" : ".TryParseDecoded<") + .Append(c.TypeFqn).Append(">(").Append(col).Append(", context").Append(parse).Append(", out ").Append(c.TypeFqn).Append(' ').Append(localName).AppendLine(fail); + break; + case ColumnKind.Enum: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryEnum<") + .Append(c.TypeFqn).Append(">(").Append(col).Append(", context").Append(parse).Append(", out ").Append(c.TypeFqn).Append(' ').Append(localName).AppendLine(fail); + break; + case ColumnKind.Converter: + sb.Append(stmt).Append("if (!").Append(RuntimeFqn).Append(".TryConvert<").Append(c.TypeFqn).Append(", ").Append(c.ConverterFqn) + .Append(">(").Append(col).Append(", context, __converter").Append(i).Append(parse).Append(", out ").Append(c.TypeFqn) + .Append(' ').Append(localName).AppendLine(fail); + break; + default: + // Unsupported kinds were reported in Emit, which then skips generation. + break; + } } - private static void AppendWriteMethod(StringBuilder sb, string body, ModelInfo model) + private static void AppendFormatMethod(StringBuilder sb, string body, ModelInfo model) { var columns = model.Columns.AsImmutableArray(); - int lineLength = 0; - foreach (var c in columns) - { - int end = c.Start + c.Length; - if (end > lineLength) - { - lineLength = end; - } - } - + int lineLength = LineLength(model); sb.Append(body) .AppendLine("[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]"); sb.Append(body) .Append("public static bool TryFormat(in ").Append(model.FullyQualifiedName) - .Append(" model, global::System.Span destination, global::System.IFormatProvider? formatProvider, out int charsWritten)").AppendLine(); + .AppendLine(" model, global::System.Span destination, in global::FixedWidthParser.FixedWidthContext context, out int bytesWritten)"); sb.Append(body).AppendLine("{"); string stmt = body + " "; - sb.Append(stmt).Append("if (destination.Length < ").Append(lineLength).AppendLine(") { charsWritten = 0; return false; }"); + sb.Append(stmt).Append("if (destination.Length < ").Append(lineLength).AppendLine(") { bytesWritten = 0; return false; }"); if (columns.Length > 0) { sb.Append(stmt).Append("var __line = destination[..").Append(lineLength).AppendLine("];"); - sb.Append(stmt).AppendLine("__line.Fill(' ');"); + sb.Append(stmt).AppendLine("__line.Fill((byte)' ');"); for (int i = 0; i < columns.Length; i++) { - var c = columns[i]; - string slice = "__line.Slice(" + c.Start + ", " + c.Length + ")"; - AppendColumnWrite(sb, stmt, c, slice, i); + AppendColumnFormat(sb, stmt, columns[i], i); } } - sb.Append(stmt).Append("charsWritten = ").Append(lineLength).AppendLine(";"); + sb.Append(stmt).Append("bytesWritten = ").Append(lineLength).AppendLine(";"); sb.Append(stmt).AppendLine("return true;"); - sb.Append(body).AppendLine("}"); } - // Writes one column: a nullable (T?) column fills its slice with the padding character when - // null, otherwise formats exactly as the non-nullable case would (via the temp local pattern - // variable from the `is { }` check). - private static void AppendColumnWrite(StringBuilder sb, string stmt, ColumnInfo c, string slice, int i) + // A nullable (T?) column fills its slice with its trim character when null. + private static void AppendColumnFormat(StringBuilder sb, string stmt, ColumnInfo c, int i) { + string slice = "__line.Slice(" + c.Start + ", " + c.Length + ")"; if (!c.IsNullable) { - AppendColumnKindWrite(sb, stmt, c, slice, i, "model." + c.Name); + AppendColumnKindFormat(sb, stmt, c, slice, i, "model." + c.Name); return; } sb.Append(stmt).Append("if (model.").Append(c.Name).Append(" is { } __u").Append(i).AppendLine(")"); sb.Append(stmt).AppendLine("{"); - AppendColumnKindWrite(sb, stmt + " ", c, slice, i, "__u" + i); + AppendColumnKindFormat(sb, stmt + " ", c, slice, i, "__u" + i); sb.Append(stmt).AppendLine("}"); sb.Append(stmt).AppendLine("else"); sb.Append(stmt).AppendLine("{"); - char fillChar = c.TrimChar != ' ' ? c.TrimChar : ' '; - sb.Append(stmt).Append(" ").Append(slice).Append(".Fill(").Append(SymbolDisplay.FormatLiteral(fillChar, true)).AppendLine(");"); + sb.Append(stmt).Append(" ").Append(slice).Append(".Fill(").Append(ByteLiteral(c.TrimChar)).AppendLine(");"); sb.Append(stmt).AppendLine("}"); } - private static void AppendColumnKindWrite(StringBuilder sb, string stmt, ColumnInfo c, string slice, int i, string valueExpr) + private static void AppendColumnKindFormat(StringBuilder sb, string stmt, ColumnInfo c, string slice, int i, string valueExpr) { + string tail = ", __options" + i + ", " + SymbolDisplay.FormatLiteral(c.Name, true) + ");"; switch (c.WKind) { case WriteKind.String: - sb.Append(stmt).Append("global::FixedWidthParser.FixedWidthRuntime.FormatString(").Append(valueExpr).Append(", ").Append(slice) - .Append(", __options").Append(i).Append(", \"").Append(c.Name).AppendLine("\");"); + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatString(").Append(valueExpr).Append(", ").Append(slice).Append(", context").AppendLine(tail); break; case WriteKind.Formattable: - sb.Append(stmt).Append("global::FixedWidthParser.FixedWidthRuntime.FormatValue(").Append(valueExpr).Append(", ").Append(slice) - .Append(", formatProvider, __options").Append(i).Append(", \"").Append(c.Name).AppendLine("\");"); + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatValue(").Append(valueExpr).Append(", ").Append(slice).Append(", context").AppendLine(tail); + break; + case WriteKind.Utf8Formattable: + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatUtf8Value(").Append(valueExpr).Append(", ").Append(slice).Append(", context").AppendLine(tail); break; case WriteKind.Converter: - sb.Append(stmt).Append("global::FixedWidthParser.FixedWidthRuntime.FormatConvert(").Append(valueExpr).Append(", ").Append(slice) - .Append(", formatProvider, __converter").Append(i).Append(", __options").Append(i).Append(", \"").Append(c.Name).AppendLine("\");"); + sb.Append(stmt).Append(RuntimeFqn).Append(".FormatConvert(").Append(valueExpr).Append(", ").Append(slice) + .Append(", context, __converter").Append(i).AppendLine(tail); break; default: - // Should not happen, already filtered in Emit with a diagnostic. + // Unsupported kinds were reported in Emit, which then skips generation. break; } } - private static void AppendMethod(StringBuilder sb, string body, ModelInfo model, string elementType, string runtimeFqn) + private static string ByteLiteral(char value) { - var columns = model.Columns.AsImmutableArray(); - sb.Append(body) - .AppendLine("[global::System.Runtime.CompilerServices.MethodImpl(global::System.Runtime.CompilerServices.MethodImplOptions.AggressiveInlining)]"); - sb.Append(body) - .Append("public static bool TryParse(global::System.ReadOnlySpan<").Append(elementType) - .Append("> line, global::System.IFormatProvider? formatProvider, global::CommunityToolkit.HighPerformance.Buffers.StringPool? stringPool, out ") - .Append(model.FullyQualifiedName).AppendLine(" model)"); - sb.Append(body).AppendLine("{"); - - string stmt = body + " "; - if (columns.Length > 0) - { - int maxEnd = 0; - foreach (var v in columns) - { - int end = v.Start + v.Length; - if (end > maxEnd) - { - maxEnd = end; - } - } - - sb.Append(stmt).Append("if (line.Length < ").Append(maxEnd).AppendLine(") { model = default!; return false; }"); - AppendParseAndReturn(sb, stmt, model, runtimeFqn, elementType, static c => "line.Slice(" + c.Start + ", " + c.Length + ")"); - } - else - { - AppendParseAndReturn(sb, stmt, model, runtimeFqn, elementType, static _ => "default"); - } - - sb.Append(body).AppendLine("}"); + return "(byte)" + SymbolDisplay.FormatLiteral(value, true); } - private static void AppendParseAndReturn( - StringBuilder sb, - string stmt, - ModelInfo model, - string runtimeFqn, - string elementType, - Func columnExpression) - { - var columns = model.Columns.AsImmutableArray(); - for (int i = 0; i < columns.Length; i++) - { - var c = columns[i]; - string col = columnExpression(c); - - if (!c.IsNullable) - { - AppendColumnKindParse(sb, stmt, c, col, runtimeFqn, elementType, i, "__v" + i); - continue; - } - - // T?: a blank (trimmed-empty) column assigns null without invoking the underlying - // parser; otherwise the underlying T parses into a temp local exactly as it would for - // a non-nullable column, then is boxed into the T? local. - string trimCall = elementType == "char" - ? "global::FixedWidthParser.FixedWidthRuntime.TrimColumn(" + col + ", " + SymbolDisplay.FormatLiteral(c.TrimChar, true) + ", global::FixedWidthParser.Attributes.TrimMode.Both).Trim(' ').IsEmpty" - : "global::FixedWidthParser.Utf8FixedWidthRuntime.TrimColumn(" + col + ", " + (c.TrimChar == ' ' ? "(byte)' '" : "__trim" + i) + ", global::FixedWidthParser.Attributes.TrimMode.Both).Trim((byte)' ').IsEmpty"; - sb.Append(stmt).Append(c.TypeFqn).Append("? __v").Append(i).AppendLine(";"); - sb.Append(stmt).Append("if (").Append(trimCall).AppendLine(")"); - sb.Append(stmt).AppendLine("{"); - sb.Append(stmt).Append(" __v").Append(i).AppendLine(" = null;"); - sb.Append(stmt).AppendLine("}"); - sb.Append(stmt).AppendLine("else"); - sb.Append(stmt).AppendLine("{"); - AppendColumnKindParse(sb, stmt + " ", c, col, runtimeFqn, elementType, i, "__u" + i); - sb.Append(stmt).Append(" __v").Append(i).Append(" = __u").Append(i).AppendLine(";"); - sb.Append(stmt).AppendLine("}"); - } - - sb.Append(stmt).Append("model = new ").Append(model.FullyQualifiedName).AppendLine(); - sb.Append(stmt).AppendLine("{"); - for (int i = 0; i < columns.Length; i++) - { - sb.Append(stmt).Append(" ").Append(columns[i].Name).Append(" = __v").Append(i).AppendLine(","); - } - sb.Append(stmt).AppendLine("};"); - sb.Append(stmt).AppendLine("return true;"); - } - - // Emits the kind-specific parse statement for one column into a local named . Shared by the non-nullable case (localName = "__v{i}") and the nullable - // case (localName = "__u{i}", a temp later boxed into the "__v{i}" nullable local) — the - // converter field is always "__converter{i}" (keyed by column index, not the local's name). - private static void AppendColumnKindParse(StringBuilder sb, string stmt, ColumnInfo c, string col, string runtimeFqn, string elementType, int i, string localName) + private static string AlignmentLiteral(int value) { - string trimArg = TrimArgSuffix(elementType, c, i); - bool isDateTimeType = IsDateTimeType(c.TypeFqn); - if (c.Format is not null && isDateTimeType) - { - string exactHelper = c.TypeFqn.Contains("DateTimeOffset") ? "TryDateTimeOffsetExact" - : c.TypeFqn.Contains("DateTime") ? "TryDateTimeExact" - : c.TypeFqn.Contains("DateOnly") ? "TryDateOnlyExact" - : "TryTimeOnlyExact"; - - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append('.').Append(exactHelper).Append('(').Append(col) - .Append(", ").Append(SymbolDisplay.FormatLiteral(c.Format, true)).Append(", formatProvider, out var ").Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - return; - } - - switch (c.Kind) - { - case ColumnKind.String: - sb.Append(stmt).Append("string ").Append(localName) - .Append(" = ").Append(runtimeFqn).Append(".String(").Append(col).Append(", stringPool").Append(trimArg).AppendLine(");"); - break; - case ColumnKind.Double: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryDouble(").Append(col) - .Append(", formatProvider, out double ").Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - case ColumnKind.Float: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryFloat(").Append(col) - .Append(", formatProvider, out float ").Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - case ColumnKind.SpanParsable: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryParse<").Append(c.TypeFqn).Append(">(").Append(col) - .Append(", formatProvider, out ").Append(c.TypeFqn).Append(' ').Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - case ColumnKind.Converter: - sb.Append(stmt).Append("if (!").Append(runtimeFqn).Append(".TryConvert<").Append(c.TypeFqn).Append(", ").Append(c.ConverterFqn) - .Append(">(").Append(col).Append(", formatProvider, __converter").Append(i).Append(", out ").Append(c.TypeFqn) - .Append(' ').Append(localName).Append(trimArg).AppendLine(")) { model = default!; return false; }"); - break; - default: - // Should not happen, already filtered in Emit with a diagnostic. - break; - } + return value == 1 + ? "global::FixedWidthParser.Alignment.Right" + : "global::FixedWidthParser.Alignment.Left"; } - // Trailing trimChar argument for a runtime parse call: omitted when the column uses the - // default (space) trim, so unconfigured columns keep emitting the exact same call shape as - // before this option existed. Char columns pass the literal directly; byte columns reference - // the precomputed __trim{i} field (AppendUtf8TrimFields) since a non-ASCII char can't be a - // byte literal — the field does that conversion once, with a clear throw if it doesn't fit. - private static string TrimArgSuffix(string elementType, ColumnInfo c, int i) + private static string OverflowLiteral(int resolvedValue) { - if (c.TrimChar == ' ' && c.TrimMode == 0) + return resolvedValue switch { - return string.Empty; - } - string trimArg = elementType == "char" - ? SymbolDisplay.FormatLiteral(c.TrimChar, true) - : "__trim" + i; - return ", " + trimArg + ", " + TrimModeLiteral(c.TrimMode); + 1 => "global::FixedWidthParser.OverflowBehavior.Truncate", + 2 => "global::FixedWidthParser.OverflowBehavior.Throw", + _ => "global::FixedWidthParser.OverflowBehavior.Default", + }; } private static string TrimModeLiteral(int value) { return value switch { - 1 => "global::FixedWidthParser.Attributes.TrimMode.Leading", - 2 => "global::FixedWidthParser.Attributes.TrimMode.Both", - _ => "global::FixedWidthParser.Attributes.TrimMode.Trailing" + 1 => "global::FixedWidthParser.TrimMode.Leading", + 2 => "global::FixedWidthParser.TrimMode.Both", + _ => "global::FixedWidthParser.TrimMode.Trailing" }; } - private enum ColumnKind { String, Double, Float, SpanParsable, Converter, Unsupported } - - private enum WriteKind { String, Formattable, Converter, Unsupported } - - private readonly struct ColumnInfo( - string name, int start, int length, ColumnKind kind, string typeFqn, string? converterFqn, bool isNullable, bool isCharParsable, bool isUtf8Parsable, - WriteKind writeKind, bool isCharFormattable, int alignment, char padding, string? format, int overflow, char trimChar, int trimMode) : IEquatable - { - public readonly string Name = name; - public readonly int Start = start; - public readonly int Length = length; - public readonly ColumnKind Kind = kind; - public readonly string TypeFqn = typeFqn; - public readonly string? ConverterFqn = converterFqn; - public readonly bool IsNullable = isNullable; - public readonly bool IsCharParsable = isCharParsable; - public readonly bool IsUtf8Parsable = isUtf8Parsable; - public readonly WriteKind WKind = writeKind; - public readonly bool IsCharFormattable = isCharFormattable; - public readonly int Alignment = alignment; - public readonly char Padding = padding; - public readonly string? Format = format; - public readonly int Overflow = overflow; - public readonly char TrimChar = trimChar; - public readonly int TrimMode = trimMode; - - public bool Equals(ColumnInfo other) - { - return Start == other.Start - && Length == other.Length - && Kind == other.Kind - && IsNullable == other.IsNullable - && IsCharParsable == other.IsCharParsable - && WKind == other.WKind - && IsCharFormattable == other.IsCharFormattable - && Alignment == other.Alignment - && Padding == other.Padding - && Overflow == other.Overflow - && TrimChar == other.TrimChar - && TrimMode == other.TrimMode - && string.Equals(Format, other.Format, StringComparison.Ordinal) - && IsUtf8Parsable == other.IsUtf8Parsable - && string.Equals(Name, other.Name, StringComparison.Ordinal) - && string.Equals(TypeFqn, other.TypeFqn, StringComparison.Ordinal) - && string.Equals(ConverterFqn, other.ConverterFqn, StringComparison.Ordinal); - } + private enum ColumnKind { String, Double, Float, Enum, Utf8Parsable, DecodedParsable, Exact, Converter, BadConverter, Unsupported } - [ExcludeFromCodeCoverage] - public override bool Equals(object? obj) - { - return obj is ColumnInfo other && Equals(other); - } + private enum WriteKind { String, Formattable, Utf8Formattable, Converter, Unsupported } - [ExcludeFromCodeCoverage] - public override int GetHashCode() - { - unchecked - { - int hash = 17; - hash = hash * 31 + StringComparer.Ordinal.GetHashCode(Name); - hash = hash * 31 + Start; - hash = hash * 31 + Length; - hash = hash * 31 + (int)Kind; - hash = hash * 31 + StringComparer.Ordinal.GetHashCode(TypeFqn); - hash = hash * 31 + (ConverterFqn is null ? 0 : StringComparer.Ordinal.GetHashCode(ConverterFqn)); - hash = hash * 31 + (IsNullable ? 1 : 0); - hash = hash * 31 + (IsCharParsable ? 1 : 0); - hash = hash * 31 + (IsUtf8Parsable ? 1 : 0); - hash = hash * 31 + (int)WKind; - hash = hash * 31 + (IsCharFormattable ? 1 : 0); - hash = hash * 31 + Alignment; - hash = hash * 31 + Padding; - hash = hash * 31 + Overflow; - hash = hash * 31 + TrimChar; - hash = hash * 31 + TrimMode; - hash = hash * 31 + (Format is null ? 0 : StringComparer.Ordinal.GetHashCode(Format)); - return hash; - } - } - } + // Value equality, so the incremental pipeline caches correctly. + private readonly record struct ColumnInfo( + string Name, int Start, int Length, ColumnKind Kind, string TypeFqn, string? ConverterFqn, bool IsNullable, + WriteKind WKind, int Alignment, char Padding, string? Format, int Overflow, char TrimChar, int TrimMode); private sealed class ModelInfo(string? ns, string typeKeywords, string typeName, string fullyQualifiedName, - EquatableArray columns, bool isPartial, bool isNested, - bool implementsChar, bool implementsUtf8, Location location) : IEquatable + EquatableArray columns, bool isPartial, bool isNested, Location location) : IEquatable { public string? Namespace { get; } = ns; public string TypeKeywords { get; } = typeKeywords; @@ -958,20 +751,16 @@ private sealed class ModelInfo(string? ns, string typeKeywords, string typeName, public EquatableArray Columns { get; } = columns; public bool IsPartial { get; } = isPartial; public bool IsNested { get; } = isNested; - public bool ImplementsChar { get; } = implementsChar; - public bool ImplementsUtf8 { get; } = implementsUtf8; public Location Location { get; } = location; // Value equality (excluding Location, which doesn't affect the emitted source) so the - // incremental pipeline caches correctly: toggling a marker or a column type re-runs Emit. + // incremental pipeline caches correctly. [ExcludeFromCodeCoverage] public bool Equals(ModelInfo? other) { return other is not null && IsPartial == other.IsPartial && IsNested == other.IsNested - && ImplementsChar == other.ImplementsChar - && ImplementsUtf8 == other.ImplementsUtf8 && string.Equals(Namespace, other.Namespace, StringComparison.Ordinal) && string.Equals(TypeKeywords, other.TypeKeywords, StringComparison.Ordinal) && string.Equals(TypeName, other.TypeName, StringComparison.Ordinal) @@ -998,8 +787,6 @@ public override int GetHashCode() hash = hash * 31 + Columns.GetHashCode(); hash = hash * 31 + (IsPartial ? 1 : 0); hash = hash * 31 + (IsNested ? 1 : 0); - hash = hash * 31 + (ImplementsChar ? 1 : 0); - hash = hash * 31 + (ImplementsUtf8 ? 1 : 0); return hash; } } @@ -1053,3 +840,13 @@ public override int GetHashCode() } } } + +namespace System.Runtime.CompilerServices +{ + // netstandard2.0 lacks it; the records' init accessors need it. +#pragma warning disable S2094 // A marker type, empty by design. + internal static class IsExternalInit + { + } +#pragma warning restore S2094 +} diff --git a/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj b/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj index 881ffc0..c90cb18 100644 --- a/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj +++ b/src/FixedWidthParser.Generator/FixedWidthParser.Generator.csproj @@ -1,11 +1,13 @@ - netstandard2.1 + netstandard2.0 latest enable disable true + true + true true false diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs new file mode 100644 index 0000000..1d646a3 --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.Stream.cs @@ -0,0 +1,165 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Arrow +{ + // The stream's GCHandle owns its state, and the state owns the reader, until the release callback runs. +#pragma warning disable CA2000, IDISP001, IDISP004, IDISP007 + internal static unsafe partial class ArrowApi + { + private const int ArrowErrno = 5; + + /// One ArrowArrayStream: a batched reader and the message its last failure left. + internal sealed class ArrowStreamState(NativeTableReader reader) : IDisposable + { + internal NativeTableReader Reader { get; } = reader; + + internal IntPtr LastErrorUtf8 { get; private set; } + + internal void SetError(string message) + { + FreeIfSet(LastErrorUtf8); + LastErrorUtf8 = AllocUtf8Z(message); + } + + public void Dispose() + { + Reader.Dispose(); + FreeIfSet(LastErrorUtf8); + LastErrorUtf8 = IntPtr.Zero; + } + } + + internal static int OpenStreamFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + *outStream = default; + int status = ReaderApi.CreateFile(path, layout, options, out var reader); + return status == NativeStatus.Ok ? Publish(reader!, outStream) : status; + } + + /// Reads lazily: the caller keeps it alive until the stream is released. + internal static int OpenStreamMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + *outStream = default; + int status = ReaderApi.CreateMemory(data, length, layout, options, out var reader); + return status == NativeStatus.Ok ? Publish(reader!, outStream) : status; + } + + /// Reads lazily and owns it: it is disposed when this fails or the stream is released. + internal static int OpenStreamStream(Stream source, NativeLayout layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + *outStream = default; + int status = ReaderApi.CreateStream(source, layout, options, out var reader); + return status == NativeStatus.Ok ? Publish(reader!, outStream) : status; + } + + private static int Publish(NativeTableReader reader, ArrowArrayStream* outStream) + { + *outStream = new ArrowArrayStream + { + GetSchema = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamGetSchema, + GetNext = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamGetNext, + GetLastError = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamGetLastError, + Release = (IntPtr)(delegate* unmanaged)&Exports.ArrowStreamRelease, + PrivateData = GCHandle.ToIntPtr(GCHandle.Alloc(new ArrowStreamState(reader))), + }; + return NativeStatus.Ok; + } + + private static ArrowStreamState? StateOf(ArrowArrayStream* stream) + { + if (stream is null || stream->PrivateData == IntPtr.Zero) + { + return null; + } + return GCHandle.FromIntPtr(stream->PrivateData).Target as ArrowStreamState; + } + + internal static int StreamGetSchema(ArrowArrayStream* stream, ArrowSchema* outSchema) + { + var state = StateOf(stream); + if (state is null || outSchema is null) + { + return ArrowErrno; + } + try + { + *outSchema = BuildSchema(state.Reader.Layout); + return 0; + } + catch (Exception ex) + { + state.SetError(ex.Message); + *outSchema = default; + return ArrowErrno; + } + } + + internal static int StreamGetNext(ArrowArrayStream* stream, ArrowArray* outArray) + { + var state = StateOf(stream); + if (state is null || outArray is null) + { + return ArrowErrno; + } + *outArray = default; + NativeTableRaw table = default; + int status = ReaderApi.NextBatch(state.Reader, &table); + if (status == NativeStatus.Eof) + { + return 0; + } + if (status != NativeStatus.Ok) + { + state.SetError(NativeApi.LastErrorText()); + return ArrowErrno; + } + try + { + *outArray = BuildArray(&table, state.Reader.Layout); + return 0; + } + catch (Exception ex) + { + state.SetError(ex.Message); + *outArray = default; + return ArrowErrno; + } + finally + { + NativeTable.Free(&table); + } + } + + internal static IntPtr StreamGetLastError(ArrowArrayStream* stream) + { + return StateOf(stream)?.LastErrorUtf8 ?? IntPtr.Zero; + } + + internal static void StreamRelease(ArrowArrayStream* stream) + { + if (stream is null || stream->Release == IntPtr.Zero) + { + return; + } + IntPtr privateData = stream->PrivateData; + stream->PrivateData = IntPtr.Zero; + stream->Release = IntPtr.Zero; + if (privateData == IntPtr.Zero) + { + return; + } + var handle = GCHandle.FromIntPtr(privateData); + try + { + (handle.Target as ArrowStreamState)?.Dispose(); + } + finally + { + handle.Free(); + } + } + } +#pragma warning restore CA2000, IDISP001, IDISP004, IDISP007 +} diff --git a/src/FixedWidthParser.Native/Arrow/ArrowApi.cs b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs new file mode 100644 index 0000000..6ba674b --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/ArrowApi.cs @@ -0,0 +1,344 @@ +using System.Globalization; +using System.Numerics; +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Arrow +{ + /// + /// fw_read_arrow_*: a columnar read handed over through the Arrow C Data Interface as a struct + /// array, one child per column. The column buffers move into the arrays without a copy; only DECIMAL + /// widens its int64 values to Arrow's 128-bit decimals. + /// + internal static unsafe partial class ArrowApi + { + private const int DecimalBytes = 16; + + internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + NativeTableRaw table = default; + int status = ReadApi.ReadFile(path, layout, options, &table, outResult); + return Export(status, layout, &table, outArray, outSchema); + } + + internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + NativeTableRaw table = default; + int status = ReadApi.ReadMemory(data, length, layout, options, &table, outResult); + return Export(status, layout, &table, outArray, outSchema); + } + + private static int Export(int status, NativeLayout layout, NativeTableRaw* table, ArrowArray* outArray, ArrowSchema* outSchema) + { + *outArray = default; + *outSchema = default; + if (status != NativeStatus.Ok) + { + return status; + } + try + { + *outSchema = BuildSchema(layout); + *outArray = BuildArray(table, layout); + return NativeStatus.Ok; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + ReleaseArrowSchema((IntPtr)outSchema); + ReleaseArrowArray((IntPtr)outArray); + *outArray = default; + *outSchema = default; + return NativeStatus.Error; + } + finally + { + NativeTable.Free(table); + } + } + + internal static void ReleaseArrowSchema(IntPtr schemaPtr) + { + var schema = (ArrowSchema*)schemaPtr; + if (schema is null || schema->Release == IntPtr.Zero) + { + return; + } + ReleaseChildren(schema->Children, schema->NChildren, &ReleaseArrowSchema); + ReleaseDictionary(schema->Dictionary, &ReleaseArrowSchema); + FreeIfSet(schema->Format); + FreeIfSet(schema->Name); + schema->Release = IntPtr.Zero; + } + + internal static void ReleaseArrowArray(IntPtr arrayPtr) + { + var array = (ArrowArray*)arrayPtr; + if (array is null || array->Release == IntPtr.Zero) + { + return; + } + ReleaseChildren(array->Children, array->NChildren, &ReleaseArrowArray); + ReleaseDictionary(array->Dictionary, &ReleaseArrowArray); + var buffers = (IntPtr*)array->Buffers; + for (long i = 0; i < array->NBuffers; i++) + { + FreeIfSet(buffers[i]); + } + FreeIfSet(array->Buffers); + array->Release = IntPtr.Zero; + } + + private static void ReleaseChildren(IntPtr children, long count, delegate* release) + { + var pointers = (IntPtr*)children; + for (long i = 0; i < count; i++) + { + release(pointers[i]); + Marshal.FreeHGlobal(pointers[i]); + } + FreeIfSet(children); + } + + private static void ReleaseDictionary(IntPtr dictionary, delegate* release) + { + if (dictionary != IntPtr.Zero) + { + release(dictionary); + Marshal.FreeHGlobal(dictionary); + } + } + + private static void FreeIfSet(IntPtr pointer) + { + if (pointer != IntPtr.Zero) + { + Marshal.FreeHGlobal(pointer); + } + } + + internal static ArrowSchema BuildSchema(NativeLayout layout) + { + var columns = layout.Columns; + var children = (IntPtr*)Marshal.AllocHGlobal(columns.Length * IntPtr.Size); + var schema = new ArrowSchema + { + Format = AllocUtf8Z("+s"), + NChildren = 0, + Children = (IntPtr)children, + Release = SchemaReleaseCallback, + }; + try + { + foreach (var column in columns) + { + var child = (ArrowSchema*)Marshal.AllocHGlobal(sizeof(ArrowSchema)); + *child = new ArrowSchema + { + Format = AllocUtf8Z(FormatCode(column)), + Name = AllocUtf8Z(column.Name), + Flags = column.Nullable ? ArrowFlags.Nullable : 0, + Dictionary = column.Dictionary ? DictionaryValuesSchema() : IntPtr.Zero, + Release = SchemaReleaseCallback, + }; + children[schema.NChildren++] = (IntPtr)child; + } + return schema; + } + catch + { + ReleaseArrowSchema((IntPtr)(&schema)); + throw; + } + } + + private static IntPtr DictionaryValuesSchema() + { + var values = (ArrowSchema*)Marshal.AllocHGlobal(sizeof(ArrowSchema)); + *values = new ArrowSchema { Format = AllocUtf8Z("u"), Release = SchemaReleaseCallback }; + return (IntPtr)values; + } + + /// Moves every column of into a struct array, leaving the table's buffers empty. + internal static ArrowArray BuildArray(NativeTableRaw* table, NativeLayout layout) + { + var children = (IntPtr*)Marshal.AllocHGlobal(Math.Max(1, table->ColumnCount) * IntPtr.Size); + var buffers = (IntPtr*)Marshal.AllocHGlobal(IntPtr.Size); + buffers[0] = IntPtr.Zero; + var array = new ArrowArray + { + Length = table->RowCount, + NBuffers = 1, + Buffers = (IntPtr)buffers, + Children = (IntPtr)children, + Release = ArrayReleaseCallback, + }; + try + { + for (int i = 0; i < table->ColumnCount; i++) + { + var child = (ArrowArray*)Marshal.AllocHGlobal(sizeof(ArrowArray)); + *child = default; + children[array.NChildren++] = (IntPtr)child; + } + // Encoding a dictionary column hashes every row, so the columns are built side by side. + nint columns = (nint)table->Columns; + nint built = (nint)children; + Parallel.For(0, table->ColumnCount, i => + *((ArrowArray**)built)[i] = BuildChild((NativeColumnRaw*)columns + i, layout.Columns[i].Dictionary)); + return array; + } + catch (AggregateException ex) + { + ReleaseArrowArray((IntPtr)(&array)); + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw(ex.InnerExceptions[0]); + throw; + } + catch + { + ReleaseArrowArray((IntPtr)(&array)); + throw; + } + } + + private static ArrowArray BuildChild(NativeColumnRaw* column, bool dictionary) + { + if (dictionary) + { + return BuildDictionaryChild(column); + } + bool isString = column->Type == NativeType.String; + int bufferCount = isString ? 3 : 2; + var buffers = (IntPtr*)Marshal.AllocHGlobal(bufferCount * IntPtr.Size); + buffers[0] = Take(ref column->Validity); + buffers[1] = column->Type == NativeType.Decimal ? WidenDecimals(column) : OrAllocate((IntPtr)column->Values, 8); + column->Values = null; + if (isString) + { + buffers[2] = OrAllocate((IntPtr)column->Data, 1); + column->Data = null; + } + return new ArrowArray + { + Length = column->Length, + NullCount = CountUnset(buffers[0], column->Length), + NBuffers = bufferCount, + Buffers = (IntPtr)buffers, + Release = ArrayReleaseCallback, + }; + } + + private static ArrowArray BuildDictionaryChild(NativeColumnRaw* column) + { + long rows = column->Length; + int* indices = DictionaryEncoder.Encode((int*)column->Values, column->Data, rows, + out int* valueOffsets, out byte* valueBytes, out long valueCount, out _); + FreeIfSet((IntPtr)column->Values); + FreeIfSet((IntPtr)column->Data); + column->Values = null; + column->Data = null; + + var valueBuffers = (IntPtr*)Marshal.AllocHGlobal(3 * IntPtr.Size); + valueBuffers[0] = IntPtr.Zero; + valueBuffers[1] = (IntPtr)valueOffsets; + valueBuffers[2] = (IntPtr)valueBytes; + var values = (ArrowArray*)Marshal.AllocHGlobal(sizeof(ArrowArray)); + *values = new ArrowArray { Length = valueCount, NBuffers = 3, Buffers = (IntPtr)valueBuffers, Release = ArrayReleaseCallback }; + + var buffers = (IntPtr*)Marshal.AllocHGlobal(2 * IntPtr.Size); + buffers[0] = Take(ref column->Validity); + buffers[1] = (IntPtr)indices; + return new ArrowArray + { + Length = rows, + NullCount = CountUnset(buffers[0], rows), + NBuffers = 2, + Buffers = (IntPtr)buffers, + Dictionary = (IntPtr)values, + Release = ArrayReleaseCallback, + }; + } + + private static IntPtr Take(ref byte* pointer) + { + IntPtr taken = (IntPtr)pointer; + pointer = null; + return taken; + } + + // A buffer of size 0 may be NULL under the C Data Interface, but some importers want a pointer. + private static IntPtr OrAllocate(IntPtr buffer, int minimum) + { + return buffer == IntPtr.Zero ? Marshal.AllocHGlobal(minimum) : buffer; + } + + private static IntPtr WidenDecimals(NativeColumnRaw* column) + { + var values = new ReadOnlySpan(column->Values, checked((int)column->Length)); + var wide = (long*)Marshal.AllocHGlobal((nint)DecimalBufferBytes(column->Length)); + for (int i = 0; i < values.Length; i++) + { + wide[2 * i] = values[i]; + wide[(2 * i) + 1] = values[i] < 0 ? -1 : 0; + } + Marshal.FreeHGlobal((IntPtr)column->Values); + column->Values = null; + return (IntPtr)wide; + } + + /// The bytes of a decimal128 buffer for values, in 64-bit arithmetic. + internal static long DecimalBufferBytes(long rows) + { + return checked(Math.Max(1, rows) * DecimalBytes); + } + + private static long CountUnset(IntPtr validity, long length) + { + if (validity == IntPtr.Zero) + { + return 0; + } + ReadOnlySpan bits = new((void*)validity, (int)((length + 7) / 8)); + long set = 0; + for (long row = 0; row < length; row += 64) + { + int bytes = (int)Math.Min(8, bits.Length - (row / 8)); + ulong packed = 0; + bits.Slice((int)(row / 8), bytes).CopyTo(MemoryMarshal.AsBytes(new Span(ref packed))); + packed &= length - row >= 64 ? ulong.MaxValue : (1UL << (int)(length - row)) - 1; + set += BitOperations.PopCount(packed); + } + return length - set; + } + + private static string FormatCode(NativeColumn column) + { + return column.Type switch + { + // A dictionary column's own format is its index type; the values' utf8 is in its dictionary schema. + NativeType.String => column.Dictionary ? "i" : "u", + NativeType.Int64 => "l", + NativeType.Float64 => "g", + NativeType.Decimal => string.Create(CultureInfo.InvariantCulture, $"d:18,{column.Scale}"), + NativeType.Date => "tdD", + NativeType.Time => "ttu", + _ => "tsu:", + }; + } + + private static IntPtr AllocUtf8Z(string value) + { + int length = Encoding.UTF8.GetByteCount(value); + var block = (byte*)Marshal.AllocHGlobal(length + 1); + Encoding.UTF8.GetBytes(value, new Span(block, length)); + block[length] = 0; + return (IntPtr)block; + } + + private static IntPtr SchemaReleaseCallback => (IntPtr)(delegate* unmanaged)&Exports.ReleaseArrowSchemaCallback; + + private static IntPtr ArrayReleaseCallback => (IntPtr)(delegate* unmanaged)&Exports.ReleaseArrowArrayCallback; + } +} diff --git a/src/FixedWidthParser.Native/Arrow/DictionaryEncoder.cs b/src/FixedWidthParser.Native/Arrow/DictionaryEncoder.cs new file mode 100644 index 0000000..08e01f5 --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/DictionaryEncoder.cs @@ -0,0 +1,125 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Arrow +{ + /// + /// Dictionary-encodes one STRING column for Arrow: an int32 index per row into the column's distinct values, in + /// the order they first appear. A null row keeps its index to the empty string; its validity bit says it is null. + /// + // ponytail: encodes the finished column at export, one thread per column: ~30 ms on COTAHIST's 1M rows with + // nine dictionary columns, bound by the most varied one. Encoding each partition while it parses, then + // remapping the partitions' indices at publish, would spread that over every thread. + internal sealed unsafe class DictionaryEncoder : IDisposable + { + private const int MinSlots = 1024; + + private readonly GrowableNativeBuffer _offsets = new(); + private readonly GrowableNativeBuffer _bytes = new(); + private int[] _slots = new int[MinSlots]; // index + 1; 0 is an empty slot + private int[] _hashes = new int[MinSlots]; + private int _count; + + private DictionaryEncoder() + { + _offsets.Add(0); + } + + /// + /// The indices of a column's values, given as into + /// , and its distinct values as Arrow utf8 offsets and bytes. Every buffer is + /// allocated with ; the caller frees them. + /// + internal static int* Encode(int* offsets, byte* data, long rows, out int* valueOffsets, out byte* valueBytes, out long valueCount, out long valueBytesLength) + { + using var encoder = new DictionaryEncoder(); + int* indices = (int*)Marshal.AllocHGlobal((nint)(Math.Max(1, rows) * sizeof(int))); + try + { + for (long row = 0; row < rows; row++) + { + indices[row] = encoder.IndexOf(new ReadOnlySpan(data + offsets[row], offsets[row + 1] - offsets[row])); + } + } + catch + { + Marshal.FreeHGlobal((IntPtr)indices); + throw; + } + valueCount = encoder._count; + valueBytesLength = encoder._bytes.Count; + valueOffsets = (int*)encoder._offsets.Detach(); + valueBytes = valueBytesLength == 0 ? (byte*)Marshal.AllocHGlobal(1) : (byte*)encoder._bytes.Detach(); + return indices; + } + + private int IndexOf(ReadOnlySpan value) + { + var hasher = default(HashCode); + hasher.AddBytes(value); + int hash = hasher.ToHashCode(); + int mask = _slots.Length - 1; + int slot = hash & mask; + // The table is at most half full, so the probe always reaches an empty slot. + while (_slots[slot] != 0) + { + int entry = _slots[slot]; + if (_hashes[slot] == hash && Value(entry - 1).SequenceEqual(value)) + { + return entry - 1; + } + slot = (slot + 1) & mask; + } + return Add(value, hash, slot); + } + + private ReadOnlySpan Value(int index) + { + var offsets = _offsets.Span; + return _bytes.Span[offsets[index]..offsets[index + 1]]; + } + + private int Add(ReadOnlySpan value, int hash, int slot) + { + int index = _count++; + _bytes.AddRange(value); + _offsets.Add(_bytes.Count); + _slots[slot] = index + 1; + _hashes[slot] = hash; + // Half full at most, so a probe ends quickly at an empty slot. + if (_count * 2 > _slots.Length) + { + Rehash(); + } + return index; + } + + private void Rehash() + { + int[] slots = new int[_slots.Length * 2]; + int[] hashes = new int[slots.Length]; + int mask = slots.Length - 1; + for (int old = 0; old < _slots.Length; old++) + { + if (_slots[old] == 0) + { + continue; + } + int slot = _hashes[old] & mask; + while (slots[slot] != 0) + { + slot = (slot + 1) & mask; + } + slots[slot] = _slots[old]; + hashes[slot] = _hashes[old]; + } + _slots = slots; + _hashes = hashes; + } + + public void Dispose() + { + _offsets.Dispose(); + _bytes.Dispose(); + } + } +} diff --git a/src/FixedWidthParser.Native/Arrow/NativeArrow.cs b/src/FixedWidthParser.Native/Arrow/NativeArrow.cs new file mode 100644 index 0000000..6dc0c82 --- /dev/null +++ b/src/FixedWidthParser.Native/Arrow/NativeArrow.cs @@ -0,0 +1,48 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Arrow +{ + internal static class ArrowFlags + { + internal const long Nullable = 2; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct ArrowSchema + { + public IntPtr Format; + public IntPtr Name; + public IntPtr Metadata; + public long Flags; + public long NChildren; + public IntPtr Children; + public IntPtr Dictionary; + public IntPtr Release; + public IntPtr PrivateData; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct ArrowArray + { + public long Length; + public long NullCount; + public long Offset; + public long NBuffers; + public long NChildren; + public IntPtr Buffers; + public IntPtr Children; + public IntPtr Dictionary; + public IntPtr Release; + public IntPtr PrivateData; + } + + [StructLayout(LayoutKind.Sequential)] + internal struct ArrowArrayStream + { + public IntPtr GetSchema; + public IntPtr GetNext; + public IntPtr GetLastError; + public IntPtr Release; + public IntPtr PrivateData; + } +} diff --git a/src/FixedWidthParser.Native/Exports.Arrow.cs b/src/FixedWidthParser.Native/Exports.Arrow.cs new file mode 100644 index 0000000..3a93d0b --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Arrow.cs @@ -0,0 +1,114 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_file")] + public static int ReadArrowFile(byte* path, int pathLength, nint layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + if (path is null || pathLength <= 0 || outArray is null || outSchema is null) + { + return InvalidArgument("fw_read_arrow_file needs a path, out_array and out_schema."); + } + *outArray = default; + *outSchema = default; + return WithLayout(layout, l => ArrowApi.ReadFile(new ReadOnlySpan(path, pathLength), l, options, outArray, outSchema, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_memory")] + public static int ReadArrowMemory(byte* data, long dataLength, nint layout, NativeReadOptionsRaw* options, ArrowArray* outArray, ArrowSchema* outSchema, NativeReadResultRaw* outResult) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outArray is null || outSchema is null) + { + return InvalidArgument("fw_read_arrow_memory needs data for a non-zero data_len, out_array and out_schema."); + } + *outArray = default; + *outSchema = default; + return WithLayout(layout, l => ArrowApi.ReadMemory(data, dataLength, l, options, outArray, outSchema, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_stream_file")] + public static int ReadArrowStreamFile(byte* path, int pathLength, nint layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + if (path is null || pathLength <= 0 || outStream is null) + { + return InvalidArgument("fw_read_arrow_stream_file needs a path and out_stream."); + } + *outStream = default; + return WithLayout(layout, l => ArrowApi.OpenStreamFile(new ReadOnlySpan(path, pathLength), l, options, outStream)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_stream_memory")] + public static int ReadArrowStreamMemory(byte* data, long dataLength, nint layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outStream is null) + { + return InvalidArgument("fw_read_arrow_stream_memory needs data for a non-zero data_len, and out_stream."); + } + *outStream = default; + return WithLayout(layout, l => ArrowApi.OpenStreamMemory(data, dataLength, l, options, outStream)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_arrow_stream_stream")] + public static int ReadArrowStreamStream(NativeStreamRaw* stream, nint layout, NativeReaderOptionsRaw* options, ArrowArrayStream* outStream) + { + if (outStream is null) + { + return InvalidArgument("fw_read_arrow_stream_stream needs out_stream."); + } + *outStream = default; +#pragma warning disable CA2000 // The reader takes the stream over, or it is disposed below. + if (!CallbackReadStream.TryCreate(stream, "fw_read_arrow_stream_stream", out var source)) + { + return NativeStatus.InvalidArgument; + } + int status = WithLayout(layout, l => ArrowApi.OpenStreamStream(source, l, options, outStream)); + if (status != NativeStatus.Ok) + { + // A layout that is not live never reached OpenStreamStream, which releases the stream on its own failures. + source.Dispose(); + } +#pragma warning restore CA2000 + return status; + } + + [UnmanagedCallersOnly] + internal static void ReleaseArrowSchemaCallback(ArrowSchema* schema) + { + ArrowApi.ReleaseArrowSchema((IntPtr)schema); + } + + [UnmanagedCallersOnly] + internal static void ReleaseArrowArrayCallback(ArrowArray* array) + { + ArrowApi.ReleaseArrowArray((IntPtr)array); + } + + [UnmanagedCallersOnly] + internal static int ArrowStreamGetSchema(ArrowArrayStream* stream, ArrowSchema* outSchema) + { + return ArrowApi.StreamGetSchema(stream, outSchema); + } + + [UnmanagedCallersOnly] + internal static int ArrowStreamGetNext(ArrowArrayStream* stream, ArrowArray* outArray) + { + return ArrowApi.StreamGetNext(stream, outArray); + } + + [UnmanagedCallersOnly] + internal static IntPtr ArrowStreamGetLastError(ArrowArrayStream* stream) + { + return ArrowApi.StreamGetLastError(stream); + } + + [UnmanagedCallersOnly] + internal static void ArrowStreamRelease(ArrowArrayStream* stream) + { + ArrowApi.StreamRelease(stream); + } + } +} diff --git a/src/FixedWidthParser.Native/Exports.Layout.cs b/src/FixedWidthParser.Native/Exports.Layout.cs new file mode 100644 index 0000000..be29098 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Layout.cs @@ -0,0 +1,48 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_layout_create")] + public static int LayoutCreate(NativeColumnSpecRaw* columns, int columnCount, NativeLayoutOptionsRaw* options, nint* outLayout) + { + if (outLayout is null || columns is null || columnCount <= 0) + { + return InvalidArgument("fw_layout_create needs columns, a positive column_count and out_layout."); + } + *outLayout = 0; + if (columns->StructSize != sizeof(NativeColumnSpecRaw)) + { + return InvalidArgument($"fw_column_spec struct_size is {columns->StructSize}; expected {sizeof(NativeColumnSpecRaw)}."); + } + try + { + int status = NativeLayout.TryCreate(new ReadOnlySpan(columns, columnCount), options, out var layout); + if (status == NativeStatus.Ok) + { + *outLayout = NativeHandleTable.Register(layout!); + } + return status; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + [UnmanagedCallersOnly(EntryPoint = "fw_layout_free")] + public static void LayoutFree(nint layout) + { + NativeHandleTable.TryUnregister(layout, out _); + } + + private static int InvalidArgument(string message) + { + NativeApi.SetLastError(message); + return NativeStatus.InvalidArgument; + } + } +} diff --git a/src/FixedWidthParser.Native/Exports.Reader.cs b/src/FixedWidthParser.Native/Exports.Reader.cs new file mode 100644 index 0000000..6a12dc3 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Reader.cs @@ -0,0 +1,97 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_reader_open_file")] + public static int ReaderOpenFile(byte* path, int pathLength, nint layout, NativeReaderOptionsRaw* options, nint* outReader) + { + if (path is null || pathLength <= 0 || outReader is null) + { + return InvalidArgument("fw_reader_open_file needs a path and out_reader."); + } + *outReader = 0; + return WithLayout(layout, l => + { + int status = ReaderApi.OpenFile(new ReadOnlySpan(path, pathLength), l, options, out nint reader); + *outReader = reader; + return status; + }); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_reader_open_memory")] + public static int ReaderOpenMemory(byte* data, long dataLength, nint layout, NativeReaderOptionsRaw* options, nint* outReader) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outReader is null) + { + return InvalidArgument("fw_reader_open_memory needs data for a non-zero data_len, and out_reader."); + } + *outReader = 0; + return WithLayout(layout, l => + { + int status = ReaderApi.OpenMemory(data, dataLength, l, options, out nint reader); + *outReader = reader; + return status; + }); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_reader_open_stream")] + public static int ReaderOpenStream(NativeStreamRaw* stream, nint layout, NativeReaderOptionsRaw* options, nint* outReader) + { + if (outReader is null) + { + return InvalidArgument("fw_reader_open_stream needs out_reader."); + } + *outReader = 0; +#pragma warning disable CA2000 // The reader takes the stream over, or it is disposed below. + if (!CallbackReadStream.TryCreate(stream, "fw_reader_open_stream", out var source)) + { + return NativeStatus.InvalidArgument; + } + int status = WithLayout(layout, l => + { + int opened = ReaderApi.OpenStream(source, l, options, out nint reader); + *outReader = reader; + return opened; + }); + if (status != NativeStatus.Ok) + { + // A layout that is not live never reached OpenStream, which releases the stream on its own failures. + source.Dispose(); + } +#pragma warning restore CA2000 + return status; + } + + [UnmanagedCallersOnly(EntryPoint = "fw_set_source_error")] + public static void SetSourceError(byte* message, int length) + { + SourceErrors.Set(message is null || length <= 0 ? null : System.Text.Encoding.UTF8.GetString(message, length)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_reader_next")] + public static int ReaderNext(nint reader, NativeTableRaw* outTable) + { + if (outTable is null) + { + return InvalidArgument("fw_reader_next needs out_table."); + } + return ReaderApi.Next(reader, outTable); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_reader_close")] + public static void ReaderClose(nint reader) + { + try + { + ReaderApi.Close(reader); + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + } + } + } +} diff --git a/src/FixedWidthParser.Native/Exports.Reading.cs b/src/FixedWidthParser.Native/Exports.Reading.cs new file mode 100644 index 0000000..ece4cc3 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Reading.cs @@ -0,0 +1,56 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_read_file")] + public static int ReadFile(byte* path, int pathLength, nint layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + { + if (path is null || pathLength <= 0 || outTable is null) + { + return InvalidArgument("fw_read_file needs a path and out_table."); + } + *outTable = default; + return WithLayout(layout, l => ReadApi.ReadFile(new ReadOnlySpan(path, pathLength), l, options, outTable, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_read_memory")] + public static int ReadMemory(byte* data, long dataLength, nint layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult) + { + if ((data is null && dataLength != 0) || dataLength < 0 || outTable is null) + { + return InvalidArgument("fw_read_memory needs data for a non-zero data_len, and out_table."); + } + *outTable = default; + return WithLayout(layout, l => ReadApi.ReadMemory(data, dataLength, l, options, outTable, outResult)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_free_table")] + public static void FreeTable(NativeTableRaw* table) + { + NativeTable.Free(table); + } + + private static int WithLayout(nint handle, Func call) + { + var layout = NativeHandleTable.Resolve(handle); + if (layout is null) + { + NativeApi.SetLastError("The layout handle is not a live fw_layout."); + return NativeStatus.InvalidHandle; + } + try + { + return call(layout); + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + } +} diff --git a/src/FixedWidthParser.Native/Exports.Writer.cs b/src/FixedWidthParser.Native/Exports.Writer.cs new file mode 100644 index 0000000..aa03a60 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.Writer.cs @@ -0,0 +1,53 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Reading; +using FixedWidthParser.Native.Writing; + +namespace FixedWidthParser.Native +{ + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_write_file")] + public static int WriteFile(byte* path, int pathLength, NativeTableRaw* table, nint layout, NativeWriteOptionsRaw* options) + { + if (path is null || pathLength <= 0 || table is null) + { + return InvalidArgument("fw_write_file needs a path and a table."); + } + return WithLayout(layout, l => WriteApi.WriteFile(new ReadOnlySpan(path, pathLength), table, l, options)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_write_memory")] + public static int WriteMemory(NativeTableRaw* table, nint layout, NativeWriteOptionsRaw* options, NativeBufferRaw* outBuffer) + { + if (table is null || outBuffer is null) + { + return InvalidArgument("fw_write_memory needs a table and out_buffer."); + } + *outBuffer = default; + return WithLayout(layout, l => WriteApi.WriteMemory(table, l, options, outBuffer)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_write_stream")] + public static int WriteStream(NativeTableRaw* table, nint layout, NativeWriteOptionsRaw* options, NativeSinkRaw* sink) + { + if (table is null) + { + return InvalidArgument("fw_write_stream needs a table."); + } +#pragma warning disable CA2000 // Disposed by the using below; the sink has nothing to release. + if (!CallbackWriteStream.TryCreate(sink, out var created)) + { + return NativeStatus.InvalidArgument; + } +#pragma warning restore CA2000 + using var stream = created; + return WithLayout(layout, l => WriteApi.WriteStream(table, l, options, stream)); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_free_buffer")] + public static void FreeBuffer(NativeBufferRaw* buffer) + { + WriteApi.FreeBuffer(buffer); + } + } +} diff --git a/src/FixedWidthParser.Native/Exports.cs b/src/FixedWidthParser.Native/Exports.cs new file mode 100644 index 0000000..7fa8b76 --- /dev/null +++ b/src/FixedWidthParser.Native/Exports.cs @@ -0,0 +1,49 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native +{ + /// + /// The [UnmanagedCallersOnly] entry points: pointer checks and marshaling only, so the logic + /// stays in the span-based layer the tests drive. One partial per feature. + /// + [ExcludeFromCodeCoverage] + internal static unsafe partial class Exports + { + [UnmanagedCallersOnly(EntryPoint = "fw_abi_version")] + public static int AbiVersion() + { + return NativeStatus.AbiVersion; + } + + [UnmanagedCallersOnly(EntryPoint = "fw_last_error")] + public static int LastError(byte* buffer, int capacity, int* outLength) + { + if (outLength is null || capacity < 0 || (buffer is null && capacity > 0)) + { + return NativeStatus.InvalidArgument; + } + return NativeApi.LastError(new Span(buffer, capacity), out *outLength); + } + + [UnmanagedCallersOnly(EntryPoint = "fw_last_parse_error")] + public static int LastParseError(NativeParseError* outError) + { + if (outError is null || outError->StructSize != sizeof(NativeParseError)) + { + return NativeStatus.InvalidArgument; + } + NativeApi.LastParseError(out outError->LineNumber, out outError->Column); + return NativeStatus.Ok; + } + } + + /// fw_parse_error. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeParseError + { + public int StructSize; + public long LineNumber; + public int Column; + } +} diff --git a/src/FixedWidthParser.Native/FixedWidthParser.Native.csproj b/src/FixedWidthParser.Native/FixedWidthParser.Native.csproj new file mode 100644 index 0000000..d0dfde9 --- /dev/null +++ b/src/FixedWidthParser.Native/FixedWidthParser.Native.csproj @@ -0,0 +1,50 @@ + + + + net10.0 + enable + enable + true + false + true + true + + true + Shared + false + + true + Speed + false + false + true + false + + x86-64-v3 + + + + $(NoWarn);RS0016;RS0026;RS0037;CA1031;S6640 + + + + false + + + + + + + + + + + + diff --git a/src/FixedWidthParser.Native/GrowableNativeBuffer.cs b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs new file mode 100644 index 0000000..e6a6b2e --- /dev/null +++ b/src/FixedWidthParser.Native/GrowableNativeBuffer.cs @@ -0,0 +1,161 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native +{ + /// + /// One native block that doubles as it fills, so a finished column can be handed over as-is instead + /// of copied. A owns the block, so a buffer dropped on an error path still + /// frees it. + /// + internal sealed unsafe class GrowableNativeBuffer : IDisposable where T : unmanaged + { + private const int InitialLength = 256; +#pragma warning disable SS066 // Disposed in Dispose, or detached by Detach. + private Block? _block; +#pragma warning restore SS066 + private T* _data; + private int _capacity; + + internal int Count { get; private set; } + + internal ref T this[int index] => ref _data[index]; + + internal ReadOnlySpan Span => new(_data, Count); + + internal T* Pointer => _data; + + /// Whether the elements still sit in memory lent by . + internal bool Borrowed => _block is null && _data is not null; + + /// + /// Fills elements at , which the caller owns and frees, and + /// copies them into a block of its own if it outgrows them. + /// + internal void Borrow(T* data, int capacity) + { + _data = data; + _capacity = capacity; + } + + /// Grows once to hold at least elements, clamped to what a buffer can hold. + internal void Reserve(long capacity) + { + capacity = Math.Min(capacity, int.MaxValue); + if (capacity > _capacity) + { + Grow(capacity); + } + } + + /// Drops every element from on. + internal void Truncate(int count) + { + Count = count; + } + + /// Appends uninitialized elements and returns them to be filled. + internal Span Append(int count) + { + if (count > _capacity - Count) + { + Grow((long)Count + count); + } + var span = new Span(_data + Count, count); + Count += count; + return span; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void Add(T value) + { + if (Count == _capacity) + { + Grow(Count + 1L); + } + _data[Count++] = value; + } + + internal void AddRange(ReadOnlySpan values) + { + if (values.Length > _capacity - Count) + { + Grow((long)Count + values.Length); + } + values.CopyTo(new Span(_data + Count, values.Length)); + Count += values.Length; + } + + /// Hands the block to the caller, who frees it with . + internal IntPtr Detach() + { + IntPtr block = (IntPtr)_data; + _block?.SetHandleAsInvalid(); + Reset(); + return block; + } + + public void Dispose() + { + _block?.Dispose(); + Reset(); + } + + private void Reset() + { +#pragma warning disable IDISP003 // Callers dispose or detach the block first. + _block = null; +#pragma warning restore IDISP003 + _data = null; + _capacity = 0; + Count = 0; + } + + /// Doubles, at least to , clamped to int.MaxValue elements; past that the buffer cannot grow. + internal static int NextCapacity(int capacity, long minimum) + { + if (minimum > int.MaxValue) + { + throw new InvalidOperationException($"A column buffer cannot hold more than {int.MaxValue} elements."); + } + return (int)Math.Min(int.MaxValue, Math.Max(minimum, Math.Max(capacity * 2L, InitialLength))); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void Grow(long minimum) + { + int capacity = NextCapacity(_capacity, minimum); + nint bytes = checked((nint)capacity * sizeof(T)); + T* lent = Borrowed ? _data : null; + _block ??= new Block(); + _data = (T*)_block.Resize(bytes); + _capacity = capacity; + if (lent is not null) + { + new ReadOnlySpan(lent, Count).CopyTo(new Span(_data, Count)); + } + } + + private sealed class Block : SafeHandle + { + internal Block() + : base(IntPtr.Zero, ownsHandle: true) + { + } + + public override bool IsInvalid => handle == IntPtr.Zero; + + internal IntPtr Resize(nint bytes) + { + SetHandle(handle == IntPtr.Zero ? Marshal.AllocHGlobal(bytes) : Marshal.ReAllocHGlobal(handle, bytes)); + return handle; + } + + protected override bool ReleaseHandle() + { + Marshal.FreeHGlobal(handle); + return true; + } + } + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs new file mode 100644 index 0000000..e52ccd3 --- /dev/null +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnRules.cs @@ -0,0 +1,101 @@ +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Native.Layouts +{ + /// The per-column rules of fw_layout_create, and the defaults the zeroed fields stand for. + internal static class NativeColumnRules + { + internal static string? Check(NativeColumnSpecRaw spec, string? format) + { + if (spec.Type is < NativeType.String or > NativeType.Timestamp) + { + return $"unknown type {spec.Type}."; + } + if (spec.Scale is < 0 or > NativeType.MaxScale) + { + return $"scale {spec.Scale} is outside 0..{NativeType.MaxScale}."; + } + if (spec.Scale != 0 && spec.Type is not (NativeType.Float64 or NativeType.Decimal)) + { + return "scale applies only to FLOAT64 and DECIMAL columns."; + } + if (format is not null && (spec.Type is NativeType.String or NativeType.Decimal || spec.Scale != 0)) + { + return "a format applies only to INT64, an unscaled FLOAT64 and date or time columns."; + } + return CheckOptions(spec); + } + + private static string? CheckOptions(NativeColumnSpecRaw spec) + { + if (!NativeOptions.TryAscii(spec.TrimChar, ' ', out _) || !NativeOptions.TryAscii(spec.Padding, ' ', out _)) + { + return "trim_char and padding must be ASCII characters."; + } + if (spec.TrimMode is < 0 or > 2 || spec.Alignment is < 0 or > 2 || spec.Overflow is < 0 or > 1) + { + return "unknown trim_mode, alignment or overflow."; + } + if (spec.Overflow == 1 && spec.Type != NativeType.String) + { + return "truncate overflow applies only to STRING columns."; + } + if (spec.Dictionary is < 0 or > 1 || (spec.Dictionary == 1 && spec.Type != NativeType.String)) + { + return "dictionary is 0 or 1, and applies only to STRING columns."; + } + return null; + } + + internal static NativeColumn Build(NativeColumnSpecRaw spec, string name, string? format) + { + NativeOptions.TryAscii(spec.TrimChar, ' ', out char trimChar); + NativeOptions.TryAscii(spec.Padding, ' ', out char padding); + return new NativeColumn + { + Name = name, + Start = spec.Start, + Length = spec.Length, + Type = spec.Type, + Nullable = spec.Nullable != 0, + Dictionary = spec.Dictionary == 1, + Scale = spec.Scale, + Parse = new ColumnParseOptions(trimChar, TrimModeOf(spec.TrimMode), format), + FormatOptions = new ColumnFormatOptions(AlignmentOf(spec.Alignment, spec.Type), padding, format ?? IsoFormat(spec.Type), + spec.Overflow == 1 ? OverflowBehavior.Truncate : OverflowBehavior.Throw), + }; + } + + // Without a format, dates and times are written as ISO 8601 to the second; reading without one accepts it back. + private static string? IsoFormat(int type) + { + return type switch + { + NativeType.Date => "yyyy-MM-dd", + NativeType.Time => "HH:mm:ss", + NativeType.Timestamp => "yyyy-MM-ddTHH:mm:ss", + _ => null, + }; + } + + private static TrimMode TrimModeOf(int value) + { + return value switch + { + 1 => TrimMode.Trailing, + 2 => TrimMode.Leading, + _ => TrimMode.Both, + }; + } + + private static Alignment AlignmentOf(int value, int type) + { + return value switch + { + 1 => Alignment.Left, + 2 => Alignment.Right, + _ => type == NativeType.String ? Alignment.Left : Alignment.Right, + }; + } + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs new file mode 100644 index 0000000..27e0d01 --- /dev/null +++ b/src/FixedWidthParser.Native/Layouts/NativeColumnSpec.cs @@ -0,0 +1,55 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Layouts +{ + /// The FW_T_* column types. + internal static class NativeType + { + internal const int String = 0; + internal const int Int64 = 1; + internal const int Float64 = 2; + internal const int Decimal = 3; + internal const int Date = 4; + internal const int Time = 5; + internal const int Timestamp = 6; + + internal const int MaxScale = 18; + + /// The day DATE values count from, as Arrow's date32 does. + internal static readonly int EpochDay = new DateOnly(1970, 1, 1).DayNumber; + + /// 10^scale for every scale a column may have. + internal static readonly double[] Pow10 = [.. Enumerable.Range(0, MaxScale + 1).Select(p => Math.Pow(10, p))]; + } + + /// fw_column_spec. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeColumnSpecRaw + { + public int StructSize; + public byte* Name; + public int NameLen; + public int Start; + public int Length; + public int Type; + public int Nullable; + public int Scale; + public byte* Format; + public int FormatLen; + public int TrimChar; + public int TrimMode; + public int Alignment; + public int Padding; + public int Overflow; + public int Dictionary; + } + + /// fw_layout_options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeLayoutOptionsRaw + { + public int StructSize; + public int LineLength; + public int Filler; + } +} diff --git a/src/FixedWidthParser.Native/Layouts/NativeLayout.cs b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs new file mode 100644 index 0000000..f8cba12 --- /dev/null +++ b/src/FixedWidthParser.Native/Layouts/NativeLayout.cs @@ -0,0 +1,126 @@ +using System.Text; +using FixedWidthParser.Attributes; +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Native.Layouts +{ + /// One validated column of a . + internal sealed class NativeColumn + { + internal required string Name { get; init; } + internal int Start { get; init; } + internal int Length { get; init; } + internal int Type { get; init; } + internal bool Nullable { get; init; } + + /// A STRING column handed to Arrow dictionary-encoded: int32 indices into its distinct values. + internal bool Dictionary { get; init; } + + internal int Scale { get; init; } + internal ColumnParseOptions Parse { get; init; } + internal ColumnFormatOptions FormatOptions { get; init; } + } + + /// + /// The C counterpart of FixedWidthLayout<T>: columns described at run time by + /// fw_column_specs, validated once and shared, immutable, by every read and write. + /// + internal sealed class NativeLayout + { + private const int MaxColumns = 16_384; + + private NativeLayout(NativeColumn[] columns, int lineLength, byte filler) + { + Columns = columns; + LineLength = lineLength; + Filler = filler; + } + + internal NativeColumn[] Columns { get; } + internal int LineLength { get; } + internal byte Filler { get; } + + /// The index of the first column named , or -1. + internal int IndexOf(string? name) + { + return Array.FindIndex(Columns, c => string.Equals(c.Name, name, StringComparison.Ordinal)); + } + + internal static unsafe int TryCreate(ReadOnlySpan specs, NativeLayoutOptionsRaw* options, out NativeLayout? layout) + { + layout = null; + int status = NativeOptions.TryRead(options, "fw_layout_options", out var settings); + if (status != NativeStatus.Ok) + { + return status; + } + string? error = specs.Length is < 1 or > MaxColumns + ? $"A layout needs at least one column and at most {MaxColumns}; got {specs.Length}." + : null; + var columns = new NativeColumn[specs.Length]; + for (int i = 0; i < specs.Length && error is null; i++) + { + error = TryColumn(specs[i], i, out columns[i]); + } + error ??= ValidatePositions(columns); + if (error is not null) + { + return NativeApi.InvalidArgument(error); + } + return TryFinish(columns, settings, out layout); + } + + private static int TryFinish(NativeColumn[] columns, NativeLayoutOptionsRaw settings, out NativeLayout? layout) + { + layout = null; + int end = columns.Max(c => c.Start + c.Length); + int lineLength = settings.LineLength == 0 ? end : settings.LineLength; + if (lineLength < end) + { + return NativeApi.InvalidArgument($"line_length {settings.LineLength} is shorter than the columns, which end at {end}."); + } + if (!NativeOptions.TryAscii(settings.Filler, ' ', out char filler)) + { + return NativeApi.InvalidArgument($"filler {settings.Filler} is not an ASCII character."); + } + layout = new NativeLayout(columns, lineLength, (byte)filler); + return NativeStatus.Ok; + } + + private static string? ValidatePositions(NativeColumn[] columns) + { + var positions = columns.Select(c => (c.Start, c.Length, c.Name)).ToArray(); + try + { + ColumnLayoutValidator.Validate(positions, "the layout"); + return null; + } + catch (InvalidOperationException ex) + { + return ex.Message; + } + } + + private static unsafe string? TryColumn(NativeColumnSpecRaw spec, int index, out NativeColumn column) + { + column = null!; + if (spec.StructSize != sizeof(NativeColumnSpecRaw)) + { + return $"Column {index}: fw_column_spec struct_size is {spec.StructSize}; expected {sizeof(NativeColumnSpecRaw)}."; + } + if (spec.Name is null || spec.NameLen <= 0 || (spec.Format is null && spec.FormatLen != 0) || spec.FormatLen < 0) + { + return $"Column {index} has no name, or a name or format pointer that does not match its length."; + } + string name = Encoding.UTF8.GetString(spec.Name, spec.NameLen); + string? format = spec.FormatLen == 0 ? null : Encoding.UTF8.GetString(spec.Format, spec.FormatLen); + string? error = NativeColumnRules.Check(spec, format); + if (error is not null) + { + return $"Column \"{name}\": {error}"; + } + column = NativeColumnRules.Build(spec, name, format); + return null; + } + } +} diff --git a/src/FixedWidthParser.Native/NativeApi.cs b/src/FixedWidthParser.Native/NativeApi.cs new file mode 100644 index 0000000..de21f6d --- /dev/null +++ b/src/FixedWidthParser.Native/NativeApi.cs @@ -0,0 +1,77 @@ +using System.Runtime.CompilerServices; +using System.Text; + +namespace FixedWidthParser.Native +{ + /// The calling thread's last failure, read back through fw_last_error and fw_last_parse_error. + internal static class NativeApi + { + [ThreadStatic] + private static string? _lastError; + + [ThreadStatic] + private static long _parseLine; + + [ThreadStatic] + private static int _parseColumn; + +#pragma warning disable CA2255 // A native library has no Main; this is its only start-up hook. + [ModuleInitializer] +#pragma warning restore CA2255 + internal static void RegisterCodePages() + { + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + } + + internal static void SetLastError(string message) + { + _lastError = message; + _parseLine = 0; + _parseColumn = -1; + } + + /// Records and returns . + internal static int InvalidArgument(string message) + { + SetLastError(message); + return NativeStatus.InvalidArgument; + } + + /// Records a parse failure; is the layout index, -1 for a short line. + internal static void SetParseError(FixedWidthParseException exception, int column) + { + _lastError = exception.Message; + _parseLine = exception.LineNumber; + _parseColumn = column; + } + + internal static void LastParseError(out long line, out int column) + { + line = _parseLine; + column = _lastError is null ? -1 : _parseColumn; + } + + internal static string LastErrorText() + { + return _lastError ?? ""; + } + + internal static int LastError(Span buffer, out int length) + { + string? message = _lastError; + if (string.IsNullOrEmpty(message)) + { + length = 0; + return NativeStatus.Ok; + } + + length = Encoding.UTF8.GetByteCount(message); + if (buffer.Length < length) + { + return NativeStatus.BufferTooSmall; + } + Encoding.UTF8.GetBytes(message, buffer); + return NativeStatus.Ok; + } + } +} diff --git a/src/FixedWidthParser.Native/NativeHandleTable.cs b/src/FixedWidthParser.Native/NativeHandleTable.cs new file mode 100644 index 0000000..86457fe --- /dev/null +++ b/src/FixedWidthParser.Native/NativeHandleTable.cs @@ -0,0 +1,40 @@ +using System.Collections.Concurrent; + +namespace FixedWidthParser.Native +{ + internal static class NativeHandleTable + { + private static readonly ConcurrentDictionary _live = new(); + private static long _counter; + + internal static nint Register(object handle) + { + while (true) + { + nint id = (nint)Interlocked.Increment(ref _counter); + if (id != 0 && _live.TryAdd(id, handle)) + { + return id; + } + } + } + + internal static T? Resolve(nint id) where T : class + { + return id != 0 && _live.TryGetValue(id, out object? handle) ? handle as T : null; + } + + internal static bool TryUnregister(nint id, out T? handle) where T : class + { + if (id != 0 && _live.TryGetValue(id, out object? existing) && existing is T typed + && _live.TryRemove(new KeyValuePair(id, existing))) + { + handle = typed; + return true; + } + + handle = null; + return false; + } + } +} diff --git a/src/FixedWidthParser.Native/NativeOptions.cs b/src/FixedWidthParser.Native/NativeOptions.cs new file mode 100644 index 0000000..9610640 --- /dev/null +++ b/src/FixedWidthParser.Native/NativeOptions.cs @@ -0,0 +1,33 @@ +namespace FixedWidthParser.Native +{ + /// + /// Reads a caller's options struct under the struct_size contract: means all + /// defaults (the zeroed struct), and any size other than the current one is refused. + /// + internal static unsafe class NativeOptions + { + internal static int TryRead(T* options, string name, out T value) where T : unmanaged + { + value = default; + if (options is null) + { + return NativeStatus.Ok; + } + int size = *(int*)options; + if (size != sizeof(T)) + { + NativeApi.SetLastError($"{name} struct_size is {size}; expected {sizeof(T)}."); + return NativeStatus.InvalidArgument; + } + value = *options; + return NativeStatus.Ok; + } + + /// An ASCII byte option where 0 means ; for anything outside ASCII. + internal static bool TryAscii(int value, char fallback, out char result) + { + result = value == 0 ? fallback : (char)value; + return value is >= 0 and <= 127; + } + } +} diff --git a/src/FixedWidthParser.Native/NativeStatus.cs b/src/FixedWidthParser.Native/NativeStatus.cs new file mode 100644 index 0000000..b948032 --- /dev/null +++ b/src/FixedWidthParser.Native/NativeStatus.cs @@ -0,0 +1,15 @@ +namespace FixedWidthParser.Native +{ + internal static class NativeStatus + { + internal const int Ok = 0; + internal const int Eof = -1; + internal const int BufferTooSmall = -2; + internal const int InvalidHandle = -3; + internal const int InvalidArgument = -4; + internal const int Error = -5; + internal const int ParseError = -6; + + internal const int AbiVersion = 1; + } +} diff --git a/src/FixedWidthParser.Native/README.md b/src/FixedWidthParser.Native/README.md new file mode 100644 index 0000000..d53b0e5 --- /dev/null +++ b/src/FixedWidthParser.Native/README.md @@ -0,0 +1,87 @@ +# FixedWidthParser.Native + +NativeAOT shared library exposing FixedWidthParser over a C ABI, so Python, C and C++ can read and write +fixed-width files without a .NET runtime. + +- ABI reference and full contract: `include/fixedwidth.h`, plus `include/fixedwidth_arrow.h` for the + Arrow C Data Interface reads +- Python binding: `python/`; C++ wrapper: `include/fixedwidth.hpp` and `cpp/` + +## Build + + dotnet publish src/FixedWidthParser.Native/FixedWidthParser.Native.csproj -c Release -r + +Output lands in `bin/Release/net10.0//publish/FixedWidthParser.Native.{dll,so,dylib}`. On x64 the +library needs AVX2 (`x86-64-v3`). + +## Layout + +| File | Role | +|---|---| +| `*Api.cs`, `Layouts/`, `Reading/`, `Writing/`, `Arrow/` | Span-based implementation. This is what the tests drive. | +| `Exports*.cs` | `[UnmanagedCallersOnly]` pointer wrappers, one partial per feature. Keep logic out of here: it cannot be called from managed code. | +| `NativeHandleTable.cs` | Maps the opaque handles callers see onto layouts and readers. A handle resolves only as its own kind and is never reissued after it is freed. | +| `NativeApi.cs` | The calling thread's last error (`fw_last_error`, `fw_last_parse_error`). | +| `include/fixedwidth.h`, `include/fixedwidth_arrow.h` | Hand-written C headers; keep them in sync with `Exports*.cs`. | + +Every export catches everything and returns a status (`FW_OK` is 0, failures are negative). Every options +struct starts with `struct_size`; `NULL` means all defaults. + +## Exports + +| Area | Functions | +|---|---| +| Errors | `fw_abi_version`, `fw_last_error`, `fw_last_parse_error` | +| Layouts | `fw_layout_create`, `fw_layout_free` | +| Columnar reads | `fw_read_file`, `fw_read_memory`, `fw_free_table` | +| Batches | `fw_reader_open_file`, `fw_reader_open_memory`, `fw_reader_open_stream`, `fw_reader_next`, `fw_reader_close` | +| Arrow | `fw_read_arrow_file`, `fw_read_arrow_memory`, `fw_read_arrow_stream_file`, `fw_read_arrow_stream_memory`, `fw_read_arrow_stream_stream` | +| Streams | `fw_set_source_error` | +| Writes | `fw_write_file`, `fw_write_memory`, `fw_write_stream`, `fw_free_buffer` | + +A batched read or an Arrow stream can pull from an `fw_stream` instead of a file or memory: a `read` callback +that fills a buffer and returns the count, 0 at the end or -1 after `fw_set_source_error`, and an optional +`release` the library calls once it is done. It reads forward only, skipping the header lines as it goes, so a +pipe, a socket or a decompressor works; the batches stay in constant memory. Whole-table reads take a file or +memory, since they split the source into byte ranges. + +`fw_write_stream` pushes the lines to an `fw_sink` instead: a `write` callback that takes every byte it is +given and returns 0, or -1 after `fw_set_source_error`. It is called in order from the calling thread, a batch at +a time, while batches format in parallel, so memory stays at a batch per thread. + +`include/fixedwidth.def` lists them for import-library generation on Windows; a test fails when it drifts +from the `[UnmanagedCallersOnly]` entry points. + +## Threads + +Any thread may call any function, and calls on different tables, readers and streams run concurrently. A layout +is immutable and may be shared by every thread; handles live in a concurrent table. `fw_last_error` and +`fw_last_parse_error` belong to the calling thread, so read them on the thread whose call failed. A reader or an +Arrow stream is used by one thread at a time, and an `fw_stream`'s `read` runs on whichever thread is reading. +A read or write with `degree_of_parallelism` 0 splits across every processor; several such calls at once share +the same thread pool, so give each `1` when the calls themselves are the parallelism. + +## Consuming from C + +Check `fw_abi_version()` against `FW_ABI_VERSION` first. NativeAOT ships no import library on Windows: load +the library dynamically (`LoadLibraryA`/`dlopen`, as `tests/FixedWidthParser.NativeSmoke` does) or generate +one from `include/fixedwidth.def` (`cpp/cmake/NativeLib.cmake` does this for CMake). + +```c +fw_column_spec columns[2] = {0}; +columns[0] = (fw_column_spec){ .struct_size = sizeof(fw_column_spec), .name = (const uint8_t*)"codneg", + .name_len = 6, .start = 12, .length = 12, .type = FW_T_STRING }; +columns[1] = (fw_column_spec){ .struct_size = sizeof(fw_column_spec), .name = (const uint8_t*)"preabe", + .name_len = 6, .start = 56, .length = 13, .type = FW_T_FLOAT64, .scale = 2 }; +fw_layout* layout; +fw_layout_create(columns, 2, NULL, &layout); + +fw_read_options options = { .struct_size = sizeof(fw_read_options), .header_lines = 1, .skip_invalid = 1 }; +fw_table table; +if (fw_read_file((const uint8_t*)path, (int32_t)strlen(path), layout, &options, &table, NULL) == FW_OK) { + const double* preabe = table.columns[1].values; + /* ... */ + fw_free_table(&table); +} +fw_layout_free(layout); +``` diff --git a/src/FixedWidthParser.Native/Reading/CallbackReadStream.cs b/src/FixedWidthParser.Native/Reading/CallbackReadStream.cs new file mode 100644 index 0000000..9560381 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/CallbackReadStream.cs @@ -0,0 +1,141 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_stream. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeStreamRaw + { + public int StructSize; + public void* UserData; + public delegate* unmanaged Read; + public delegate* unmanaged Release; + } + + /// + /// The caller's fw_stream as a forward-only : ExcelReader's xl_stream. Disposing it + /// calls release once. + /// + internal sealed unsafe class CallbackReadStream : Stream + { + private readonly delegate* unmanaged _read; + private readonly delegate* unmanaged _release; + private readonly void* _userData; + private int _released; + + private CallbackReadStream(NativeStreamRaw raw) + { + _read = raw.Read; + _release = raw.Release; + _userData = raw.UserData; + } + + /// Wraps , or explains why it cannot; release is not called then. + internal static bool TryCreate(NativeStreamRaw* raw, string function, [NotNullWhen(true)] out CallbackReadStream? stream) + { + stream = null; + if (raw is null || raw->StructSize != sizeof(NativeStreamRaw) || raw->Read is null) + { + NativeApi.SetLastError($"{function} needs a non-NULL stream whose struct_size is sizeof(fw_stream) and whose read is set."); + return false; + } + stream = new CallbackReadStream(*raw); + return true; + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override int Read(Span buffer) + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _released) != 0, this); + if (buffer.IsEmpty) + { + return 0; + } + SourceErrors.Clear(); + long got; + fixed (byte* destination = buffer) + { + got = _read(_userData, destination, buffer.Length); + } + if (got < 0 || got > buffer.Length) + { + throw SourceErrors.Failed("fw_stream.read", got, buffer.Length); + } + return (int)got; + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override void Flush() + { + // Nothing to flush: the stream only reads. + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + if (Interlocked.Exchange(ref _released, 1) == 0 && _release is not null) + { + _release(_userData); + } + base.Dispose(disposing); + } + } + + /// The message a stream callback leaves through fw_set_source_error before returning -1. + internal static class SourceErrors + { + [ThreadStatic] + private static string? _pending; + + internal static void Set(string? message) + { + _pending = string.IsNullOrEmpty(message) ? null : message; + } + + internal static void Clear() + { + _pending = null; + } + + internal static IOException Failed(string where, long got, long requested) + { + string? message = _pending; + _pending = null; + return got == -1 + ? new IOException(message ?? $"{where} failed.") + : new IOException($"{where} returned {got} for a request of {requested} bytes, which is not a valid result."); + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs new file mode 100644 index 0000000..c0168a8 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ColumnBuilder.cs @@ -0,0 +1,513 @@ +using System.Buffers; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.Unicode; +using FixedWidthParser.Infrastructure; +using FixedWidthParser.Native.Layouts; + +namespace FixedWidthParser.Native.Reading +{ + /// + /// One column of one partition, parsed straight into native buffers that are handed to the caller as they + /// are. Values: long for INT64, DECIMAL, TIME and TIMESTAMP, double for FLOAT64, int + /// for DATE and for STRING offsets into UTF-8 bytes. The validity bitmap exists only once a null arrives. + /// + internal sealed unsafe class ColumnBuilder : IDisposable + { + private const int StackChars = 256; + private const long DecimalLimit = 1_000_000_000_000_000_000; + private const DateTimeStyles Utc = DateTimeStyles.AdjustToUniversal | DateTimeStyles.AssumeUniversal; + + private readonly NativeColumn _column; + private readonly long _maxTextBytes; + private readonly GrowableNativeBuffer _longs = new(); + private readonly GrowableNativeBuffer _doubles = new(); + private readonly GrowableNativeBuffer _ints = new(); + private readonly GrowableNativeBuffer _bytes = new(); +#pragma warning disable SS066 // Disposed in Dispose, or detached by Publish. + private GrowableNativeBuffer? _validity; +#pragma warning restore SS066 + + /// + /// Initializes a new instance of the class. + /// + /// The native column this builder is for. + /// The maximum number of text bytes expected for this column. + /// How many rows to size the buffers for up front, so a partition does not grow them row by row. + /// Room for values that a owns, or null. + /// For a STRING column, room for its + 1 offsets, or null. + /// Whether starts the column: only the first slice writes its leading 0. + internal ColumnBuilder(NativeColumn column, long maxTextBytes, long expectedRows, void* lent = null, int* lentOffsets = null, bool firstSlice = true) + { + _column = column; + _maxTextBytes = maxTextBytes; + switch (column.Type) + { + case NativeType.String: + Size(_ints, lentOffsets, expectedRows + 1); + Size(_bytes, lent, Math.Min(expectedRows * column.Length, maxTextBytes)); + // A later slice's index 0 is the previous slice's last offset, which that slice writes: this one + // claims it without writing, and reads it nowhere (Truncate knows row 0 starts at 0). + if (lentOffsets is null || firstSlice) + { + _ints.Add(0); + } + else + { + _ints.Append(1); + } + break; + case NativeType.Float64: + Size(_doubles, lent, expectedRows); + break; + case NativeType.Date: + Size(_ints, lent, expectedRows); + break; + default: + Size(_longs, lent, expectedRows); + break; + } + } + + private static void Size(GrowableNativeBuffer values, void* lent, long rows) where T : unmanaged + { + if (lent is null) + { + values.Reserve(rows); + return; + } + values.Borrow((T*)lent, (int)rows); + } + + /// The bytes of one value of a column of ; 0 for STRING, which has no fixed size. + internal static int ValueSize(int type) + { + return type switch + { + NativeType.String => 0, + NativeType.Date => sizeof(int), + _ => sizeof(long), + }; + } + + internal int Rows { get; private set; } + + /// The UTF-8 bytes a STRING column holds. + internal long TextBytes => _bytes.Count; + + internal bool TryAppend(ReadOnlySpan raw, in FixedWidthContext context) + { + var parse = _column.Parse; + if (_column.Nullable && FixedWidthRuntime.IsBlank(raw, in parse)) + { + AppendNull(); + return true; + } + bool parsed = _column.Type switch + { + NativeType.String => AppendString(FixedWidthRuntime.TrimColumn(raw, in parse), in context), + NativeType.Int64 => Add(FixedWidthRuntime.TryParse(raw, in context, in parse, out long integer), integer), + NativeType.Float64 => AppendFloat(raw, in context, in parse), + NativeType.Decimal => Add(TryDecimal(FixedWidthRuntime.TrimColumn(raw, in parse), _column.Scale, out long unscaled), unscaled), + _ => AppendTemporal(raw, in context, in parse), + }; + if (parsed) + { + SetValid(Rows, true); + Rows++; + } + return parsed; + } + + /// Drops every row from on, undoing the columns of a line rejected midway. + internal void Truncate(int rows) + { + Rows = rows; + _longs.Truncate(Math.Min(_longs.Count, rows)); + _doubles.Truncate(Math.Min(_doubles.Count, rows)); + if (_column.Type != NativeType.String) + { + _ints.Truncate(Math.Min(_ints.Count, rows)); + return; + } + _ints.Truncate(rows + 1); + // Index 0 of a lent slice belongs to the slice before; row 0 always starts at 0. + _bytes.Truncate(rows == 0 ? 0 : _ints[rows]); + } + + internal void Publish(NativeColumnRaw* target) + { + target->Type = _column.Type; + target->Scale = _column.Scale; + target->Length = Rows; + target->DataLen = _bytes.Count; + target->Validity = _validity is null ? null : (byte*)_validity.Detach(); + target->Data = _column.Type == NativeType.String ? (byte*)_bytes.Detach() : null; + target->Values = (void*)(_column.Type switch + { + NativeType.Float64 => _doubles.Detach(), + NativeType.String or NativeType.Date => _ints.Detach(), + _ => _longs.Detach(), + }); + } + + /// + /// Publishes the same column of several partitions, in order, as one column. Partitions that parsed into slices + /// of are moved down in it to close the gaps between them, and it becomes the column; + /// otherwise, or once a partition outgrew its slice, they are copied into fresh memory. + /// + /// The block a STRING column's offsets were lent from, closed up and handed over the same way. + internal static void Concat(ReadOnlySpan parts, NativeColumnRaw* target, GrowableNativeBuffer? lent = null, + GrowableNativeBuffer? lentOffsets = null) + { + long rows = 0; + foreach (ref readonly var part in parts) + { + rows += part.Rows; + lent = part.Lent ? lent : null; + lentOffsets = part._ints.Borrowed ? lentOffsets : null; + } + if (parts.Length == 1 && lent is null && lentOffsets is null) + { + parts[0].Publish(target); + return; + } + var column = parts[0]._column; + target->Type = column.Type; + target->Scale = column.Scale; + target->Length = rows; + target->Validity = ConcatValidity(parts, rows); + target->Data = null; + target->DataLen = 0; + if (column.Type == NativeType.String) + { + ConcatStrings(parts, rows, target, lent, lentOffsets); + return; + } + target->Values = ConcatValues(parts, lent); + } + + /// Whether this partition's values, or a STRING column's bytes, still sit where a lent them. + private bool Lent => _column.Type switch + { + NativeType.String => _bytes.Borrowed, + NativeType.Float64 => _doubles.Borrowed, + NativeType.Date => _ints.Borrowed, + _ => _longs.Borrowed, + }; + + private ReadOnlySpan ValueBytes => _column.Type switch + { + NativeType.Float64 => MemoryMarshal.AsBytes(_doubles.Span), + NativeType.Date => MemoryMarshal.AsBytes(_ints.Span), + _ => MemoryMarshal.AsBytes(_longs.Span), + }; + + private static void* ConcatValues(ReadOnlySpan parts, GrowableNativeBuffer? lent) + { + long total = 0; + foreach (ref readonly var part in parts) + { + total += part.ValueBytes.Length; + } + byte* values = lent is null ? (byte*)Marshal.AllocHGlobal((nint)total) : lent.Pointer; + long at = 0; + foreach (ref readonly var part in parts) + { + var bytes = part.ValueBytes; + MoveTo(bytes, values + at); + at += bytes.Length; + } + return lent is null ? values : (void*)lent.Detach(); + } + + private static void ConcatStrings(ReadOnlySpan parts, long rows, NativeColumnRaw* target, GrowableNativeBuffer? lent, + GrowableNativeBuffer? lentOffsets) + { + long total = 0; + foreach (ref readonly var part in parts) + { + total += part._bytes.Count; + } + if (total > int.MaxValue) + { + throw new InvalidOperationException( + $"Column '{parts[0]._column.Name}' holds more than 2 GiB of text; read in batches (fw_reader_open_*)."); + } + byte* data = lent is null ? (byte*)Marshal.AllocHGlobal((nint)total) : lent.Pointer; + // Lent offsets are rebased in place, moving down in order as values do; one partition of its own already + // starts at 0; several of their own are copied into fresh memory. + int* offsets = null; + if (lentOffsets is not null) + { + offsets = lentOffsets.Pointer; + } + else if (parts.Length > 1) + { + offsets = (int*)Marshal.AllocHGlobal((nint)((rows + 1) * sizeof(int))); + } + if (offsets is not null) + { + offsets[0] = 0; + } + long row = 1; + int baseOffset = 0; + foreach (ref readonly var part in parts) + { + if (offsets is not null) + { + foreach (ref readonly int offset in part._ints.Span[1..]) + { + offsets[row++] = baseOffset + offset; + } + } + MoveTo(part._bytes.Span, data + baseOffset); + baseOffset += part._bytes.Count; + } + if (lentOffsets is not null) + { + offsets = (int*)lentOffsets.Detach(); + } + target->Values = offsets is null ? (void*)parts[0]._ints.Detach() : offsets; + target->Data = lent is null ? data : (byte*)lent.Detach(); + target->DataLen = total; + } + + /// + /// Copies to , which may overlap them: partitions lent slices of one + /// buffer move down in order, each staying inside its own slice, so none overwrites one not yet moved. + /// + private static void MoveTo(ReadOnlySpan bytes, byte* to) + { + fixed (byte* from = bytes) + { + if (from != to) + { + bytes.CopyTo(new Span(to, bytes.Length)); + } + } + } + + private static byte* ConcatValidity(ReadOnlySpan parts, long rows) + { + bool any = false; + foreach (ref readonly var part in parts) + { + any |= part._validity is not null; + } + if (!any) + { + return null; + } + long length = (rows + 7) / 8; + byte* bits = (byte*)Marshal.AllocHGlobal((nint)length); + new Span(bits, (int)length).Fill(0xFF); + long row = 0; + foreach (ref readonly var part in parts) + { + var partBits = part._validity is null ? default : part._validity.Span; + for (int r = 0; r < part.Rows && !partBits.IsEmpty; r++) + { + if ((partBits[r >> 3] & (1 << (r & 7))) == 0) + { + bits[(row + r) >> 3] &= (byte)~(1 << (int)((row + r) & 7)); + } + } + row += part.Rows; + } + return bits; + } + + public void Dispose() + { + _longs.Dispose(); + _doubles.Dispose(); + _ints.Dispose(); + _bytes.Dispose(); + _validity?.Dispose(); + } + + private bool Add(bool parsed, long value) + { + if (parsed) + { + _longs.Add(value); + } + return parsed; + } + + private bool AppendFloat(ReadOnlySpan raw, in FixedWidthContext context, in ColumnParseOptions parse) + { + var text = FixedWidthRuntime.TrimColumn(raw, in parse); + if (_column.Scale > 0 && TryInteger(text, out long digits)) + { + _doubles.Add(digits / NativeType.Pow10[_column.Scale]); + return true; + } + if (!FixedWidthRuntime.TryDouble(raw, in context, in parse, out double value)) + { + return false; + } + // Digits too many for a long still carry the implied decimals. + bool impliedDecimals = _column.Scale > 0 && text.IndexOfAny(".,eE"u8) < 0; + _doubles.Add(impliedDecimals ? value / NativeType.Pow10[_column.Scale] : value); + return true; + } + + private static bool TryInteger(ReadOnlySpan text, out long value) + { + return FastInteger.TryParse(text.Trim((byte)' '), allowMinus: true, maxDigits: 18, out value) + || long.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out value); + } + + private static bool TryDecimal(ReadOnlySpan text, int scale, out long unscaled) + { + if (TryInteger(text, out unscaled)) + { + return unscaled is > -DecimalLimit and < DecimalLimit; + } + if (!decimal.TryParse(text, NumberStyles.Number, CultureInfo.InvariantCulture, out decimal value)) + { + return false; + } + decimal scaled = value * (decimal)NativeType.Pow10[scale]; + if (scaled != decimal.Truncate(scaled) || scaled <= -DecimalLimit || scaled >= DecimalLimit) + { + return false; + } + unscaled = (long)scaled; + return true; + } + + private bool AppendTemporal(ReadOnlySpan raw, in FixedWidthContext context, in ColumnParseOptions parse) + { + bool exact = parse.Format is not null; + switch (_column.Type) + { + case NativeType.Date: + bool dateOk = exact ? FixedWidthRuntime.TryDateOnlyExact(raw, in context, in parse, out var date) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out date); + if (dateOk) + { + _ints.Add(date.DayNumber - NativeType.EpochDay); + } + return dateOk; + case NativeType.Time: + bool timeOk = exact ? FixedWidthRuntime.TryTimeOnlyExact(raw, in context, in parse, out var time) : FixedWidthRuntime.TryParseDecoded(raw, in context, in parse, out time); + return Add(timeOk, time.Ticks / TimeSpan.TicksPerMicrosecond); + default: + // A fast parse has no zone, which the slow path would read as UTC too. + var text = FixedWidthRuntime.TrimColumn(raw, in parse); + bool stampOk = (parse.FastDate is { HasDate: true } fast && fast.TryParse(text, in context, out var stamp)) + || TryTimestamp(text, context.Encoding, parse.Format, out stamp); + return Add(stampOk, (stamp.Ticks - DateTime.UnixEpoch.Ticks) / TimeSpan.TicksPerMicrosecond); + } + } + + /// A timestamp in UTC: a zone or offset in the text is applied, and text without one is taken as UTC. + private static bool TryTimestamp(ReadOnlySpan text, Encoding encoding, string? format, out DateTime value) + { + value = default; + if (encoding.GetMaxCharCount(text.Length) > StackChars && encoding.GetCharCount(text) > StackChars) + { + return false; + } + Span chars = stackalloc char[StackChars]; + chars = chars[..encoding.GetChars(text, chars)]; + return format is null + ? DateTime.TryParse(chars, CultureInfo.InvariantCulture, Utc, out value) + : DateTime.TryParseExact(chars, format, CultureInfo.InvariantCulture, Utc, out value); + } + + private bool AppendString(ReadOnlySpan text, in FixedWidthContext context) + { + // Invalid UTF-8 (a split character, or another encoding read as UTF-8) is decoded with U+FFFD, as the core does. + if (!context.IsUtf8 || !Utf8.IsValid(text)) + { + AppendTranscoded(text, context.IsUtf8 ? Encoding.UTF8 : context.Encoding); + _ints.Add(_bytes.Count); + return true; + } + EnsureTextRoom(text.Length); + _bytes.AddRange(text); + _ints.Add(_bytes.Count); + return true; + } + + private void AppendTranscoded(ReadOnlySpan text, Encoding encoding) + { + int charCount = encoding.GetCharCount(text); + char[]? rented = charCount > StackChars ? ArrayPool.Shared.Rent(charCount) : null; + try + { + Span chars = rented ?? stackalloc char[StackChars]; + chars = chars[..encoding.GetChars(text, chars)]; + int byteCount = Encoding.UTF8.GetByteCount(chars); + EnsureTextRoom(byteCount); + Encoding.UTF8.GetBytes(chars, _bytes.Append(byteCount)); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } + } + } + + private void EnsureTextRoom(int bytes) + { + if (_bytes.Count + (long)bytes > _maxTextBytes) + { + throw TooMuchText(_column.Name, _maxTextBytes); + } + } + + internal static InvalidOperationException TooMuchText(string column, long limit) + { + return new InvalidOperationException( + $"Column '{column}' holds more than {limit} bytes of text; read in batches (fw_reader_open_*)."); + } + + private void AppendNull() + { + switch (_column.Type) + { + case NativeType.String: + _ints.Add(_bytes.Count); + break; + case NativeType.Float64: + _doubles.Add(0); + break; + case NativeType.Date: + _ints.Add(0); + break; + default: + _longs.Add(0); + break; + } + SetValid(Rows, false); + Rows++; + } + + private void SetValid(int row, bool valid) + { + if (_validity is null) + { + if (valid) + { + return; + } + _validity = new GrowableNativeBuffer(); + _validity.Append((row >> 3) + 1).Fill(0xFF); + } + while (_validity.Count <= row >> 3) + { + _validity.Add(0xFF); + } + ref byte bits = ref _validity[row >> 3]; + int mask = 1 << (row & 7); + bits = (byte)(valid ? bits | mask : bits & ~mask); + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/NativeTable.cs b/src/FixedWidthParser.Native/Reading/NativeTable.cs new file mode 100644 index 0000000..516ed05 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/NativeTable.cs @@ -0,0 +1,69 @@ +using System.Runtime.InteropServices; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_column: one column of a published table. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeColumnRaw + { + public int Type; + public int Scale; + public long Length; + public void* Values; + public byte* Validity; + public byte* Data; + public long DataLen; + } + + /// fw_table. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeTableRaw + { + public int ColumnCount; + public long RowCount; + public NativeColumnRaw* Columns; + } + + /// fw_read_options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeReadOptionsRaw + { + public int StructSize; + public long HeaderLines; + public int SkipInvalid; + public int DegreeOfParallelism; + public int Encoding; + public int MaxLineLength; + public int SkipByteOrderMark; + } + + /// fw_read_result: an optional output, so its struct_size is checked, not read as options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeReadResultRaw + { + public int StructSize; + public long LinesRead; + public long LinesSkipped; + } + + internal static unsafe class NativeTable + { + /// Frees every buffer of a table published by the library and zeroes it; safe on a zeroed table. + internal static void Free(NativeTableRaw* table) + { + if (table is null || table->Columns is null) + { + return; + } + for (int i = 0; i < table->ColumnCount; i++) + { + var column = table->Columns[i]; + Marshal.FreeHGlobal((IntPtr)column.Values); + Marshal.FreeHGlobal((IntPtr)column.Validity); + Marshal.FreeHGlobal((IntPtr)column.Data); + } + NativeMemory.Free(table->Columns); + *table = default; + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/NativeTableReader.cs b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs new file mode 100644 index 0000000..f1a274c --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/NativeTableReader.cs @@ -0,0 +1,116 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_reader_options. + [StructLayout(LayoutKind.Sequential)] + internal struct NativeReaderOptionsRaw + { + public int StructSize; + public NativeReadOptionsRaw Read; + public long BatchRows; + } + + /// + /// A sequential read handed out a table of at most batch_rows rows at a time, behind + /// fw_reader_open_*. Used by one thread at a time; it keeps its layout alive after + /// fw_layout_free. + /// + internal sealed unsafe class NativeTableReader : IDisposable + { + private const long DefaultBatchRows = 65_536; + + private readonly Stream _stream; + private readonly NativeLayout _layout; + private readonly FixedWidthReaderOptions _options; + private readonly FixedWidthContext _context; + private readonly long _batchRows; + private LineBufferState _lines; + private bool _done; + + /// Positioned at the first data line, past header lines. + /// Whether nothing of the source was skipped, so a byte order mark may begin it. + internal NativeTableReader(Stream stream, bool atStart, long headerLines, NativeLayout layout, ReadSettings settings, long batchRows) + { + _stream = stream; + _layout = layout; + _options = settings.ReaderOptions(new StrongBox(), degreeOfParallelism: 1, chunkSize: 0); + _context = _options.Context; + _batchRows = batchRows == 0 ? DefaultBatchRows : batchRows; + _lines = default; + _lines.Rent(_options, atStart && _options.SkipsByteOrderMark); + _lines.LineNumber = headerLines; + } + + /// Publishes the next batch into , or returns ; a failure ends the read. + internal int Next(NativeTableRaw* table) + { + if (_done) + { + return NativeStatus.Eof; + } + var sink = new TableSink(_layout, TableLimits.Default); + try + { +#pragma warning disable IDISP003 // Fill appends to the sink; it never replaces it. + Fill(ref sink); +#pragma warning restore IDISP003 + if (sink.Rows == 0) + { + return NativeStatus.Eof; + } + sink.Publish(table); + return NativeStatus.Ok; + } + catch + { + _done = true; + throw; + } + finally + { + sink.Dispose(); + } + } + + private void Fill(ref TableSink sink) + { + while (sink.Rows < _batchRows) + { + var status = _lines.TryGetLine(out var line); + if (status == LineStatus.End) + { + _done = true; + return; + } + if (status == LineStatus.NeedData) + { + _lines.Fill(_stream); + continue; + } + if (!sink.TryAdd(line, in _context)) + { + if (_options.OnSkip is { } skip) + { + skip(); + continue; + } + ParseErrors.Reject(sink.Diagnose(line, _lines.LineNumber, in _context), _options.OnParseError); + } + } + } + + internal NativeLayout Layout => _layout; + + public void Dispose() + { + _lines.Return(); +#pragma warning disable IDISP007 // The reader owns the stream it was opened over. + _stream.Dispose(); +#pragma warning restore IDISP007 + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/PointerSource.cs b/src/FixedWidthParser.Native/Reading/PointerSource.cs new file mode 100644 index 0000000..b851978 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/PointerSource.cs @@ -0,0 +1,29 @@ +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// The caller's memory as a random-access source. The memory must stay valid while it is read. + internal sealed unsafe class PointerSource : IRandomAccessSource + { + private readonly byte* _data; + + internal PointerSource(byte* data, long length) + { + _data = data; + Length = length; + } + + public long Length { get; } + + public int Read(Span destination, long offset) + { + if (offset >= Length) + { + return 0; + } + int count = (int)Math.Min(destination.Length, Length - offset); + new ReadOnlySpan(_data + offset, count).CopyTo(destination); + return count; + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/ReadApi.cs b/src/FixedWidthParser.Native/Reading/ReadApi.cs new file mode 100644 index 0000000..8595308 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ReadApi.cs @@ -0,0 +1,141 @@ +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_read_file and fw_read_memory: a whole source into one columnar table. + internal static unsafe class ReadApi + { + internal static int ReadFile(ReadOnlySpan path, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, + int chunkSize = 0, TableLimits? limits = null) + { + try + { + using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read); + return Read(new FileSource(handle), layout, options, outTable, outResult, chunkSize, limits ?? TableLimits.Default); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + // chunkSize: bytes per partition, 0 sizes them from the source; tests set it to cut every boundary. + internal static int ReadMemory(byte* data, long length, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, + int chunkSize = 0, TableLimits? limits = null) + { + return Read(new PointerSource(data, length), layout, options, outTable, outResult, chunkSize, limits ?? TableLimits.Default); + } + + private static int Read(IRandomAccessSource source, NativeLayout layout, NativeReadOptionsRaw* options, NativeTableRaw* outTable, NativeReadResultRaw* outResult, + int chunkSize, TableLimits limits) + { + *outTable = default; + int status = CheckResult(outResult); + if (status != NativeStatus.Ok) + { + return status; + } + status = ReadSettings.TryCreate(options, out var settings); + if (status != NativeStatus.Ok) + { + return status; + } + try + { + long dataStart = HeaderEnd(source, settings!.HeaderLines, out long headerLines); + var skipped = new StrongBox(headerLines); + var readerOptions = settings.ReaderOptions(skipped, settings.DegreeOfParallelism, chunkSize); +#pragma warning disable VSTHRD002 // An export is synchronous and runs with no synchronization context. + using var arena = TableArena.TryCreate(source, dataStart, layout, limits); + using var table = ParallelAggregator.RunAsync(source, dataStart, (start, end) => new TableSink(layout, limits, arena, start, end), readerOptions, CancellationToken.None) + .GetAwaiter().GetResult(); +#pragma warning restore VSTHRD002 + table.Publish(outTable); + WriteResult(outResult, table.TotalRows, skipped.Value); + return NativeStatus.Ok; + } + catch (FixedWidthParseException ex) + { + NativeApi.SetParseError(ex, layout.IndexOf(ex.ColumnName)); + return NativeStatus.ParseError; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + private static int CheckResult(NativeReadResultRaw* result) + { + if (result is not null && result->StructSize != sizeof(NativeReadResultRaw)) + { + NativeApi.SetLastError($"fw_read_result struct_size is {result->StructSize}; expected {sizeof(NativeReadResultRaw)}."); + return NativeStatus.InvalidArgument; + } + return NativeStatus.Ok; + } + + private static void WriteResult(NativeReadResultRaw* result, long rows, long skipped) + { + if (result is null) + { + return; + } + result->LinesRead = rows; + result->LinesSkipped = skipped; + } + + /// The offset just past the first physical lines, and how many lines that was. + internal static long HeaderEnd(IRandomAccessSource source, long count, out long skipped) + { + skipped = 0; + if (count == 0) + { + return 0; + } + byte[] buffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + long offset = 0; + int read; + while ((read = source.Read(buffer, offset)) > 0) + { + var block = buffer.AsSpan(0, read); + for (int at = block.IndexOf((byte)'\n'); at >= 0; at = NextBreak(block, at)) + { + if (++skipped == count) + { + return offset + at + 1; + } + } + offset += read; + } + skipped += offset > 0 && LastByte(source, offset) != (byte)'\n' ? 1 : 0; + return offset; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static int NextBreak(ReadOnlySpan block, int at) + { + int next = block[(at + 1)..].IndexOf((byte)'\n'); + return next < 0 ? -1 : at + 1 + next; + } + + private static byte LastByte(IRandomAccessSource source, long length) + { + Span last = stackalloc byte[1]; + source.Read(last, length - 1); + return last[0]; + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/ReadSettings.cs b/src/FixedWidthParser.Native/Reading/ReadSettings.cs new file mode 100644 index 0000000..090f448 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ReadSettings.cs @@ -0,0 +1,85 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text; + +namespace FixedWidthParser.Native.Reading +{ + /// A validated fw_read_options, turned into the core's reader options. + internal sealed class ReadSettings + { + private const int OptDefault = 0; + private const int OptTrue = 2; + + private ReadSettings(NativeReadOptionsRaw raw, Encoding? encoding) + { + HeaderLines = raw.HeaderLines; + SkipInvalid = raw.SkipInvalid != 0; + DegreeOfParallelism = raw.DegreeOfParallelism; + MaxLineLength = raw.MaxLineLength == 0 ? 1024 * 1024 : raw.MaxLineLength; + ByteOrderMark = raw.SkipByteOrderMark == OptDefault ? null : raw.SkipByteOrderMark == OptTrue; + Encoding = encoding; + } + + internal long HeaderLines { get; } + internal bool SkipInvalid { get; } + internal int DegreeOfParallelism { get; } + internal int MaxLineLength { get; } + internal bool? ByteOrderMark { get; } + internal Encoding? Encoding { get; } + + internal static unsafe int TryCreate(NativeReadOptionsRaw* options, out ReadSettings? settings) + { + settings = null; + int status = NativeOptions.TryRead(options, "fw_read_options", out var raw); + if (status != NativeStatus.Ok) + { + return status; + } + if (raw.HeaderLines < 0 || raw.DegreeOfParallelism < 0 || raw.MaxLineLength < 0 || raw.SkipByteOrderMark is < 0 or > OptTrue) + { + NativeApi.SetLastError("fw_read_options: header_lines, degree_of_parallelism and max_line_length must not be negative, and skip_byte_order_mark must be an FW_OPT_* value."); + return NativeStatus.InvalidArgument; + } + if (!TryEncoding(raw.Encoding, out var encoding)) + { + NativeApi.SetLastError($"fw_read_options: encoding {raw.Encoding} is not a code page this library knows."); + return NativeStatus.InvalidArgument; + } + settings = new ReadSettings(raw, encoding); + return NativeStatus.Ok; + } + + /// The core's options; a skipped line is counted in . + internal FixedWidthReaderOptions ReaderOptions(StrongBox skipped, int degreeOfParallelism, int chunkSize) + { + return FixedWidthReaderOptions.Validate(new FixedWidthReaderOptions + { + FormatProvider = CultureInfo.InvariantCulture, + Encoding = Encoding, + MaxLineLength = MaxLineLength, + DegreeOfParallelism = degreeOfParallelism, + ChunkSize = chunkSize, + ByteOrderMark = ByteOrderMark, + OnSkip = SkipInvalid ? () => Interlocked.Increment(ref skipped.Value) : null, + }); + } + + internal static bool TryEncoding(int codePage, out Encoding? encoding) + { + encoding = null; + if (codePage == 0) + { + return true; + } + try + { + encoding = Encoding.GetEncoding(codePage); + return true; + } + catch (Exception ex) when (ex is ArgumentException or NotSupportedException) + { + return false; + } + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/ReaderApi.cs b/src/FixedWidthParser.Native/Reading/ReaderApi.cs new file mode 100644 index 0000000..c954ede --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/ReaderApi.cs @@ -0,0 +1,188 @@ +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// fw_reader_*: batches of a sequential read, behind a handle. + internal static unsafe class ReaderApi + { +#pragma warning disable CA2000, IDISP001 // The handle table owns a reader until fw_reader_close. + internal static int OpenFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + { + int status = CreateFile(path, layout, options, out var created); + reader = status == NativeStatus.Ok ? NativeHandleTable.Register(created!) : 0; + return status; + } + + /// Reads in place: the caller keeps it alive until the reader is closed. + internal static int OpenMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + { + int status = CreateMemory(data, length, layout, options, out var created); + reader = status == NativeStatus.Ok ? NativeHandleTable.Register(created!) : 0; + return status; + } + + /// + internal static int OpenStream(Stream source, NativeLayout layout, NativeReaderOptionsRaw* options, out nint reader) + { + int status = CreateStream(source, layout, options, out var created); + reader = status == NativeStatus.Ok ? NativeHandleTable.Register(created!) : 0; + return status; + } + +#pragma warning restore CA2000, IDISP001 + + internal static int CreateFile(ReadOnlySpan path, NativeLayout layout, NativeReaderOptionsRaw* options, out NativeTableReader? reader) + { + reader = null; + int status = TrySettings(options, out var settings, out long batchRows); + if (status != NativeStatus.Ok) + { + return status; + } + FileStream? stream = null; + try + { + stream = new FileStream(Encoding.UTF8.GetString(path), FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); + long dataStart = ReadApi.HeaderEnd(new FileSource(stream.SafeFileHandle), settings!.HeaderLines, out long headerLines); + stream.Position = dataStart; + reader = new NativeTableReader(stream, dataStart == 0, headerLines, layout, settings, batchRows); + return NativeStatus.Ok; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + stream?.Dispose(); + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + internal static int CreateMemory(byte* data, long length, NativeLayout layout, NativeReaderOptionsRaw* options, out NativeTableReader? reader) + { + reader = null; + int status = TrySettings(options, out var settings, out long batchRows); + if (status != NativeStatus.Ok) + { + return status; + } + long dataStart = ReadApi.HeaderEnd(new PointerSource(data, length), settings!.HeaderLines, out long headerLines); +#pragma warning disable CA2000 // The reader owns and disposes the stream. + var stream = new UnmanagedMemoryStream(data, length) { Position = dataStart }; +#pragma warning restore CA2000 + reader = new NativeTableReader(stream, dataStart == 0, headerLines, layout, settings, batchRows); + return NativeStatus.Ok; + } + + /// + /// Reads forward from where it is, skipping the header lines as it goes. Takes ownership: + /// is disposed, so its release runs, if this fails, or with the reader. + /// + internal static int CreateStream(Stream source, NativeLayout layout, NativeReaderOptionsRaw* options, out NativeTableReader? reader) + { + reader = null; + int status = TrySettings(options, out var settings, out long batchRows); + if (status != NativeStatus.Ok) + { +#pragma warning disable IDISP007 // CreateStream owns the source it is handed. + source.Dispose(); +#pragma warning restore IDISP007 + return status; + } + // Buffered so the header skip reads a byte at a time from memory, not from the caller's callback. +#pragma warning disable CA2000 // The reader owns and disposes the stream, and with it the source. + var stream = new BufferedStream(source, StreamBufferSize); +#pragma warning restore CA2000 + try + { + long headerLines = SkipLines(stream, settings!.HeaderLines); + reader = new NativeTableReader(stream, settings.HeaderLines == 0, headerLines, layout, settings, batchRows); + return NativeStatus.Ok; + } + catch (IOException ex) + { + stream.Dispose(); + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + private const int StreamBufferSize = 64 * 1024; + + /// Skips physical lines, as counts them, and returns how many there were. + private static long SkipLines(Stream stream, long count) + { + long skipped = 0; + bool partial = false; + while (skipped < count) + { + int b = stream.ReadByte(); + if (b < 0) + { + return skipped + (partial ? 1 : 0); + } + partial = b != '\n'; + skipped += partial ? 0 : 1; + } + return skipped; + } + + internal static int Next(nint handle, NativeTableRaw* table) + { + *table = default; + var reader = NativeHandleTable.Resolve(handle); + if (reader is null) + { + NativeApi.SetLastError("The reader handle is not a live fw_reader."); + return NativeStatus.InvalidHandle; + } + return NextBatch(reader, table); + } + + /// The next batch of , with every failure turned into a status and the last error. + internal static int NextBatch(NativeTableReader reader, NativeTableRaw* table) + { + try + { + return reader.Next(table); + } + catch (FixedWidthParseException ex) + { + NativeApi.SetParseError(ex, reader.Layout.IndexOf(ex.ColumnName)); + return NativeStatus.ParseError; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + internal static void Close(nint handle) + { + if (NativeHandleTable.TryUnregister(handle, out var reader)) + { + reader!.Dispose(); + } + } + + private static int TrySettings(NativeReaderOptionsRaw* options, out ReadSettings? settings, out long batchRows) + { + settings = null; + batchRows = 0; + int status = NativeOptions.TryRead(options, "fw_reader_options", out var raw); + if (status != NativeStatus.Ok) + { + return status; + } + if (raw.BatchRows is < 0 or > int.MaxValue || (raw.Read.StructSize != 0 && raw.Read.StructSize != sizeof(NativeReadOptionsRaw))) + { + NativeApi.SetLastError("fw_reader_options: batch_rows must be 0..2147483647, and read.struct_size must be 0 or sizeof(fw_read_options)."); + return NativeStatus.InvalidArgument; + } + batchRows = raw.BatchRows; + var read = raw.Read with { StructSize = sizeof(NativeReadOptionsRaw) }; + return ReadSettings.TryCreate(&read, out settings); + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/TableArena.cs b/src/FixedWidthParser.Native/Reading/TableArena.cs new file mode 100644 index 0000000..2cea9ed --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/TableArena.cs @@ -0,0 +1,131 @@ +using System.Buffers; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// + /// The final buffers of a whole-source read, lent a slice per partition so the partitions parse straight into them + /// and publishing has nothing to copy. Every line is taken to be as long as the first, so a partition's first row + /// is its offset over that length. A partition with fewer rows leaves a gap that publishing closes in place; one + /// with more copies its column out of the slice, and that column is copied as before. A STRING column is lent + /// room for its whole width on every row, for its bytes, and its offsets: a slice's index 0 is the previous slice's + /// last offset, so only the first slice writes the column's leading 0. + /// + internal sealed unsafe class TableArena : IDisposable + { + private const int ProbeBytes = 64 * 1024; + + private readonly long _dataStart; + private readonly long _stride; + private readonly int _rows; + private readonly int[] _sizes; + private readonly GrowableNativeBuffer?[] _blocks; + private readonly GrowableNativeBuffer?[] _offsetBlocks; + + private TableArena(long dataStart, long stride, int rows, NativeLayout layout, TableLimits limits) + { + _dataStart = dataStart; + _stride = stride; + _rows = rows; + _sizes = [.. layout.Columns.Select(c => BytesPerRow(c, rows, limits))]; + _blocks = new GrowableNativeBuffer?[_sizes.Length]; + for (int i = 0; i < _blocks.Length; i++) + { + if (_sizes[i] > 0) + { + var block = new GrowableNativeBuffer(); + block.Reserve((long)rows * _sizes[i]); + _blocks[i] = block; + } + } + _offsetBlocks = new GrowableNativeBuffer?[_sizes.Length]; + for (int i = 0; i < _offsetBlocks.Length; i++) + { + if (layout.Columns[i].Type == NativeType.String) + { + var offsets = new GrowableNativeBuffer(); + offsets.Reserve(rows + 1L); + _offsetBlocks[i] = offsets; + } + } + } + + /// + /// An arena sized from the length of the first line at ; null when there is no line + /// break in the first 64 KiB, or the rows would not fit one buffer or . + /// + internal static TableArena? TryCreate(IRandomAccessSource source, long dataStart, NativeLayout layout, TableLimits limits) + { + byte[] probe = ArrayPool.Shared.Rent(ProbeBytes); + try + { + int read = source.Read(probe.AsSpan(0, ProbeBytes), dataStart); + long stride = probe.AsSpan(0, read).IndexOf((byte)'\n') + 1L; + if (stride == 0) + { + return null; + } + long rows = (source.Length - dataStart + stride - 1) / stride; + bool fits = rows > 0 && rows <= limits.MaxRows && rows * sizeof(long) <= int.MaxValue; + return fits ? new TableArena(dataStart, stride, (int)rows, layout, limits) : null; + } + finally + { + ArrayPool.Shared.Return(probe); + } + } + + /// What the arena lends a column per row; 0 for a STRING column whose full width would break the text limit. + private static int BytesPerRow(NativeColumn column, int rows, TableLimits limits) + { + if (column.Type != NativeType.String) + { + return ColumnBuilder.ValueSize(column.Type); + } + return (long)rows * column.Length <= Math.Min(limits.MaxTextBytes, int.MaxValue) ? column.Length : 0; + } + + /// The row of the first line starting at or after , if every line is as long as the first. + internal int RowAt(long offset) + { + return (int)Math.Min(_rows, (offset - _dataStart + _stride - 1) / _stride); + } + + /// Where column 's value for goes; null for a column the arena does not hold. + internal void* Lend(int column, int row) + { + return _blocks[column] is { } block ? block.Pointer + ((long)row * _sizes[column]) : null; + } + + internal GrowableNativeBuffer? Block(int column) + { + return _blocks[column]; + } + + /// Where a STRING column's offset at index goes, the start of the row's text; null for other columns. + internal int* LendOffsets(int column, int row) + { + return _offsetBlocks[column] is { } block ? block.Pointer + row : null; + } + + internal GrowableNativeBuffer? OffsetsBlock(int column) + { + return _offsetBlocks[column]; + } + + /// Where a STRING column's offset at index + /// What one published table may hold: its row count and each string column's bytes must fit the int32 + /// lengths and offsets of fw_table and Arrow utf8. Tests lower them to reach the limits. + /// + [StructLayout(LayoutKind.Auto)] + internal readonly record struct TableLimits(long MaxTextBytes, long MaxRows) + { + internal static TableLimits Default => new(int.MaxValue, int.MaxValue); + + internal InvalidOperationException TooManyRows() + { + return new InvalidOperationException($"The table has more than {MaxRows} rows; read in batches (fw_reader_open_*)."); + } + } +} diff --git a/src/FixedWidthParser.Native/Reading/TableSink.cs b/src/FixedWidthParser.Native/Reading/TableSink.cs new file mode 100644 index 0000000..0c98497 --- /dev/null +++ b/src/FixedWidthParser.Native/Reading/TableSink.cs @@ -0,0 +1,193 @@ +using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Native.Reading +{ + /// One partition's columns: each accepted line appends one value to every . + internal struct TableSink : ILineSink, IDisposable + { + private readonly NativeLayout _layout; + private readonly ColumnBuilder[] _columns; + private readonly int _requiredLength; + private readonly TableLimits _limits; +#pragma warning disable SS066 // Shared by every partition; the read that made it disposes it. + private readonly TableArena? _arena; +#pragma warning restore SS066 + private int _failedColumn; + private List? _following; + + /// + /// Initializes a new instance of the struct. + /// + /// The final buffers to parse into, shared by every partition; null for buffers of its own. + /// Where the bytes this sink will see start, if known: fixed-width lines make them a row count. + /// Where those bytes end. + internal unsafe TableSink(NativeLayout layout, TableLimits limits, TableArena? arena = null, long start = 0, long end = 0) + { + _layout = layout; + _limits = limits; + _arena = arena; + int firstRow = arena?.RowAt(start) ?? 0; + long expectedRows = arena is null + ? Math.Min((end - start) / (layout.LineLength + 1), limits.MaxRows) + : arena.RowAt(end) - firstRow; + _columns = new ColumnBuilder[layout.Columns.Length]; + for (int i = 0; i < _columns.Length; i++) + { + _columns[i] = arena is null + ? new ColumnBuilder(layout.Columns[i], limits.MaxTextBytes, expectedRows) + : new ColumnBuilder(layout.Columns[i], limits.MaxTextBytes, expectedRows, arena.Lend(i, firstRow), arena.LendOffsets(i, firstRow), firstRow == 0); + } + _requiredLength = layout.Columns.Max(c => c.Start + c.Length); + _failedColumn = -1; + _following = null; + Rows = 0; + } + + /// The rows of this partition only. + internal int Rows { get; private set; } + + /// The rows of this partition and every partition merged into it. + internal readonly long TotalRows => Rows + (_following?.Sum(f => (long)f.Rows) ?? 0); + + public bool TryAdd(ReadOnlySpan line, in FixedWidthContext context) + { + if (line.Length < _requiredLength) + { + _failedColumn = -1; + return false; + } + if (Rows >= _limits.MaxRows) + { + throw _limits.TooManyRows(); + } + var columns = _layout.Columns; + for (int i = 0; i < _columns.Length; i++) + { + if (!_columns[i].TryAppend(line.Slice(columns[i].Start, columns[i].Length), in context)) + { + Reject(i); + return false; + } + } + Rows++; + return true; + } + + public readonly FixedWidthParseException Diagnose(ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + { + if (_failedColumn < 0) + { + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed: it is {line.Length} bytes long and the layout needs {_requiredLength}.", + lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + var column = _layout.Columns[_failedColumn]; + string raw = context.Encoding.GetString(line.Slice(column.Start, column.Length)); + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed: column '{column.Name}' (start {column.Start}, length {column.Length}) holds \"{raw}\".", + lineNumber, column.Name, column.Start, column.Length, raw); + } + + /// Adopts the following partition without copying; concatenates once. + public void Merge(TableSink following) + { + (_following ??= []).Add(following); + } + + /// This partition's columns, then every merged partition's, in file order. + internal readonly IEnumerable Chunks => _following is null ? [_columns] : [_columns, .. _following.Select(f => f._columns)]; + + /// Hands every column's buffers to ; the caller frees them with . + /// Fails before allocating when the merged partitions exceed the limits; frees what it built if anything else fails. + internal readonly unsafe void Publish(NativeTableRaw* table) + { + var chunks = Chunks.ToArray(); + CheckLimits(chunks); + var columns = (NativeColumnRaw*)NativeMemory.AllocZeroed((nuint)_columns.Length, (nuint)sizeof(NativeColumnRaw)); + table->Columns = columns; + table->ColumnCount = _columns.Length; + try + { + if (chunks.Length == 1 && _arena is null) + { + for (int i = 0; i < _columns.Length; i++) + { + chunks[0][i].Publish(columns + i); + } + } + else + { + ConcatInParallel(chunks, columns, _arena); + } + table->RowCount = TotalRows; + } + catch + { + NativeTable.Free(table); + throw; + } + } + + // A concatenated column is all fresh memory, and its page faults are most of the copy's cost: one + // column per thread spreads them out. + private static unsafe void ConcatInParallel(ColumnBuilder[][] chunks, NativeColumnRaw* columns, TableArena? arena) + { + nint target = (nint)columns; + try + { + Parallel.For(0, chunks[0].Length, i => + { + var parts = new ColumnBuilder[chunks.Length]; + for (int c = 0; c < chunks.Length; c++) + { + parts[c] = chunks[c][i]; + } + ColumnBuilder.Concat(parts, (NativeColumnRaw*)target + i, arena?.Block(i), arena?.OffsetsBlock(i)); + }); + } + catch (AggregateException ex) + { + ExceptionDispatchInfo.Throw(ex.InnerExceptions[0]); + } + } + + private readonly void CheckLimits(ColumnBuilder[][] chunks) + { + if (TotalRows > _limits.MaxRows) + { + throw _limits.TooManyRows(); + } + for (int i = 0; i < _columns.Length; i++) + { + long text = chunks.Sum(chunk => chunk[i].TextBytes); + if (text > _limits.MaxTextBytes) + { + throw ColumnBuilder.TooMuchText(_layout.Columns[i].Name, _limits.MaxTextBytes); + } + } + } + + public readonly void Dispose() + { + foreach (var chunk in Chunks) + { + foreach (var column in chunk) + { + column.Dispose(); + } + } + } + + private void Reject(int failed) + { + _failedColumn = failed; + foreach (ref readonly var column in _columns.AsSpan(0, failed)) + { + column.Truncate(Rows); + } + } + } +} diff --git a/src/FixedWidthParser.Native/Writing/CallbackWriteStream.cs b/src/FixedWidthParser.Native/Writing/CallbackWriteStream.cs new file mode 100644 index 0000000..49ce0ae --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/CallbackWriteStream.cs @@ -0,0 +1,98 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// fw_sink. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeSinkRaw + { + public int StructSize; + public void* UserData; + public delegate* unmanaged Write; + } + + /// The caller's fw_sink as a write-only : every byte handed to it, in order. + internal sealed unsafe class CallbackWriteStream : Stream + { + private readonly delegate* unmanaged _write; + private readonly void* _userData; + + private CallbackWriteStream(NativeSinkRaw raw) + { + _write = raw.Write; + _userData = raw.UserData; + } + + /// Wraps , or explains why it cannot. + internal static bool TryCreate(NativeSinkRaw* raw, [NotNullWhen(true)] out CallbackWriteStream? stream) + { + stream = null; + if (raw is null || raw->StructSize != sizeof(NativeSinkRaw) || raw->Write is null) + { + NativeApi.SetLastError("fw_write_stream needs a non-NULL sink whose struct_size is sizeof(fw_sink) and whose write is set."); + return false; + } + stream = new CallbackWriteStream(*raw); + return true; + } + + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Write(ReadOnlySpan buffer) + { + if (buffer.IsEmpty) + { + return; + } + SourceErrors.Clear(); + int status; + fixed (byte* source = buffer) + { + status = _write(_userData, source, buffer.Length); + } + if (status != 0) + { + throw SourceErrors.Failed("fw_sink.write", -1, buffer.Length); + } + } + + public override void Write(byte[] buffer, int offset, int count) + { + Write(buffer.AsSpan(offset, count)); + } + + public override void Flush() + { + // Every write has reached the sink when its callback returns. + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + } +} diff --git a/src/FixedWidthParser.Native/Writing/TableFormatter.cs b/src/FixedWidthParser.Native/Writing/TableFormatter.cs new file mode 100644 index 0000000..297664f --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/TableFormatter.cs @@ -0,0 +1,163 @@ +using System.Globalization; +using System.Text; +using FixedWidthParser.Infrastructure; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// + /// Formats the rows of a caller's fw_table through a layout, one line at a time into a slice the + /// caller owns. Read-only over the table, so several threads may format different rows at once. + /// + internal sealed unsafe class TableFormatter + { + private readonly NativeColumn[] _columns; + private readonly NativeColumnRaw* _data; + private readonly byte _filler; + private readonly FixedWidthContext _context; + + private TableFormatter(NativeLayout layout, NativeTableRaw* table, Encoding? encoding) + { + _columns = layout.Columns; + _data = table->Columns; + _filler = layout.Filler; + _context = new FixedWidthContext { FormatProvider = CultureInfo.InvariantCulture, Encoding = encoding! }; + Rows = table->RowCount; + } + + internal long Rows { get; } + + internal static int TryCreate(NativeTableRaw* table, NativeLayout layout, Encoding? encoding, out TableFormatter? formatter) + { + formatter = null; + string? error = Check(table, layout); + if (error is not null) + { + NativeApi.SetLastError(error); + return NativeStatus.InvalidArgument; + } + formatter = new TableFormatter(layout, table, encoding); + return NativeStatus.Ok; + } + + private static string? Check(NativeTableRaw* table, NativeLayout layout) + { + var columns = layout.Columns; + if (table->ColumnCount != columns.Length || table->RowCount < 0 || (table->Columns is null && table->ColumnCount > 0)) + { + return $"The table has {table->ColumnCount} columns and the layout {columns.Length}; they must match by position."; + } + for (int i = 0; i < columns.Length; i++) + { + var raw = table->Columns[i]; + var column = columns[i]; + if (raw.Type != column.Type || (column.Type == NativeType.Decimal && raw.Scale != column.Scale)) + { + return $"Column '{column.Name}' has type {raw.Type} (scale {raw.Scale}) in the table and {column.Type} (scale {column.Scale}) in the layout."; + } + if (raw.Length != table->RowCount || (raw.Values is null && raw.Length > 0) || (raw.Data is null && raw.DataLen > 0)) + { + return $"Column '{column.Name}' has {raw.Length} values for {table->RowCount} rows, or a buffer that does not match its length."; + } + } + return null; + } + + /// Writes row into , whose length is the layout's line length. + internal void FormatRow(long row, Span line) + { + line.Fill(_filler); + for (int i = 0; i < _columns.Length; i++) + { + var column = _columns[i]; + var slice = line.Slice(column.Start, column.Length); + var raw = _data + i; + if (raw->Validity is not null && (raw->Validity[row >> 3] & (1 << (int)(row & 7))) == 0) + { + slice.Fill((byte)column.Parse.TrimChar); + continue; + } + FormatValue(column, raw, row, slice); + } + } + + private void FormatValue(NativeColumn column, NativeColumnRaw* raw, long row, Span slice) + { + var options = column.FormatOptions; + switch (column.Type) + { + case NativeType.String: + var offsets = (int*)raw->Values; + FormatText(column, new ReadOnlySpan(raw->Data + offsets[row], offsets[row + 1] - offsets[row]), slice); + return; + case NativeType.Float64: + FormatFloat(column, ((double*)raw->Values)[row], slice); + return; + case NativeType.Date: + FixedWidthRuntime.FormatUtf8Value(DateOnly.FromDayNumber(NativeType.EpochDay + ((int*)raw->Values)[row]), slice, in _context, options, column.Name); + return; + case NativeType.Time: + FixedWidthRuntime.FormatUtf8Value(new TimeOnly(((long*)raw->Values)[row] * TimeSpan.TicksPerMicrosecond), slice, in _context, options, column.Name); + return; + case NativeType.Timestamp: + var ticks = DateTime.UnixEpoch.Ticks + (((long*)raw->Values)[row] * TimeSpan.TicksPerMicrosecond); + FixedWidthRuntime.FormatUtf8Value(new DateTime(ticks, DateTimeKind.Unspecified), slice, in _context, options, column.Name); + return; + default: + FixedWidthRuntime.FormatUtf8Value(((long*)raw->Values)[row], slice, in _context, options, column.Name); + return; + } + } + + private void FormatFloat(NativeColumn column, double value, Span slice) + { + if (column.Scale == 0) + { + FixedWidthRuntime.FormatUtf8Value(value, slice, in _context, column.FormatOptions, column.Name); + return; + } + double scaled = Math.Round(value * NativeType.Pow10[column.Scale]); + if (!(Math.Abs(scaled) < 9e18)) + { + throw new InvalidOperationException($"Value {value} of column \"{column.Name}\" does not fit a scaled 64-bit integer."); + } + FixedWidthRuntime.FormatUtf8Value((long)scaled, slice, in _context, column.FormatOptions, column.Name); + } + + private void FormatText(NativeColumn column, ReadOnlySpan utf8, Span slice) + { + var options = column.FormatOptions; + if (!_context.IsUtf8) + { + FixedWidthRuntime.FormatString(Encoding.UTF8.GetString(utf8), slice, in _context, options, column.Name); + return; + } + if (utf8.Length > slice.Length && options.Overflow == OverflowBehavior.Truncate) + { + utf8 = WholeCharacters(utf8, slice.Length, options.Alignment == Alignment.Right); + } + options.WriteInto(utf8, slice, column.Name); + } + + /// At most bytes of , from the aligned side, never splitting a character. + private static ReadOnlySpan WholeCharacters(ReadOnlySpan utf8, int width, bool keepEnd) + { + if (keepEnd) + { + int start = utf8.Length - width; + while (start < utf8.Length && (utf8[start] & 0xC0) == 0x80) + { + start++; + } + return utf8[start..]; + } + int end = width; + while (end > 0 && (utf8[end] & 0xC0) == 0x80) + { + end--; + } + return utf8[..end]; + } + } +} diff --git a/src/FixedWidthParser.Native/Writing/WriteApi.cs b/src/FixedWidthParser.Native/Writing/WriteApi.cs new file mode 100644 index 0000000..351b13c --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/WriteApi.cs @@ -0,0 +1,206 @@ +using System.Buffers; +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// + /// fw_write_file and fw_write_memory. Every line has the same length, so the output size and + /// each batch's offset are known up front: batches format in parallel straight into place, and the bytes + /// equal a serial write. + /// + internal static unsafe class WriteApi + { + internal static int WriteMemory(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options, NativeBufferRaw* outBuffer) + { + *outBuffer = default; + int status = Prepare(table, layout, options, out var formatter, out var settings, out long lineBytes); + if (status != NativeStatus.Ok) + { + return status; + } + long total = formatter!.Rows * lineBytes; + var data = (byte*)Marshal.AllocHGlobal((nint)Math.Max(1, total)); + status = Run(formatter, settings!, lineBytes, (first, count) => + { + FormatBatch(formatter, settings!, layout, first, new Span(data + (first * lineBytes), (int)(count * lineBytes))); + }); + if (status != NativeStatus.Ok) + { + Marshal.FreeHGlobal((IntPtr)data); + return status; + } + outBuffer->Data = data; + outBuffer->Length = total; + return NativeStatus.Ok; + } + + internal static int WriteFile(ReadOnlySpan path, NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options) + { + int status = Prepare(table, layout, options, out var formatter, out var settings, out long lineBytes); + if (status != NativeStatus.Ok) + { + return status; + } + try + { + long total = formatter!.Rows * lineBytes; + // Asynchronous, though every write waits: Windows runs one synchronous I/O at a time on a handle, + // which would serialize the parallel batches. + using var handle = File.OpenHandle(Encoding.UTF8.GetString(path), FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.Asynchronous, total); + RandomAccess.SetLength(handle, total); + return Run(formatter, settings!, lineBytes, (first, count) => WriteBatch(handle, formatter, settings!, layout, first, count, lineBytes)); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + + /// + /// Writes to in order, from the calling thread. A window of one batch per thread formats + /// in parallel, then goes to the sink, so memory stays at a window of batches whatever the table's size. + /// + // ponytail: the sink waits while a window formats, and the window while the sink writes; overlapping them + // with two windows would hide the slower of the two. + internal static int WriteStream(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options, Stream sink) + { + int status = Prepare(table, layout, options, out var formatter, out var settings, out long lineBytes); + if (status != NativeStatus.Ok) + { + return status; + } + long rows = formatter!.Rows; + long batchRows = Math.Max(1, Math.Min(settings!.BatchRows, int.MaxValue / lineBytes)); + long batches = (rows + batchRows - 1) / batchRows; + int window = (int)Math.Min(settings.DegreeOfParallelism, Math.Max(1, batches)); + var buffers = new byte[window][]; + try + { + foreach (ref byte[] buffer in buffers.AsSpan()) + { + buffer = ArrayPool.Shared.Rent((int)(Math.Min(batchRows, rows) * lineBytes)); + } + for (long first = 0; first < batches; first += window) + { + int count = (int)Math.Min(window, batches - first); + Parallel.For(0, count, new ParallelOptions { MaxDegreeOfParallelism = window }, i => + { + long row = (first + i) * batchRows; + FormatBatch(formatter, settings, layout, row, buffers[i].AsSpan(0, (int)(Math.Min(batchRows, rows - row) * lineBytes))); + }); + for (int i = 0; i < count; i++) + { + long row = (first + i) * batchRows; + sink.Write(buffers[i], 0, (int)(Math.Min(batchRows, rows - row) * lineBytes)); + } + } + sink.Flush(); + return NativeStatus.Ok; + } + catch (AggregateException ex) + { + NativeApi.SetLastError(ex.InnerExceptions[0].Message); + return NativeStatus.Error; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + finally + { + foreach (byte[]? buffer in buffers.Where(b => b is not null)) + { + ArrayPool.Shared.Return(buffer); + } + } + } + + internal static void FreeBuffer(NativeBufferRaw* buffer) + { + if (buffer is null) + { + return; + } + Marshal.FreeHGlobal((IntPtr)buffer->Data); + *buffer = default; + } + + private static int Prepare(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw* options, + out TableFormatter? formatter, out WriteSettings? settings, out long lineBytes) + { + formatter = null; + lineBytes = 0; + int status = WriteSettings.TryCreate(options, out settings); + if (status != NativeStatus.Ok) + { + return status; + } + lineBytes = layout.LineLength + settings!.NewLine.Length; + return TableFormatter.TryCreate(table, layout, settings.Encoding, out formatter); + } + + private static void FormatBatch(TableFormatter formatter, WriteSettings settings, NativeLayout layout, long first, Span destination) + { + int lineLength = layout.LineLength; + int lineBytes = lineLength + settings.NewLine.Length; + for (int at = 0; at < destination.Length; at += lineBytes) + { + formatter.FormatRow(first + (at / lineBytes), destination.Slice(at, lineLength)); + settings.NewLine.CopyTo(destination.Slice(at + lineLength)); + } + } + + private static void WriteBatch(Microsoft.Win32.SafeHandles.SafeFileHandle handle, TableFormatter formatter, WriteSettings settings, NativeLayout layout, + long first, long count, long lineBytes) + { + int length = (int)(count * lineBytes); + byte[] buffer = ArrayPool.Shared.Rent(length); + try + { + FormatBatch(formatter, settings, layout, first, buffer.AsSpan(0, length)); + RandomAccess.Write(handle, buffer.AsSpan(0, length), first * lineBytes); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// Runs over every batch of rows, in parallel, and turns a failure into a status. + private static int Run(TableFormatter formatter, WriteSettings settings, long lineBytes, Action batch) + { + long rows = formatter.Rows; + long batchRows = Math.Max(1, Math.Min(settings.BatchRows, int.MaxValue / lineBytes)); + long batches = (rows + batchRows - 1) / batchRows; + try + { + if (settings.DegreeOfParallelism == 1) + { + for (long b = 0; b < batches; b++) + { + batch(b * batchRows, Math.Min(batchRows, rows - (b * batchRows))); + } + return NativeStatus.Ok; + } + var parallel = new ParallelOptions { MaxDegreeOfParallelism = settings.DegreeOfParallelism }; + Parallel.For(0, batches, parallel, b => batch(b * batchRows, Math.Min(batchRows, rows - (b * batchRows)))); + return NativeStatus.Ok; + } + catch (AggregateException ex) + { + NativeApi.SetLastError(ex.InnerExceptions[0].Message); + return NativeStatus.Error; + } + catch (Exception ex) + { + NativeApi.SetLastError(ex.Message); + return NativeStatus.Error; + } + } + } +} diff --git a/src/FixedWidthParser.Native/Writing/WriteSettings.cs b/src/FixedWidthParser.Native/Writing/WriteSettings.cs new file mode 100644 index 0000000..007c59c --- /dev/null +++ b/src/FixedWidthParser.Native/Writing/WriteSettings.cs @@ -0,0 +1,66 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Native.Writing +{ + /// fw_write_options. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeWriteOptionsRaw + { + public int StructSize; + public int Encoding; + public byte* NewLine; + public int NewLineLen; + public int DegreeOfParallelism; + public long BatchRows; + } + + /// fw_buffer: bytes the library allocated, released with fw_free_buffer. + [StructLayout(LayoutKind.Sequential)] + internal unsafe struct NativeBufferRaw + { + public byte* Data; + public long Length; + } + + /// A validated fw_write_options. + internal sealed class WriteSettings + { + private const long DefaultBatchRows = 8192; + + private WriteSettings(Encoding? encoding, byte[] newLine, int degreeOfParallelism, long batchRows) + { + Encoding = encoding; + NewLine = newLine; + DegreeOfParallelism = degreeOfParallelism == 0 ? Environment.ProcessorCount : degreeOfParallelism; + BatchRows = batchRows == 0 ? DefaultBatchRows : batchRows; + } + + internal Encoding? Encoding { get; } + internal byte[] NewLine { get; } + internal int DegreeOfParallelism { get; } + internal long BatchRows { get; } + + internal static unsafe int TryCreate(NativeWriteOptionsRaw* options, out WriteSettings? settings) + { + settings = null; + int status = NativeOptions.TryRead(options, "fw_write_options", out var raw); + if (status != NativeStatus.Ok) + { + return status; + } + if (raw.DegreeOfParallelism < 0 || raw.BatchRows < 0 || raw.NewLineLen < 0 || (raw.NewLine is null && raw.NewLineLen > 0)) + { + return NativeApi.InvalidArgument("fw_write_options: degree_of_parallelism, batch_rows and newline_len must not be negative, and a newline_len needs a newline."); + } + if (!ReadSettings.TryEncoding(raw.Encoding, out var encoding)) + { + return NativeApi.InvalidArgument($"fw_write_options: encoding {raw.Encoding} is not a code page this library knows."); + } + byte[] newLine = raw.NewLine is null ? [(byte)'\n'] : new ReadOnlySpan(raw.NewLine, raw.NewLineLen).ToArray(); + settings = new WriteSettings(encoding, newLine, raw.DegreeOfParallelism, raw.BatchRows); + return NativeStatus.Ok; + } + } +} diff --git a/src/FixedWidthParser.Native/include/fixedwidth.def b/src/FixedWidthParser.Native/include/fixedwidth.def new file mode 100644 index 0000000..6ca0717 --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth.def @@ -0,0 +1,24 @@ +EXPORTS + fw_abi_version + fw_free_buffer + fw_free_table + fw_last_error + fw_last_parse_error + fw_layout_create + fw_layout_free + fw_read_arrow_file + fw_read_arrow_memory + fw_read_arrow_stream_file + fw_read_arrow_stream_memory + fw_read_arrow_stream_stream + fw_read_file + fw_read_memory + fw_reader_close + fw_reader_next + fw_reader_open_file + fw_reader_open_memory + fw_reader_open_stream + fw_set_source_error + fw_write_file + fw_write_memory + fw_write_stream diff --git a/src/FixedWidthParser.Native/include/fixedwidth.h b/src/FixedWidthParser.Native/include/fixedwidth.h new file mode 100644 index 0000000..6f3627f --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth.h @@ -0,0 +1,245 @@ +/* FixedWidthParser C ABI: read and write fixed-width files into a columnar table. + * + * Conventions: + * - Every function returns an int32_t status: FW_OK (0) or a negative FW_* code. On failure, + * fw_last_error() gives the calling thread's message; after FW_PARSE_ERROR, fw_last_parse_error() + * also gives the 1-based line number and the layout column. + * - Strings crossing the ABI are UTF-8 pointer + length, never NUL-terminated. + * - Every options struct starts with int32_t struct_size, which the caller sets to sizeof(the struct). + * A NULL options pointer, or a zeroed struct with struct_size set, means every default. Any other + * struct_size is refused with FW_INVALID_ARGUMENT. New fields only ever arrive at the end. + * - Handles (fw_layout, fw_reader) are opaque. A freed handle stays invalid (FW_INVALID_HANDLE); it + * is never reissued. + * - Threads: a layout is immutable and may be used by several threads at once. A reader is used by + * one thread at a time. Reads and writes may run concurrently on different tables. + * - Column positions and lengths are bytes. Text is decoded with the read encoding and handed out as + * UTF-8; numbers and dates use the invariant culture. + * + * Call fw_abi_version() first and refuse to continue when it differs from FW_ABI_VERSION. */ +#ifndef FIXEDWIDTH_H +#define FIXEDWIDTH_H + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define FW_ABI_VERSION 1 + +#define FW_OK 0 +#define FW_EOF -1 +#define FW_BUFFER_TOO_SMALL -2 +#define FW_INVALID_HANDLE -3 +#define FW_INVALID_ARGUMENT -4 +#define FW_ERROR -5 +#define FW_PARSE_ERROR -6 + +/* Column types and their fw_column values. */ +#define FW_T_STRING 0 /* int32 offsets (length + 1) into data, valid UTF-8: bytes invalid in the read + encoding become U+FFFD */ +#define FW_T_INT64 1 /* int64 */ +#define FW_T_FLOAT64 2 /* double */ +#define FW_T_DECIMAL 3 /* int64 unscaled value: 10^-scale per unit; at most 18 digits */ +#define FW_T_DATE 4 /* int32 days since 1970-01-01 */ +#define FW_T_TIME 5 /* int64 microseconds since midnight */ +#define FW_T_TIMESTAMP 6 /* int64 microseconds since 1970-01-01T00:00:00Z: a zone or offset in the text is + applied, text without one is taken as UTC */ + +#define FW_TRIM_BOTH 0 +#define FW_TRIM_TRAILING 1 +#define FW_TRIM_LEADING 2 + +#define FW_ALIGN_DEFAULT 0 /* left for STRING, right for everything else */ +#define FW_ALIGN_LEFT 1 +#define FW_ALIGN_RIGHT 2 + +#define FW_OVERFLOW_THROW 0 /* a value wider than its column fails the write */ +#define FW_OVERFLOW_TRUNCATE 1 /* STRING only: keep the aligned side, whole characters */ + +#define FW_OPT_DEFAULT 0 +#define FW_OPT_FALSE 1 +#define FW_OPT_TRUE 2 + +int32_t fw_abi_version(void); + +/* Copies the calling thread's last error message into buffer (not NUL-terminated) and its length into + * *out_len. FW_BUFFER_TOO_SMALL when capacity is short; *out_len is then the length needed. */ +int32_t fw_last_error(uint8_t* buffer, int32_t capacity, int32_t* out_len); + +typedef struct fw_parse_error { + int32_t struct_size; + int64_t line_number; /* 1-based, counting every line of the source as a sequential read does */ + int32_t column; /* index in the layout; -1 when the line is shorter than the layout */ +} fw_parse_error; + +/* The parts of the calling thread's last FW_PARSE_ERROR. */ +int32_t fw_last_parse_error(fw_parse_error* out_error); + +/* ---- Layouts ---------------------------------------------------------------------------------- */ + +typedef struct fw_layout fw_layout; + +typedef struct fw_column_spec { + int32_t struct_size; /* sizeof(fw_column_spec); the same in every element */ + const uint8_t* name; /* UTF-8, required */ + int32_t name_len; + int32_t start; /* zero-based byte offset */ + int32_t length; /* bytes, at least 1 */ + int32_t type; /* FW_T_* */ + int32_t nullable; /* a blank column (only trim_char and spaces) reads as null */ + int32_t scale; /* 0..18: FLOAT64 divides digits-only text by 10^scale; DECIMAL records it */ + const uint8_t* format; /* exact date/time format, or a number format on write; NULL for none */ + int32_t format_len; + int32_t trim_char; /* ASCII; 0 = space. A null is written as this character. */ + int32_t trim_mode; /* FW_TRIM_* */ + int32_t alignment; /* FW_ALIGN_*, on write */ + int32_t padding; /* ASCII; 0 = space, on write. '0' keeps a sign leftmost. */ + int32_t overflow; /* FW_OVERFLOW_*, on write */ + int32_t dictionary; /* 1: a STRING column reaches Arrow dictionary-encoded, as int32 indices into its + * distinct values (pandas category, polars Categorical); fw_table keeps strings */ +} fw_column_spec; + +typedef struct fw_layout_options { + int32_t struct_size; + int32_t line_length; /* 0 = the end of the last column; longer pads lines with filler on write */ + int32_t filler; /* ASCII written in gaps between columns; 0 = space */ +} fw_layout_options; + +/* Validates the columns (positions, overlaps, types, scale, formats, ASCII options) and creates an + * immutable layout. The specs and their strings may be freed after the call. Free with fw_layout_free; + * readers and Arrow streams opened from it keep working after that. */ +int32_t fw_layout_create(const fw_column_spec* columns, int32_t column_count, + const fw_layout_options* options, fw_layout** out_layout); +void fw_layout_free(fw_layout* layout); + +/* ---- Reading ---------------------------------------------------------------------------------- */ + +typedef struct fw_read_options { + int32_t struct_size; + int64_t header_lines; /* lines skipped first, never parsed; counted by line breaks */ + int32_t skip_invalid; /* a line that fails to parse is skipped instead of an error */ + int32_t degree_of_parallelism; /* 0 = one thread per processor, 1 = sequential */ + int32_t encoding; /* code page of the text; 0 = UTF-8, 28591 = Latin-1, 1252, ... */ + int32_t max_line_length; /* bytes; 0 = 1 MiB. A longer line fails with FW_ERROR. */ + int32_t skip_byte_order_mark; /* FW_OPT_*; by default only a UTF-8 read skips a leading BOM */ +} fw_read_options; + +typedef struct fw_column { + int32_t type; /* FW_T_* */ + int32_t scale; + int64_t length; /* rows */ + const void* values; /* see FW_T_*; may be NULL when length is 0 */ + const uint8_t* validity; /* LSB-first bitmap, 1 = valid; NULL when nothing is null */ + const uint8_t* data; /* STRING bytes; NULL otherwise or when empty */ + int64_t data_len; +} fw_column; + +typedef struct fw_table { + int32_t column_count; + int64_t row_count; + fw_column* columns; /* in layout order */ +} fw_table; + +typedef struct fw_read_result { + int32_t struct_size; + int64_t lines_read; /* rows in the table */ + int64_t lines_skipped; /* header lines plus invalid lines skipped; empty lines are not counted */ +} fw_read_result; + +/* Reads a whole file or buffer into *out_table, which the caller frees with fw_free_table. out_result + * may be NULL. A read with degree_of_parallelism other than 1 splits the source into byte ranges and + * gives the same table and error line numbers as a sequential read. fw_read_memory does not keep + * data past the call. A string column holding more than 2 GiB of text fails: read in batches. */ +int32_t fw_read_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_read_options* options, fw_table* out_table, fw_read_result* out_result); +int32_t fw_read_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_read_options* options, fw_table* out_table, fw_read_result* out_result); + +/* Frees a table the library published and zeroes it. Safe on a zeroed table. */ +void fw_free_table(fw_table* table); + +typedef struct fw_reader fw_reader; + +typedef struct fw_reader_options { + int32_t struct_size; + fw_read_options read; /* read.struct_size 0 means its defaults; degree_of_parallelism is ignored */ + int64_t batch_rows; /* rows per table; 0 = 65,536 */ +} fw_reader_options; + +/* A sequential reader handing out tables of at most batch_rows rows. fw_reader_open_memory reads data + * in place: keep it alive until fw_reader_close. */ +int32_t fw_reader_open_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_reader_options* options, fw_reader** out_reader); +int32_t fw_reader_open_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_reader_options* options, fw_reader** out_reader); + +/* A forward-only source the library pulls from, for a pipe, a socket, a decompressor or a language's own stream + * objects. read copies up to len bytes into buf and returns how many, 0 at the end, or -1 on failure, after + * fw_set_source_error to say why; it is called from the thread reading. release, which may be NULL, is called + * once the library is done with the stream: when the reader is closed or the Arrow stream released, or right + * away if opening fails after the stream was accepted. A stream rejected as an argument is not released. */ +typedef struct fw_stream { + int32_t struct_size; + void* user_data; + int64_t (*read)(void* user_data, uint8_t* buf, int64_t len); + void (*release)(void* user_data); +} fw_stream; + +/* Reads the stream from where it stands, skipping read.header_lines first; it never seeks. */ +int32_t fw_reader_open_stream(const fw_stream* stream, const fw_layout* layout, + const fw_reader_options* options, fw_reader** out_reader); + +/* Called by a read callback before it returns -1: the message becomes the failure's fw_last_error. */ +void fw_set_source_error(const uint8_t* message, int32_t len); + +/* FW_OK with a table the caller frees with fw_free_table, FW_EOF at the end. After a failure the + * reader is finished: later calls return FW_EOF. */ +int32_t fw_reader_next(fw_reader* reader, fw_table* out_table); +void fw_reader_close(fw_reader* reader); + +/* ---- Writing ---------------------------------------------------------------------------------- */ + +typedef struct fw_write_options { + int32_t struct_size; + int32_t encoding; /* code page for text; 0 = UTF-8 */ + const uint8_t* newline; /* NULL = "\n" */ + int32_t newline_len; + int32_t degree_of_parallelism; /* 0 = one thread per processor, 1 = serial */ + int64_t batch_rows; /* lines formatted per task; 0 = 8,192 */ +} fw_write_options; + +typedef struct fw_buffer { + uint8_t* data; + int64_t length; +} fw_buffer; + +/* Writes every row of table through layout. Columns match the layout by position and type (and scale + * for DECIMAL), else FW_INVALID_ARGUMENT. A null writes as its column filled with trim_char; a date or + * time without a format writes as ISO 8601 to the second. The output is byte-identical whatever + * degree_of_parallelism is. The table is read from several threads and must not change during the + * call. fw_write_memory's buffer is freed with fw_free_buffer. */ +int32_t fw_write_file(const uint8_t* path, int32_t path_len, const fw_table* table, + const fw_layout* layout, const fw_write_options* options); +int32_t fw_write_memory(const fw_table* table, const fw_layout* layout, + const fw_write_options* options, fw_buffer* out_buffer); +void fw_free_buffer(fw_buffer* buffer); + +/* A destination the library pushes the lines to, in order: a pipe, a socket, a compressor, a language's own + * stream objects. write consumes all len bytes and returns 0, or -1 after fw_set_source_error to say why. It is + * called from the thread that called fw_write_stream, a batch per call, while batches format in parallel. */ +typedef struct fw_sink { + int32_t struct_size; + void* user_data; + int32_t (*write)(void* user_data, const uint8_t* buf, int64_t len); +} fw_sink; + +/* The same lines as fw_write_file, to a sink; memory stays at one batch per thread whatever the table's size. */ +int32_t fw_write_stream(const fw_table* table, const fw_layout* layout, const fw_write_options* options, + const fw_sink* sink); + +#ifdef __cplusplus +} +#endif + +#endif /* FIXEDWIDTH_H */ diff --git a/src/FixedWidthParser.Native/include/fixedwidth.hpp b/src/FixedWidthParser.Native/include/fixedwidth.hpp new file mode 100644 index 0000000..79d2cf6 --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth.hpp @@ -0,0 +1,1037 @@ +// Header-only C++23 wrapper over fixedwidth.h: RAII handles, options structs with defaults, +// std::expected results, and structs mapped to columns (FixedWidthMapper). Link against the native +// library (fw::fixedwidth in cpp/CMakeLists.txt). +#pragma once + +#include "fixedwidth.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace fw +{ + enum class Type : int32_t + { + String = FW_T_STRING, + Int64 = FW_T_INT64, + Float64 = FW_T_FLOAT64, + Decimal = FW_T_DECIMAL, + Date = FW_T_DATE, + Time = FW_T_TIME, + Timestamp = FW_T_TIMESTAMP, + }; + + enum class Trim : int32_t { Both = FW_TRIM_BOTH, Trailing = FW_TRIM_TRAILING, Leading = FW_TRIM_LEADING }; + enum class Align : int32_t { Default = FW_ALIGN_DEFAULT, Left = FW_ALIGN_LEFT, Right = FW_ALIGN_RIGHT }; + enum class Overflow : int32_t { Throw = FW_OVERFLOW_THROW, Truncate = FW_OVERFLOW_TRUNCATE }; + + /// A failed call: its FW_* status and the library's message; line and column after FW_PARSE_ERROR. + struct Error + { + int32_t status = FW_ERROR; + std::string message; + int64_t line_number = 0; + int32_t column = -1; + }; + + template + using Result = std::expected; + + namespace detail + { + inline Error last_error(int32_t status) + { + Error error; + error.status = status; + int32_t length = 0; + std::string buffer(1024, '\0'); + if (fw_last_error(reinterpret_cast(buffer.data()), static_cast(buffer.size()), &length) == FW_BUFFER_TOO_SMALL) + { + buffer.resize(static_cast(length)); + fw_last_error(reinterpret_cast(buffer.data()), length, &length); + } + buffer.resize(static_cast(length)); + error.message = std::move(buffer); + if (status == FW_PARSE_ERROR) + { + fw_parse_error parse{.struct_size = sizeof(fw_parse_error), .line_number = 0, .column = -1}; + fw_last_parse_error(&parse); + error.line_number = parse.line_number; + error.column = parse.column; + } + return error; + } + + inline Result check(int32_t status) + { + if (status == FW_OK) + { + return {}; + } + return std::unexpected(last_error(status)); + } + + inline const uint8_t* bytes(std::string_view text) + { + return reinterpret_cast(text.data()); + } + + // fw_stream.read over a std::istream: a short count at the end, -1 once the stream is bad. An exception + // must not unwind into the library, so it fails the read too. + inline int64_t read_istream(void* user_data, uint8_t* buffer, int64_t length) noexcept + { + constexpr std::string_view failed = "the std::istream failed"; + try + { + auto& in = *static_cast(user_data); + in.read(reinterpret_cast(buffer), static_cast(length)); + if (!in.bad()) + { + return static_cast(in.gcount()); + } + } + catch (...) + { + } + fw_set_source_error(bytes(failed), static_cast(failed.size())); + return -1; + } + + /// An fw_stream reading `in`, which the caller keeps alive; nothing to release. + inline fw_stream istream_source(std::istream& in) + { + return {.struct_size = sizeof(fw_stream), .user_data = &in, .read = &read_istream, .release = nullptr}; + } + } + + /// One column: name, zero-based byte start, byte length and type, plus everything optional. + struct ColumnSpec + { + std::string name; + int32_t start = 0; + int32_t length = 0; + Type type = Type::String; + bool nullable = false; + int32_t scale = 0; + std::optional format; + char trim_char = 0; + Trim trim_mode = Trim::Both; + Align alignment = Align::Default; + char padding = 0; + Overflow overflow = Overflow::Throw; + /// A String column reaches Arrow dictionary-encoded (int32 indices into its distinct values); fw::Table keeps strings. + bool dictionary = false; + }; + + struct LayoutOptions + { + int32_t line_length = 0; + char filler = 0; + }; + + /// An immutable, validated layout; move-only, shareable across threads by reference. + class Layout + { + public: + static Result create(std::span columns, const LayoutOptions& options = {}) + { + std::vector specs; + specs.reserve(columns.size()); + for (const ColumnSpec& column : columns) + { + specs.push_back(fw_column_spec{ + .struct_size = sizeof(fw_column_spec), + .name = detail::bytes(column.name), + .name_len = static_cast(column.name.size()), + .start = column.start, + .length = column.length, + .type = static_cast(column.type), + .nullable = column.nullable ? 1 : 0, + .scale = column.scale, + .format = column.format ? detail::bytes(*column.format) : nullptr, + .format_len = column.format ? static_cast(column.format->size()) : 0, + .trim_char = static_cast(column.trim_char), + .trim_mode = static_cast(column.trim_mode), + .alignment = static_cast(column.alignment), + .padding = static_cast(column.padding), + .overflow = static_cast(column.overflow), + .dictionary = column.dictionary ? 1 : 0, + }); + } + const fw_layout_options raw{.struct_size = sizeof(fw_layout_options), .line_length = options.line_length, + .filler = static_cast(options.filler)}; + fw_layout* handle = nullptr; + if (auto ok = detail::check(fw_layout_create(specs.data(), static_cast(specs.size()), &raw, &handle)); !ok) + { + return std::unexpected(ok.error()); + } + return Layout(handle); + } + + Layout(Layout&& other) noexcept : handle_(std::exchange(other.handle_, nullptr)) {} + Layout& operator=(Layout&& other) noexcept + { + std::swap(handle_, other.handle_); + return *this; + } + Layout(const Layout&) = delete; + Layout& operator=(const Layout&) = delete; + ~Layout() + { + if (handle_ != nullptr) + { + fw_layout_free(handle_); + } + } + + const fw_layout* get() const noexcept { return handle_; } + + private: + explicit Layout(fw_layout* handle) : handle_(handle) {} + fw_layout* handle_; + }; + + struct ReadOptions + { + int64_t header_lines = 0; + bool skip_invalid = false; + int32_t degree_of_parallelism = 0; + int32_t encoding = 0; + int32_t max_line_length = 0; + std::optional skip_byte_order_mark; + + fw_read_options to_c() const + { + return { + .struct_size = sizeof(fw_read_options), + .header_lines = header_lines, + .skip_invalid = skip_invalid ? 1 : 0, + .degree_of_parallelism = degree_of_parallelism, + .encoding = encoding, + .max_line_length = max_line_length, + .skip_byte_order_mark = !skip_byte_order_mark ? FW_OPT_DEFAULT : (*skip_byte_order_mark ? FW_OPT_TRUE : FW_OPT_FALSE), + }; + } + }; + + struct ReaderOptions + { + ReadOptions read; + int64_t batch_rows = 0; + + fw_reader_options to_c() const + { + return {.struct_size = sizeof(fw_reader_options), .read = read.to_c(), .batch_rows = batch_rows}; + } + }; + + struct WriteOptions + { + int32_t encoding = 0; + std::string newline = "\n"; + int32_t degree_of_parallelism = 0; + int64_t batch_rows = 0; + }; + + /// A table the library published; move-only, freed on destruction. + class Table + { + public: + explicit Table(fw_table table, fw_read_result result = {}) noexcept : table_(table), result_(result) {} + Table(Table&& other) noexcept : table_(std::exchange(other.table_, {})), result_(other.result_) {} + Table& operator=(Table&& other) noexcept + { + std::swap(table_, other.table_); + std::swap(result_, other.result_); + return *this; + } + Table(const Table&) = delete; + Table& operator=(const Table&) = delete; + ~Table() { fw_free_table(&table_); } + + int64_t row_count() const noexcept { return table_.row_count; } + int32_t column_count() const noexcept { return table_.column_count; } + Type type(int32_t column) const noexcept { return static_cast(table_.columns[column].type); } + int64_t lines_read() const noexcept { return result_.lines_read; } + int64_t lines_skipped() const noexcept { return result_.lines_skipped; } + const fw_table& get() const noexcept { return table_; } + + /// The values of a non-string column as stored (see FW_T_*): int64_t, double or int32_t. + template + std::span values(int32_t column) const noexcept + { + const fw_column& raw = table_.columns[column]; + return {static_cast(raw.values), static_cast(raw.length)}; + } + + std::string_view string(int32_t column, int64_t row) const noexcept + { + const fw_column& raw = table_.columns[column]; + const int32_t* offsets = static_cast(raw.values); + const char* data = reinterpret_cast(raw.data); + return {data == nullptr ? "" : data + offsets[row], static_cast(offsets[row + 1] - offsets[row])}; + } + + bool is_valid(int32_t column, int64_t row) const noexcept + { + const uint8_t* bits = table_.columns[column].validity; + return bits == nullptr || (bits[row >> 3] & (1 << (row & 7))) != 0; + } + + private: + fw_table table_; + fw_read_result result_; + }; + + inline Result read_file(std::string_view path, const Layout& layout, const ReadOptions& options = {}) + { + fw_table table{}; + fw_read_result result{.struct_size = sizeof(fw_read_result), .lines_read = 0, .lines_skipped = 0}; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, &table, &result)); !ok) + { + return std::unexpected(ok.error()); + } + return Table(table, result); + } + + inline Result
read_memory(std::string_view data, const Layout& layout, const ReadOptions& options = {}) + { + fw_table table{}; + fw_read_result result{.struct_size = sizeof(fw_read_result), .lines_read = 0, .lines_skipped = 0}; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, &table, &result)); !ok) + { + return std::unexpected(ok.error()); + } + return Table(table, result); + } + + /// A sequential read in tables of at most batch_rows rows. Use from one thread at a time. + class Reader + { + public: + static Result open_file(std::string_view path, const Layout& layout, const ReaderOptions& options = {}) + { + const fw_reader_options raw = options.to_c(); + fw_reader* reader = nullptr; + if (auto ok = detail::check(fw_reader_open_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, &reader)); !ok) + { + return std::unexpected(ok.error()); + } + return Reader(reader); + } + + /// Reads data in place: it must outlive the reader. + static Result open_memory(std::string_view data, const Layout& layout, const ReaderOptions& options = {}) + { + const fw_reader_options raw = options.to_c(); + fw_reader* reader = nullptr; + if (auto ok = detail::check(fw_reader_open_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, &reader)); !ok) + { + return std::unexpected(ok.error()); + } + return Reader(reader); + } + + /// Reads `in` forward, a batch at a time, without seeking: it must outlive the reader. + static Result open_stream(std::istream& in, const Layout& layout, const ReaderOptions& options = {}) + { + const fw_reader_options raw = options.to_c(); + const fw_stream stream = detail::istream_source(in); + fw_reader* reader = nullptr; + if (auto ok = detail::check(fw_reader_open_stream(&stream, layout.get(), &raw, &reader)); !ok) + { + return std::unexpected(ok.error()); + } + return Reader(reader); + } + + Reader(Reader&& other) noexcept : reader_(std::exchange(other.reader_, nullptr)) {} + Reader& operator=(Reader&& other) noexcept + { + std::swap(reader_, other.reader_); + return *this; + } + Reader(const Reader&) = delete; + Reader& operator=(const Reader&) = delete; + ~Reader() + { + if (reader_ != nullptr) + { + fw_reader_close(reader_); + } + } + + /// The next batch, or an empty optional at the end. + Result> next() + { + fw_table table{}; + const int32_t status = fw_reader_next(reader_, &table); + if (status == FW_EOF) + { + return std::optional
{}; + } + if (auto ok = detail::check(status); !ok) + { + return std::unexpected(ok.error()); + } + return std::optional
(Table(table)); + } + + private: + explicit Reader(fw_reader* reader) : reader_(reader) {} + fw_reader* reader_; + }; + + namespace detail + { + inline fw_write_options write_options(const WriteOptions& options) + { + return { + .struct_size = sizeof(fw_write_options), + .encoding = options.encoding, + .newline = bytes(options.newline), + .newline_len = static_cast(options.newline.size()), + .degree_of_parallelism = options.degree_of_parallelism, + .batch_rows = options.batch_rows, + }; + } + } + + inline Result write_file(std::string_view path, const fw_table& table, const Layout& layout, const WriteOptions& options = {}) + { + const fw_write_options raw = detail::write_options(options); + return detail::check(fw_write_file(detail::bytes(path), static_cast(path.size()), &table, layout.get(), &raw)); + } + + inline Result write_file(std::string_view path, const Table& table, const Layout& layout, const WriteOptions& options = {}) + { + return write_file(path, table.get(), layout, options); + } + + inline Result> write_memory(const fw_table& table, const Layout& layout, const WriteOptions& options = {}) + { + const fw_write_options raw = detail::write_options(options); + fw_buffer buffer{}; + if (auto ok = detail::check(fw_write_memory(&table, layout.get(), &raw, &buffer)); !ok) + { + return std::unexpected(ok.error()); + } + std::vector bytes(buffer.data, buffer.data + buffer.length); + fw_free_buffer(&buffer); + return bytes; + } + + inline Result> write_memory(const Table& table, const Layout& layout, const WriteOptions& options = {}) + { + return write_memory(table.get(), layout, options); + } + + namespace detail + { + // fw_sink.write over a std::ostream: -1 once the stream is bad, or if it throws. + inline int32_t write_ostream(void* user_data, const uint8_t* buffer, int64_t length) noexcept + { + constexpr std::string_view failed = "the std::ostream failed"; + try + { + auto& out = *static_cast(user_data); + out.write(reinterpret_cast(buffer), static_cast(length)); + if (!out.bad()) + { + return 0; + } + } + catch (...) + { + } + fw_set_source_error(bytes(failed), static_cast(failed.size())); + return -1; + } + } + + /// Writes to `out` in order, a batch at a time, while batches format in parallel; memory stays at a batch per thread. + inline Result write_stream(std::ostream& out, const fw_table& table, const Layout& layout, const WriteOptions& options = {}) + { + const fw_write_options raw = detail::write_options(options); + const fw_sink sink{.struct_size = sizeof(fw_sink), .user_data = &out, .write = &detail::write_ostream}; + return detail::check(fw_write_stream(&table, layout.get(), &raw, &sink)); + } + + inline Result write_stream(std::ostream& out, const Table& table, const Layout& layout, const WriteOptions& options = {}) + { + return write_stream(out, table.get(), layout, options); + } + + // ---- Struct mapping ---------------------------------------------------------------------------- + // + // Specialize FixedWidthMapper with a `static constexpr auto get_bindings()` returning a std::tuple of + // make_field results; read_file, read_memory, TypedReader, write_file and write_memory all use + // that one mapping. A field's type picks its column type: std::string or std::string_view -> String, + // an integer -> Int64 (Decimal, unscaled, when the field sets a scale), float or double -> Float64, + // std::chrono::sys_days or year_month_day -> Date, std::chrono::microseconds or + // hh_mm_ss -> Time, std::chrono::system_clock::time_point -> Timestamp. std::optional + // makes the column nullable. + + /// The optional parts of a mapped column; see ColumnSpec. + struct FieldOptions + { + int32_t scale = 0; + const char* format = nullptr; + char trim_char = 0; + Trim trim_mode = Trim::Both; + Align alignment = Align::Default; + char padding = 0; + Overflow overflow = Overflow::Throw; + }; + + namespace detail + { + template + struct FieldBinding + { + const char* name; + int32_t start; + int32_t length; + T Class::*member; + FieldOptions options; + using FieldType = T; + }; + } + + /// Binds a member to the column at bytes [start, start + length). + template + constexpr detail::FieldBinding make_field(const char* name, int32_t start, int32_t length, T Class::*member, + FieldOptions options = {}) + { + return {name, start, length, member, options}; + } + + /// Maps a row type to columns: specialize with `static constexpr auto get_bindings()`. + template + struct FixedWidthMapper; + + namespace detail + { + template + struct IsOptional : std::false_type + { + using Inner = T; + }; + + template + struct IsOptional> : std::true_type + { + using Inner = T; + }; + + template + using unwrap_optional_t = typename IsOptional::Inner; + + template + inline constexpr bool always_false = false; + + template + using standard_integer_t = std::conditional_t, std::make_signed_t, std::make_unsigned_t>; + + using micros = std::chrono::microseconds; + + template + constexpr Type type_of(int32_t scale) + { + if constexpr (std::is_same_v || std::is_same_v) + { + return Type::String; + } + else if constexpr (std::is_integral_v && !std::is_same_v) + { + return scale > 0 ? Type::Decimal : Type::Int64; + } + else if constexpr (std::is_floating_point_v) + { + return Type::Float64; + } + else if constexpr (std::is_same_v || std::is_same_v) + { + return Type::Date; + } + else if constexpr (std::is_same_v || std::is_same_v>) + { + return Type::Time; + } + else if constexpr (std::is_same_v) + { + return Type::Timestamp; + } + else + { + static_assert(always_false, "no fixed-width column type for this field type"); + } + } + + template + constexpr auto bindings_of() + { + return FixedWidthMapper::get_bindings(); + } + + template + inline constexpr size_t field_count = std::tuple_size_v())>; + + template + ColumnSpec spec_of(const FieldBinding& binding) + { + const FieldOptions& options = binding.options; + return ColumnSpec{ + .name = binding.name, + .start = binding.start, + .length = binding.length, + .type = type_of>(options.scale), + .nullable = IsOptional::value, + .scale = options.scale, + .format = options.format == nullptr ? std::nullopt : std::optional(options.format), + .trim_char = options.trim_char, + .trim_mode = options.trim_mode, + .alignment = options.alignment, + .padding = options.padding, + .overflow = options.overflow, + }; + } + + template + Field value_at(const Table& table, int32_t column, int64_t row) + { + if constexpr (std::is_same_v || std::is_same_v) + { + return Field(table.string(column, row)); + } + else if constexpr (std::is_integral_v) + { + return static_cast(table.values(column)[row]); + } + else if constexpr (std::is_floating_point_v) + { + return static_cast(table.values(column)[row]); + } + else if constexpr (std::is_same_v || std::is_same_v) + { + return Field(std::chrono::sys_days(std::chrono::days(table.values(column)[row]))); + } + else if constexpr (std::is_same_v || std::is_same_v>) + { + return Field(micros(table.values(column)[row])); + } + else + { + return std::chrono::time_point_cast( + std::chrono::sys_time(micros(table.values(column)[row]))); + } + } + + template + T row_at(const Table& table, int64_t row, std::index_sequence) + { + static constexpr auto bindings = bindings_of(); + T instance{}; + // A null leaves a plain field at its default and an optional one empty. + (..., (table.is_valid(static_cast(Is), row) + ? void(instance.*(std::get(bindings).member) = + value_at>::FieldType>>( + table, static_cast(Is), row)) + : void())); + return instance; + } + + /// The int64 a narrower or unsigned integer field cannot hold is refused once, when the table is + /// handed over, instead of wrapping on every read. + template + Result check_ranges(const Table& table, std::index_sequence) + { + static constexpr auto bindings = bindings_of(); + Result result; + auto check = [&](auto index) + { + constexpr size_t I = decltype(index)::value; + using Field = unwrap_optional_t>::FieldType>; + if constexpr (std::is_integral_v && !std::is_same_v) + { + const auto values = table.values(static_cast(I)); + for (int64_t row = 0; row < table.row_count(); ++row) + { + if (table.is_valid(static_cast(I), row) && !std::in_range>(values[row])) + { + result = std::unexpected(Error{ + .status = FW_INVALID_ARGUMENT, + .message = "column '" + std::string(std::get(bindings).name) + "' holds " + std::to_string(values[row]) + + " at row " + std::to_string(row) + ", which its field's type cannot represent", + }); + return false; + } + } + } + return true; + }; + (void)(... && check(std::integral_constant{})); + return result; + } + } + + /// The layout a FixedWidthMapper describes. + template + Result layout_for(const LayoutOptions& options = {}) + { + static constexpr auto bindings = detail::bindings_of(); + const auto columns = std::apply([](const auto&... binding) { return std::array{detail::spec_of(binding)...}; }, + bindings); + return Layout::create(columns, options); + } + + /// The rows of a read, held in the library's columnar memory and converted to T on access; a + /// random-access range of T. A std::string_view field points into this view. + template + class TableView + { + public: + class iterator + { + public: + using iterator_category = std::random_access_iterator_tag; + using iterator_concept = std::random_access_iterator_tag; + using value_type = T; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = T; + + iterator() = default; + + T operator*() const { return (*view_)[row_]; } + T operator[](difference_type n) const { return (*view_)[row_ + n]; } + + iterator& operator++() { ++row_; return *this; } + iterator operator++(int) { iterator copy = *this; ++row_; return copy; } + iterator& operator--() { --row_; return *this; } + iterator operator--(int) { iterator copy = *this; --row_; return copy; } + iterator& operator+=(difference_type n) { row_ += n; return *this; } + iterator& operator-=(difference_type n) { row_ -= n; return *this; } + + friend iterator operator+(iterator it, difference_type n) { return it += n; } + friend iterator operator+(difference_type n, iterator it) { return it += n; } + friend iterator operator-(iterator it, difference_type n) { return it -= n; } + friend difference_type operator-(const iterator& lhs, const iterator& rhs) { return lhs.row_ - rhs.row_; } + friend bool operator==(const iterator& lhs, const iterator& rhs) { return lhs.row_ == rhs.row_; } + friend std::strong_ordering operator<=>(const iterator& lhs, const iterator& rhs) { return lhs.row_ <=> rhs.row_; } + + private: + friend class TableView; + iterator(const TableView* view, int64_t row) : view_(view), row_(row) {} + + const TableView* view_ = nullptr; + int64_t row_ = 0; + }; + + /// Takes over a table read through layout_for(), checking that every integer fits its field. + static Result adopt(Table table) + { + if (auto ok = detail::check_ranges(table, std::make_index_sequence>{}); !ok) + { + return std::unexpected(ok.error()); + } + return TableView(std::move(table)); + } + + iterator begin() const { return iterator(this, 0); } + iterator end() const { return iterator(this, size()); } + int64_t size() const noexcept { return table_.row_count(); } + bool empty() const noexcept { return size() == 0; } + + /// Row `row`, unchecked; at() is the checked read. + T operator[](int64_t row) const { return detail::row_at(table_, row, std::make_index_sequence>{}); } + + std::optional at(int64_t row) const + { + if (row < 0 || row >= size()) + { + return std::nullopt; + } + return (*this)[row]; + } + + /// The columns underneath, with the read's line counts. + const Table& table() const noexcept { return table_; } + + private: + explicit TableView(Table table) noexcept : table_(std::move(table)) {} + Table table_; + }; + + /// Reads a whole file as T. + template + Result> read_file(std::string_view path, const ReadOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return read_file(path, layout, options); }) + .and_then([](Table table) { return TableView::adopt(std::move(table)); }); + } + + /// Reads a whole buffer as T. data need not outlive the call, but std::string_view fields point into the view. + template + Result> read_memory(std::string_view data, const ReadOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return read_memory(data, layout, options); }) + .and_then([](Table table) { return TableView::adopt(std::move(table)); }); + } + + /// A sequential read as T in views of at most batch_rows rows. Use from one thread at a time. + template + class TypedReader + { + public: + static Result open_file(std::string_view path, const ReaderOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return Reader::open_file(path, layout, options); }) + .transform([](Reader reader) { return TypedReader(std::move(reader)); }); + } + + /// Reads data in place: it must outlive the reader. + static Result open_memory(std::string_view data, const ReaderOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return Reader::open_memory(data, layout, options); }) + .transform([](Reader reader) { return TypedReader(std::move(reader)); }); + } + + /// Reads `in` forward without seeking: it must outlive the reader. + static Result open_stream(std::istream& in, const ReaderOptions& options = {}) + { + return layout_for() + .and_then([&](const Layout& layout) { return Reader::open_stream(in, layout, options); }) + .transform([](Reader reader) { return TypedReader(std::move(reader)); }); + } + + /// The next batch, or an empty optional at the end. + Result>> next() + { + auto batch = reader_.next(); + if (!batch) + { + return std::unexpected(std::move(batch.error())); + } + if (!batch->has_value()) + { + return std::optional>{}; + } + return TableView::adopt(std::move(**batch)).transform([](TableView view) { return std::optional>(std::move(view)); }); + } + + private: + explicit TypedReader(Reader reader) : reader_(std::move(reader)) {} + Reader reader_; + }; + + namespace detail + { + /// One field's column on the way out: the values as the library stores them, and a validity + /// bitmap once a null appears. + template + struct ColumnBuilder + { + using Field = unwrap_optional_t; + static constexpr bool is_string = std::is_same_v || std::is_same_v; + using Stored = std::conditional_t, double, + std::conditional_t || + std::is_same_v, + int32_t, int64_t>>; + + std::vector values; + std::vector offsets{0}; + std::string data; + std::vector validity; + int64_t rows = 0; + bool any_null = false; + bool out_of_range = false; + + void push(const T& value) + { + if (validity.size() * 8 <= static_cast(rows)) + { + validity.push_back(0); + } + if constexpr (IsOptional::value) + { + if (!value.has_value()) + { + any_null = true; + if constexpr (is_string) + { + offsets.push_back(offsets.back()); + } + else + { + values.push_back(Stored{}); + } + ++rows; + return; + } + append(*value); + } + else + { + append(value); + } + validity[static_cast(rows >> 3)] |= static_cast(1 << (rows & 7)); + ++rows; + } + + void append(const Field& value) + { + if constexpr (is_string) + { + data.append(value); + offsets.push_back(static_cast(std::min(data.size(), std::numeric_limits::max()))); + } + else if constexpr (std::is_integral_v) + { + out_of_range |= !std::in_range(static_cast>(value)); + values.push_back(static_cast(value)); + } + else if constexpr (std::is_floating_point_v) + { + values.push_back(static_cast(value)); + } + else if constexpr (std::is_same_v || std::is_same_v) + { + values.push_back(static_cast(std::chrono::sys_days(value).time_since_epoch().count())); + } + else if constexpr (std::is_same_v) + { + values.push_back(value.count()); + } + else if constexpr (std::is_same_v>) + { + values.push_back(value.to_duration().count()); + } + else + { + values.push_back(std::chrono::time_point_cast(value).time_since_epoch().count()); + } + } + + bool overflowed() const { return is_string && data.size() > static_cast(std::numeric_limits::max()); } + + fw_column column(const ColumnSpec& spec) const + { + return fw_column{ + .type = static_cast(spec.type), + .scale = spec.scale, + .length = rows, + .values = is_string ? static_cast(offsets.data()) : static_cast(values.data()), + .validity = any_null ? validity.data() : nullptr, + .data = is_string && !data.empty() ? reinterpret_cast(data.data()) : nullptr, + .data_len = is_string ? static_cast(data.size()) : 0, + }; + } + }; + + template + auto make_builders(std::index_sequence) + { + static constexpr auto bindings = bindings_of(); + return std::tuple>::FieldType>...>{}; + } + + /// Builds the columns of rows, then hands write the table and its layout; both live only for the call. + template + Result write_rows(R&& rows, Write&& write) + { + using T = std::remove_cvref_t>; + static constexpr auto bindings = bindings_of(); + constexpr auto indices = std::make_index_sequence>{}; + auto layout = layout_for(); + if (!layout) + { + return std::unexpected(std::move(layout.error())); + } + auto builders = make_builders(indices); + for (const auto& row : rows) + { + std::apply([&](auto&... builder) + { std::apply([&](const auto&... binding) { (..., builder.push(row.*(binding.member))); }, bindings); }, + builders); + } + const bool overflowed = std::apply([](const auto&... builder) { return (... || builder.overflowed()); }, builders); + if (overflowed) + { + return std::unexpected(Error{.status = FW_INVALID_ARGUMENT, .message = "a string column holds more than 2 GiB, which int32 offsets cannot address"}); + } + const bool out_of_range = std::apply([](const auto&... builder) { return (... || builder.out_of_range); }, builders); + if (out_of_range) + { + return std::unexpected(Error{.status = FW_INVALID_ARGUMENT, .message = "an integer field holds a value above INT64_MAX, which an int64 column cannot represent"}); + } + auto columns = std::apply( + [&](const auto&... builder) + { + return std::apply([&](const auto&... binding) { return std::array{builder.column(spec_of(binding))...}; }, + bindings); + }, + builders); + const fw_table table{.column_count = static_cast(columns.size()), .row_count = std::get<0>(builders).rows, .columns = columns.data()}; + return write(table, *layout); + } + } + + template + concept mapped_range = std::ranges::input_range && requires { FixedWidthMapper>>::get_bindings(); }; + + /// Writes rows, a range of a type with a FixedWidthMapper, to a file. An empty std::optional writes as + /// its column filled with trim_char. + template + Result write_file(std::string_view path, R&& rows, const WriteOptions& options = {}) + { + return detail::write_rows(std::forward(rows), [&](const fw_table& table, const Layout& layout) { return write_file(path, table, layout, options); }); + } + + /// write_file into memory. + template + Result> write_memory(R&& rows, const WriteOptions& options = {}) + { + std::vector bytes; + auto written = detail::write_rows(std::forward(rows), + [&](const fw_table& table, const Layout& layout) -> Result + { + auto result = write_memory(table, layout, options); + if (!result) + { + return std::unexpected(std::move(result.error())); + } + bytes = std::move(*result); + return {}; + }); + if (!written) + { + return std::unexpected(std::move(written.error())); + } + return bytes; + } +} diff --git a/src/FixedWidthParser.Native/include/fixedwidth_arrow.h b/src/FixedWidthParser.Native/include/fixedwidth_arrow.h new file mode 100644 index 0000000..0818192 --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth_arrow.h @@ -0,0 +1,91 @@ +/* Optional Arrow C Data Interface export for FixedWidthParser. Separate from fixedwidth.h so a caller + * that does not want Arrow never pulls these declarations in. + * + * struct ArrowSchema, ArrowArray and ArrowArrayStream are the Arrow C Data and C Stream Interface + * structs, reproduced from the Apache Arrow specification (Apache License 2.0). */ +#ifndef FIXEDWIDTH_ARROW_H +#define FIXEDWIDTH_ARROW_H + +#include "fixedwidth.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef ARROW_C_DATA_INTERFACE +#define ARROW_C_DATA_INTERFACE + +#define ARROW_FLAG_DICTIONARY_ORDERED 1 +#define ARROW_FLAG_NULLABLE 2 +#define ARROW_FLAG_MAP_KEYS_SORTED 4 + +struct ArrowSchema { + const char* format; + const char* name; + const char* metadata; + int64_t flags; + int64_t n_children; + struct ArrowSchema** children; + struct ArrowSchema* dictionary; + + void (*release)(struct ArrowSchema*); + void* private_data; +}; + +struct ArrowArray { + int64_t length; + int64_t null_count; + int64_t offset; + int64_t n_buffers; + int64_t n_children; + const void** buffers; + struct ArrowArray** children; + struct ArrowArray* dictionary; + + void (*release)(struct ArrowArray*); + void* private_data; +}; + +#endif /* ARROW_C_DATA_INTERFACE */ + +#ifndef ARROW_C_STREAM_INTERFACE +#define ARROW_C_STREAM_INTERFACE + +struct ArrowArrayStream { + int (*get_schema)(struct ArrowArrayStream*, struct ArrowSchema* out); + int (*get_next)(struct ArrowArrayStream*, struct ArrowArray* out); + const char* (*get_last_error)(struct ArrowArrayStream*); + void (*release)(struct ArrowArrayStream*); + void* private_data; +}; + +#endif /* ARROW_C_STREAM_INTERFACE */ + +/* A struct array ("+s"), one child per layout column, named from the layout and flagged nullable when + * the column is: STRING "u", INT64 "l", FLOAT64 "g", DECIMAL "d:18,", DATE "tdD", + * TIME "ttu", TIMESTAMP "tsu:". The column buffers move into the arrays without a copy, except + * DECIMAL, widened to 128 bits. A parallel read (degree_of_parallelism other than 1) is concatenated + * into one array. Release both with their release callbacks. */ +int32_t fw_read_arrow_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_read_options* options, + struct ArrowArray* out_array, struct ArrowSchema* out_schema, fw_read_result* out_result); +int32_t fw_read_arrow_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_read_options* options, + struct ArrowArray* out_array, struct ArrowSchema* out_schema, fw_read_result* out_result); + +/* The batches of a sequential reader (fw_reader_options) as an ArrowArrayStream; a failure makes + * get_next return a non-zero errno and get_last_error describe it. fw_read_arrow_stream_memory reads + * data lazily: keep it alive until the stream is released. */ +int32_t fw_read_arrow_stream_file(const uint8_t* path, int32_t path_len, const fw_layout* layout, + const fw_reader_options* options, struct ArrowArrayStream* out_stream); +int32_t fw_read_arrow_stream_memory(const uint8_t* data, int64_t data_len, const fw_layout* layout, + const fw_reader_options* options, struct ArrowArrayStream* out_stream); +/* The same over an fw_stream, which is released with the Arrow stream. */ +int32_t fw_read_arrow_stream_stream(const fw_stream* stream, const fw_layout* layout, + const fw_reader_options* options, struct ArrowArrayStream* out_stream); + +#ifdef __cplusplus +} +#endif + +#endif /* FIXEDWIDTH_ARROW_H */ diff --git a/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp b/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp new file mode 100644 index 0000000..c3df05e --- /dev/null +++ b/src/FixedWidthParser.Native/include/fixedwidth_arrow.hpp @@ -0,0 +1,136 @@ +// Arrow C Data Interface helpers for fixedwidth.hpp: RAII owners of the arrays and streams the library +// publishes. Hand get() to any Arrow consumer, which then takes ownership (set its release to null). +#pragma once + +#include "fixedwidth.hpp" +#include "fixedwidth_arrow.h" + +#include + +namespace fw +{ + /// A struct array and its schema, one child per layout column; released on destruction. + class ArrowResult + { + public: + ArrowResult() noexcept : array_{}, schema_{} {} + ArrowResult(ArrowResult&& other) noexcept : array_(std::exchange(other.array_, {})), schema_(std::exchange(other.schema_, {})) {} + ArrowResult& operator=(ArrowResult&& other) noexcept + { + std::swap(array_, other.array_); + std::swap(schema_, other.schema_); + return *this; + } + ArrowResult(const ArrowResult&) = delete; + ArrowResult& operator=(const ArrowResult&) = delete; + ~ArrowResult() + { + if (array_.release != nullptr) + { + array_.release(&array_); + } + if (schema_.release != nullptr) + { + schema_.release(&schema_); + } + } + + ArrowArray& array() noexcept { return array_; } + ArrowSchema& schema() noexcept { return schema_; } + + private: + ArrowArray array_; + ArrowSchema schema_; + }; + + inline Result read_arrow_file(std::string_view path, const Layout& layout, const ReadOptions& options = {}) + { + ArrowResult result; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, + &result.array(), &result.schema(), nullptr)); + !ok) + { + return std::unexpected(ok.error()); + } + return result; + } + + inline Result read_arrow_memory(std::string_view data, const Layout& layout, const ReadOptions& options = {}) + { + ArrowResult result; + const fw_read_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, + &result.array(), &result.schema(), nullptr)); + !ok) + { + return std::unexpected(ok.error()); + } + return result; + } + + /// An ArrowArrayStream over a sequential read; released on destruction. + class ArrowStream + { + public: + ArrowStream() noexcept : stream_{} {} + ArrowStream(ArrowStream&& other) noexcept : stream_(std::exchange(other.stream_, {})) {} + ArrowStream& operator=(ArrowStream&& other) noexcept + { + std::swap(stream_, other.stream_); + return *this; + } + ArrowStream(const ArrowStream&) = delete; + ArrowStream& operator=(const ArrowStream&) = delete; + ~ArrowStream() + { + if (stream_.release != nullptr) + { + stream_.release(&stream_); + } + } + + ArrowArrayStream* get() noexcept { return &stream_; } + + private: + ArrowArrayStream stream_; + }; + + inline Result read_arrow_stream_file(std::string_view path, const Layout& layout, const ReaderOptions& options = {}) + { + ArrowStream stream; + const fw_reader_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_stream_file(detail::bytes(path), static_cast(path.size()), layout.get(), &raw, stream.get())); + !ok) + { + return std::unexpected(ok.error()); + } + return stream; + } + + /// Reads data lazily: it must outlive the stream. + inline Result read_arrow_stream_memory(std::string_view data, const Layout& layout, const ReaderOptions& options = {}) + { + ArrowStream stream; + const fw_reader_options raw = options.to_c(); + if (auto ok = detail::check(fw_read_arrow_stream_memory(detail::bytes(data), static_cast(data.size()), layout.get(), &raw, stream.get())); + !ok) + { + return std::unexpected(ok.error()); + } + return stream; + } + + /// Reads `in` lazily, without seeking: it must outlive the stream. + inline Result read_arrow_stream(std::istream& in, const Layout& layout, const ReaderOptions& options = {}) + { + ArrowStream stream; + const fw_reader_options raw = options.to_c(); + const fw_stream source = detail::istream_source(in); + if (auto ok = detail::check(fw_read_arrow_stream_stream(&source, layout.get(), &raw, stream.get())); !ok) + { + return std::unexpected(ok.error()); + } + return stream; + } +} diff --git a/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs b/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs index cb0811a..9af3a65 100644 --- a/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs +++ b/src/FixedWidthParser/Attributes/ColumnLayoutValidator.cs @@ -7,25 +7,19 @@ namespace FixedWidthParser.Attributes /// internal static class ColumnLayoutValidator { - public static void Validate(Span<(int Start, int Length, string Name)> columns, Type modelType) + public static void Validate(Span<(int Start, int Length, string Name)> columns, string modelName) { - if (columns.Length == 0) - { - throw new InvalidOperationException( - $"Type {modelType.Name} has no [FixedColumn] members; a fixed-width model must define at least one column."); - } - foreach (var (Start, Length, Name) in columns) { if (Start < 0) { throw new InvalidOperationException( - $"Column \"{Name}\" in {modelType.Name} has a negative Start ({Start})."); + $"Column \"{Name}\" in {modelName} has a negative Start ({Start})."); } if (Length < 1) { throw new InvalidOperationException( - $"Column \"{Name}\" in {modelType.Name} has an invalid Length ({Length}); it must be >= 1."); + $"Column \"{Name}\" in {modelName} has an invalid Length ({Length}); it must be >= 1."); } } @@ -43,7 +37,7 @@ public static void Validate(Span<(int Start, int Length, string Name)> columns, if (current.Start < maxEnd) { throw new InvalidOperationException( - $"Overlapping columns in {modelType.Name}: \"{farthest.Name}\" [{farthest.Start}, {farthest.Start + farthest.Length}) " + + $"Overlapping columns in {modelName}: \"{farthest.Name}\" [{farthest.Start}, {farthest.Start + farthest.Length}) " + $"and \"{current.Name}\" [{current.Start}, {current.Start + current.Length})."); } int end = current.Start + current.Length; diff --git a/src/FixedWidthParser/Attributes/ModelColumns.cs b/src/FixedWidthParser/Attributes/ModelColumns.cs index ae1658d..9699fa4 100644 --- a/src/FixedWidthParser/Attributes/ModelColumns.cs +++ b/src/FixedWidthParser/Attributes/ModelColumns.cs @@ -28,7 +28,12 @@ public static void ForEachColumn(Type modelType, Action(); diff --git a/src/FixedWidthParser/Delegates.cs b/src/FixedWidthParser/Delegates.cs index cc4786a..169e07b 100644 --- a/src/FixedWidthParser/Delegates.cs +++ b/src/FixedWidthParser/Delegates.cs @@ -1,10 +1,8 @@ namespace FixedWidthParser { -#if NET9_0_OR_GREATER - public delegate void RefAction(ref TModel model, TProperty value) where TModel : allows ref struct; - public delegate TProperty RefGetter(in TModel model) where TModel : allows ref struct; -#else - public delegate void RefAction(ref TModel model, TProperty value); - public delegate TProperty RefGetter(in TModel model); -#endif -} \ No newline at end of file + /// Assigns a parsed column value to its member of . + public delegate void ColumnSetter(ref TModel model, TValue value) where TModel : allows ref struct; + + /// Reads the value a column is written from. + public delegate TValue ColumnGetter(in TModel model) where TModel : allows ref struct; +} diff --git a/src/FixedWidthParser/Attributes/FixedColumnAttribute.cs b/src/FixedWidthParser/FixedColumnAttribute.cs similarity index 72% rename from src/FixedWidthParser/Attributes/FixedColumnAttribute.cs rename to src/FixedWidthParser/FixedColumnAttribute.cs index 0d663bd..91a07af 100644 --- a/src/FixedWidthParser/Attributes/FixedColumnAttribute.cs +++ b/src/FixedWidthParser/FixedColumnAttribute.cs @@ -1,4 +1,4 @@ -namespace FixedWidthParser.Attributes +namespace FixedWidthParser { /// Alignment of the content within the column when writing. public enum Alignment @@ -39,10 +39,9 @@ public sealed class FixedColumnAttribute(int start, int length) : Attribute public char Padding { get; set; } = ' '; /// - /// Character trimmed from the end of the column when parsing (e.g. '0' for a zero-padded - /// numeric column written by another producer). Ignored for string columns' interning - /// identity in the same way as the default — it only changes which character is stripped - /// before the value is handed to the parser/converter. Default: space. + /// ASCII character trimmed from the column before it is parsed (e.g. '0' for a zero-padded numeric + /// column), from the ends picks, and written for a + /// value. Default: space. /// public char TrimChar { get; set; } = ' '; @@ -56,10 +55,8 @@ public sealed class FixedColumnAttribute(int start, int length) : Attribute public OverflowBehavior Overflow { get; set; } = OverflowBehavior.Default; /// - /// A type implementing IFixedWidthConverter<T> (char parser/writer) and/or - /// IUtf8FixedWidthConverter<T> (UTF-8 parser/writer), where T is this - /// column's member type. When set, it is used instead of the built-in ISpanParsable - /// fallback, for both parsing and writing. A single instance is created once per model type. + /// A type implementing , where T is this column's member + /// type, used instead of the built-in parsing and formatting. One instance serves every line. /// public Type? Converter { get; set; } } diff --git a/src/FixedWidthParser/FixedWidth.cs b/src/FixedWidthParser/FixedWidth.cs index ace9596..e041363 100644 --- a/src/FixedWidthParser/FixedWidth.cs +++ b/src/FixedWidthParser/FixedWidth.cs @@ -1,135 +1,297 @@ -using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser.Infrastructure; using FixedWidthParser.Readers; -using System.Text; +using FixedWidthParser.Writers; namespace FixedWidthParser { /// - /// Entry points for the source-generated, reflection-free parse path. Dispatches to the model's - /// generated through the type parameter, so the call - /// is statically resolved (no reflection, no delegates, JIT-devirtualized). + /// Parses, reads, aggregates, formats and writes fixed-width lines. Every operation comes in two + /// overloads: one for a source-generated model (), resolved + /// statically through the type parameter, and one that takes a + /// built from attributes or fluently. + /// Column positions are bytes. /// public static class FixedWidth { - /// Parses a single fixed-width line into using its generated parser. - public static bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) -#if NET9_0_OR_GREATER + /// Parses a single line, without its line break, into . + public static bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) where TModel : IFixedWidthModel, allows ref struct -#else - where TModel : IFixedWidthModel -#endif { - return TModel.TryParse(line, formatProvider, stringPool, out model); + return TModel.TryParse(line, in context, out model); + } + + /// + /// Parses a single line, without its line break, into through + /// . Returns when the line is shorter than the layout + /// or a non-string column fails to parse, after reporting why through . + /// + /// has a column without a setter. + public static bool TryParse(ReadOnlySpan line, FixedWidthLayout layout, in FixedWidthContext context, out TModel model) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(layout); + layout.EnsureCanRead(); + return layout.TryParse(line, in context, out model); } - /// Formats into using its generated writer. - public static bool TryFormat(in TModel model, Span destination, IFormatProvider? formatProvider, out int charsWritten) -#if NET9_0_OR_GREATER + /// Formats into , without a line break. + public static bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten) where TModel : IFixedWidthModel, allows ref struct -#else - where TModel : IFixedWidthModel -#endif { - return TModel.TryFormat(in model, destination, formatProvider, out charsWritten); + return TModel.TryFormat(in model, destination, in context, out bytesWritten); } - /// Reads source-generated models from an existing . - public static FixedWidthRecordEnumerable> Read( - TextReader reader, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IFixedWidthModel + /// + /// Formats into through , + /// without a line break. Returns only when is + /// shorter than the line. + /// + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static bool TryFormat(in TModel model, FixedWidthLayout layout, Span destination, in FixedWidthContext context, out int bytesWritten) + where TModel : allows ref struct + { + return WritableLayout(layout).TryFormat(in model, destination, in context, out bytesWritten); + } + + /// Reads from a stream, once. controls whether the stream is disposed when iteration ends. + public static FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + where TModel : IFixedWidthModel, allows ref struct + { + ArgumentNullException.ThrowIfNull(stream); + return new(default, stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); + } + + /// Reads from a stream through , once. controls whether the stream is disposed when iteration ends. + /// has a column without a setter. + public static FixedWidthRecordEnumerable> Read(Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(stream); + return new(ReadableLayout(layout), stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); + } + + /// Reads from a file, reopening it on each enumeration. + public static FixedWidthRecordEnumerable> ReadFile(string path, FixedWidthReaderOptions? options = null) + where TModel : IFixedWidthModel, allows ref struct + { + ArgumentException.ThrowIfNullOrEmpty(path); + return new(default, path, FixedWidthReaderOptions.Validate(options)); + } + + /// Reads from a file through , reopening it on each enumeration. + /// has a column without a setter. + public static FixedWidthRecordEnumerable> ReadFile(string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct + { + ArgumentException.ThrowIfNullOrEmpty(path); + return new(ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options)); + } + + /// Reads from a stream via await foreach, once. controls whether the stream is disposed when iteration ends. + public static FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + where TModel : IFixedWidthModel, allows ref struct + { + ArgumentNullException.ThrowIfNull(stream); + return new(default, stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); + } + + /// Reads from a stream through via await foreach, once. controls whether the stream is disposed when iteration ends. + /// has a column without a setter. + public static FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(stream); + return new(ReadableLayout(layout), stream, ownsStream: !leaveOpen, FixedWidthReaderOptions.Validate(options)); + } + + /// Reads from a file via await foreach with asynchronous I/O, reopening it on each enumeration. + public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthReaderOptions? options = null) + where TModel : IFixedWidthModel, allows ref struct + { + ArgumentException.ThrowIfNullOrEmpty(path); + return new(default, path, FixedWidthReaderOptions.Validate(options)); + } + + /// Reads from a file through via await foreach with asynchronous I/O, reopening it on each enumeration. + /// has a column without a setter. + public static FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null) + where TModel : allows ref struct + { + ArgumentException.ThrowIfNullOrEmpty(path); + return new(ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options)); + } + + /// + /// Reads a file across several threads and folds every record into a , + /// one instance per partition, merged in file order. + /// + /// + /// The result equals one sequential pass when is + /// associative. Every line is parsed exactly once, and errors report the same line number as a + /// sequential read. runs on the worker threads, + /// possibly concurrently, and the first failure in file order is the one thrown. + /// + public static Task AggregateFileAsync( + string path, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) + where TModel : IFixedWidthModel, allows ref struct + where TAccumulator : IRecordAccumulator, new() + { + ArgumentException.ThrowIfNullOrEmpty(path); + return ParallelAggregator.RunAsync, TAccumulator>( + default, path, FixedWidthReaderOptions.Validate(options), cancellationToken); + } + + /// + /// Reads a file through across several threads and folds every record into + /// a , one instance per partition, merged in file order. + /// + /// Carries the same contract as . + /// has a column without a setter. + public static Task AggregateFileAsync( + string path, FixedWidthLayout layout, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) + where TModel : allows ref struct + where TAccumulator : IRecordAccumulator, new() + { + ArgumentException.ThrowIfNullOrEmpty(path); + return ParallelAggregator.RunAsync, TAccumulator>( + ReadableLayout(layout), path, FixedWidthReaderOptions.Validate(options), cancellationToken); + } + + /// + /// Reads a stream once and folds every record into a across + /// several threads. controls whether the stream is disposed afterwards. + /// + /// Carries the same contract as . + public static Task AggregateAsync( + Stream stream, bool leaveOpen = false, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) + where TModel : IFixedWidthModel, allows ref struct + where TAccumulator : IRecordAccumulator, new() { - ArgumentNullException.ThrowIfNull(reader); - ValidateBufferSize(bufferSize); - return new FixedWidthRecordEnumerable>( - default, TextReaderSource.FromReader(reader), formatProvider, stringPool, bufferSize); + ArgumentNullException.ThrowIfNull(stream); + return ParallelStreamAggregator.RunAsync, TAccumulator>( + default, stream, leaveOpen, FixedWidthReaderOptions.Validate(options), cancellationToken); } - /// Reads source-generated models from a . - public static FixedWidthRecordEnumerable> Read( - Stream stream, - Encoding? encoding = null, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// + /// Reads a stream once through and folds every record into a + /// across several threads. controls + /// whether the stream is disposed afterwards. + /// + /// Carries the same contract as . + /// has a column without a setter. + public static Task AggregateAsync( + Stream stream, FixedWidthLayout layout, bool leaveOpen = false, FixedWidthReaderOptions? options = null, CancellationToken cancellationToken = default) + where TModel : allows ref struct + where TAccumulator : IRecordAccumulator, new() + { + ArgumentNullException.ThrowIfNull(stream); + return ParallelStreamAggregator.RunAsync, TAccumulator>( + ReadableLayout(layout), stream, leaveOpen, FixedWidthReaderOptions.Validate(options), cancellationToken); + } + + /// Writes one record and its line break. + /// A value does not fit a column whose overflow throws. + public static void Write(Stream stream, in TModel model, FixedWidthWriterOptions? options = null) where TModel : IFixedWidthModel { ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - default, TextReaderSource.FromStream(stream, enc, leaveOpen), formatProvider, stringPool, bufferSize); + RecordWriter.Write(stream, in model, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options)); } - /// Reads source-generated models from a file, reopening it for each enumeration. - public static FixedWidthRecordEnumerable> ReadFile( - string path, - Encoding? encoding = null, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// Writes one record and its line break through . + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static void Write(Stream stream, in TModel model, FixedWidthLayout layout, FixedWidthWriterOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + RecordWriter.Write(stream, in model, WritableLayout(layout), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes every record, each followed by a line break. + /// A value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, IEnumerable models, FixedWidthWriterOptions? options = null) where TModel : IFixedWidthModel { - ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - default, TextReaderSource.FromFile(path, enc, useAsync: false), formatProvider, stringPool, bufferSize); - } - - /// Reads source-generated models from an existing via await foreach. - public static FixedWidthAsyncRecordEnumerable> ReadAsync( - TextReader reader, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + RecordWriter.WriteMany(stream, models, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes every record through , each followed by a line break. + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, IEnumerable models, FixedWidthLayout layout, FixedWidthWriterOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + RecordWriter.WriteMany(stream, models, WritableLayout(layout), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes every record, each followed by a line break, reading each one by reference. + /// A value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, ReadOnlySpan models, FixedWidthWriterOptions? options = null) where TModel : IFixedWidthModel { - ArgumentNullException.ThrowIfNull(reader); - ValidateBufferSize(bufferSize); - return new FixedWidthAsyncRecordEnumerable>( - default, TextReaderSource.FromReader(reader), formatProvider, stringPool, bufferSize); + ArgumentNullException.ThrowIfNull(stream); + RecordWriter.WriteMany(stream, models, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options)); } - /// Reads source-generated models from a via await foreach. - public static FixedWidthAsyncRecordEnumerable> ReadAsync( - Stream stream, - Encoding? encoding = null, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// Writes every record through , each followed by a line break, reading each one by reference. + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static void WriteMany(Stream stream, ReadOnlySpan models, FixedWidthLayout layout, FixedWidthWriterOptions? options = null) + { + ArgumentNullException.ThrowIfNull(stream); + RecordWriter.WriteMany(stream, models, WritableLayout(layout), FixedWidthWriterOptions.Validate(options)); + } + + /// Writes one record and its line break. + /// A value does not fit a column whose overflow throws. + public static Task WriteAsync(Stream stream, TModel model, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) where TModel : IFixedWidthModel { ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - default, TextReaderSource.FromStream(stream, enc, leaveOpen), formatProvider, stringPool, bufferSize); + return RecordWriter.WriteAsync(stream, model, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options), cancellationToken); + } + + /// Writes one record and its line break through . + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static Task WriteAsync(Stream stream, TModel model, FixedWidthLayout layout, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(stream); + return RecordWriter.WriteAsync(stream, model, WritableLayout(layout), FixedWidthWriterOptions.Validate(options), cancellationToken); } - /// Reads source-generated models from a file asynchronously, reopening it for each enumeration. - public static FixedWidthAsyncRecordEnumerable> ReadFileAsync( - string path, - Encoding? encoding = null, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) + /// Writes every record, each followed by a line break. + /// A value does not fit a column whose overflow throws. + public static Task WriteManyAsync(Stream stream, IEnumerable models, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) where TModel : IFixedWidthModel { - ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - default, TextReaderSource.FromFile(path, enc, useAsync: true), formatProvider, stringPool, bufferSize); + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + return RecordWriter.WriteManyAsync(stream, models, default(GeneratedRecordFormatter), FixedWidthWriterOptions.Validate(options), cancellationToken); + } + + /// Writes every record through , each followed by a line break. + /// has a column without a getter, or a value does not fit a column whose overflow throws. + public static Task WriteManyAsync(Stream stream, IEnumerable models, FixedWidthLayout layout, FixedWidthWriterOptions? options = null, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(stream); + ArgumentNullException.ThrowIfNull(models); + return RecordWriter.WriteManyAsync(stream, models, WritableLayout(layout), FixedWidthWriterOptions.Validate(options), cancellationToken); + } + + private static LayoutRecordFormatter WritableLayout(FixedWidthLayout layout) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(layout); + layout.EnsureCanWrite(); + return new LayoutRecordFormatter(layout); } - private static void ValidateBufferSize(int bufferSize) + private static LayoutLineParser ReadableLayout(FixedWidthLayout layout) + where TModel : allows ref struct { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); + ArgumentNullException.ThrowIfNull(layout); + layout.EnsureCanRead(); + return new LayoutLineParser(layout); } } } diff --git a/src/FixedWidthParser/FixedWidthContext.cs b/src/FixedWidthParser/FixedWidthContext.cs new file mode 100644 index 0000000..22d98bc --- /dev/null +++ b/src/FixedWidthParser/FixedWidthContext.cs @@ -0,0 +1,113 @@ +using System.ComponentModel; +using System.Globalization; +using System.Text; +using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser.Readers; + +namespace FixedWidthParser +{ + /// + /// Settings a model parser, model writer or converter needs for one line, handed to every + /// generated TryParse/TryFormat and every call. + /// + /// + /// Passed by reference. New settings arrive as new properties, so a converter + /// written against this type keeps compiling. means the invariant culture, + /// UTF-8 and no string pool; construct one with an object initializer to unit-test a converter. + /// + public readonly record struct FixedWidthContext + { + private readonly IFormatProvider? _formatProvider; + + /// The format provider for numbers and dates. Defaults to . + public IFormatProvider FormatProvider + { + get => _formatProvider ?? CultureInfo.InvariantCulture; + init + { + _formatProvider = value; + DecimalSeparator = PlainDecimalSeparator(value); + if (value is not null) + { + var dates = DateTimeFormatInfo.GetInstance(value); + GregorianDates = dates.Calendar is GregorianCalendar; + StandardDateSeparators = string.Equals(dates.DateSeparator, "/", StringComparison.Ordinal) + && string.Equals(dates.TimeSeparator, ":", StringComparison.Ordinal); + } + } + } + + /// True when the provider's calendar is Gregorian, so a date's digits are its year, month and day. + internal bool GregorianDates { get => _formatProvider is null || field; private init; } + + /// True when the provider's date and time separators are "/" and ":". + internal bool StandardDateSeparators { get => _formatProvider is null || field; private init; } + + /// + /// The provider's decimal separator as one ASCII byte when its negative sign is "-", so a plain + /// number can be written without going through the provider; 0 otherwise. + /// + // ponytail: read once, so a NumberFormatInfo mutated after this context was built is not seen. + internal byte DecimalSeparator { get => _formatProvider is null ? (byte)'.' : field; private init; } + + private static byte PlainDecimalSeparator(IFormatProvider? provider) + { + if (provider is null) + { + return (byte)'.'; + } + var info = NumberFormatInfo.GetInstance(provider); + if (!string.Equals(info.NegativeSign, "-", StringComparison.Ordinal) || info.NumberDecimalSeparator is not [var separator] || !char.IsAscii(separator)) + { + return 0; + } + return (byte)separator; + } + + /// + /// The encoding of the text in the line, used to decode and encode string columns, dates parsed + /// with a format, and types that only parse from characters. Must be ASCII-compatible (UTF-8, + /// Latin-1, Windows-1252, …), since column positions are bytes. Defaults to UTF-8. + /// + public Encoding Encoding + { + get => field ?? Encoding.UTF8; + init + { + field = value; + IsUtf8 = value is null || value.CodePage == Encoding.UTF8.CodePage; + } + } + + /// True when is UTF-8, so UTF-8 bytes can be written as they are. + // The backing field stores the negation, so default(FixedWidthContext), which means UTF-8, reads as UTF-8. + internal bool IsUtf8 { get => !field; private init => field = !value; } + + /// Interns string columns when set; otherwise each string column is a new string. + public StringPool? StringPool { get; init; } + + internal ParseFailure? Failure { get; init; } + + /// + /// Records that rejected the line, so a reader can name it in its + /// . Generated parsers call this before returning + /// ; outside a reader's diagnosis it does nothing. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public void ReportFailedColumn(string columnName, int start, int length) + { + Failure?.SetColumn(columnName, start, length); + } + + /// + /// Records that the line is shorter than the the layout needs. + /// Generated parsers call this before returning ; outside a reader's + /// diagnosis it does nothing. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public void ReportShortLine(int requiredLength) + { + Failure?.SetShortLine(requiredLength); + } + } +} diff --git a/src/FixedWidthParser/FixedWidthParseException.cs b/src/FixedWidthParser/FixedWidthParseException.cs new file mode 100644 index 0000000..7292c1e --- /dev/null +++ b/src/FixedWidthParser/FixedWidthParseException.cs @@ -0,0 +1,60 @@ +using System.Diagnostics.CodeAnalysis; + +namespace FixedWidthParser +{ + /// + /// Thrown by a reader when a line does not parse into the model, or handed to + /// . Derives from , so + /// existing catch (FormatException) blocks still catch it. + /// + public sealed class FixedWidthParseException : FormatException + { + /// Creates an exception with no message. + [ExcludeFromCodeCoverage] + public FixedWidthParseException() + { + } + + /// Creates an exception with the given message. + [ExcludeFromCodeCoverage] + public FixedWidthParseException(string message) + : base(message) + { + } + + /// Creates an exception with the given message and inner exception. + [ExcludeFromCodeCoverage] + public FixedWidthParseException(string message, Exception innerException) + : base(message, innerException) + { + } + + internal FixedWidthParseException(string message, long lineNumber, string? columnName, int columnStart, int columnLength, string? rawValue) + : base(message) + { + LineNumber = lineNumber; + ColumnName = columnName; + ColumnStart = columnStart; + ColumnLength = columnLength; + RawValue = rawValue; + } + + /// The 1-based number of the line that failed. Empty lines count. + public long LineNumber { get; } + + /// + /// The column that rejected the line, or when the line is shorter than the + /// layout or the model's parser did not report a column. + /// + public string? ColumnName { get; } + + /// The 0-based start of , or -1 when there is no column. + public int ColumnStart { get; } = -1; + + /// The width of , or 0 when there is no column. + public int ColumnLength { get; } + + /// The untrimmed text of , or when there is no column. + public string? RawValue { get; } + } +} diff --git a/src/FixedWidthParser/FixedWidthParser.csproj b/src/FixedWidthParser/FixedWidthParser.csproj index 6669b4a..e5816e2 100644 --- a/src/FixedWidthParser/FixedWidthParser.csproj +++ b/src/FixedWidthParser/FixedWidthParser.csproj @@ -1,7 +1,7 @@ - net8.0;net10.0 + net10.0 latest enable enable @@ -11,14 +11,27 @@ true - $(NoWarn);CS1591 + true + true + + $([System.String]::Copy('$(NoWarn)').Replace('RS0016', '').Replace('RS0037', ''));CS1591;S2743;CA1720;S3011 + + + + + false FixedWidthParser.NET - 1.0.0 + 2.0.0-dev Gabriel Matte FixedWidthParser High-performance, low-allocation parsing and writing of fixed-width (flat) files for .NET. Attribute-driven column mapping, sync/async batch reading (IEnumerable/IAsyncEnumerable), configurable formatting, culture awareness and ref struct support. @@ -46,10 +59,22 @@ + + + + + - - + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + /// Settings for the parse, read and aggregate methods. Every property has a default, so + /// options and new FixedWidthReaderOptions() read the same way. + /// + public sealed class FixedWidthReaderOptions + { + internal static readonly FixedWidthReaderOptions Default = new(); + + /// The format provider for numbers and dates. means the invariant culture. + public IFormatProvider? FormatProvider { get; init; } + + /// + /// The encoding of the file. Must be ASCII-compatible, since column positions are bytes. + /// means UTF-8, and only then is a leading byte order mark skipped. + /// + public Encoding? Encoding { get; init; } + + /// Interns string columns when set; otherwise each string column is a new string. + public StringPool? StringPool { get; init; } + + /// The number of bytes read from the source at a time. Defaults to 65,536. + public int BufferSize { get; init; } = 65536; + + /// + /// The longest line accepted, in bytes, without its line break. A longer line throws + /// , which bounds the memory a source without line breaks can + /// make the reader buffer. Defaults to 1,048,576. + /// + public int MaxLineLength { get; init; } = 1024 * 1024; + + /// + /// Called when a line fails to parse. Return to skip the line and keep + /// reading, or to throw the exception. When , the + /// reader throws. + /// + public Func? OnParseError { get; init; } + + /// + /// The maximum number of threads AggregateFileAsync and AggregateAsync read with. + /// 0, the default, means ; 1 reads in one pass. + /// Other reads ignore it. + /// + public int DegreeOfParallelism { get; init; } + + // Bytes per partition or chunk; 0 sizes them from the source. Tests set it to cut every boundary. + internal int ChunkSize { get; init; } + + internal int Workers + { + get + { + if (DegreeOfParallelism == 0) + { + return Environment.ProcessorCount; + } + return DegreeOfParallelism; + } + } + + internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding!, StringPool = StringPool }; + + // Skips a line that does not parse and calls this, instead of building the exception OnParseError gets: + // the native library's skip_invalid only counts lines, and numbering one in a partition re-reads the + // source from its start. + internal Action? OnSkip { get; init; } + + // Overrides whether a leading UTF-8 byte order mark is skipped; null decides from the encoding. + internal bool? ByteOrderMark { get; init; } + + internal bool SkipsByteOrderMark => ByteOrderMark ?? (Encoding is null || Encoding.CodePage == Encoding.UTF8.CodePage); + + internal static FixedWidthReaderOptions Validate(FixedWidthReaderOptions? options) + { + options ??= Default; + if (options.BufferSize < 1) + { + throw new ArgumentOutOfRangeException(nameof(options), options.BufferSize, "BufferSize must be at least 1."); + } + if (options.MaxLineLength < 1) + { + throw new ArgumentOutOfRangeException(nameof(options), options.MaxLineLength, "MaxLineLength must be at least 1."); + } + if (options.DegreeOfParallelism < 0) + { + throw new ArgumentOutOfRangeException(nameof(options), options.DegreeOfParallelism, "DegreeOfParallelism must not be negative."); + } + return options; + } + } +} diff --git a/src/FixedWidthParser/FixedWidthRuntime.cs b/src/FixedWidthParser/FixedWidthRuntime.cs deleted file mode 100644 index 7b4eee1..0000000 --- a/src/FixedWidthParser/FixedWidthRuntime.cs +++ /dev/null @@ -1,191 +0,0 @@ -using System.Buffers; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Runtime.CompilerServices; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Formatters; -using FixedWidthParser.Processors; - -namespace FixedWidthParser -{ - /// - /// Low-level helpers called by the source-generated - /// and implementations. They centralize the column - /// slicing, string pooling, numeric parsing/formatting so the generated code stays small and - /// matches the runtime parser/writer's semantics exactly. - /// Public because the generated code lives in the consumer assembly; not intended for direct use. - /// - public static class FixedWidthRuntime - { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ReadOnlySpan TrimColumn(ReadOnlySpan column, char trimChar, Attributes.TrimMode trimMode) - { - return trimMode switch - { - Attributes.TrimMode.Leading => column.TrimStart(trimChar), - Attributes.TrimMode.Both => column.Trim(trimChar), - _ => column.TrimEnd(trimChar), - }; - } - - /// Materializes a string column: trims spaces and interns via the pool when supplied. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static string String(ReadOnlySpan column, StringPool? stringPool, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - return stringPool is null ? trimmed.ToString() : stringPool.GetOrAdd(trimmed); - } - - /// Parses a column, honoring the provider's decimal separator (see ). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return CultureHelpers.TryParseDouble(column, formatProvider, out value, trimChar, trimMode); - } - - /// Parses a column, honoring the provider's decimal separator (see ). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return CultureHelpers.TryParseFloat(column, formatProvider, out value, trimChar, trimMode); - } - - /// - /// Parses any column (int, decimal, DateTime, …), trimming - /// spaces. A means - /// (not the BCL's own default of - /// ) — a fixed-width file is a machine layout, and this - /// keeps every column type in a record agreeing on what "no provider" means, matching the - /// double/float columns (see ), which already treat null - /// as invariant. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryParse(ReadOnlySpan column, IFormatProvider? formatProvider, [MaybeNullWhen(false)] out TValue value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - where TValue : ISpanParsable - { - return TValue.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out value); - } - - /// Parses a column via a FixedColumnAttribute.Converter instance, trimming spaces. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryConvert( - ReadOnlySpan column, IFormatProvider? formatProvider, TConverter converter, [MaybeNullWhen(false)] out TValue value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - where TConverter : IFixedWidthConverter - { - return converter.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTime value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return DateTime.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateOnly value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return DateOnly.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryTimeOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out TimeOnly value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return TimeOnly.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeOffsetExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTimeOffset value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return DateTimeOffset.TryParseExact(TrimColumn(column, trimChar, trimMode), format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - /// Formats a string column: fills per (alignment/padding/overflow). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void FormatString(string value, Span slice, ColumnFormatOptions options, string columnName) - { - options.WriteInto(value.AsSpan(), slice, columnName); - } - - /// - /// Formats an column into per - /// : tries a stack buffer first, then grows via a bounded - /// loop for the rare value that needs more room. - /// - [SkipLocalsInit] - public static void FormatValue(TValue value, Span slice, IFormatProvider? formatProvider, ColumnFormatOptions options, string columnName) - where TValue : ISpanFormattable - { - // Null means invariant here too (see TryParse above) — otherwise a null-provider - // write would use CurrentCulture while a null-provider parse of that same value uses - // invariant, breaking the default round-trip on any non-invariant machine. - formatProvider ??= CultureInfo.InvariantCulture; - Span stack = stackalloc char[64]; - if (value.TryFormat(stack, out int written, options.Format, formatProvider)) - { - options.WriteInto(stack[..written], slice, columnName); - return; - } - - int maxSize = Math.Max(1024, slice.Length * 16); - for (int size = 512; size <= maxSize; size *= 2) - { - char[] rented = ArrayPool.Shared.Rent(size); - try - { - if (value.TryFormat(rented, out int w, options.Format, formatProvider)) - { - options.WriteInto(rented.AsSpan(0, w), slice, columnName); - return; - } - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - - throw new InvalidOperationException( - $"Value of type \"{typeof(TValue)}\" for column \"{columnName}\" could not be formatted " + - $"within {maxSize} characters (format \"{options.Format}\")."); - } - - /// - /// Formats a column via a FixedColumnAttribute.Converter instance into - /// per , with the same stack-then-ArrayPool growth as . - /// - [SkipLocalsInit] - public static void FormatConvert( - TValue value, Span slice, IFormatProvider? formatProvider, TConverter converter, ColumnFormatOptions options, string columnName) - where TConverter : IFixedWidthConverter - { - Span stack = stackalloc char[64]; - if (converter.TryFormat(value, stack, formatProvider, out int written)) - { - options.WriteInto(stack[..written], slice, columnName); - return; - } - - int maxSize = Math.Max(1024, slice.Length * 16); - for (int size = 512; size <= maxSize; size *= 2) - { - char[] rented = ArrayPool.Shared.Rent(size); - try - { - if (converter.TryFormat(value, rented, formatProvider, out int w)) - { - options.WriteInto(rented.AsSpan(0, w), slice, columnName); - return; - } - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - - throw new InvalidOperationException( - $"Value of type \"{typeof(TValue)}\" for column \"{columnName}\" could not be formatted by " + - $"converter \"{typeof(TConverter)}\" within {maxSize} characters."); - } - } -} diff --git a/src/FixedWidthParser/FixedWidthUtf8.cs b/src/FixedWidthParser/FixedWidthUtf8.cs deleted file mode 100644 index 35ba694..0000000 --- a/src/FixedWidthParser/FixedWidthUtf8.cs +++ /dev/null @@ -1,109 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; - -namespace FixedWidthParser -{ - /// - /// UTF-8 / byte counterpart of : entry points for the source-generated, - /// reflection-free byte parse path. Dispatches to the model's generated - /// through the type parameter, so the call is - /// statically resolved (no reflection, no delegates, JIT-devirtualized). Reads bytes straight from a - /// — no , no transcode, no string per line. - /// Column offsets are measured in bytes (see ). - /// - public static class FixedWidthUtf8 - { - /// Parses a single UTF-8 fixed-width line into using its generated parser. - public static bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) -#if NET9_0_OR_GREATER - where TModel : IUtf8FixedWidthModel, allows ref struct -#else - where TModel : IUtf8FixedWidthModel -#endif - { - return TModel.TryParse(line, formatProvider, stringPool, out model); - } - - - /// Reads source-generated models from a as raw bytes (single pass). - public static Utf8FixedWidthRecordEnumerable> Read( - Stream stream, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthRecordEnumerable>( - default, stream, ownsStream: !leaveOpen, formatProvider, stringPool, bufferSize); - } - - /// Reads source-generated models from a file as raw bytes, reopening it for each enumeration. - public static Utf8FixedWidthRecordEnumerable> ReadFile( - string path, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthRecordEnumerable>( - default, path, formatProvider, stringPool, bufferSize); - } - - /// Reads source-generated models from a via await foreach (single pass). - public static Utf8FixedWidthAsyncRecordEnumerable> ReadAsync( - Stream stream, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentNullException.ThrowIfNull(stream); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthAsyncRecordEnumerable>( - default, stream, ownsStream: !leaveOpen, formatProvider, stringPool, bufferSize); - } - - /// Reads source-generated models from a file via await foreach (true async I/O), reopening it for each enumeration. - public static Utf8FixedWidthAsyncRecordEnumerable> ReadFileAsync( - string path, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null, - int bufferSize = 4096) - where TModel : IUtf8FixedWidthModel - { - ArgumentException.ThrowIfNullOrEmpty(path); - ValidateBufferSize(bufferSize); - return new Utf8FixedWidthAsyncRecordEnumerable>(default, path, formatProvider, stringPool, bufferSize); - } - - /// - /// Reads source-generated models from a via await foreach (single - /// pass), letting the pipe own buffering and read-ahead. Use this when the source is already a pipe - /// (e.g. a Kestrel request body, a socket, or an upstream System.IO.Pipelines stage). - /// controls whether the reader is completed when iteration ends. - /// - public static GeneratedUtf8FixedWidthPipeRecordEnumerable ReadAsync( - PipeReader reader, - bool leaveOpen = false, - IFormatProvider? formatProvider = null, - StringPool? stringPool = null) - where TModel : IUtf8FixedWidthModel - { - ArgumentNullException.ThrowIfNull(reader); - return new GeneratedUtf8FixedWidthPipeRecordEnumerable( - reader, completeReader: !leaveOpen, formatProvider, stringPool); - } - - private static void ValidateBufferSize(int bufferSize) - { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); - } - } -} diff --git a/src/FixedWidthParser/FixedWidthWriterOptions.cs b/src/FixedWidthParser/FixedWidthWriterOptions.cs new file mode 100644 index 0000000..d567cae --- /dev/null +++ b/src/FixedWidthParser/FixedWidthWriterOptions.cs @@ -0,0 +1,34 @@ +using System.Text; + +namespace FixedWidthParser +{ + /// Settings for the write methods. Every property has a default. + public sealed class FixedWidthWriterOptions + { + internal static readonly FixedWidthWriterOptions Default = new(); + + /// The format provider for numbers and dates. means the invariant culture. + public IFormatProvider? FormatProvider { get; init; } + + /// The encoding of the file. Must be ASCII-compatible, since column positions are bytes. means UTF-8, written without a byte order mark. + public Encoding? Encoding { get; init; } + + /// The line break written after each record. Defaults to . + public string NewLine { get; init; } = Environment.NewLine; + + internal FixedWidthContext Context => new() { FormatProvider = FormatProvider!, Encoding = Encoding! }; + + /// in the file's encoding, encoded once per options instance. + internal byte[] NewLineBytes => field ??= Context.Encoding.GetBytes(NewLine); + + internal static FixedWidthWriterOptions Validate(FixedWidthWriterOptions? options) + { + options ??= Default; + if (options.NewLine is null) + { + throw new ArgumentException("NewLine must not be null.", nameof(options)); + } + return options; + } + } +} diff --git a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs b/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs deleted file mode 100644 index c715d5a..0000000 --- a/src/FixedWidthParser/Formatters/ColumnFormatOptions.cs +++ /dev/null @@ -1,64 +0,0 @@ -using FixedWidthParser.Attributes; - -namespace FixedWidthParser.Formatters -{ - /// - /// Resolved formatting options for a column (alignment, padding, format string and an overflow - /// policy already resolved to or - /// ). Centralizes placing the content into the line slice, - /// shared by all formatters. - /// - public readonly record struct ColumnFormatOptions(Alignment Alignment, char Padding, string? Format, OverflowBehavior Overflow) - { - /// - /// Writes into the column slice , - /// applying alignment, padding and overflow. - /// - public void WriteInto(ReadOnlySpan content, Span slice, string columnName) - { - int width = slice.Length; - int length = content.Length; - - if (length <= width) - { - if (Alignment == Alignment.Right) - { - int pad = width - length; - if (Padding != ' ' && content.Length > 0 && (content[0] == '-' || content[0] == '+')) - { - slice[0] = content[0]; - slice[1..(pad + 1)].Fill(Padding); - content[1..].CopyTo(slice[(pad + 1)..]); - } - else - { - slice[..pad].Fill(Padding); - content.CopyTo(slice[pad..]); - } - } - else - { - content.CopyTo(slice); - slice[length..].Fill(Padding); - } - return; - } - - if (Overflow == OverflowBehavior.Throw) - { - throw new InvalidOperationException( - $"Value \"{content}\" ({length} chars) exceeds the width {width} of column \"{columnName}\"."); - } - - // Truncate: keep the characters on the alignment side. - if (Alignment == Alignment.Right) - { - content[(length - width)..].CopyTo(slice); - } - else - { - content[..width].CopyTo(slice); - } - } - } -} diff --git a/src/FixedWidthParser/Formatters/ColumnFormatters.cs b/src/FixedWidthParser/Formatters/ColumnFormatters.cs new file mode 100644 index 0000000..51d0300 --- /dev/null +++ b/src/FixedWidthParser/Formatters/ColumnFormatters.cs @@ -0,0 +1,74 @@ +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Formatters +{ + /// Formats one column of a model into its slice of the line, for the runtime writer. + internal interface IColumnFormatter + where TModel : allows ref struct + { + void Format(in TModel model, Span line, in FixedWidthContext context); + } + + // The attribute path instantiates the formatters through reflection (Activator.CreateInstance). +#pragma warning disable CA1812 // Avoid uninstantiated internal classes + internal sealed class StringColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter) + : IColumnFormatter where TModel : allows ref struct + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatString(getter(in model), line.Slice(start, length), in context, options, columnName); + } + } + + internal sealed class SpanFormattableColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter) + : IColumnFormatter where TProperty : ISpanFormattable + where TModel : allows ref struct + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatValue(getter(in model), line.Slice(start, length), in context, options, columnName); + } + } + + internal sealed class Utf8SpanFormattableColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter) + : IColumnFormatter where TProperty : IUtf8SpanFormattable, ISpanFormattable + where TModel : allows ref struct + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatUtf8Value(getter(in model), line.Slice(start, length), in context, options, columnName); + } + } + + internal sealed class ConverterColumnFormatter( + int start, int length, ColumnFormatOptions options, string columnName, ColumnGetter getter, TConverter converter) + : IColumnFormatter where TConverter : IFixedWidthConverter + where TModel : allows ref struct + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + FixedWidthRuntime.FormatConvert(getter(in model), line.Slice(start, length), in context, converter, options, columnName); + } + } + + /// A T? column: fills the column with its trim character; a value formats through the T formatter. + internal sealed class NullableColumnFormatter( + int start, int length, byte fill, ColumnGetter getter, IColumnFormatter inner) + : IColumnFormatter where TUnderlying : struct + where TModel : allows ref struct + { + public void Format(in TModel model, Span line, in FixedWidthContext context) + { + if (!getter(in model).HasValue) + { + line.Slice(start, length).Fill(fill); + return; + } + inner.Format(in model, line, in context); + } + } +#pragma warning restore CA1812 +} diff --git a/src/FixedWidthParser/Formatters/ConverterColumnFormatter.cs b/src/FixedWidthParser/Formatters/ConverterColumnFormatter.cs deleted file mode 100644 index 69899df..0000000 --- a/src/FixedWidthParser/Formatters/ConverterColumnFormatter.cs +++ /dev/null @@ -1,16 +0,0 @@ -using FixedWidthParser.Processors; - -namespace FixedWidthParser.Formatters -{ - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class ConverterColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter, TConverter converter) - : IColumnFormatter where TConverter : IFixedWidthConverter - { - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - FixedWidthRuntime.FormatConvert(getter(in model), buffer.Slice(start, length), formatProvider, converter, options, columnName); - } - } -} diff --git a/src/FixedWidthParser/Formatters/FormatterBuilder.cs b/src/FixedWidthParser/Formatters/FormatterBuilder.cs new file mode 100644 index 0000000..8505b49 --- /dev/null +++ b/src/FixedWidthParser/Formatters/FormatterBuilder.cs @@ -0,0 +1,104 @@ +using System.Linq.Expressions; +using System.Reflection; +using FixedWidthParser.Attributes; +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Formatters +{ + /// Builds the attribute path's column formatters: a compiled getter per [FixedColumn] member, wrapped in the formatter its type needs. + internal static class FormatterBuilder + { + public static IColumnFormatter[] Build() + where TModel : allows ref struct + { + var formatters = new List>(); + ModelColumns.ForEachColumn(typeof(TModel), (member, attribute) => formatters.Add(CreateFormatter(member, attribute))); + return [.. formatters]; + } + + private static IColumnFormatter CreateFormatter(MemberInfo member, FixedColumnAttribute attribute) + where TModel : allows ref struct + { + var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); + var options = ResolveOptions(member.Name, memberType, attribute.Alignment, attribute.Padding, attribute.Format, attribute.Overflow); + byte fill = FixedWidthRuntime.ToAsciiByte(attribute.TrimChar, member.Name, nameof(FixedColumnAttribute.TrimChar)); + var getter = Expression.Lambda(typeof(ColumnGetter<,>).MakeGenericType(typeof(TModel), memberType), access, model).Compile(); + + var underlyingType = Nullable.GetUnderlyingType(memberType); + if (underlyingType is null) + { + return BuildValueFormatter(member, attribute, memberType, options, getter); + } + + var inner = BuildValueFormatter(member, attribute, underlyingType, options, BuildUnderlyingGetter(underlyingType, getter)); + return (IColumnFormatter)Activator.CreateInstance( + typeof(NullableColumnFormatter<,>).MakeGenericType(typeof(TModel), underlyingType), + attribute.Start, attribute.Length, fill, getter, inner)!; + } + + /// Validates a column's write options and resolves its overflow; shared by attribute and fluent layouts. + internal static ColumnFormatOptions ResolveOptions(string columnName, Type valueType, Alignment alignment, char padding, string? format, OverflowBehavior overflow) + { + FixedWidthRuntime.ToAsciiByte(padding, columnName, nameof(FixedColumnAttribute.Padding)); + if (overflow == OverflowBehavior.Truncate && valueType != typeof(string)) + { + throw new InvalidOperationException( + $"OverflowBehavior.Truncate is not supported for non-string column '{columnName}' of type '{valueType.Name}'."); + } + // Default overflow resolves per type: string truncates, everything else throws. + var resolved = overflow switch + { + OverflowBehavior.Default when valueType == typeof(string) => OverflowBehavior.Truncate, + OverflowBehavior.Default => OverflowBehavior.Throw, + _ => overflow, + }; + return new ColumnFormatOptions(alignment, padding, format, resolved); + } + + // Adapts a ColumnGetter into a ColumnGetter; only called after HasValue is confirmed. + private static Delegate BuildUnderlyingGetter(Type underlyingType, Delegate nullableGetter) + where TModel : allows ref struct + { + var modelParam = Expression.Parameter(typeof(TModel).MakeByRefType(), "model"); + var value = Expression.Property(Expression.Invoke(Expression.Constant(nullableGetter), modelParam), "Value"); + return Expression.Lambda(typeof(ColumnGetter<,>).MakeGenericType(typeof(TModel), underlyingType), value, modelParam).Compile(); + } + + private static IColumnFormatter BuildValueFormatter( + MemberInfo member, FixedColumnAttribute attribute, Type valueType, ColumnFormatOptions options, Delegate getter) + where TModel : allows ref struct + { + if (attribute.Converter is { } converterType) + { + var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); + if (!requiredInterface.IsAssignableFrom(converterType)) + { + throw new InvalidOperationException( + $"Converter '{converterType}' for column '{member.Name}' must implement '{requiredInterface}'."); + } + var converter = Activator.CreateInstance(converterType) + ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{member.Name}'."); + return (IColumnFormatter)Activator.CreateInstance( + typeof(ConverterColumnFormatter<,,>).MakeGenericType(typeof(TModel), valueType, converterType), + attribute.Start, attribute.Length, options, member.Name, getter, converter)!; + } + + if (valueType == typeof(string)) + { + return new StringColumnFormatter(attribute.Start, attribute.Length, options, member.Name, (ColumnGetter)getter); + } + + if (!typeof(ISpanFormattable).IsAssignableFrom(valueType)) + { + throw new InvalidOperationException( + $"Column '{member.Name}' has type '{valueType}', which is not string or ISpanFormattable; set a Converter for it."); + } + var formatterType = typeof(IUtf8SpanFormattable).IsAssignableFrom(valueType) + ? typeof(Utf8SpanFormattableColumnFormatter<,>) + : typeof(SpanFormattableColumnFormatter<,>); + return (IColumnFormatter)Activator.CreateInstance( + formatterType.MakeGenericType(typeof(TModel), valueType), + attribute.Start, attribute.Length, options, member.Name, getter)!; + } + } +} diff --git a/src/FixedWidthParser/Formatters/IColumnFormatter.cs b/src/FixedWidthParser/Formatters/IColumnFormatter.cs deleted file mode 100644 index 6885cd9..0000000 --- a/src/FixedWidthParser/Formatters/IColumnFormatter.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - public interface IColumnFormatter - { - void Format(in TModel model, Span buffer, IFormatProvider? formatProvider); - } -} \ No newline at end of file diff --git a/src/FixedWidthParser/Formatters/NullableColumnFormatter.cs b/src/FixedWidthParser/Formatters/NullableColumnFormatter.cs deleted file mode 100644 index 347310b..0000000 --- a/src/FixedWidthParser/Formatters/NullableColumnFormatter.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - /// - /// Wraps another (built for the underlying T of a - /// T? column) so a value writes as a blank (padding-filled) column - /// instead of delegating to the inner formatter — no duplicated formatting logic per type. - /// - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class NullableColumnFormatter( - int start, int length, char trimChar, RefGetter getter, IColumnFormatter inner) - : IColumnFormatter where TUnderlying : struct - { - private readonly char _fillChar = trimChar; - - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - if (!getter(in model).HasValue) - { - buffer.Slice(start, length).Fill(_fillChar); - return; - } - inner.Format(in model, buffer, formatProvider); - } - } -} diff --git a/src/FixedWidthParser/Formatters/SpanFormattableColumnFormatter.cs b/src/FixedWidthParser/Formatters/SpanFormattableColumnFormatter.cs deleted file mode 100644 index 5988c6c..0000000 --- a/src/FixedWidthParser/Formatters/SpanFormattableColumnFormatter.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class SpanFormattableColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) - : IColumnFormatter where TProperty : ISpanFormattable - { - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - FixedWidthRuntime.FormatValue(getter(in model), buffer.Slice(start, length), formatProvider, options, columnName); - } - } -} diff --git a/src/FixedWidthParser/Formatters/StringColumnFormatter.cs b/src/FixedWidthParser/Formatters/StringColumnFormatter.cs deleted file mode 100644 index 6c4c077..0000000 --- a/src/FixedWidthParser/Formatters/StringColumnFormatter.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace FixedWidthParser.Formatters -{ - [System.Diagnostics.CodeAnalysis.SuppressMessage("Performance", "CA1812:Avoid uninstantiated internal classes", - Justification = "Instantiated via reflection (Activator.CreateInstance) in FixedWidthWriter.CreateFormatter.")] - internal sealed class StringColumnFormatter( - int start, int length, ColumnFormatOptions options, string columnName, RefGetter getter) - : IColumnFormatter - { - public void Format(in TModel model, Span buffer, IFormatProvider? formatProvider) - { - FixedWidthRuntime.FormatString(getter(in model), buffer.Slice(start, length), options, columnName); - } - } -} diff --git a/src/FixedWidthParser/IFixedWidthConverter.cs b/src/FixedWidthParser/IFixedWidthConverter.cs new file mode 100644 index 0000000..9aba327 --- /dev/null +++ b/src/FixedWidthParser/IFixedWidthConverter.cs @@ -0,0 +1,26 @@ +namespace FixedWidthParser +{ + /// + /// Custom conversion for a fixed-width column, wired through + /// or a fluent ConvertedColumn. One instance serves every line of the layout, on every thread + /// that reads or writes in parallel, so implementations must be stateless. + /// One interface covers both directions: a fixed-width column always round-trips. + /// + /// A later version never adds a member to this interface; new settings arrive on . + public interface IFixedWidthConverter + { + /// + /// Parses the already-sliced, trimmed column bytes. Decode text with + /// . Returns to reject the line. + /// + bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out T value); + + /// + /// Formats into , encoding text with + /// , and returns the number of bytes written. Returns + /// when is too small; the writer retries + /// with a larger buffer. + /// + bool TryFormat(T value, Span destination, in FixedWidthContext context, out int bytesWritten); + } +} diff --git a/src/FixedWidthParser/IFixedWidthModel.cs b/src/FixedWidthParser/IFixedWidthModel.cs index ffed152..20c856b 100644 --- a/src/FixedWidthParser/IFixedWidthModel.cs +++ b/src/FixedWidthParser/IFixedWidthModel.cs @@ -1,34 +1,28 @@ -using CommunityToolkit.HighPerformance.Buffers; - namespace FixedWidthParser { /// - /// Implemented by models that opt into the compile-time generated parser. Declaring this - /// interface on a model is the trigger for the source generator, which - /// fills in with fully-inlined, reflection-free parsing (mirroring the - /// runtime semantics). Mirrors the shape of - /// so a generic, devirtualized read path is possible. + /// Implemented by models that opt into the compile-time generated parser and writer. Declaring + /// this interface on a model is the trigger for the source generator, + /// which fills in and with reflection-free code + /// that reads and writes exactly like the model's layout. + /// Column positions are bytes. /// -#if NET9_0_OR_GREATER public interface IFixedWidthModel where TSelf : IFixedWidthModel, allows ref struct -#else - public interface IFixedWidthModel where TSelf : IFixedWidthModel -#endif { /// - /// Parses a single fixed-width line into . Returns - /// (rejecting the line) when a non-string column fails to parse. + /// Parses a single fixed-width line, without its line break, into . + /// Returns (rejecting the line) when the line is shorter than the layout + /// or a non-string column fails to parse, after reporting why through + /// or . /// - static abstract bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TSelf model); + static abstract bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TSelf model); /// - /// Formats into , mirroring - /// : fills unused portions of each column with its - /// configured padding character, honors per-column alignment/format/overflow, and returns - /// only when is shorter than the model's - /// fixed-width line length (a column value that doesn't fit throws or truncates per its - /// FixedColumnAttribute.Overflow, it does not make this return ). + /// Formats into , without a line break, + /// mirroring the model's attribute layout. Returns only + /// when is shorter than the line; a value that does not fit its + /// column throws or truncates per its FixedColumnAttribute.Overflow. /// - static abstract bool TryFormat(in TSelf model, Span destination, IFormatProvider? formatProvider, out int charsWritten); + static abstract bool TryFormat(in TSelf model, Span destination, in FixedWidthContext context, out int bytesWritten); } } diff --git a/src/FixedWidthParser/IRecordAccumulator.cs b/src/FixedWidthParser/IRecordAccumulator.cs new file mode 100644 index 0000000..0bd62e8 --- /dev/null +++ b/src/FixedWidthParser/IRecordAccumulator.cs @@ -0,0 +1,33 @@ +namespace FixedWidthParser +{ + /// + /// Folds the records of a file read by AggregateFileAsync into a result. Each partition of the + /// file gets its own instance. + /// + /// The implementing type. + /// The record type. + /// + /// + /// runs concurrently across instances on worker threads, and must mutate only the + /// instance it is called on. Every line is parsed and added exactly once, by the partition its first + /// byte falls in. + /// + /// + /// runs on a single thread, left to right in file order, folding each later + /// partition into the accumulator of everything before it. It may receive an instance that saw no + /// records. A file read as a single partition never calls it. + /// + /// + public interface IRecordAccumulator + where TSelf : IRecordAccumulator + where TModel : allows ref struct + { + /// Folds one record into this instance. + /// The record. It is valid only for the duration of the call. + void Add(in TModel model); + + /// Folds the records of the partition that immediately follows this one into this instance. + /// The accumulator of the following partition. It is not used again afterwards. + void Merge(TSelf following); + } +} diff --git a/src/FixedWidthParser/IUtf8FixedWidthModel.cs b/src/FixedWidthParser/IUtf8FixedWidthModel.cs deleted file mode 100644 index ed7af64..0000000 --- a/src/FixedWidthParser/IUtf8FixedWidthModel.cs +++ /dev/null @@ -1,30 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser -{ - /// - /// UTF-8 / byte counterpart of : implemented by models that - /// opt into the compile-time generated byte parser. Declaring this interface on a - /// model is the trigger for the source generator, which fills in - /// with fully-inlined, reflection-free parsing straight from raw UTF-8 bytes - /// (mirroring the runtime semantics). A model may - /// implement this, , or both — the two generated - /// TryParse methods are distinct overloads (one over , one over - /// ) and coexist in the same partial type. - /// Column offsets are measured in bytes, exact for the single-byte/ASCII payloads - /// typical of flat files (see ). - /// -#if NET9_0_OR_GREATER - public interface IUtf8FixedWidthModel where TSelf : IUtf8FixedWidthModel, allows ref struct -#else - public interface IUtf8FixedWidthModel where TSelf : IUtf8FixedWidthModel -#endif - { - /// - /// Parses a single UTF-8 fixed-width line into . Returns - /// (rejecting the line) when the line is shorter (in bytes) than the - /// layout or a non-string column fails to parse. - /// - static abstract bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TSelf model); - } -} diff --git a/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs new file mode 100644 index 0000000..48a5cd7 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/AsyncRecordEnumerator.cs @@ -0,0 +1,103 @@ +using System.ComponentModel; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// Asynchronous record enumerator over a stream. The line scan stays synchronous; only the refill + /// awaits the stream. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class AsyncRecordEnumerator : IAsyncEnumerator + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + private readonly TParser _parser; + private readonly FixedWidthContext _context; + private readonly Func? _onParseError; + private readonly Stream _stream; + private readonly bool _ownsStream; + private readonly CancellationToken _cancellationToken; + private LineBufferState _lines; + private bool _disposed; + + internal AsyncRecordEnumerator(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options, CancellationToken cancellationToken) + { + _parser = parser; + _context = options.Context; + _onParseError = options.OnParseError; + _stream = stream; + _ownsStream = ownsStream; + _cancellationToken = cancellationToken; + _lines = default; + _lines.Rent(options); + } + + /// The record on the line the enumerator sits on, parsed on access. + /// + /// Parsed here rather than by because the enumerator cannot hold a field + /// of a ref struct . await foreach reads it once per line, + /// so each line is still parsed once; reading it twice parses twice. + /// + /// The line does not parse. + public TModel Current => ParseErrors.Parse(_parser, _lines.CurrentLine, _lines.LineNumber, in _context); + + /// + public ValueTask MoveNextAsync() + { + ObjectDisposedException.ThrowIf(_disposed, nameof(AsyncRecordEnumerator)); + _cancellationToken.ThrowIfCancellationRequested(); + var status = TryReadFromBuffer(); + if (status != LineStatus.NeedData) + { + return new ValueTask(status == LineStatus.Line); + } + + return MoveNextSlowAsync(); + } + + private async ValueTask MoveNextSlowAsync() + { + while (true) + { + _lines.Compact(); + _lines.GrowIfFull(); + _lines.Advance(await _stream.ReadAsync(_lines.Buffer.AsMemory(_lines.End), _cancellationToken).ConfigureAwait(false)); + _cancellationToken.ThrowIfCancellationRequested(); + + var status = TryReadFromBuffer(); + if (status != LineStatus.NeedData) + { + return status == LineStatus.Line; + } + } + } + + private LineStatus TryReadFromBuffer() + { + while (true) + { + var status = _lines.TryGetLine(out var line); + if (status != LineStatus.Line || ParseErrors.Accepts(_parser, line, _lines.LineNumber, in _context, _onParseError)) + { + return status; + } + } + } + + /// + public ValueTask DisposeAsync() + { + if (_disposed) + { + return ValueTask.CompletedTask; + } + + _lines.Return(); + _disposed = true; +#pragma warning disable IDISP007 // Disposed only when the enumerable was told it owns the stream. + return _ownsStream ? _stream.DisposeAsync() : ValueTask.CompletedTask; +#pragma warning restore IDISP007 + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs new file mode 100644 index 0000000..450c055 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/ColumnFormatOptions.cs @@ -0,0 +1,98 @@ +using System.ComponentModel; +using System.Text; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// Resolved formatting options for a column (alignment, padding, format string and an overflow + /// policy already resolved to or + /// ), passed by generated code and layouts to + /// . must be ASCII. + /// Public because generated code constructs it; not intended for direct use. + /// + /// + /// Generated code in consumer assemblies calls the constructor, so a new setting arrives as a new + /// constructor overload and a new property; the existing constructor is never changed. + /// +#pragma warning disable CA1815 // Never compared; value equality is meaningless for it. + [EditorBrowsable(EditorBrowsableState.Never)] + public readonly struct ColumnFormatOptions +#pragma warning restore CA1815 + { + /// Creates the options for one column. + public ColumnFormatOptions(Alignment alignment, char padding, string? format, OverflowBehavior overflow) + { + Alignment = alignment; + Padding = padding; + Format = format; + Overflow = overflow; + FastDate = FastDate.TryCreate(format); + } + + /// Which side a written value is aligned to. + public Alignment Alignment { get; } + + /// The ASCII character that fills the column around its value. + public char Padding { get; } + + /// The format string for the value, or for the type's default. + public string? Format { get; } + + /// The resolved overflow policy: or . + public OverflowBehavior Overflow { get; } + + /// compiled for , or when it cannot be. + internal FastDate? FastDate { get; } + + /// + /// Writes the encoded into the column slice , + /// applying alignment, padding and overflow. + /// + internal void WriteInto(ReadOnlySpan content, Span slice, string columnName) + { + int width = slice.Length; + int length = content.Length; + if (length > width) + { + if (Overflow == OverflowBehavior.Throw) + { + throw new InvalidOperationException( + $"Value \"{Encoding.UTF8.GetString(content)}\" ({length} bytes) exceeds the width {width} of column \"{columnName}\"."); + } + (Alignment == Alignment.Right ? content[(length - width)..] : content[..width]).CopyTo(slice); + return; + } + content.CopyTo(slice); + AlignInPlace(slice, length); + } + + /// + /// Aligns and pads bytes already formatted at the start of + /// , exactly as would place them. + /// + internal void AlignInPlace(Span slice, int length) + { + byte padding = (byte)Padding; + int pad = slice.Length - length; + if (Alignment != Alignment.Right) + { + slice[length..].Fill(padding); + return; + } + if (pad == 0) + { + return; + } + byte first = slice[0]; + slice[..length].CopyTo(slice[pad..]); + if (Padding != ' ' && length > 0 && (first == (byte)'-' || first == (byte)'+')) + { + // The sign stays leftmost, ahead of the padding. + slice[..(pad + 1)].Fill(padding); + slice[0] = first; + return; + } + slice[..pad].Fill(padding); + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs new file mode 100644 index 0000000..957e4d1 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/ColumnParseOptions.cs @@ -0,0 +1,42 @@ +using System.ComponentModel; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// How a column is trimmed and, for a date or time, the exact format it is parsed with, passed by + /// generated code and layouts to . must be ASCII. + /// Public because generated code constructs it; not intended for direct use. + /// + /// + /// Generated code in consumer assemblies calls the constructor, so a new setting arrives as a new + /// constructor overload and a new property; the existing constructor is never changed. + /// +#pragma warning disable CA1815 // Never compared; value equality is meaningless for it. + [EditorBrowsable(EditorBrowsableState.Never)] + public readonly struct ColumnParseOptions +#pragma warning restore CA1815 + { + /// Creates the options for one column. + public ColumnParseOptions(char trimChar, TrimMode trimMode, string? format) + { + TrimChar = trimChar; + TrimMode = trimMode; + Format = format; + FastDate = FastDate.TryCreate(format); + } + + /// The ASCII character trimmed from the column before it is parsed, and that a value is written with. + public char TrimChar { get; } + + /// Which ends of the column is trimmed from. + public TrimMode TrimMode { get; } + + /// The exact format of a date or time column, or . + public string? Format { get; } + + internal byte TrimByte => (byte)TrimChar; + + /// compiled for , or when it cannot be. + internal FastDate? FastDate { get; } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FastDate.cs b/src/FixedWidthParser/Infrastructure/FastDate.cs new file mode 100644 index 0000000..7175756 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastDate.cs @@ -0,0 +1,170 @@ +namespace FixedWidthParser.Infrastructure +{ + /// + /// A date or time format made only of fixed-width digit fields (yyyy, MM, dd, HH, + /// mm, ss) and the literals - . T / :, read straight from the bytes the way ExcelReader's + /// FastDate reads ISO 8601, and written the same way. compiles it once per column + /// and returns for any other format; and + /// return for anything they cannot do exactly as TryParseExact and ToString + /// would, and the caller falls back to those. + /// + internal sealed class FastDate + { + private readonly string _format; + private readonly int _year = -1; + private readonly int _month = -1; + private readonly int _day = -1; + private readonly int _hour = -1; + private readonly int _minute = -1; + private readonly int _second = -1; + // "/" and ":" stand for the provider's date and time separators, not for themselves. + private readonly bool _usesSeparators; + private readonly bool _valid; + + private FastDate(string format) + { + _format = format; + for (int i = 0; i < format.Length;) + { + char c = format[i]; + int run = 1; + while (i + run < format.Length && format[i + run] == c) + { + run++; + } + switch (c, run) + { + case ('y', 4) when _year < 0: + _year = i; + break; + case ('M', 2) when _month < 0: + _month = i; + break; + case ('d', 2) when _day < 0: + _day = i; + break; + case ('H', 2) when _hour < 0: + _hour = i; + break; + case ('m', 2) when _minute < 0: + _minute = i; + break; + case ('s', 2) when _second < 0: + _second = i; + break; + case ('-' or '.' or 'T', _): + break; + case ('/' or ':', _): + _usesSeparators = true; + break; + default: + return; + } + i += run; + } + // A date needs all of year, month and day, and a time its hour: TryParseExact fills a missing date + // part from today, which a fixed result cannot match. + bool date = _year >= 0 && _month >= 0 && _day >= 0; + bool noDate = _year < 0 && _month < 0 && _day < 0; + bool time = _hour >= 0 && (_second < 0 || _minute >= 0); + bool noTime = _hour < 0 && _minute < 0 && _second < 0; + _valid = (date || noDate) && (time || noTime) && (date || time); + } + + internal bool HasDate => _year >= 0; + + internal bool HasTime => _hour >= 0; + + internal static FastDate? TryCreate(string? format) + { + if (format is null) + { + return null; + } + var fast = new FastDate(format); + return fast._valid ? fast : null; + } + + /// The date and time in , already trimmed; 0001-01-01 when the format has no date. + internal bool TryParse(ReadOnlySpan s, in FixedWidthContext context, out DateTime value) + { + value = default; + if (s.Length != _format.Length || !context.GregorianDates || (_usesSeparators && !context.StandardDateSeparators)) + { + return false; + } + for (int i = 0; i < s.Length; i++) + { + char f = _format[i]; + bool matches = f is 'y' or 'M' or 'd' or 'H' or 'm' or 's' ? (uint)(s[i] - '0') <= 9 : s[i] == f; + if (!matches) + { + return false; + } + } + int year = _year < 0 ? 1 : Digits(s, _year, 4); + int month = _month < 0 ? 1 : Digits(s, _month, 2); + int day = _day < 0 ? 1 : Digits(s, _day, 2); + int hour = _hour < 0 ? 0 : Digits(s, _hour, 2); + int minute = _minute < 0 ? 0 : Digits(s, _minute, 2); + int second = _second < 0 ? 0 : Digits(s, _second, 2); + if (year < 1 || month is < 1 or > 12 || day < 1 || day > DateTime.DaysInMonth(year, month) + || hour > 23 || minute > 59 || second > 59) + { + return false; + } + // Unspecified, as TryParseExact returns text without a zone. + value = new DateTime(year, month, day, hour, minute, second, DateTimeKind.Unspecified); + return true; + } + + /// + /// Writes as ToString with the format would, at the start of + /// ; when it does not fit or the provider's calendar or + /// separators rule the fast path out. + /// + internal bool TryFormat(DateTime value, Span destination, in FixedWidthContext context, out int written) + { + written = 0; + if (destination.Length < _format.Length || !context.GregorianDates || (_usesSeparators && !context.StandardDateSeparators)) + { + return false; + } + for (int i = 0; i < _format.Length; i++) + { + destination[i] = (byte)_format[i]; + } + WriteDigits(destination, _year, 4, value.Year); + WriteDigits(destination, _month, 2, value.Month); + WriteDigits(destination, _day, 2, value.Day); + WriteDigits(destination, _hour, 2, value.Hour); + WriteDigits(destination, _minute, 2, value.Minute); + WriteDigits(destination, _second, 2, value.Second); + written = _format.Length; + return true; + } + + private static void WriteDigits(Span destination, int offset, int count, int value) + { + if (offset < 0) + { + return; + } + for (int i = offset + count - 1; i >= offset; i--) + { + destination[i] = (byte)('0' + (value % 10)); + value /= 10; + } + } + + private static int Digits(ReadOnlySpan s, int offset, int count) + { + int value = 0; + foreach (ref readonly byte b in s.Slice(offset, count)) + { + value = (value * 10) + (b - '0'); + } + return value; + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FastDecimal.cs b/src/FixedWidthParser/Infrastructure/FastDecimal.cs new file mode 100644 index 0000000..30c40de --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastDecimal.cs @@ -0,0 +1,64 @@ +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// ExcelReader's FastDecimal with the decimal separator as a parameter: [sign]digits[separator digits], + /// at most 19 digits, scale kept as written. for anything else, and the caller falls + /// back to the BCL. + /// + internal static class FastDecimal + { + private const int MaxMantissaDigits = 19; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, byte decimalSeparator, out decimal value) + { + value = 0; + if (s.IsEmpty) + { + return false; + } + int i = 0; + bool neg = s[0] == (byte)'-'; + if (neg || s[0] == (byte)'+') + { + i = 1; + } + + ulong mantissa = 0; + int integerStart = i; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + int integerDigits = i - integerStart; + int scale = 0; + if (i < s.Length && s[i] == decimalSeparator) + { + int fractionStart = ++i; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + scale = i - fractionStart; + } + if (i != s.Length || integerDigits + scale is 0 or > MaxMantissaDigits) + { + return false; + } + value = new decimal((int)mantissa, (int)(mantissa >> 32), 0, neg, (byte)scale); + return true; + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FastDouble.cs b/src/FixedWidthParser/Infrastructure/FastDouble.cs new file mode 100644 index 0000000..fb3a318 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastDouble.cs @@ -0,0 +1,377 @@ +using System.Buffers.Binary; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// ExcelReader's FastDouble (fast_float's algorithm) with the decimal separator as a parameter. + /// All or nothing: for anything it cannot parse exactly (thousands separators, + /// trailing text, too many digits, an exponent out of its range), and the caller falls back to the BCL. + /// + internal static class FastDouble + { + private const int MaxExponentDigits = 3; + private const int MaxMantissaDigits = 19; + private const int MinPowerOfFive = -64; + private const int MaxPowerOfFive = 64; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, byte decimalSeparator, out double value) + { + value = 0; + if (!TryScan(s, decimalSeparator, out ulong mantissa, out int scale, out bool neg)) + { + return false; + } + if (scale is < -22 or > 22 || mantissa > 1UL << 53) + { + return EiselLemire(mantissa, -scale, neg, out value); + } + + double result = mantissa; + if (scale > 0) + { + result /= Pow10(scale); + } + else if (scale < 0) + { + result *= Pow10(-scale); + } + value = neg ? -result : result; + return true; + } + + /// + /// fast_float's binary32 fast path only: a mantissa and a power of ten that are both exact floats, so + /// the one division or multiplication rounds once. Rounding the double result instead can be one ulp off. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, byte decimalSeparator, out float value) + { + value = 0; + if (!TryScan(s, decimalSeparator, out ulong mantissa, out int scale, out bool neg) || scale is < -10 or > 10 || mantissa > 1UL << 24) + { + return false; + } + float result = mantissa; + if (scale > 0) + { + result /= (float)Pow10(scale); + } + else if (scale < 0) + { + result *= (float)Pow10(-scale); + } + value = neg ? -result : result; + return true; + } + + // Reads [sign]digits[separator digits][e[sign]digits] as mantissa * 10^-scale; false for any other + // text, or more than 19 significant digits. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool TryScan(ReadOnlySpan s, byte decimalSeparator, out ulong mantissa, out int scale, out bool neg) + { + mantissa = 0; + scale = 0; + neg = false; + if (s.IsEmpty) + { + return false; + } + int i = 0; + neg = s[0] == (byte)'-'; + if (neg || s[0] == (byte)'+') + { + i = 1; + } + + int integerStart = i; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + int integerDigits = i - integerStart; + if (i < s.Length && s[i] == decimalSeparator) + { + int fractionStart = ++i; + i = ReadDigits(s, i, ref mantissa); + scale = i - fractionStart; + } + int allDigits = integerDigits + scale; + if (allDigits == 0 || HasTooManyDigits(s[integerStart..i], allDigits, decimalSeparator)) + { + return false; + } + int exponent = 0; + if (i < s.Length && (s[i] is not ((byte)'e' or (byte)'E') || !TryExponent(s[(i + 1)..], out exponent))) + { + return false; + } + + scale -= exponent; + return true; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool EiselLemire(ulong w, int q, bool neg, out double value) + { + value = 0; + if (q is < MinPowerOfFive or > MaxPowerOfFive) + { + return false; + } + if (w == 0) + { + value = neg ? -0.0 : 0.0; + return true; + } + int lz = BitOperations.LeadingZeroCount(w); + w <<= lz; + int index = 2 * (q - MinPowerOfFive); + ReadOnlySpan powers = PowersOfFive; + ulong hi = Math.BigMul(w, powers[index], out ulong lo); + const ulong PrecisionMask = ulong.MaxValue >> 55; + if ((hi & PrecisionMask) == PrecisionMask) + { + ulong hi2 = Math.BigMul(w, powers[index + 1], out _); + lo += hi2; + if (hi2 > lo) + { + hi++; + } + if (lo == ulong.MaxValue && q is < -27 or > 55) + { + return false; + } + } + int upperBit = (int)(hi >> 63); + int shift = upperBit + 9; + ulong m = hi >> shift; + int biasedExponent = ((217706 * q) >> 16) + 63 + upperBit - lz + 1023; + if (biasedExponent <= 0) + { + return false; + } + if (lo <= 1 && q is >= -4 and <= 23 && (m & 3) == 1 && (m << shift) == hi) + { + m &= ~1UL; + } + m += m & 1; + m >>= 1; + if (m >= 2UL << 52) + { + m = 1UL << 52; + biasedExponent++; + } + m &= ~(1UL << 52); + if (biasedExponent >= 0x7FF) + { + return false; + } + value = BitConverter.UInt64BitsToDouble(m | ((ulong)biasedExponent << 52) | (neg ? 1UL << 63 : 0)); + return true; + } + + // fast_float's digit loop: eight digits per step while the next eight bytes are all digits, then one + // four-digit step. Integer parts stay byte by byte, since most are too short for a step to pay off. + // FastInteger reads with it too, as zero-padded fixed-width integers are long. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static int ReadDigits(ReadOnlySpan s, int i, ref ulong mantissa) + { + while (s.Length - i >= 8) + { + ulong eight = BinaryPrimitives.ReadUInt64LittleEndian(s[i..]); + if ((((eight + 0x4646464646464646) | (eight - 0x3030303030303030)) & 0x8080808080808080) != 0) + { + break; + } + mantissa = (mantissa * 100_000_000) + ParseEightDigits(eight); + i += 8; + } + if (s.Length - i >= 4) + { + uint four = BinaryPrimitives.ReadUInt32LittleEndian(s[i..]); + if ((((four + 0x46464646) | (four - 0x30303030)) & 0x80808080) == 0) + { + mantissa = (mantissa * 10_000) + ParseFourDigits(four); + i += 4; + } + } + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + break; + } + mantissa = (mantissa * 10) + d; + } + return i; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ulong ParseEightDigits(ulong eight) + { + eight -= 0x3030303030303030; + eight = (eight * 10) + (eight >> 8); + return (((eight & 0x000000FF000000FF) * 0x000F424000000064) + (((eight >> 16) & 0x000000FF000000FF) * 0x0000271000000001)) >> 32; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static uint ParseFourDigits(uint four) + { + four -= 0x30303030; + four = (four * 10) + (four >> 8); + return (((four & 0x00FF00FF) * 0x00640001) >> 16) & 0xFFFF; + } + + // A leading non-zero digit makes every digit significant, so only leading zeros need the count. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool HasTooManyDigits(ReadOnlySpan number, int allDigits, byte decimalSeparator) + { + if (allDigits <= MaxMantissaDigits) + { + return false; + } + if (number[0] is >= (byte)'1' and <= (byte)'9') + { + return true; + } + return SignificantDigits(number, decimalSeparator) > MaxMantissaDigits; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static int SignificantDigits(ReadOnlySpan number, byte decimalSeparator) + { + int digits = 0; + foreach (ref readonly byte c in number) + { + if (c != decimalSeparator && (digits > 0 || c != (byte)'0')) + { + digits++; + } + } + return digits; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static bool TryExponent(ReadOnlySpan s, out int exponent) + { + exponent = 0; + if (s.IsEmpty) + { + return false; + } + int i = 0; + bool neg = s[0] == (byte)'-'; + if (neg || s[0] == (byte)'+') + { + i = 1; + } + if (i >= s.Length || s.Length - i > MaxExponentDigits) + { + return false; + } + int value = 0; + for (; i < s.Length; i++) + { + uint d = (uint)(s[i] - (byte)'0'); + if (d > 9) + { + return false; + } + value = (value * 10) + (int)d; + } + exponent = neg ? -value : value; + return true; + } + + private static double Pow10(int scale) + { + return PowersOfTen[scale]; + } + + private static ReadOnlySpan PowersOfTen => + [ + 1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9, 1e10, 1e11, + 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19, 1e20, 1e21, 1e22, + ]; + + // ponytail: fast_float's 128-bit truncated 5^q table cut to q in [-64, 64]; widen to its full + // [-342, 308] range if values outside 1e±64 need the fast path (they fall back to the BCL now). + private static ReadOnlySpan PowersOfFive => + [ + 0xA87FEA27A539E9A5, 0x3F2398D747B36224, 0xD29FE4B18E88640E, 0x8EEC7F0D19A03AAD, + 0x83A3EEEEF9153E89, 0x1953CF68300424AC, 0xA48CEAAAB75A8E2B, 0x5FA8C3423C052DD7, + 0xCDB02555653131B6, 0x3792F412CB06794D, 0x808E17555F3EBF11, 0xE2BBD88BBEE40BD0, + 0xA0B19D2AB70E6ED6, 0x5B6ACEAEAE9D0EC4, 0xC8DE047564D20A8B, 0xF245825A5A445275, + 0xFB158592BE068D2E, 0xEED6E2F0F0D56712, 0x9CED737BB6C4183D, 0x55464DD69685606B, + 0xC428D05AA4751E4C, 0xAA97E14C3C26B886, 0xF53304714D9265DF, 0xD53DD99F4B3066A8, + 0x993FE2C6D07B7FAB, 0xE546A8038EFE4029, 0xBF8FDB78849A5F96, 0xDE98520472BDD033, + 0xEF73D256A5C0F77C, 0x963E66858F6D4440, 0x95A8637627989AAD, 0xDDE7001379A44AA8, + 0xBB127C53B17EC159, 0x5560C018580D5D52, 0xE9D71B689DDE71AF, 0xAAB8F01E6E10B4A6, + 0x9226712162AB070D, 0xCAB3961304CA70E8, 0xB6B00D69BB55C8D1, 0x3D607B97C5FD0D22, + 0xE45C10C42A2B3B05, 0x8CB89A7DB77C506A, 0x8EB98A7A9A5B04E3, 0x77F3608E92ADB242, + 0xB267ED1940F1C61C, 0x55F038B237591ED3, 0xDF01E85F912E37A3, 0x6B6C46DEC52F6688, + 0x8B61313BBABCE2C6, 0x2323AC4B3B3DA015, 0xAE397D8AA96C1B77, 0xABEC975E0A0D081A, + 0xD9C7DCED53C72255, 0x96E7BD358C904A21, 0x881CEA14545C7575, 0x7E50D64177DA2E54, + 0xAA242499697392D2, 0xDDE50BD1D5D0B9E9, 0xD4AD2DBFC3D07787, 0x955E4EC64B44E864, + 0x84EC3C97DA624AB4, 0xBD5AF13BEF0B113E, 0xA6274BBDD0FADD61, 0xECB1AD8AEACDD58E, + 0xCFB11EAD453994BA, 0x67DE18EDA5814AF2, 0x81CEB32C4B43FCF4, 0x80EACF948770CED7, + 0xA2425FF75E14FC31, 0xA1258379A94D028D, 0xCAD2F7F5359A3B3E, 0x096EE45813A04330, + 0xFD87B5F28300CA0D, 0x8BCA9D6E188853FC, 0x9E74D1B791E07E48, 0x775EA264CF55347E, + 0xC612062576589DDA, 0x95364AFE032A819E, 0xF79687AED3EEC551, 0x3A83DDBD83F52205, + 0x9ABE14CD44753B52, 0xC4926A9672793543, 0xC16D9A0095928A27, 0x75B7053C0F178294, + 0xF1C90080BAF72CB1, 0x5324C68B12DD6339, 0x971DA05074DA7BEE, 0xD3F6FC16EBCA5E04, + 0xBCE5086492111AEA, 0x88F4BB1CA6BCF585, 0xEC1E4A7DB69561A5, 0x2B31E9E3D06C32E6, + 0x9392EE8E921D5D07, 0x3AFF322E62439FD0, 0xB877AA3236A4B449, 0x09BEFEB9FAD487C3, + 0xE69594BEC44DE15B, 0x4C2EBE687989A9B4, 0x901D7CF73AB0ACD9, 0x0F9D37014BF60A11, + 0xB424DC35095CD80F, 0x538484C19EF38C95, 0xE12E13424BB40E13, 0x2865A5F206B06FBA, + 0x8CBCCC096F5088CB, 0xF93F87B7442E45D4, 0xAFEBFF0BCB24AAFE, 0xF78F69A51539D749, + 0xDBE6FECEBDEDD5BE, 0xB573440E5A884D1C, 0x89705F4136B4A597, 0x31680A88F8953031, + 0xABCC77118461CEFC, 0xFDC20D2B36BA7C3E, 0xD6BF94D5E57A42BC, 0x3D32907604691B4D, + 0x8637BD05AF6C69B5, 0xA63F9A49C2C1B110, 0xA7C5AC471B478423, 0x0FCF80DC33721D54, + 0xD1B71758E219652B, 0xD3C36113404EA4A9, 0x83126E978D4FDF3B, 0x645A1CAC083126EA, + 0xA3D70A3D70A3D70A, 0x3D70A3D70A3D70A4, 0xCCCCCCCCCCCCCCCC, 0xCCCCCCCCCCCCCCCD, + 0x8000000000000000, 0x0000000000000000, 0xA000000000000000, 0x0000000000000000, + 0xC800000000000000, 0x0000000000000000, 0xFA00000000000000, 0x0000000000000000, + 0x9C40000000000000, 0x0000000000000000, 0xC350000000000000, 0x0000000000000000, + 0xF424000000000000, 0x0000000000000000, 0x9896800000000000, 0x0000000000000000, + 0xBEBC200000000000, 0x0000000000000000, 0xEE6B280000000000, 0x0000000000000000, + 0x9502F90000000000, 0x0000000000000000, 0xBA43B74000000000, 0x0000000000000000, + 0xE8D4A51000000000, 0x0000000000000000, 0x9184E72A00000000, 0x0000000000000000, + 0xB5E620F480000000, 0x0000000000000000, 0xE35FA931A0000000, 0x0000000000000000, + 0x8E1BC9BF04000000, 0x0000000000000000, 0xB1A2BC2EC5000000, 0x0000000000000000, + 0xDE0B6B3A76400000, 0x0000000000000000, 0x8AC7230489E80000, 0x0000000000000000, + 0xAD78EBC5AC620000, 0x0000000000000000, 0xD8D726B7177A8000, 0x0000000000000000, + 0x878678326EAC9000, 0x0000000000000000, 0xA968163F0A57B400, 0x0000000000000000, + 0xD3C21BCECCEDA100, 0x0000000000000000, 0x84595161401484A0, 0x0000000000000000, + 0xA56FA5B99019A5C8, 0x0000000000000000, 0xCECB8F27F4200F3A, 0x0000000000000000, + 0x813F3978F8940984, 0x4000000000000000, 0xA18F07D736B90BE5, 0x5000000000000000, + 0xC9F2C9CD04674EDE, 0xA400000000000000, 0xFC6F7C4045812296, 0x4D00000000000000, + 0x9DC5ADA82B70B59D, 0xF020000000000000, 0xC5371912364CE305, 0x6C28000000000000, + 0xF684DF56C3E01BC6, 0xC732000000000000, 0x9A130B963A6C115C, 0x3C7F400000000000, + 0xC097CE7BC90715B3, 0x4B9F100000000000, 0xF0BDC21ABB48DB20, 0x1E86D40000000000, + 0x96769950B50D88F4, 0x1314448000000000, 0xBC143FA4E250EB31, 0x17D955A000000000, + 0xEB194F8E1AE525FD, 0x5DCFAB0800000000, 0x92EFD1B8D0CF37BE, 0x5AA1CAE500000000, + 0xB7ABC627050305AD, 0xF14A3D9E40000000, 0xE596B7B0C643C719, 0x6D9CCD05D0000000, + 0x8F7E32CE7BEA5C6F, 0xE4820023A2000000, 0xB35DBF821AE4F38B, 0xDDA2802C8A800000, + 0xE0352F62A19E306E, 0xD50B2037AD200000, 0x8C213D9DA502DE45, 0x4526F422CC340000, + 0xAF298D050E4395D6, 0x9670B12B7F410000, 0xDAF3F04651D47B4C, 0x3C0CDD765F114000, + 0x88D8762BF324CD0F, 0xA5880A69FB6AC800, 0xAB0E93B6EFEE0053, 0x8EEA0D047A457A00, + 0xD5D238A4ABE98068, 0x72A4904598D6D880, 0x85A36366EB71F041, 0x47A6DA2B7F864750, + 0xA70C3C40A64E6C51, 0x999090B65F67D924, 0xD0CF4B50CFE20765, 0xFFF4B4E3F741CF6D, + 0x82818F1281ED449F, 0xBFF8F10E7A8921A4, 0xA321F2D7226895C7, 0xAFF72D52192B6A0D, + 0xCBEA6F8CEB02BB39, 0x9BF4F8A69F764490, 0xFEE50B7025C36A08, 0x02F236D04753D5B4, + 0x9F4F2726179A2245, 0x01D762422C946590, 0xC722F0EF9D80AAD6, 0x424D3AD2B7B97EF5, + 0xF8EBAD2B84E0D58B, 0xD2E0898765A7DEB2, 0x9B934C3B330C8577, 0x63CC55F49F88EB2F, + 0xC2781F49FFCFA6D5, 0x3CBF6B71C76B25FB + ]; + } +} diff --git a/src/FixedWidthParser/Infrastructure/FastInteger.cs b/src/FixedWidthParser/Infrastructure/FastInteger.cs new file mode 100644 index 0000000..db50bdb --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FastInteger.cs @@ -0,0 +1,76 @@ +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// ExcelReader's ASCII-digit fast path for integers: digits only, after a "-" when the provider's negative + /// sign is "-", and few enough digits that the value cannot overflow. for anything + /// else, and the caller falls back to the BCL, which reads such text the same way in every culture. + /// + internal static class FastInteger + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan s, bool allowMinus, int maxDigits, out long value) + { + value = 0; + bool negative = allowMinus && !s.IsEmpty && s[0] == (byte)'-'; + if (negative) + { + s = s[1..]; + } + if (s.IsEmpty || s.Length > maxDigits) + { + return false; + } + ulong accumulated = 0; + if (FastDouble.ReadDigits(s, 0, ref accumulated) != s.Length) + { + return false; + } + value = negative ? -(long)accumulated : (long)accumulated; + return true; + } + + /// + /// Writes right-aligned into the whole of in one pass, as + /// the default format and would together: digits from the + /// right, then , with "-" ahead of the padding unless it is a space. + /// when it does not fit, leaving partly written. + /// + public static bool TryFormatRight(long value, Span slice, byte padding) + { + // Unsigned, so long.MinValue negates. + ulong magnitude = value < 0 ? 0 - (ulong)value : (ulong)value; + int i = slice.Length; + do + { + if (i == 0) + { + return false; + } + ulong quotient = magnitude / 10; + slice[--i] = (byte)('0' + (magnitude - (quotient * 10))); + magnitude = quotient; + } + while (magnitude != 0); + if (value >= 0) + { + slice[..i].Fill(padding); + return true; + } + if (i == 0) + { + return false; + } + if (padding == (byte)' ') + { + slice[--i] = (byte)'-'; + slice[..i].Fill(padding); + return true; + } + slice[0] = (byte)'-'; + slice[1..i].Fill(padding); + return true; + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs new file mode 100644 index 0000000..0de00f7 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FixedWidthAsyncRecordEnumerable.cs @@ -0,0 +1,52 @@ +using System.ComponentModel; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// Asynchronous version of for + /// await foreach. A stream is read once; a file is reopened, with asynchronous I/O, on each + /// enumeration. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class FixedWidthAsyncRecordEnumerable : IAsyncEnumerable + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + private readonly TParser _parser; + private readonly Stream? _stream; + private readonly string? _path; + private readonly bool _ownsStream; + private readonly FixedWidthReaderOptions _options; + + internal FixedWidthAsyncRecordEnumerable(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options) + { + _parser = parser; + _stream = stream; + _ownsStream = ownsStream; + _options = options; + } + + internal FixedWidthAsyncRecordEnumerable(TParser parser, string path, FixedWidthReaderOptions options) + { + _parser = parser; + _path = path; + _ownsStream = true; + _options = options; + } + +#pragma warning disable HLQ006, VSTHRD200 // GetEnumerator() or GetAsyncEnumerator() should return a value type + public AsyncRecordEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) +#pragma warning restore HLQ006, VSTHRD200 + { +#pragma warning disable CA2000, IDISP001 // The enumerator owns and disposes the file stream. + var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.Asynchronous | FileOptions.SequentialScan); +#pragma warning restore CA2000, IDISP001 + return new(_parser, stream, _ownsStream, _options, cancellationToken); + } + + IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) + { + return GetAsyncEnumerator(cancellationToken); + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs new file mode 100644 index 0000000..9f3f828 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRecordEnumerable.cs @@ -0,0 +1,60 @@ +using System.Collections; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// A lazily-read sequence of models from a stream or a file, without a string per line: bytes are + /// read in blocks into a pooled buffer and each line is parsed in place. Exposes a + /// enumerator for allocation-free foreach. A stream is read once; a + /// file is reopened on each enumeration. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class FixedWidthRecordEnumerable : IEnumerable + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + private readonly TParser _parser; + private readonly Stream? _stream; + private readonly string? _path; + private readonly bool _ownsStream; + private readonly FixedWidthReaderOptions _options; + + internal FixedWidthRecordEnumerable(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options) + { + _parser = parser; + _stream = stream; + _ownsStream = ownsStream; + _options = options; + } + + internal FixedWidthRecordEnumerable(TParser parser, string path, FixedWidthReaderOptions options) + { + _parser = parser; + _path = path; + _ownsStream = true; + _options = options; + } + + /// Struct enumerator: foreach iteration without heap allocation. + public RecordEnumerator GetEnumerator() + { +#pragma warning disable CA2000, IDISP001 // The enumerator owns and disposes the file stream. + var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); +#pragma warning restore CA2000, IDISP001 + return new(_parser, stream, _ownsStream, _options); + } + + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + [ExcludeFromCodeCoverage] + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs new file mode 100644 index 0000000..5150050 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/FixedWidthRuntime.cs @@ -0,0 +1,566 @@ +using System.Buffers.Text; +using System.Buffers; +using System.Collections.Frozen; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Text.Unicode; +using System.Numerics; +using System.Text; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// Low-level helpers called by the source-generated + /// and implementations, and by the runtime parser + /// and writer, so both paths share one set of semantics. + /// Public because the generated code lives in the consumer assembly; not intended for direct use. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public static class FixedWidthRuntime + { + private const int StackBufferSize = 256; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static ReadOnlySpan TrimColumn(ReadOnlySpan column, in ColumnParseOptions options) + { + return options.TrimMode switch + { + TrimMode.Leading => column.TrimStart(options.TrimByte), + TrimMode.Both => column.Trim(options.TrimByte), + _ => column.TrimEnd(options.TrimByte), + }; + } + + /// True when the column holds only the trim character and spaces, so a nullable column reads as . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool IsBlank(ReadOnlySpan column, in ColumnParseOptions options) + { + return column.Trim(options.TrimByte).Trim((byte)' ').IsEmpty; + } + + /// Decodes a string column with the context's encoding, interning it through the context's pool when there is one. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static string String(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options) + { + var trimmed = TrimColumn(column, in options); + return context.StringPool is { } pool ? Utf8StringCache.GetOrAdd(pool, trimmed, context.Encoding) : context.Encoding.GetString(trimmed); + } + + // The fast parsers need the provider's decimal separator as one ASCII byte and "-" as its negative sign + // (FixedWidthContext.DecimalSeparator). What they refuse, thousands separators included, goes to the + // BCL over the decoded text: its UTF-8 parser does not take a space for a no-break-space group + // separator, as fr-FR writes one. Spaces are trimmed too, so a right-aligned column stays fast. + private const NumberStyles FloatStyles = NumberStyles.Float | NumberStyles.AllowThousands; + + /// Parses a column, honoring the provider's separators. + public static bool TryDouble(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out double value) + { + var trimmed = TrimColumn(column, in options).Trim((byte)' '); + byte separator = context.DecimalSeparator; + if (separator != 0 && FastDouble.TryParse(trimmed, separator, out value)) + { + return true; + } + return ParseDecoded(trimmed, in context, null, static (text, _, provider, out result) => double.TryParse(text, FloatStyles, provider, out result), out value); + } + + /// Parses a column, honoring the provider's separators. + public static bool TryFloat(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out float value) + { + var trimmed = TrimColumn(column, in options).Trim((byte)' '); + byte separator = context.DecimalSeparator; + if (separator != 0 && FastDouble.TryParse(trimmed, separator, out value)) + { + return true; + } + return ParseDecoded(trimmed, in context, null, static (text, _, provider, out result) => float.TryParse(text, FloatStyles, provider, out result), out value); + } + + /// Parses an column straight from its bytes. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryParse(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) + where TValue : IUtf8SpanParsable + { + // The BCL's default styles for these types allow surrounding white space, so trimming spaces for + // the fast paths changes no result. + var trimmed = TrimColumn(column, in options); + if (typeof(TValue) == typeof(long) || typeof(TValue) == typeof(int)) + { + bool isInt = typeof(TValue) == typeof(int); + if (FastInteger.TryParse(trimmed.Trim((byte)' '), context.DecimalSeparator != 0, isInt ? 9 : 18, out long integer)) + { + if (isInt) + { + int narrow = (int)integer; + value = Unsafe.As(ref narrow); + return true; + } + value = Unsafe.As(ref integer); + return true; + } + } + if (typeof(TValue) == typeof(decimal) && context.DecimalSeparator is var separator and not 0 + && FastDecimal.TryParse(trimmed.Trim((byte)' '), separator, out decimal fast)) + { + value = Unsafe.As(ref fast); + return true; + } + return TValue.TryParse(trimmed, context.FormatProvider, out value); + } + + /// Parses an column that has no UTF-8 parser, decoding it with the context's encoding first. + public static bool TryParseDecoded(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) + where TValue : ISpanParsable + { + return ParseDecoded(TrimColumn(column, in options), in context, null, static (text, _, provider, [MaybeNullWhen(false)] out result) => TValue.TryParse(text, provider, out result), out value); + } + + /// + /// Parses an enum column, as ExcelReader does: a member's name in any case, or its number (2, 02). + /// A number no member has, or a combination of flags, does not parse. + /// + [SkipLocalsInit] + public static bool TryEnum(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out TEnum value) + where TEnum : struct, Enum + { + var trimmed = TrimColumn(column, in options); + // ASCII reads the same in every supported encoding, and widening it skips the decoder. + Span chars = stackalloc char[64]; + if (trimmed.Length <= chars.Length && Ascii.ToUtf16(trimmed, chars, out int written) == OperationStatus.Done) + { + return EnumNames.TryLookup(chars[..written], out value); + } + return ParseDecoded(trimmed, in context, null, static (text, _, _, out result) => EnumNames.TryLookup(text, out result), out value); + } + + // ExcelReader's EnumCache: every member under its name and its number, in one case-insensitive frozen map. + // With the ASCII widening in TryEnum, a column costs what an int column does (~17 ns), where Enum.TryParse + // with Enum.IsDefined took ~29 ns. + private static class EnumNames + where TEnum : struct, Enum + { + private static readonly FrozenDictionary.AlternateLookup> Lookup = Build(); + + private static FrozenDictionary.AlternateLookup> Build() + { + var map = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (TEnum member in Enum.GetValues()) + { + map[member.ToString()] = member; + map[member.ToString("D")] = member; + } + return map.ToFrozenDictionary(StringComparer.OrdinalIgnoreCase).GetAlternateLookup>(); + } + + public static bool TryLookup(ReadOnlySpan text, out TEnum value) + { + text = text.Trim(); + if (Lookup.TryGetValue(text, out value)) + { + return true; + } + // A zero-padded code is its number: 02 is 2, and 00 is 0. + int digits = text.IndexOfAnyExcept('0'); + if (digits == 0 || text.IsEmpty) + { + return false; + } + var number = digits < 0 ? "0" : text[digits..]; + return !number.ContainsAnyExceptInRange('0', '9') && Lookup.TryGetValue(number, out value); + } + } + + /// Parses a column through an instance. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static bool TryConvert( + ReadOnlySpan column, in FixedWidthContext context, TConverter converter, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value) + where TConverter : IFixedWidthConverter + { + return converter.TryParse(TrimColumn(column, in options), in context, out value); + } + + /// Parses a column with exactly , decoding it first. + public static bool TryDateTimeExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateTime value) + { + var trimmed = TrimColumn(column, in options); + if (options.FastDate is { HasDate: true } fast && fast.TryParse(trimmed, in context, out value)) + { + return true; + } + return ParseDecoded(trimmed, in context, options.Format, static (text, format, provider, out result) => DateTime.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + } + + /// Parses a column with exactly , decoding it first. + public static bool TryDateOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateOnly value) + { + var trimmed = TrimColumn(column, in options); + if (options.FastDate is { HasDate: true, HasTime: false } fast && fast.TryParse(trimmed, in context, out var date)) + { + value = DateOnly.FromDateTime(date); + return true; + } + return ParseDecoded(trimmed, in context, options.Format, static (text, format, provider, out result) => DateOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + } + + /// Parses a column with exactly , decoding it first. + public static bool TryTimeOnlyExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out TimeOnly value) + { + var trimmed = TrimColumn(column, in options); + if (options.FastDate is { HasDate: false } fast && fast.TryParse(trimmed, in context, out var time)) + { + value = TimeOnly.FromDateTime(time); + return true; + } + return ParseDecoded(trimmed, in context, options.Format, static (text, format, provider, out result) => TimeOnly.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + } + + /// Parses a column with exactly , decoding it first. + public static bool TryDateTimeOffsetExact(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, out DateTimeOffset value) + { + return ParseDecoded(TrimColumn(column, in options), in context, options.Format, static (text, format, provider, out result) => DateTimeOffset.TryParseExact(text, format, provider, DateTimeStyles.None, out result), out value); + } + + private delegate bool CharParser(ReadOnlySpan text, string? format, IFormatProvider provider, [MaybeNullWhen(false)] out TValue value); + + // Decodes already-trimmed bytes with the context's encoding, on the stack when they fit. + [SkipLocalsInit] + private static bool ParseDecoded(ReadOnlySpan bytes, in FixedWidthContext context, string? format, CharParser parse, [MaybeNullWhen(false)] out TValue value) + { + Span stack = stackalloc char[StackBufferSize]; + var text = Decode(bytes, context.Encoding, stack, out char[]? rented); + try + { + return parse(text, format, context.FormatProvider, out value); + } + finally + { + Return(rented); + } + } + + /// + /// Encodes a string column into per . A string + /// truncated to fit keeps whole characters, so a multi-byte character is never split. + /// + public static void FormatString(string value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) + { + // Left-aligned UTF-8 transcodes straight into the slice; FromUtf16 stops at a whole character, + // which is exactly the truncation. Overflow that throws falls through to report it. + // ponytail: lone surrogates become U+FFFD here even for a throwing UTF8Encoding. + if (context.IsUtf8 && options.Alignment == Alignment.Left) + { + var status = Utf8.FromUtf16(value, slice, out _, out int written); + if (status == OperationStatus.Done || options.Overflow == OverflowBehavior.Truncate) + { + slice[written..].Fill((byte)options.Padding); + return; + } + } + + ReadOnlySpan text = value; + var encoding = context.Encoding; + if (options.Overflow == OverflowBehavior.Truncate && encoding.GetByteCount(text) > slice.Length) + { + text = FitText(text, slice.Length, encoding, options.Alignment); + } + WriteText(text, slice, encoding, options, columnName); + } + + /// + /// Formats an column into per + /// : tries a stack buffer first, then grows via a bounded + /// loop for the rare value that needs more room. + /// + [SkipLocalsInit] + public static void FormatValue(TValue value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) + where TValue : ISpanFormattable + { + var formatProvider = context.FormatProvider; + Span stack = stackalloc char[64]; + if (value.TryFormat(stack, out int written, options.Format, formatProvider)) + { + WriteText(stack[..written], slice, context.Encoding, options, columnName); + return; + } + + int maxSize = Math.Max(1024, slice.Length * 16); + for (int size = 512; size <= maxSize; size *= 2) + { + char[] rented = ArrayPool.Shared.Rent(size); + try + { + if (value.TryFormat(rented, out int w, options.Format, formatProvider)) + { + WriteText(rented.AsSpan(0, w), slice, context.Encoding, options, columnName); + return; + } + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + throw new InvalidOperationException( + $"Value of type \"{typeof(TValue)}\" for column \"{columnName}\" could not be formatted " + + $"within {maxSize} characters (format \"{options.Format}\")."); + } + + /// + /// Formats an column straight into , with + /// no intermediate buffer or transcode. The UTF-8 result is kept when the encoding is UTF-8 or the + /// result is ASCII (numbers and dates, almost always); otherwise, and for a value wider than the + /// column, it falls back to , which re-encodes or reports the overflow. + /// + public static void FormatUtf8Value(TValue value, Span slice, in FixedWidthContext context, ColumnFormatOptions options, string columnName) + where TValue : IUtf8SpanFormattable, ISpanFormattable + { + // The default format writes ASCII digits after the provider's negative sign, which DecimalSeparator + // being set guarantees is "-". + if ((typeof(TValue) == typeof(long) || typeof(TValue) == typeof(int)) && options.Format is null + && options.Alignment == Alignment.Right && context.DecimalSeparator != 0) + { + long integer = typeof(TValue) == typeof(int) ? Unsafe.As(ref value) : Unsafe.As(ref value); + if (FastInteger.TryFormatRight(integer, slice, (byte)options.Padding)) + { + return; + } + } + if (TryFormatInto(value, slice, in context, options, out int written) && (context.IsUtf8 || Ascii.IsValid(slice[..written]))) + { + options.AlignInPlace(slice, written); + return; + } + FormatValue(value, slice, in context, options, columnName); + } + + private static bool TryFormatInto(TValue value, Span destination, in FixedWidthContext context, ColumnFormatOptions options, out int written) + where TValue : IUtf8SpanFormattable + { + if (typeof(TValue) == typeof(double) && options.Format is null) + { + return TryFormatDouble(Unsafe.As(ref value), destination, in context, out written); + } + if (typeof(TValue) == typeof(DateOnly) && options.FastDate is { HasDate: true, HasTime: false } dateFormat) + { + return dateFormat.TryFormat(Unsafe.As(ref value).ToDateTime(TimeOnly.MinValue), destination, in context, out written); + } + if (typeof(TValue) == typeof(TimeOnly) && options.FastDate is { HasDate: false } timeFormat) + { + return timeFormat.TryFormat(DateTime.MinValue + Unsafe.As(ref value).ToTimeSpan(), destination, in context, out written); + } + if (typeof(TValue) == typeof(DateTime) && options.FastDate is { } dateTimeFormat) + { + return dateTimeFormat.TryFormat(Unsafe.As(ref value), destination, in context, out written); + } + return value.TryFormat(destination, out written, options.Format, context.FormatProvider); + } + + /// + /// Byte-identical to + /// with no format. A value with at most four fractional digits and a magnitude under 1e9 is written + /// as a scaled integer; the smallest scale whose quotient reproduces the value exactly is the + /// shortest round-trip form. + /// + internal static bool TryFormatDouble(double value, Span destination, in FixedWidthContext context, out int written) + { + byte separator = context.DecimalSeparator; + if (separator != 0 && value != 0 && Math.Abs(value) < 1e9) + { + ReadOnlySpan pow10 = [1, 10, 100, 1000, 10000]; + for (int scale = 0; scale < pow10.Length; scale++) + { + double scaled = Math.Round(value * pow10[scale]); + if (scaled / pow10[scale] == value) + { + return TryFormatScaled((long)scaled, scale, separator, destination, out written); + } + } + } + return value.TryFormat(destination, out written, default, context.FormatProvider); + } + + private static bool TryFormatScaled(long scaled, int scale, byte separator, Span destination, out int written) + { + written = 0; + if (scaled < 0) + { + if (destination.IsEmpty) + { + return false; + } + destination[written++] = (byte)'-'; + } + ulong magnitude = (ulong)Math.Abs(scaled); + ulong divisor = scale switch { 0 => 1, 1 => 10, 2 => 100, 3 => 1000, _ => 10000 }; + if (!Utf8Formatter.TryFormat(magnitude / divisor, destination[written..], out int n)) + { + return false; + } + written += n; + if (scale == 0) + { + return true; + } + if (destination.Length < written + 1 + scale) + { + return false; + } + destination[written++] = separator; + Utf8Formatter.TryFormat(magnitude % divisor, destination[written..], out n, new StandardFormat('D', (byte)scale)); + written += n; + return true; + } + + /// + /// Formats a column via a FixedColumnAttribute.Converter instance into + /// per , with the same stack-then-ArrayPool growth as . + /// + [SkipLocalsInit] + public static void FormatConvert( + TValue value, Span slice, in FixedWidthContext context, TConverter converter, ColumnFormatOptions options, string columnName) + where TConverter : IFixedWidthConverter + { + Span stack = stackalloc byte[StackBufferSize]; + if (converter.TryFormat(value, stack, in context, out int written)) + { + options.WriteInto(stack[..written], slice, columnName); + return; + } + + int maxSize = Math.Max(4096, slice.Length * 16); + for (int size = 1024; size <= maxSize; size *= 2) + { + byte[] rented = ArrayPool.Shared.Rent(size); + try + { + if (converter.TryFormat(value, rented, in context, out int w)) + { + options.WriteInto(rented.AsSpan(0, w), slice, columnName); + return; + } + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + throw new InvalidOperationException( + $"Value of type \"{typeof(TValue)}\" for column \"{columnName}\" could not be formatted by " + + $"converter \"{typeof(TConverter)}\" within {maxSize} bytes."); + } + + internal static byte ToAsciiByte(char value, string columnName, string setting) + { + if (value > '\x7F') + { + throw new InvalidOperationException( + $"The {setting} '{value}' (U+{(int)value:X4}) of column \"{columnName}\" is not an ASCII character; column positions are bytes, so it must be one byte in every supported encoding."); + } + return (byte)value; + } + + // Numbers and dates are ASCII in almost every culture, so they narrow straight into bytes; + // anything else goes through the encoding. + [SkipLocalsInit] + private static void WriteText(ReadOnlySpan text, Span slice, Encoding encoding, ColumnFormatOptions options, string columnName) + { + bool ascii = Ascii.IsValid(text); + int byteCount = ascii ? text.Length : encoding.GetByteCount(text); + byte[]? rented = byteCount <= StackBufferSize ? null : ArrayPool.Shared.Rent(byteCount); + Span bytes = rented is null ? stackalloc byte[StackBufferSize] : rented; + try + { + int written; + if (ascii) + { + Ascii.FromUtf16(text, bytes, out written); + } + else + { + written = encoding.GetBytes(text, bytes); + } + options.WriteInto(bytes[..written], slice, columnName); + } + finally + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } + } + } + + // The longest run of whole characters, from the alignment side, whose encoding fits in width bytes. + private static ReadOnlySpan FitText(ReadOnlySpan text, int width, Encoding encoding, Alignment alignment) + { + int bytes = 0; + if (alignment == Alignment.Right) + { + int start = text.Length; + while (start > 0) + { + Rune.DecodeLastFromUtf16(text[..start], out _, out int consumed); + int size = encoding.GetByteCount(text.Slice(start - consumed, consumed)); + if (bytes + size > width) + { + break; + } + bytes += size; + start -= consumed; + } + return text[start..]; + } + + int end = 0; + while (end < text.Length) + { + Rune.DecodeFromUtf16(text[end..], out _, out int consumed); + int size = encoding.GetByteCount(text.Slice(end, consumed)); + if (bytes + size > width) + { + break; + } + bytes += size; + end += consumed; + } + return text[..end]; + } + + private static ReadOnlySpan Decode(ReadOnlySpan bytes, Encoding encoding, Span stack, out char[]? rented) + { + rented = null; + Span chars = stack; + int max = encoding.GetMaxCharCount(bytes.Length); + if (max > stack.Length) + { + rented = ArrayPool.Shared.Rent(max); + chars = rented; + } + int written; + try + { + written = encoding.GetChars(bytes, chars); + } + catch + { + // A strict decoder throws on invalid bytes, before the caller owns the rental. + Return(rented); + rented = null; + throw; + } + return chars[..written]; + } + + private static void Return(char[]? rented) + { + if (rented is not null) + { + ArrayPool.Shared.Return(rented); + } + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs new file mode 100644 index 0000000..c6b088a --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/RecordEnumerator.cs @@ -0,0 +1,100 @@ +using System.Collections; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// Synchronous record enumerator over a stream. A specialized by + /// , so foreach allocates nothing and parsing devirtualizes. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public struct RecordEnumerator : IEnumerator + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + private readonly TParser _parser; + private readonly FixedWidthContext _context; + private readonly Func? _onParseError; + private readonly Stream _stream; + private readonly bool _ownsStream; + private LineBufferState _lines; + private bool _disposed; + + internal RecordEnumerator(TParser parser, Stream stream, bool ownsStream, FixedWidthReaderOptions options) + { + _parser = parser; + _context = options.Context; + _onParseError = options.OnParseError; + _stream = stream; + _ownsStream = ownsStream; + _lines = default; + _lines.Rent(options); + _disposed = false; + } + + /// The record on the line the enumerator sits on, parsed on access. + /// + /// Parsed here rather than by because the enumerator cannot hold a field of a + /// ref struct . foreach reads it once per line, so each + /// line is still parsed once; reading it twice parses twice. + /// + /// The line does not parse. + // AsRef: a parser never keeps a reference to the context, so the record cannot capture this struct. + public readonly TModel Current => ParseErrors.Parse(_parser, _lines.CurrentLine, _lines.LineNumber, in Unsafe.AsRef(in _context)); + + [ExcludeFromCodeCoverage] + readonly object IEnumerator.Current => throw new NotSupportedException("Use the generic Current."); + + /// + public bool MoveNext() + { + ObjectDisposedException.ThrowIf(_disposed, nameof(RecordEnumerator)); + while (true) + { + var status = _lines.TryGetLine(out var line); + if (status == LineStatus.Line) + { + if (ParseErrors.Accepts(_parser, line, _lines.LineNumber, in _context, _onParseError)) + { + return true; + } + continue; + } + if (status == LineStatus.End) + { + return false; + } + + _lines.Fill(_stream); + } + } + + /// + public void Dispose() + { + if (_disposed) + { + return; + } + + _lines.Return(); + if (_ownsStream) + { +#pragma warning disable IDISP007 // Disposed only when the enumerable was told it owns the stream. + _stream.Dispose(); +#pragma warning restore IDISP007 + } + _disposed = true; + } + + /// + [ExcludeFromCodeCoverage] + public readonly void Reset() + { + throw new NotSupportedException("Reading is single-pass."); + } + } +} diff --git a/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs new file mode 100644 index 0000000..64bd8c3 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/RecordLineParsers.cs @@ -0,0 +1,52 @@ +using System.ComponentModel; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// Parse strategy the record enumerators are specialized by. Implemented by + /// types so the JIT monomorphizes each (TModel, TParser) pair and + /// devirtualizes into a direct call. Public only because it appears in the + /// enumerator types; the library is its only implementer. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public interface IRecordLineParser where TModel : allows ref struct + { + /// Parses a buffered line into a model. + bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model); + } + + // Strategy structs are plumbing for the generic enumerators; they are never compared, so + // value equality is meaningless for them. +#pragma warning disable CA1815 // Override equals and operator equals on value types + /// Layout parse strategy: forwards to a , built from attributes or fluently. + [EditorBrowsable(EditorBrowsableState.Never)] + public readonly struct LayoutLineParser : IRecordLineParser + where TModel : allows ref struct + { + private readonly FixedWidthLayout _layout; + + internal LayoutLineParser(FixedWidthLayout layout) + { + _layout = layout; + } + + /// + public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) + { + return _layout.TryParse(line, in context, out model); + } + } + + /// Source-generated parse strategy: forwards to the model's static . + [EditorBrowsable(EditorBrowsableState.Never)] + public readonly struct GeneratedLineParser : IRecordLineParser + where TModel : IFixedWidthModel, allows ref struct + { + /// + public bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) + { + return TModel.TryParse(line, in context, out model); + } + } +#pragma warning restore CA1815 +} diff --git a/src/FixedWidthParser/Infrastructure/Utf8StringCache.cs b/src/FixedWidthParser/Infrastructure/Utf8StringCache.cs new file mode 100644 index 0000000..86c8752 --- /dev/null +++ b/src/FixedWidthParser/Infrastructure/Utf8StringCache.cs @@ -0,0 +1,100 @@ +using System.Runtime.CompilerServices; +using System.Text; +using CommunityToolkit.HighPerformance.Buffers; + +namespace FixedWidthParser.Infrastructure +{ + /// + /// A lock-free cache of UTF-8 columns in front of a , one per thread, because the + /// pool's own lookup is slow (docs/reader-performance.md). + /// Fixed-size and 4-way set-associative, as in ExcelReader's Utf8StringCache: a full set evicts its + /// oldest entry, and every hit is re-verified against the bytes, so a collision costs a lookup in the pool, + /// never a wrong string. A miss goes to the pool, so every string returned was handed out by it. + /// + internal sealed class Utf8StringCache + { + private const int SetCount = 1024; + private const int Ways = 4; + private const int MaxKeyLength = 64; + + [ThreadStatic] + private static Utf8StringCache? t_current; + + private readonly StringPool _pool; + private readonly string?[] _values = new string[SetCount * Ways]; + private readonly int[] _hashes = new int[SetCount * Ways]; + + private Utf8StringCache(StringPool pool) + { + _pool = pool; + } + + /// The string holds for , decoded with . + internal static string GetOrAdd(StringPool pool, ReadOnlySpan column, Encoding encoding) + { + if (column.Length > MaxKeyLength || encoding.CodePage != Encoding.UTF8.CodePage) + { + return pool.GetOrAdd(column, encoding); + } + var cache = t_current; + if (cache is null || !ReferenceEquals(cache._pool, pool)) + { + // ponytail: one pool per thread; a thread alternating between pools rebuilds its cache each switch. + t_current = cache = new Utf8StringCache(pool); + } + return cache.GetOrAdd(column, encoding); + } + + private string GetOrAdd(ReadOnlySpan utf8, Encoding encoding) + { + if (utf8.IsEmpty) + { + return string.Empty; + } + int hash = ComputeHash(utf8); + int start = (hash & (SetCount - 1)) * Ways; + for (int i = start; i < start + Ways; i++) + { + string? cached = _values[i]; + if (cached is null) + { + break; + } + if (_hashes[i] == hash && Matches(cached, utf8)) + { + return cached; + } + } + // Decoded with the context's encoding, so a strict decoder still rejects invalid bytes. + string value = _pool.GetOrAdd(utf8, encoding); + Array.Copy(_values, start, _values, start + 1, Ways - 1); + Array.Copy(_hashes, start, _hashes, start + 1, Ways - 1); + _values[start] = value; + _hashes[start] = hash; + return value; + } + + [SkipLocalsInit] + private static bool Matches(string cached, ReadOnlySpan utf8) + { + if (cached.Length > utf8.Length) + { + return false; + } + if (cached.Length == utf8.Length && Ascii.Equals(cached, utf8)) + { + return true; + } + Span buffer = stackalloc byte[MaxKeyLength]; + return Encoding.UTF8.TryGetBytes(cached, buffer, out int written) + && buffer[..written].SequenceEqual(utf8); + } + + private static int ComputeHash(ReadOnlySpan utf8) + { + HashCode hash = default; + hash.AddBytes(utf8); + return hash.ToHashCode(); + } + } +} diff --git a/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs b/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs new file mode 100644 index 0000000..e19d853 --- /dev/null +++ b/src/FixedWidthParser/Layouts/FixedWidthColumnOptions.cs @@ -0,0 +1,29 @@ +namespace FixedWidthParser +{ + /// + /// How one column of a layout is trimmed, parsed and + /// written: the settings carries, with the same defaults. + /// + public sealed class FixedWidthColumnOptions + { + internal static readonly FixedWidthColumnOptions Default = new(); + + /// Which side a written value is aligned to. Defaults to . + public Alignment Alignment { get; init; } = Alignment.Left; + + /// The ASCII character that fills a written column around its value. Defaults to a space. + public char Padding { get; init; } = ' '; + + /// The ASCII character trimmed from a column before it is parsed, and written for a value. Defaults to a space. + public char TrimChar { get; init; } = ' '; + + /// Which ends of a column is trimmed from. Defaults to . + public TrimMode TrimMode { get; init; } = TrimMode.Trailing; + + /// The format for writing, and for parsing a date or time exactly. means the type's default. + public string? Format { get; init; } + + /// What happens when a written value is wider than its column. truncates a string and throws for anything else. + public OverflowBehavior Overflow { get; init; } = OverflowBehavior.Default; + } +} diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayout.cs b/src/FixedWidthParser/Layouts/FixedWidthLayout.cs new file mode 100644 index 0000000..e16dfe9 --- /dev/null +++ b/src/FixedWidthParser/Layouts/FixedWidthLayout.cs @@ -0,0 +1,213 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; +using FixedWidthParser.Attributes; +using FixedWidthParser.Formatters; +using FixedWidthParser.Parsers; + +namespace FixedWidthParser +{ + /// + /// How maps to the columns of a fixed-width line, for reading and for + /// writing. Immutable and safe to share across threads. Create one with + /// and pass it to the methods. + /// Column positions are bytes. + /// + /// The model type. + public sealed class FixedWidthLayout + where TModel : allows ref struct + { + private readonly ColumnParserInfo[]? _parsers; + private readonly Func? _factory; + private readonly IColumnFormatter[]? _formatters; + private readonly ExceptionDispatchInfo? _readError; + private readonly ExceptionDispatchInfo? _writeError; + + internal FixedWidthLayout(ColumnParserInfo[]? parsers, Func? factory, IColumnFormatter[]? formatters, int lineLength, + ExceptionDispatchInfo? readError = null, ExceptionDispatchInfo? writeError = null) + { + _parsers = parsers; + _factory = factory; + _formatters = formatters; + LineLength = lineLength; + _readError = readError; + _writeError = writeError; + } + + /// The end of the last column: the length of a written line, and the shortest line that parses. + internal int LineLength { get; } + + internal bool TryParse(ReadOnlySpan line, in FixedWidthContext context, out TModel model) + { + if (line.Length < LineLength) + { + context.ReportShortLine(LineLength); + model = default!; + return false; + } + + model = _factory!(); + foreach (ref readonly var processor in _parsers.AsSpan()) + { + if (!processor.Parse(line.Slice(processor.Start, processor.Length), in context, ref model)) + { + context.ReportFailedColumn(processor.Name, processor.Start, processor.Length); + return false; + } + } + return true; + } + + internal void FormatLine(in TModel model, Span line, in FixedWidthContext context) + { + line.Fill((byte)' '); + foreach (var formatter in _formatters!) + { + formatter.Format(in model, line, in context); + } + } + + internal void EnsureCanRead() + { + _readError?.Throw(); + if (_parsers is null || _factory is null) + { + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has a column without a setter; it cannot be used to read."); + } + } + + internal void EnsureCanWrite() + { + _writeError?.Throw(); + if (_formatters is null) + { + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has a column without a getter; it cannot be used to write."); + } + } + } + + /// Creates instances. + public static class FixedWidthLayout + { + /// + /// Builds a layout column by column through , binding each column with + /// delegates. Trimming and AOT safe. + /// + /// + /// For a type you cannot annotate, or a layout known only at run time. Each call builds its own + /// layout, so differently configured layouts for the same coexist. + /// + /// is . + /// The layout is invalid: no columns, a negative start, a length below 1, overlapping columns, a non-ASCII padding or trim character, a truncated non-string column, or a reference-type model with setters and no factory. + public static FixedWidthLayout Build(Action> configure) + where TModel : allows ref struct + { + ArgumentNullException.ThrowIfNull(configure); + var builder = new FixedWidthLayoutBuilder(); + configure(builder); + return builder.Build(); + } + + /// + /// Returns the layout declared by 's [FixedColumn] properties + /// and fields, built once per type by reflection. + /// + /// + /// Not compatible with Native AOT, and trimming can remove the members it binds to. A + /// model's layout reads but cannot write. + /// + /// The layout is invalid: no columns, a negative start, a length below 1, overlapping columns, a non-ASCII padding or trim character, or a column type with no parser or formatter. + [RequiresUnreferencedCode("Attribute layouts reflect over TModel's public properties and fields, which trimming may remove.")] + [RequiresDynamicCode("Attribute layouts compile member accessors at runtime (Expression.Compile / MakeGenericType).")] + public static FixedWidthLayout FromAttributes() + where TModel : new(), allows ref struct + { + AttributeLayout.BuildError?.Throw(); + return AttributeLayout.Layout!; + } + + private static class AttributeLayout + where TModel : new(), allows ref struct + { + internal static readonly FixedWidthLayout? Layout; + internal static readonly ExceptionDispatchInfo? BuildError; + +#pragma warning disable CA1810, S3963 // The try/catch needs a static constructor. + static AttributeLayout() + { + try + { + Layout = BuildLayout(); + } +#pragma warning disable CA1031 // Rethrown from FromAttributes on every call. + catch (Exception ex) +#pragma warning restore CA1031 + { + BuildError = ExceptionDispatchInfo.Capture(ex); + } + } +#pragma warning restore CA1810, S3963 + + // A model may be readable and not writable (or the reverse): one side's error waits for that + // side's first use. Only a model neither side can use fails here. + private static FixedWidthLayout BuildLayout() + { + var readError = TryBuildReader(out var parsers, out var factory); + IColumnFormatter[]? formatters = null; + var writeError = ExceptionDispatchInfo.Capture(new InvalidOperationException( + $"{typeof(TModel).Name} is a ref struct; an attribute layout can read it but not write it.")); + if (!typeof(TModel).IsByRefLike) + { + writeError = TryBuildWriter(out formatters); + } + if (readError is not null && formatters is null) + { + readError.Throw(); + } + return new(parsers, factory, formatters, LineLength(), readError, writeError); + } + + private static ExceptionDispatchInfo? TryBuildReader(out ColumnParserInfo[]? parsers, out Func? factory) + { + parsers = null; + factory = null; + try + { + parsers = ParserBuilder.BuildProcessors(); + factory = ParserBuilder.BuildModelFactory(); + return null; + } +#pragma warning disable CA1031 // Rethrown when the layout is first used to read. + catch (Exception ex) +#pragma warning restore CA1031 + { + return ExceptionDispatchInfo.Capture(ex); + } + } + + private static ExceptionDispatchInfo? TryBuildWriter(out IColumnFormatter[]? formatters) + { + formatters = null; + try + { + formatters = FormatterBuilder.Build(); + return null; + } +#pragma warning disable CA1031 // Rethrown when the layout is first used to write. + catch (Exception ex) +#pragma warning restore CA1031 + { + return ExceptionDispatchInfo.Capture(ex); + } + } + + private static int LineLength() + { + int lineLength = 0; + ModelColumns.ForEachColumn(typeof(TModel), (_, attribute) => lineLength = Math.Max(lineLength, attribute.Start + attribute.Length)); + return lineLength; + } + } + } +} diff --git a/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs new file mode 100644 index 0000000..a41f837 --- /dev/null +++ b/src/FixedWidthParser/Layouts/FixedWidthLayoutBuilder.cs @@ -0,0 +1,344 @@ +using System.Runtime.InteropServices; +using FixedWidthParser.Attributes; +using FixedWidthParser.Formatters; +using FixedWidthParser.Infrastructure; +using FixedWidthParser.Parsers; +using FixedWidthParser.Processors; + +namespace FixedWidthParser +{ + /// + /// Describes a column by column, for + /// . Each column binds to the model through a getter, used + /// to write, and a setter, used to read; either may be , which leaves the layout + /// unable to write or to read. No reflection: trimming and AOT safe. + /// Column positions are bytes. + /// + /// The model type. + public sealed class FixedWidthLayoutBuilder + where TModel : allows ref struct + { + private readonly List<(int Start, int Length, string Name)> _columns = []; + private readonly List> _parsers = []; + private readonly List> _formatters = []; + private bool _missingSetter; + private bool _missingGetter; + private Func? _factory; + + internal FixedWidthLayoutBuilder() + { + } + + /// Adds a column, decoded with the configured encoding. + /// The column's name in error messages. + /// The 0-based offset of the column, in bytes. + /// The width of the column, in bytes. + /// Reads the value to write; for a layout that only reads. + /// Assigns the parsed value; for a layout that only writes. + /// Trimming, alignment, padding, format and overflow. uses every default. + /// This builder. + public FixedWidthLayoutBuilder Column( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + { + var column = Begin(name, start, length, typeof(string), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForString(set, column.Parse); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new StringColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column of a type that parses from and formats to UTF-8: the numeric types, + /// and the like. A date or time with a parses exactly + /// with that format. + /// + /// + public FixedWidthLayoutBuilder Column( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : IUtf8SpanParsable, IUtf8SpanFormattable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForUtf8Parsable(set, column.Parse); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new Utf8SpanFormattableColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column of a type that parses only from characters: the column is decoded with the + /// configured encoding first. A date or time with a + /// parses exactly with that format. + /// + /// + public FixedWidthLayoutBuilder DecodedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : ISpanParsable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForDecodedParsable(set, column.Parse); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new SpanFormattableColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds an enum column: it reads a member's name in any case or its number, and writes the name, or the + /// number with "D". A number no member has does not read. + /// + /// + public FixedWidthLayoutBuilder EnumColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TEnum : struct, Enum + { + var column = Begin(name, start, length, typeof(TEnum), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForEnum(set, column.Parse); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new SpanFormattableColumnFormatter(start, length, column.FormatOptions, name, get); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a enum column: blank reads as , and + /// writes the column filled with its trim character. + /// + /// + public FixedWidthLayoutBuilder NullableEnumColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TEnum : struct, Enum + { + var column = Begin(name, start, length, typeof(TEnum?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForEnum(adapted, column.Parse)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new SpanFormattableColumnFormatter( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column: a column holding only its trim character reads as + /// , and writes the column filled with it. + /// + /// + public FixedWidthLayoutBuilder NullableColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : struct, IUtf8SpanParsable, IUtf8SpanFormattable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForUtf8Parsable(adapted, column.Parse)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new Utf8SpanFormattableColumnFormatter( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column of a type that parses only from characters, such as a + /// date or time: blank reads as , and writes the column + /// filled with its trim character. A parses a date or + /// time exactly. + /// + /// + public FixedWidthLayoutBuilder NullableDecodedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, FixedWidthColumnOptions? options = null) + where TValue : struct, ISpanParsable, ISpanFormattable + { + var column = Begin(name, start, length, typeof(TValue?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForDecodedParsable(adapted, column.Parse)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new SpanFormattableColumnFormatter( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault()); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); + } + return Add(column, parser, formatter); + } + + /// + /// Adds a column parsed and formatted by : + /// blank reads as without calling the converter, and + /// writes the column filled with its trim character. + /// + /// + public FixedWidthLayoutBuilder NullableConvertedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, + IFixedWidthConverter converter, FixedWidthColumnOptions? options = null) + where TValue : struct + { + ArgumentNullException.ThrowIfNull(converter); + var column = Begin(name, start, length, typeof(TValue?), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForNullable(set, column.Parse, + adapted => ColumnParserFactory.ForConverter(adapted, converter, column.Parse)); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + var inner = new ConverterColumnFormatter>( + start, length, column.FormatOptions, name, (in TModel model) => get(in model).GetValueOrDefault(), converter); + formatter = new NullableColumnFormatter(start, length, column.Parse.TrimByte, get, inner); + } + return Add(column, parser, formatter); + } + + /// Adds a column parsed and formatted by . + /// The column's name in error messages. + /// The 0-based offset of the column, in bytes. + /// The width of the column, in bytes. + /// Reads the value to write; for a layout that only reads. + /// Assigns the parsed value; for a layout that only writes. + /// Parses and formats the column's value; shared by every line, and by every thread reading in parallel. + /// Trimming, alignment, padding and overflow. uses every default. + /// This builder. + public FixedWidthLayoutBuilder ConvertedColumn( + string name, int start, int length, ColumnGetter? get, ColumnSetter? set, + IFixedWidthConverter converter, FixedWidthColumnOptions? options = null) + { + ArgumentNullException.ThrowIfNull(converter); + var column = Begin(name, start, length, typeof(TValue), options); + ColumnParser? parser = null; + if (set is not null) + { + parser = ColumnParserFactory.ForConverter(set, converter, column.Parse); + } + IColumnFormatter? formatter = null; + if (get is not null) + { + formatter = new ConverterColumnFormatter>(start, length, column.FormatOptions, name, get, converter); + } + return Add(column, parser, formatter); + } + + /// + /// Sets how a read creates the model before its columns are assigned. Required to read a reference + /// type; a value type without one starts from . + /// + /// Returns a new model for each line. + /// This builder. + public FixedWidthLayoutBuilder Factory(Func factory) + { + ArgumentNullException.ThrowIfNull(factory); + _factory = factory; + return this; + } + + internal FixedWidthLayout Build() + { + if (_columns.Count == 0) + { + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has no columns; a fixed-width layout must define at least one column."); + } + ColumnLayoutValidator.Validate(CollectionsMarshal.AsSpan(_columns), typeof(TModel).Name); + + ColumnParserInfo[]? parsers = null; + Func? factory = null; + if (!_missingSetter) + { + parsers = [.. _parsers]; + factory = ResolveFactory(); + } + IColumnFormatter[]? formatters = null; + if (!_missingGetter) + { + formatters = [.. _formatters]; + } + return new(parsers, factory, formatters, _columns.Max(static column => column.Start + column.Length)); + } + + private Func ResolveFactory() + { + if (_factory is not null) + { + return _factory; + } + if (typeof(TModel).IsValueType) + { + return static () => default!; + } + throw new InvalidOperationException( + $"The layout for {typeof(TModel).Name} has no factory: {typeof(TModel).Name} is a reference type, so reading needs a new instance per line. Call Factory(...)."); + } + + private PendingColumn Begin(string name, int start, int length, Type valueType, FixedWidthColumnOptions? options) + { + ArgumentException.ThrowIfNullOrEmpty(name); + options ??= FixedWidthColumnOptions.Default; + var formatOptions = FormatterBuilder.ResolveOptions(name, valueType, options.Alignment, options.Padding, options.Format, options.Overflow); + FixedWidthRuntime.ToAsciiByte(options.TrimChar, name, nameof(FixedWidthColumnOptions.TrimChar)); + _columns.Add((start, length, name)); + return new(name, start, length, new ColumnParseOptions(options.TrimChar, options.TrimMode, options.Format), formatOptions); + } + + private FixedWidthLayoutBuilder Add(in PendingColumn column, ColumnParser? parser, IColumnFormatter? formatter) + { + _missingSetter |= parser is null; + _missingGetter |= formatter is null; + if (parser is not null) + { + _parsers.Add(new(column.Start, column.Length, column.Name, parser)); + } + if (formatter is not null) + { + _formatters.Add(formatter); + } + return this; + } + + [StructLayout(LayoutKind.Auto)] + private readonly record struct PendingColumn(string Name, int Start, int Length, ColumnParseOptions Parse, ColumnFormatOptions FormatOptions); + } +} diff --git a/src/FixedWidthParser/Parsers/FixedWidthParser.cs b/src/FixedWidthParser/Parsers/FixedWidthParser.cs deleted file mode 100644 index f7d31f8..0000000 --- a/src/FixedWidthParser/Parsers/FixedWidthParser.cs +++ /dev/null @@ -1,62 +0,0 @@ -using System.Runtime.ExceptionServices; -using FixedWidthParser.Processors; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Parsers -{ -#if NET9_0_OR_GREATER - public sealed class FixedWidthParser where TModel : new(), allows ref struct -#else - public sealed class FixedWidthParser where TModel : new() -#endif - { - private static readonly Func _modelFactory; - private static readonly ColumnParserInfo>[] _processors; - private static readonly int _requiredLineLength; - private static readonly ExceptionDispatchInfo? _buildError; - - // The build is static (once per type). Any layout/configuration error is captured and - // rethrown from the instance constructor, so the caller gets a clean exception instead of - // a TypeInitializationException on first use. - static FixedWidthParser() - { - try - { - _modelFactory = ParserBuilder.BuildModelFactory(); - _processors = ParserBuilder.BuildProcessors>(ColumnParserFactory.Create); - _requiredLineLength = ParserBuilder.ComputeRequiredLineLength(_processors); - _buildError = null; - } - catch (Exception ex) - { - _modelFactory = null!; - _processors = []; - _requiredLineLength = 0; - _buildError = ExceptionDispatchInfo.Capture(ex); - } - } - public FixedWidthParser() - { - _buildError?.Throw(); - } - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - if (line.Length < _requiredLineLength) - { - model = default!; - return false; - } - - model = _modelFactory(); - foreach (ref readonly var processor in _processors.AsSpan()) - { - var column = line.Slice(processor.Start, processor.Length); - if (!processor.Parse(column, formatProvider, stringPool, ref model)) - { - return false; - } - } - return true; - } - } -} diff --git a/src/FixedWidthParser/Parsers/ParserBuilder.cs b/src/FixedWidthParser/Parsers/ParserBuilder.cs index 6ac3967..8cc4705 100644 --- a/src/FixedWidthParser/Parsers/ParserBuilder.cs +++ b/src/FixedWidthParser/Parsers/ParserBuilder.cs @@ -1,65 +1,37 @@ using System.Linq.Expressions; using System.Runtime.InteropServices; using FixedWidthParser.Attributes; +using FixedWidthParser.Processors; namespace FixedWidthParser.Parsers { - /// - /// One column of a built parser layout: the column's range plus the compiled parse-and-assign - /// delegate ( is for - /// the char parser or for the byte parser). - /// + /// One column of a built parser layout: the column's range and name plus its compiled parse-and-assign delegate. [StructLayout(LayoutKind.Auto)] - internal readonly record struct ColumnParserInfo(int Start, int Length, TParser Parse); + internal readonly record struct ColumnParserInfo(int Start, int Length, string Name, ColumnParser Parse) + where TModel : allows ref struct; - /// - /// Shared static-build logic for and - /// : the compiled model factory, the per-column setter - /// compilation, and the required line length. The two parsers differ only in the factory that - /// turns a (member type, compiled setter) pair into a column parser delegate, so that factory is - /// passed into as a delegate. - /// + /// Static-build logic for : the compiled model factory and the per-column setters. internal static class ParserBuilder { -#if NET9_0_OR_GREATER public static Func BuildModelFactory() where TModel : new(), allows ref struct -#else - public static Func BuildModelFactory() where TModel : new() -#endif { var lambda = Expression.Lambda>(Expression.New(typeof(TModel))); return lambda.Compile(); } - public static ColumnParserInfo[] BuildProcessors(Func createParser) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif + public static ColumnParserInfo[] BuildProcessors() where TModel : allows ref struct { - List> processors = []; + List> processors = []; ModelColumns.ForEachColumn(typeof(TModel), (member, attribute) => { var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); var value = Expression.Parameter(memberType, "value"); - var actionType = typeof(RefAction<,>).MakeGenericType(typeof(TModel), memberType); + var actionType = typeof(ColumnSetter<,>).MakeGenericType(typeof(TModel), memberType); var setter = Expression.Lambda(actionType, Expression.Assign(access, value), model, value).Compile(); - processors.Add(new(attribute.Start, attribute.Length, createParser(memberType, setter, attribute.Converter, member.Name, attribute.TrimChar, attribute.TrimMode, attribute.Format))); + var parse = ColumnParserFactory.Create(memberType, setter, attribute.Converter, member.Name, attribute.TrimChar, attribute.TrimMode, attribute.Format); + processors.Add(new(attribute.Start, attribute.Length, member.Name, parse)); }); return [.. processors]; } - - public static int ComputeRequiredLineLength(ReadOnlySpan> processors) - { - int required = 0; - foreach (ref readonly var processor in processors) - { - int end = processor.Start + processor.Length; - if (end > required) - { - required = end; - } - } - return required; - } } } diff --git a/src/FixedWidthParser/Parsers/Utf8FixedWidthParser.cs b/src/FixedWidthParser/Parsers/Utf8FixedWidthParser.cs deleted file mode 100644 index b3c31b7..0000000 --- a/src/FixedWidthParser/Parsers/Utf8FixedWidthParser.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System.Runtime.ExceptionServices; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Processors; - -namespace FixedWidthParser.Parsers -{ - /// - /// UTF-8 counterpart of : parses a fixed-width line directly - /// from raw of (UTF-8), skipping the - /// UTF-8 → UTF-16 transcode that the char parser pays via a . - /// - /// Column offsets are measured in bytes. This is exact for the single-byte (ASCII) data - /// that fixed-width/flat files almost always use; for content with multi-byte UTF-8 (e.g. accented - /// characters) a byte offset is not the same as a character offset, so this parser is intended for - /// single-byte payloads. Non-string columns parse via a FixedColumnAttribute.Converter, csFastFloat - /// (double/float) or ; string columns are UTF-8 decoded and, when a - /// is supplied, interned through it (otherwise a fresh string per value). - /// - /// -#if NET9_0_OR_GREATER - public sealed class Utf8FixedWidthParser where TModel : new(), allows ref struct -#else - public sealed class Utf8FixedWidthParser where TModel : new() -#endif - { - private static readonly Func _modelFactory; - private static readonly ColumnParserInfo>[] _processors; - private static readonly int _requiredLineLength; - private static readonly ExceptionDispatchInfo? _buildError; - - // The build is static (once per type). Any layout/configuration error is captured and - // rethrown from the instance constructor, so the caller gets a clean exception instead of - // a TypeInitializationException on first use. Mirrors FixedWidthParser. - static Utf8FixedWidthParser() - { - try - { - _modelFactory = ParserBuilder.BuildModelFactory(); - _processors = ParserBuilder.BuildProcessors>(Utf8ColumnParserFactory.Create); - _requiredLineLength = ParserBuilder.ComputeRequiredLineLength(_processors); - _buildError = null; - } - catch (Exception ex) - { - _modelFactory = null!; - _processors = []; - _requiredLineLength = 0; - _buildError = ExceptionDispatchInfo.Capture(ex); - } - } - public Utf8FixedWidthParser() - { - _buildError?.Throw(); - } - /// - /// Parses a single UTF-8 fixed-width line into . Returns - /// when the line is shorter (in bytes) than the configured layout or - /// when a non-string column fails to parse. When is supplied, - /// string columns are interned through it (decoded as UTF-8). - /// - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - if (line.Length < _requiredLineLength) - { - model = default!; - return false; - } - - model = _modelFactory(); - foreach (ref readonly var processor in _processors.AsSpan()) - { - var column = line.Slice(processor.Start, processor.Length); - if (!processor.Parse(column, formatProvider, stringPool, ref model)) - { - return false; - } - } - return true; - } - } -} diff --git a/src/FixedWidthParser/Processors/ColumnParser.cs b/src/FixedWidthParser/Processors/ColumnParser.cs index 7a77146..c6908b9 100644 --- a/src/FixedWidthParser/Processors/ColumnParser.cs +++ b/src/FixedWidthParser/Processors/ColumnParser.cs @@ -1,20 +1,15 @@ -using CommunityToolkit.HighPerformance.Buffers; +using System.Diagnostics.CodeAnalysis; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Processors { /// - /// Parses the already-sliced text of a single column and assigns it to the model. The parser - /// slices each column out of the line (honoring start/length) and hands the resulting span here, - /// so this delegate never deals with offsets. Replaces the old IColumnProcessor - /// abstraction: one delegate per column. Returns to reject the line. + /// Parses the already-sliced bytes of a single column and assigns the value to the model. Returns + /// to reject the line. /// - public delegate bool ColumnParser( - ReadOnlySpan column, - IFormatProvider? formatProvider, - StringPool? stringPool, -#if NET9_0_OR_GREATER - ref TModel model) where TModel : allows ref struct; -#else - ref TModel model); -#endif + internal delegate bool ColumnParser(ReadOnlySpan column, in FixedWidthContext context, ref TModel model) + where TModel : allows ref struct; + + /// Parses the already-sliced bytes of a single column: the shape of the parse helpers. + internal delegate bool ColumnValueParser(ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions options, [MaybeNullWhen(false)] out TValue value); } diff --git a/src/FixedWidthParser/Processors/ColumnParserFactory.cs b/src/FixedWidthParser/Processors/ColumnParserFactory.cs index 4713474..617c948 100644 --- a/src/FixedWidthParser/Processors/ColumnParserFactory.cs +++ b/src/FixedWidthParser/Processors/ColumnParserFactory.cs @@ -1,207 +1,215 @@ -using System.Globalization; +using System.Diagnostics.CodeAnalysis; using System.Reflection; -using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Processors { /// - /// Combines a column's compiled setter and a value parser into a single - /// closure. The column span is sliced by the parser and passed - /// in, so these closures only parse and assign. Resolution order: a FixedColumnAttribute.Converter - /// when set, then string (special-cased: never fails, uses the ), then - /// double/float (csFastFloat), then the fallback. + /// Combines a column's setter and a value parser into a single + /// closure that calls the same helpers as generated code. Resolution + /// order: a converter, a date or time with a Format, string, double/float, + /// an enum, then , then over the + /// decoded text. Attribute layouts reach it by reflection (), fluent layouts through + /// the typed For* entry points. /// internal static class ColumnParserFactory { - private static readonly MethodInfo BuildParsableMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildParsable), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildConverterMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildConverter), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildDoubleMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildDouble), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildFloatMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildFloat), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildNullableMethod = - typeof(ColumnParserFactory).GetMethod(nameof(BuildNullable), BindingFlags.NonPublic | BindingFlags.Static)!; - - /// - /// Creates a for the given member type and setter. The setter is a compiled expression assigning the parsed value to the member. - /// When is a , a blank (trimmed-empty) column assigns - /// without invoking the underlying parser; otherwise resolution for the underlying T - /// proceeds exactly as for a non-nullable column. When is set (from - /// FixedColumnAttribute.Converter), it takes priority over the fallback. - /// - /// must be a RefAction<TModel, valueType> assigning the parsed value to the member. - public static ColumnParser Create(Type valueType, Delegate setter, Type? converterType, string memberName, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing, string? format = null) -#if NET9_0_OR_GREATER + private const BindingFlags Private = BindingFlags.NonPublic | BindingFlags.Static; + + public static ColumnParser Create(Type valueType, Delegate setter, Type? converterType, string memberName, char trimChar, TrimMode trimMode, string? format) where TModel : allows ref struct -#endif { + FixedWidthRuntime.ToAsciiByte(trimChar, memberName, nameof(FixedColumnAttribute.TrimChar)); + var options = new ColumnParseOptions(trimChar, trimMode, format); var underlyingType = Nullable.GetUnderlyingType(valueType); if (underlyingType is not null) { - return (ColumnParser)BuildNullableMethod.MakeGenericMethod(typeof(TModel), underlyingType) - .Invoke(null, [setter, converterType, memberName, trimChar, trimMode, format])!; + return (ColumnParser)Method(nameof(BuildNullable), typeof(TModel), underlyingType) + .Invoke(null, [setter, converterType, memberName, options])!; } - - return ColumnParserFactoryShared.Create>( - valueType, setter, converterType, memberName, trimChar, trimMode, format, typeof(IFixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); + return CreateValue(valueType, setter, converterType, memberName, options); } - // A nullable column (T?): a blank (trimmed-empty) column assigns null without invoking the - // underlying parser. Otherwise, the underlying T resolves exactly as a non-nullable column - // would (converter → double/float → ISpanParsable fallback), via an adapter setter that boxes - // the result into the T? setter — no duplicated parse logic per type. - private static ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, char trimChar, Attributes.TrimMode trimMode, string? format) -#if NET9_0_OR_GREATER + private static ColumnParser CreateValue(Type valueType, Delegate setter, Type? converterType, string memberName, ColumnParseOptions options) where TModel : allows ref struct -#endif - where TUnderlying : struct { - var nullableSetter = (RefAction)setter; - RefAction adapted = (ref TModel model, TUnderlying value) => nullableSetter(ref model, value); - - var inner = ColumnParserFactoryShared.Create>( - typeof(TUnderlying), adapted, converterType, memberName, trimChar, trimMode, format, typeof(IFixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); - - return (column, formatProvider, stringPool, ref model) => + if (converterType is not null) { - if (column.Trim(trimChar).Trim(' ').IsEmpty) + var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); + if (!requiredInterface.IsAssignableFrom(converterType)) { - nullableSetter(ref model, null); - return true; + throw new InvalidOperationException( + $"Converter '{converterType}' for column '{memberName}' must implement '{requiredInterface}'."); } - return inner(column, formatProvider, stringPool, ref model); - }; + var converter = Activator.CreateInstance(converterType) + ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{memberName}'."); + return (ColumnParser)Method(nameof(ForConverter), typeof(TModel), valueType, converterType) + .Invoke(null, [setter, converter, options])!; + } + + var special = TryBuildSpecial(valueType, setter, options); + if (special is not null) + { + return special; + } + if (valueType == typeof(string)) + { + return ForString((ColumnSetter)setter, options); + } + if (valueType.IsEnum) + { + return (ColumnParser)Method(nameof(ForEnum), typeof(TModel), valueType).Invoke(null, [setter, options])!; + } + if (Implements(valueType, typeof(IUtf8SpanParsable<>))) + { + return (ColumnParser)Method(nameof(ForUtf8Parsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; + } + if (Implements(valueType, typeof(ISpanParsable<>))) + { + return (ColumnParser)Method(nameof(ForDecodedParsable), typeof(TModel), valueType).Invoke(null, [setter, options])!; + } + + throw new InvalidOperationException( + $"Column '{memberName}' has type '{valueType}', which is not string, double, float, an enum, IUtf8SpanParsable or ISpanParsable; set a Converter for it."); } - // Fallback for types with no dedicated builder: call the static-abstract ISpanParsable.TryParse - // directly inside the column closure. This is a constrained call the JIT devirtualizes to the - // concrete TValue.TryParse (and can inline), saving one indirect call per column and no - // value-parser delegate allocated. - private static ColumnParser BuildParsable(RefAction setter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + internal static ColumnParser ForEnum(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct -#endif - where TValue : ISpanParsable + where TEnum : struct, Enum { - return (column, formatProvider, _, ref model) => - { - // Null means invariant (matches FixedWidthRuntime.TryParse and the double/float - // columns) rather than the BCL's own CurrentCulture default. - if (!TValue.TryParse(FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; + return Bind(setter, options, FixedWidthRuntime.TryEnum); } - private static ColumnParser BuildDouble(RefAction setter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + internal static ColumnParser ForString(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct -#endif { - return (column, formatProvider, _, ref model) => + return (column, in context, ref model) => { - if (!CultureHelpers.TryParseDouble(column, formatProvider, out double value, trimChar, trimMode)) - { - return false; - } - setter(ref model, value); + setter(ref model, FixedWidthRuntime.String(column, in context, in options)); return true; }; } - private static ColumnParser BuildFloat(RefAction setter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + internal static ColumnParser ForUtf8Parsable(ColumnSetter setter, ColumnParseOptions options) where TModel : allows ref struct -#endif + where TValue : IUtf8SpanParsable { - return (column, formatProvider, _, ref model) => - { - if (!CultureHelpers.TryParseFloat(column, formatProvider, out float value, trimChar, trimMode)) - { - return false; - } - setter(ref model, value); - return true; - }; + return TryBuildSpecial(typeof(TValue), setter, options) ?? Bind(setter, options, FixedWidthRuntime.TryParse); + } + + internal static ColumnParser ForDecodedParsable(ColumnSetter setter, ColumnParseOptions options) + where TModel : allows ref struct + where TValue : ISpanParsable + { + return TryBuildSpecial(typeof(TValue), setter, options) ?? Bind(setter, options, FixedWidthRuntime.TryParseDecoded); } - // Attribute-driven custom converter (FixedColumnAttribute.Converter): the instance is created - // once (by the caller) and reused for every row, so it must be stateless. - private static ColumnParser BuildConverter( - RefAction setter, TConverter converter, char trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + // TConverter is the concrete type on the attribute path, so its calls devirtualize there. + internal static ColumnParser ForConverter(ColumnSetter setter, TConverter converter, ColumnParseOptions options) where TModel : allows ref struct -#endif where TConverter : IFixedWidthConverter { - return (column, formatProvider, _, ref model) => - { - if (!converter.TryParse(FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; + return Bind(setter, options, (ReadOnlySpan column, in FixedWidthContext context, in ColumnParseOptions o, [MaybeNullWhen(false)] out TValue value) => + FixedWidthRuntime.TryConvert(column, in context, converter, in o, out value)); + } + + internal static ColumnParser ForNullable( + ColumnSetter setter, ColumnParseOptions options, Func, ColumnParser> buildInner) + where TModel : allows ref struct + where TValue : struct + { + ColumnSetter adapted = (ref TModel model, TValue value) => setter(ref model, value); + return WrapNullable(setter, buildInner(adapted), options); } - private static ColumnParser BuildString(RefAction setter, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + // The types the attribute path special-cases ahead of the generic parsers, in its order: a date or + // time with a Format, then double and float. + private static ColumnParser? TryBuildSpecial(Type valueType, Delegate setter, ColumnParseOptions options) where TModel : allows ref struct -#endif { - char trim = (char)trimChar; - return (column, _, stringPool, ref model) => + if (options.Format is not null) { - var slice = FixedWidthRuntime.TrimColumn(column, trim, trimMode); - setter(ref model, stringPool is null ? slice.ToString() : stringPool.GetOrAdd(slice)); - return true; - }; + var date = TryBuildDate(valueType, setter, options); + if (date is not null) + { + return date; + } + } + if (valueType == typeof(double)) + { + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDouble); + } + if (valueType == typeof(float)) + { + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryFloat); + } + return null; } - private static ColumnParser BuildExact(Type valueType, Delegate setter, string format, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER + private static ColumnParser? TryBuildDate(Type valueType, Delegate setter, ColumnParseOptions options) where TModel : allows ref struct -#endif { - char trim = (char)trimChar; if (valueType == typeof(DateTime)) { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTime.TryParseExact); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDateTimeExact); } if (valueType == typeof(DateOnly)) { - return BuildExact((RefAction)setter, format, trim, trimMode, DateOnly.TryParseExact); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDateOnlyExact); } if (valueType == typeof(TimeOnly)) { - return BuildExact((RefAction)setter, format, trim, trimMode, TimeOnly.TryParseExact); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryTimeOnlyExact); } if (valueType == typeof(DateTimeOffset)) { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTimeOffset.TryParseExact); + return Bind((ColumnSetter)setter, options, FixedWidthRuntime.TryDateTimeOffsetExact); } - throw new InvalidOperationException($"Unsupported TryParseExact type {valueType}"); + return null; } - private delegate bool ExactParser(ReadOnlySpan value, ReadOnlySpan format, IFormatProvider? formatProvider, DateTimeStyles styles, out TValue result); + private static bool Implements(Type valueType, Type openInterface) + { + // Not MakeGenericType: it throws for a type that breaks the interface's self-constraint. + return Array.Exists(valueType.GetInterfaces(), i => + i.IsGenericType && i.GetGenericTypeDefinition() == openInterface && i.GenericTypeArguments[0] == valueType); + } + + private static MethodInfo Method(string name, params Type[] typeArguments) + { + return typeof(ColumnParserFactory).GetMethod(name, Private)!.MakeGenericMethod(typeArguments); + } + + // A nullable column (T?): a blank column assigns null without invoking the underlying parser. + private static ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, ColumnParseOptions options) + where TModel : allows ref struct + where TUnderlying : struct + { + return ForNullable((ColumnSetter)setter, options, + adapted => CreateValue(typeof(TUnderlying), adapted, converterType, memberName, options)); + } + + private static ColumnParser WrapNullable(ColumnSetter nullableSetter, ColumnParser inner, ColumnParseOptions options) + where TModel : allows ref struct + where TUnderlying : struct + { + return (column, in context, ref model) => + { + if (FixedWidthRuntime.IsBlank(column, in options)) + { + nullableSetter(ref model, null); + return true; + } + return inner(column, in context, ref model); + }; + } - private static ColumnParser BuildExact(RefAction setter, string format, char trim, Attributes.TrimMode trimMode, ExactParser parser) -#if NET9_0_OR_GREATER + private static ColumnParser Bind(ColumnSetter setter, ColumnParseOptions options, ColumnValueParser parse) where TModel : allows ref struct -#endif { - return (column, fp, _, ref model) => + return (column, in context, ref model) => { - var trimmed = FixedWidthRuntime.TrimColumn(column, trim, trimMode); - if (!parser(trimmed, format, fp ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out var value)) + if (!parse(column, in context, in options, out var value)) { return false; } diff --git a/src/FixedWidthParser/Processors/ColumnParserFactoryShared.cs b/src/FixedWidthParser/Processors/ColumnParserFactoryShared.cs deleted file mode 100644 index 49c30ef..0000000 --- a/src/FixedWidthParser/Processors/ColumnParserFactoryShared.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System.Reflection; - -namespace FixedWidthParser.Processors -{ - /// - /// Shared build-time orchestration for (char) and - /// (byte): the identical "converter → string special-case → - /// double/float (csFastFloat) special-case → / - /// fallback → wrap setter+parser" flow. The element-specific pieces (string/double/float/converter - /// builders and the reflected build-parsable method) are passed in, so the spans-of-different-element-type - /// closures stay in each factory while the reflection plumbing lives in one place. - /// - internal static class ColumnParserFactoryShared - { - public static TColumnParser Create( - Type valueType, - Delegate setter, - Type? converterType, - string memberName, - object trimChar, - Attributes.TrimMode trimMode, - string? format, - Type converterInterfaceDefinition, - MethodInfo buildParsableMethod, - MethodInfo buildConverterMethod, - MethodInfo buildDoubleMethod, - MethodInfo buildFloatMethod, - Func, object, Attributes.TrimMode, TColumnParser> buildString, - Func buildExact) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TColumnParser : Delegate - { - if (converterType is not null) - { - var requiredInterface = converterInterfaceDefinition.MakeGenericType(valueType); - if (!requiredInterface.IsAssignableFrom(converterType)) - { - throw new InvalidOperationException( - $"Converter '{converterType}' for column '{memberName}' must implement '{requiredInterface}'."); - } - var converterInstance = Activator.CreateInstance(converterType) - ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{memberName}'."); - return (TColumnParser)buildConverterMethod.MakeGenericMethod(typeof(TModel), valueType, converterType) - .Invoke(null, [setter, converterInstance, trimChar, trimMode])!; - } - - if (format is not null && (valueType == typeof(DateTime) || valueType == typeof(DateOnly) || valueType == typeof(TimeOnly) || valueType == typeof(DateTimeOffset))) - { - return buildExact(valueType, setter, format, trimChar, trimMode); - } - - if (valueType == typeof(string)) - { - return buildString((RefAction)setter, trimChar, trimMode); - } - - // double/float take the csFastFloat fast path ahead of the ISpanParsable fallback (both - // implement ISpanParsable/IUtf8SpanParsable too, but csFastFloat is faster). - if (valueType == typeof(double)) - { - return (TColumnParser)buildDoubleMethod.MakeGenericMethod(typeof(TModel)).Invoke(null, [setter, trimChar, trimMode])!; - } - if (valueType == typeof(float)) - { - return (TColumnParser)buildFloatMethod.MakeGenericMethod(typeof(TModel)).Invoke(null, [setter, trimChar, trimMode])!; - } - - return (TColumnParser)buildParsableMethod.MakeGenericMethod(typeof(TModel), valueType) - .Invoke(null, [setter, trimChar, trimMode])!; - } - } -} diff --git a/src/FixedWidthParser/Processors/CultureHelpers.cs b/src/FixedWidthParser/Processors/CultureHelpers.cs deleted file mode 100644 index 9d3e65e..0000000 --- a/src/FixedWidthParser/Processors/CultureHelpers.cs +++ /dev/null @@ -1,224 +0,0 @@ -using System.Buffers; -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Runtime.CompilerServices; -using System.Text; -using csFastFloat; - -namespace FixedWidthParser.Processors -{ - /// - /// Culture-aware double/float parsing for the double/float column builders (char and - /// UTF-8). csFastFloat is only trusted for the invariant '.' decimal separator, and even then only - /// after checking every character of the (trimmed) field was consumed — csFastFloat does not fail - /// on trailing content it doesn't recognize, it silently stops and returns the leading prefix it - /// did parse (so e.g. a thousands separator it wasn't told about would otherwise truncate "1.234,56" - /// down to "1" and still report success). Any other separator falls back to the real, - /// -aware , - /// which csFastFloat's single decimal_separator override cannot express correctly (thousands - /// separators, sign placement, …) and which validates the whole input natively. - /// - internal static class CultureHelpers - { - // Decimal separator cache keyed by IFormatProvider identity. A ConditionalWeakTable ties each - // entry's lifetime to its provider's own lifetime: no manual eviction, no unbounded growth if a - // caller passes many distinct (e.g. per-request) providers, and — unlike a single-entry memo — - // no thrash when two providers are used alternately in the same process. - private static readonly ConditionalWeakTable> _separatorCache = new(); - - private sealed class CacheEntry - { - public readonly IFormatProvider Provider; - public readonly char Separator; - public CacheEntry(IFormatProvider provider, char separator) - { - Provider = provider; - Separator = separator; - } - } - private static CacheEntry? _cache; - - /// - /// Decimal separator derived from the IFormatProvider (dot when null), memoized per provider. - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static char GetDecimalSeparator(IFormatProvider? formatProvider) - { - if (formatProvider is null) - { - return '.'; - } - var cache = _cache; - if (cache is not null && ReferenceEquals(formatProvider, cache.Provider)) - { - return cache.Separator; - } - char separator = _separatorCache.GetValue(formatProvider, static fp => new StrongBox(Resolve(fp))).Value; - _cache = new CacheEntry(formatProvider, separator); - return separator; - } - - /// - /// Decimal separator as a single UTF-8 byte, for the byte-based numeric parsers. Only an ASCII - /// separator (<= 0x7F) can be represented as one byte; a non-ASCII separator would be a - /// multi-byte sequence in the UTF-8 data and is rejected with a clear error rather than silently - /// truncated to the wrong byte. - /// - /// The culture's decimal separator is not a single ASCII character. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static byte GetDecimalSeparatorByte(IFormatProvider? formatProvider) - { - char separator = GetDecimalSeparator(formatProvider); - if (separator > '\x7F') - { - ThrowNonAsciiSeparator(separator); - } - return (byte)separator; - } - - [MethodImpl(MethodImplOptions.NoInlining)] - [DoesNotReturn] - private static void ThrowNonAsciiSeparator(char separator) - { - throw new NotSupportedException( - $"The decimal separator '{separator}' (U+{(int)separator:X4}) is not a single ASCII byte. " + - "The UTF-8 byte parser only supports ASCII decimal separators; use the char-based parser for this culture."); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static char Resolve(IFormatProvider formatProvider) - { - return NumberFormatInfo.GetInstance(formatProvider).NumberDecimalSeparator[0]; - } - - /// - /// Parses a (not yet trimmed) column as , honoring . - /// A provider means — a fixed-width - /// file is a machine layout, not UI text, so a caller that doesn't pass a provider should not be at - /// the mercy of the running thread's (which every other - /// column type, and this method's own BCL fallback below, would otherwise resolve to). - /// - public static bool TryParseDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - // Also trim leading whitespace: csFastFloat's characters_consumed does not reliably include - // skipped leading whitespace across its double/float overloads (verified empirically — - // FastDoubleParser counts it, FastFloatParser doesn't), so a right-aligned (space-padded) - // column would otherwise fail the full-consumption check below even though the BCL fallback - // (which does allow leading whitespace) would have accepted it. - var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart(); - if (GetDecimalSeparator(formatProvider) == '.') - { - if (FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: '.') - && consumed == trimmed.Length) - { - return true; - } - // csFastFloat's single decimal_separator override can't express thousands separators or - // sign placement; a dot-culture value using them (e.g. "1,234.56") fails the fast path - // above even though it's valid, so fall back to the real NumberFormatInfo-aware parser, - // which validates the whole input natively (still rejects genuine garbage like "12x"). - return double.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider ?? CultureInfo.InvariantCulture, out value); - } - return double.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - - /// Parses a (not yet trimmed) column as , honoring (see for the null-provider contract). - public static bool TryParseFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart(); - if (GetDecimalSeparator(formatProvider) == '.') - { - if (FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: '.') - && consumed == trimmed.Length) - { - return true; - } - return float.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider ?? CultureInfo.InvariantCulture, out value); - } - return float.TryParse(trimmed, NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - - /// - /// UTF-8 counterpart of . - /// A non-ASCII separator still throws (see ) — the byte path - /// only supports separators representable as a single UTF-8 byte. - /// - public static bool TryParseDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - byte separator = GetDecimalSeparatorByte(formatProvider); - var trimmed = Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); - if (separator == (byte)'.' - && FastDoubleParser.TryParseDouble(trimmed, out int consumed, out value, decimal_separator: separator) - && consumed == trimmed.Length) - { - return true; - } - return TryParseDoubleViaTranscode(trimmed, formatProvider, out value); - } - - /// UTF-8 counterpart of . - public static bool TryParseFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - byte separator = GetDecimalSeparatorByte(formatProvider); - var trimmed = Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode).TrimStart((byte)' '); - if (separator == (byte)'.' - && FastFloatParser.TryParseFloat(trimmed, out int consumed, out value, decimal_separator: separator) - && consumed == trimmed.Length) - { - return true; - } - return TryParseFloatViaTranscode(trimmed, formatProvider, out value); - } - - // Non-dot ASCII separators (e.g. ',') still need NumberFormatInfo-aware parsing (thousands - // separators, sign placement); there is no BCL span-based double.TryParse that both takes an - // IFormatProvider and operates on UTF-8 bytes, so the (already-trimmed, always-short) numeric - // field is transcoded to a small char buffer first. Also used as the dot-path's thousands-separator - // fallback above, so a null provider is normalized to invariant here too (see TryParseDouble's doc). - [SkipLocalsInit] - private static bool TryParseDoubleViaTranscode(ReadOnlySpan field, IFormatProvider? formatProvider, out double value) - { - formatProvider ??= CultureInfo.InvariantCulture; - const int stackThreshold = 128; - if (field.Length <= stackThreshold) - { - Span chars = stackalloc char[field.Length]; - int written = Encoding.UTF8.GetChars(field, chars); - return double.TryParse(chars[..written], NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - char[] rented = ArrayPool.Shared.Rent(field.Length); - try - { - int written = Encoding.UTF8.GetChars(field, rented); - return double.TryParse(rented.AsSpan(0, written), NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - - [SkipLocalsInit] - private static bool TryParseFloatViaTranscode(ReadOnlySpan field, IFormatProvider? formatProvider, out float value) - { - formatProvider ??= CultureInfo.InvariantCulture; - const int stackThreshold = 128; - if (field.Length <= stackThreshold) - { - Span chars = stackalloc char[field.Length]; - int written = Encoding.UTF8.GetChars(field, chars); - return float.TryParse(chars[..written], NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - char[] rented = ArrayPool.Shared.Rent(field.Length); - try - { - int written = Encoding.UTF8.GetChars(field, rented); - return float.TryParse(rented.AsSpan(0, written), NumberStyles.Float | NumberStyles.AllowThousands, formatProvider, out value); - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - } -} diff --git a/src/FixedWidthParser/Processors/IFixedWidthConverter.cs b/src/FixedWidthParser/Processors/IFixedWidthConverter.cs deleted file mode 100644 index eddea80..0000000 --- a/src/FixedWidthParser/Processors/IFixedWidthConverter.cs +++ /dev/null @@ -1,27 +0,0 @@ -namespace FixedWidthParser.Processors -{ - /// - /// Per-property custom conversion for a fixed-width column, wired via - /// FixedColumnAttribute.Converter. A single instance is created once (when the owning - /// model's parser/writer is built) and reused for every row, so implementations must be stateless. - /// One interface covers both directions: a fixed-width column always round-trips. - /// - public interface IFixedWidthConverter - { - /// Parses the already-sliced, trimmed column text. Returns to reject the line. - bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out T value); - - /// Formats into , returning the number of characters written. - bool TryFormat(T value, Span destination, IFormatProvider? formatProvider, out int written); - } - - /// UTF-8 counterpart of , used by the byte reader/writer. - public interface IUtf8FixedWidthConverter - { - /// Parses the already-sliced, trimmed column bytes. Returns to reject the line. - bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out T value); - - /// Formats into , returning the number of bytes written. - bool TryFormat(T value, Span destination, IFormatProvider? formatProvider, out int written); - } -} diff --git a/src/FixedWidthParser/Processors/Utf8ColumnParser.cs b/src/FixedWidthParser/Processors/Utf8ColumnParser.cs deleted file mode 100644 index 4775cb9..0000000 --- a/src/FixedWidthParser/Processors/Utf8ColumnParser.cs +++ /dev/null @@ -1,26 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Processors -{ - /// - /// UTF-8 counterpart of : parses the already-sliced - /// bytes of a single column and assigns it to the model. The byte parser slices each - /// column out of the raw UTF-8 line (honoring start/length, measured in bytes) and hands - /// the resulting span here, so this delegate never deals with offsets. Returns - /// to reject the line. - /// - /// A may be supplied to intern string columns: they are decoded via - /// (UTF-8), so a - /// repeated value is decoded once and the same instance is reused. - /// - /// - public delegate bool Utf8ColumnParser( - ReadOnlySpan column, - IFormatProvider? formatProvider, - StringPool? stringPool, -#if NET9_0_OR_GREATER - ref TModel model) where TModel : allows ref struct; -#else - ref TModel model); -#endif -} diff --git a/src/FixedWidthParser/Processors/Utf8ColumnParserFactory.cs b/src/FixedWidthParser/Processors/Utf8ColumnParserFactory.cs deleted file mode 100644 index 0f0c07f..0000000 --- a/src/FixedWidthParser/Processors/Utf8ColumnParserFactory.cs +++ /dev/null @@ -1,219 +0,0 @@ -using System.Globalization; -using System.Reflection; -using System.Text; - -namespace FixedWidthParser.Processors -{ - /// - /// UTF-8 counterpart of : combines a column's compiled setter and - /// a UTF-8 value parser into a single closure. The column - /// span is sliced (in bytes) by the parser and passed in, so these closures only parse and assign. - /// Resolution order: a FixedColumnAttribute.Converter when set, then string (special-cased: - /// never fails, UTF-8 decoded, interned through the supplied - /// when one is provided), then double/float (csFastFloat), then the - /// fallback. - /// - internal static class Utf8ColumnParserFactory - { - private static readonly MethodInfo BuildParsableMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildParsable), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildConverterMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildConverter), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildDoubleMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildDouble), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildFloatMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildFloat), BindingFlags.NonPublic | BindingFlags.Static)!; - private static readonly MethodInfo BuildNullableMethod = - typeof(Utf8ColumnParserFactory).GetMethod(nameof(BuildNullable), BindingFlags.NonPublic | BindingFlags.Static)!; - - /// - /// Creates a for the given member type and setter. The - /// setter is a compiled expression assigning the parsed value to the member. - /// When is a , a blank (trimmed-empty) column assigns - /// without invoking the underlying parser; otherwise resolution for the underlying T - /// proceeds exactly as for a non-nullable column. When is set (from - /// FixedColumnAttribute.Converter), it takes priority over the fallback. - /// - /// must be a RefAction<TModel, valueType> assigning the parsed value to the member. - public static Utf8ColumnParser Create(Type valueType, Delegate setter, Type? converterType, string memberName, char trimChar = ' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing, string? format = null) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - byte trimByte = Utf8FixedWidthRuntime.ToAsciiByte(trimChar, memberName); - var underlyingType = Nullable.GetUnderlyingType(valueType); - if (underlyingType is not null) - { - return (Utf8ColumnParser)BuildNullableMethod.MakeGenericMethod(typeof(TModel), underlyingType) - .Invoke(null, [setter, converterType, memberName, trimByte, trimMode, format])!; - } - - return ColumnParserFactoryShared.Create>( - valueType, setter, converterType, memberName, trimByte, trimMode, format, typeof(IUtf8FixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); - } - - // A nullable column (T?): a blank (trimmed-empty) column assigns null without invoking the - // underlying parser. Otherwise, the underlying T resolves exactly as a non-nullable column - // would (converter → double/float → IUtf8SpanParsable fallback), via an adapter setter that - // boxes the result into the T? setter — no duplicated parse logic per type. - private static Utf8ColumnParser BuildNullable(Delegate setter, Type? converterType, string memberName, byte trimChar, Attributes.TrimMode trimMode, string? format) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TUnderlying : struct - { - var nullableSetter = (RefAction)setter; - RefAction adapted = (ref TModel model, TUnderlying value) => nullableSetter(ref model, value); - - var inner = ColumnParserFactoryShared.Create>( - typeof(TUnderlying), adapted, converterType, memberName, trimChar, trimMode, format, typeof(IUtf8FixedWidthConverter<>), - BuildParsableMethod, BuildConverterMethod, BuildDoubleMethod, BuildFloatMethod, BuildString, BuildExact); - - return (column, formatProvider, stringPool, ref model) => - { - if (column.Trim(trimChar).Trim((byte)' ').IsEmpty) - { - nullableSetter(ref model, null); - return true; - } - return inner(column, formatProvider, stringPool, ref model); - }; - } - - // Fallback for types with no dedicated builder: call the static-abstract IUtf8SpanParsable.TryParse - // directly inside the column closure. This is a constrained call the JIT devirtualizes to the - // concrete TValue.TryParse (and can inline), saving one indirect call per column and no - // value-parser delegate allocated. - private static Utf8ColumnParser BuildParsable(RefAction setter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TValue : IUtf8SpanParsable - { - return (column, formatProvider, _, ref model) => - { - // Null means invariant (matches Utf8FixedWidthRuntime.TryParse and the double/float - // columns) rather than the BCL's own CurrentCulture default. - if (!TValue.TryParse(Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static Utf8ColumnParser BuildDouble(RefAction setter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - return (column, formatProvider, _, ref model) => - { - if (!CultureHelpers.TryParseDouble(column, formatProvider, out double value, trimChar, trimMode)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static Utf8ColumnParser BuildFloat(RefAction setter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - return (column, formatProvider, _, ref model) => - { - if (!CultureHelpers.TryParseFloat(column, formatProvider, out float value, trimChar, trimMode)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - // Attribute-driven custom converter (FixedColumnAttribute.Converter): the instance is created - // once (by the caller) and reused for every row, so it must be stateless. - private static Utf8ColumnParser BuildConverter( - RefAction setter, TConverter converter, byte trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - where TConverter : IUtf8FixedWidthConverter - { - return (column, formatProvider, _, ref model) => - { - if (!converter.TryParse(Utf8FixedWidthRuntime.TrimColumn(column, trimChar, trimMode), formatProvider, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - - private static Utf8ColumnParser BuildString(RefAction setter, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - byte trim = (byte)trimChar; - return (column, _, stringPool, ref model) => - { - var slice = Utf8FixedWidthRuntime.TrimColumn(column, trim, trimMode); - setter(ref model, stringPool is null ? Encoding.UTF8.GetString(slice) : stringPool.GetOrAdd(slice, Encoding.UTF8)); - return true; - }; - } - - private static Utf8ColumnParser BuildExact(Type valueType, Delegate setter, string format, object trimChar, Attributes.TrimMode trimMode) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - byte trim = (byte)trimChar; - if (valueType == typeof(DateTime)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTime.TryParseExact); - } - if (valueType == typeof(DateOnly)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, DateOnly.TryParseExact); - } - if (valueType == typeof(TimeOnly)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, TimeOnly.TryParseExact); - } - if (valueType == typeof(DateTimeOffset)) - { - return BuildExact((RefAction)setter, format, trim, trimMode, DateTimeOffset.TryParseExact); - } - throw new InvalidOperationException($"Unsupported TryParseExact type {valueType}"); - } - - private delegate bool ExactParser(ReadOnlySpan value, ReadOnlySpan format, IFormatProvider? formatProvider, DateTimeStyles styles, out TValue result); - - private static Utf8ColumnParser BuildExact(RefAction setter, string format, byte trim, Attributes.TrimMode trimMode, ExactParser parser) -#if NET9_0_OR_GREATER - where TModel : allows ref struct -#endif - { - return (column, fp, _, ref model) => - { - var trimmed = Utf8FixedWidthRuntime.TrimColumn(column, trim, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - if (!parser(chars[..written], format, fp ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out var value)) - { - return false; - } - setter(ref model, value); - return true; - }; - } - } -} diff --git a/src/FixedWidthParser/PublicAPI.Shipped.txt b/src/FixedWidthParser/PublicAPI.Shipped.txt new file mode 100644 index 0000000..7dc5c58 --- /dev/null +++ b/src/FixedWidthParser/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/FixedWidthParser/PublicAPI.Unshipped.txt b/src/FixedWidthParser/PublicAPI.Unshipped.txt new file mode 100644 index 0000000..c66e75a --- /dev/null +++ b/src/FixedWidthParser/PublicAPI.Unshipped.txt @@ -0,0 +1,200 @@ +#nullable enable +FixedWidthParser.FixedWidth +FixedWidthParser.FixedWidthContext +FixedWidthParser.FixedWidthContext.Encoding.get -> System.Text.Encoding! +FixedWidthParser.FixedWidthContext.Encoding.init -> void +FixedWidthParser.FixedWidthContext.Equals(FixedWidthParser.FixedWidthContext other) -> bool +FixedWidthParser.FixedWidthContext.FixedWidthContext() -> void +FixedWidthParser.FixedWidthContext.FormatProvider.get -> System.IFormatProvider! +FixedWidthParser.FixedWidthContext.FormatProvider.init -> void +FixedWidthParser.FixedWidthContext.ReportFailedColumn(string! columnName, int start, int length) -> void +FixedWidthParser.FixedWidthContext.ReportShortLine(int requiredLength) -> void +FixedWidthParser.FixedWidthContext.StringPool.get -> CommunityToolkit.HighPerformance.Buffers.StringPool? +FixedWidthParser.FixedWidthContext.StringPool.init -> void +FixedWidthParser.FixedWidthParseException +FixedWidthParser.FixedWidthParseException.ColumnLength.get -> int +FixedWidthParser.FixedWidthParseException.ColumnName.get -> string? +FixedWidthParser.FixedWidthParseException.ColumnStart.get -> int +FixedWidthParser.FixedWidthParseException.FixedWidthParseException() -> void +FixedWidthParser.FixedWidthParseException.FixedWidthParseException(string! message) -> void +FixedWidthParser.FixedWidthParseException.FixedWidthParseException(string! message, System.Exception! innerException) -> void +FixedWidthParser.FixedWidthParseException.LineNumber.get -> long +FixedWidthParser.FixedWidthParseException.RawValue.get -> string? +FixedWidthParser.FixedWidthReaderOptions +FixedWidthParser.FixedWidthReaderOptions.BufferSize.get -> int +FixedWidthParser.FixedWidthReaderOptions.BufferSize.init -> void +FixedWidthParser.FixedWidthReaderOptions.Encoding.get -> System.Text.Encoding? +FixedWidthParser.FixedWidthReaderOptions.Encoding.init -> void +FixedWidthParser.FixedWidthReaderOptions.FixedWidthReaderOptions() -> void +FixedWidthParser.FixedWidthReaderOptions.FormatProvider.get -> System.IFormatProvider? +FixedWidthParser.FixedWidthReaderOptions.FormatProvider.init -> void +FixedWidthParser.FixedWidthReaderOptions.MaxLineLength.get -> int +FixedWidthParser.FixedWidthReaderOptions.MaxLineLength.init -> void +FixedWidthParser.FixedWidthReaderOptions.OnParseError.get -> System.Func? +FixedWidthParser.FixedWidthReaderOptions.OnParseError.init -> void +FixedWidthParser.FixedWidthReaderOptions.StringPool.get -> CommunityToolkit.HighPerformance.Buffers.StringPool? +FixedWidthParser.FixedWidthReaderOptions.StringPool.init -> void +FixedWidthParser.FixedWidthWriterOptions +FixedWidthParser.FixedWidthWriterOptions.Encoding.get -> System.Text.Encoding? +FixedWidthParser.FixedWidthWriterOptions.Encoding.init -> void +FixedWidthParser.FixedWidthWriterOptions.FixedWidthWriterOptions() -> void +FixedWidthParser.FixedWidthWriterOptions.FormatProvider.get -> System.IFormatProvider? +FixedWidthParser.FixedWidthWriterOptions.FormatProvider.init -> void +FixedWidthParser.FixedWidthWriterOptions.NewLine.get -> string! +FixedWidthParser.FixedWidthWriterOptions.NewLine.init -> void +FixedWidthParser.IFixedWidthModel +FixedWidthParser.IFixedWidthModel.TryFormat(in TSelf model, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +FixedWidthParser.IFixedWidthModel.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TSelf model) -> bool +override FixedWidthParser.FixedWidthContext.GetHashCode() -> int +static FixedWidthParser.FixedWidth.TryFormat(in TModel model, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +static FixedWidthParser.FixedWidthContext.operator !=(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool +static FixedWidthParser.FixedWidthContext.operator ==(FixedWidthParser.FixedWidthContext left, FixedWidthParser.FixedWidthContext right) -> bool +~override FixedWidthParser.FixedWidthContext.Equals(object obj) -> bool +~override FixedWidthParser.FixedWidthContext.ToString() -> string +FixedWidthParser.IRecordAccumulator +FixedWidthParser.IRecordAccumulator.Add(in TModel model) -> void +FixedWidthParser.IRecordAccumulator.Merge(TSelf following) -> void +FixedWidthParser.ColumnGetter +FixedWidthParser.ColumnSetter +FixedWidthParser.FixedWidthLayout +FixedWidthParser.FixedWidthLayout +static FixedWidthParser.FixedWidth.TryFormat(in TModel model, FixedWidthParser.FixedWidthLayout! layout, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +static FixedWidthParser.FixedWidth.TryParse(System.ReadOnlySpan line, FixedWidthParser.FixedWidthLayout! layout, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +static FixedWidthParser.FixedWidthLayout.FromAttributes() -> FixedWidthParser.FixedWidthLayout! +virtual FixedWidthParser.ColumnGetter.Invoke(in TModel model) -> TValue +virtual FixedWidthParser.ColumnSetter.Invoke(ref TModel model, TValue value) -> void +static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.Write(System.IO.Stream! stream, in TModel model, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteAsync(System.IO.Stream! stream, TModel model, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.WriteAsync(System.IO.Stream! stream, TModel model, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.ReadOnlySpan models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteMany(System.IO.Stream! stream, System.ReadOnlySpan models, FixedWidthParser.FixedWidthWriterOptions? options = null) -> void +static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.WriteManyAsync(System.IO.Stream! stream, System.Collections.Generic.IEnumerable! models, FixedWidthParser.FixedWidthWriterOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.FixedWidthColumnOptions +FixedWidthParser.FixedWidthColumnOptions.Alignment.init -> void +FixedWidthParser.FixedWidthColumnOptions.FixedWidthColumnOptions() -> void +FixedWidthParser.FixedWidthColumnOptions.Format.get -> string? +FixedWidthParser.FixedWidthColumnOptions.Format.init -> void +FixedWidthParser.FixedWidthColumnOptions.Overflow.init -> void +FixedWidthParser.FixedWidthColumnOptions.Padding.get -> char +FixedWidthParser.FixedWidthColumnOptions.Padding.init -> void +FixedWidthParser.FixedWidthColumnOptions.TrimChar.get -> char +FixedWidthParser.FixedWidthColumnOptions.TrimChar.init -> void +FixedWidthParser.FixedWidthColumnOptions.TrimMode.init -> void +FixedWidthParser.FixedWidthLayoutBuilder +FixedWidthParser.FixedWidthLayoutBuilder.Column(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.Column(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.DecodedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.Factory(System.Func! factory) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.NullableColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +static FixedWidthParser.FixedWidthLayout.Build(System.Action!>! configure) -> FixedWidthParser.FixedWidthLayout! +FixedWidthParser.FixedWidthLayoutBuilder.NullableDecodedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.Alignment +FixedWidthParser.Alignment.Left = 0 -> FixedWidthParser.Alignment +FixedWidthParser.Alignment.Right = 1 -> FixedWidthParser.Alignment +FixedWidthParser.FixedColumnAttribute +FixedWidthParser.FixedColumnAttribute.Alignment.get -> FixedWidthParser.Alignment +FixedWidthParser.FixedColumnAttribute.Alignment.set -> void +FixedWidthParser.FixedColumnAttribute.Converter.get -> System.Type? +FixedWidthParser.FixedColumnAttribute.Converter.set -> void +FixedWidthParser.FixedColumnAttribute.FixedColumnAttribute(int start, int length) -> void +FixedWidthParser.FixedColumnAttribute.Format.get -> string? +FixedWidthParser.FixedColumnAttribute.Format.set -> void +FixedWidthParser.FixedColumnAttribute.Length.get -> int +FixedWidthParser.FixedColumnAttribute.Overflow.get -> FixedWidthParser.OverflowBehavior +FixedWidthParser.FixedColumnAttribute.Overflow.set -> void +FixedWidthParser.FixedColumnAttribute.Padding.get -> char +FixedWidthParser.FixedColumnAttribute.Padding.set -> void +FixedWidthParser.FixedColumnAttribute.Start.get -> int +FixedWidthParser.FixedColumnAttribute.TrimChar.get -> char +FixedWidthParser.FixedColumnAttribute.TrimChar.set -> void +FixedWidthParser.FixedColumnAttribute.TrimMode.get -> FixedWidthParser.TrimMode +FixedWidthParser.FixedColumnAttribute.TrimMode.set -> void +FixedWidthParser.FixedWidthColumnOptions.Alignment.get -> FixedWidthParser.Alignment +FixedWidthParser.FixedWidthColumnOptions.Overflow.get -> FixedWidthParser.OverflowBehavior +FixedWidthParser.FixedWidthColumnOptions.TrimMode.get -> FixedWidthParser.TrimMode +FixedWidthParser.FixedWidthLayoutBuilder.ConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.NullableConvertedColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.IFixedWidthConverter! converter, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.IFixedWidthConverter +FixedWidthParser.Infrastructure.AsyncRecordEnumerator +FixedWidthParser.Infrastructure.AsyncRecordEnumerator.Current.get -> TModel +FixedWidthParser.Infrastructure.AsyncRecordEnumerator.DisposeAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Infrastructure.AsyncRecordEnumerator.MoveNextAsync() -> System.Threading.Tasks.ValueTask +FixedWidthParser.Infrastructure.ColumnFormatOptions +FixedWidthParser.Infrastructure.ColumnFormatOptions.Alignment.get -> FixedWidthParser.Alignment +FixedWidthParser.Infrastructure.ColumnFormatOptions.ColumnFormatOptions() -> void +FixedWidthParser.Infrastructure.ColumnFormatOptions.Format.get -> string? +FixedWidthParser.Infrastructure.ColumnFormatOptions.Overflow.get -> FixedWidthParser.OverflowBehavior +FixedWidthParser.Infrastructure.ColumnFormatOptions.Padding.get -> char +FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable +FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable.GetAsyncEnumerator(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> FixedWidthParser.Infrastructure.AsyncRecordEnumerator! +FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable +FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable.GetEnumerator() -> FixedWidthParser.Infrastructure.RecordEnumerator +FixedWidthParser.Infrastructure.FixedWidthRuntime +FixedWidthParser.Infrastructure.GeneratedLineParser +FixedWidthParser.Infrastructure.GeneratedLineParser.GeneratedLineParser() -> void +FixedWidthParser.Infrastructure.GeneratedLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Infrastructure.IRecordLineParser +FixedWidthParser.Infrastructure.IRecordLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Infrastructure.LayoutLineParser +FixedWidthParser.Infrastructure.LayoutLineParser.LayoutLineParser() -> void +FixedWidthParser.Infrastructure.LayoutLineParser.TryParse(System.ReadOnlySpan line, in FixedWidthParser.FixedWidthContext context, out TModel model) -> bool +FixedWidthParser.Infrastructure.RecordEnumerator +FixedWidthParser.Infrastructure.RecordEnumerator.Current.get -> TModel +FixedWidthParser.Infrastructure.RecordEnumerator.Dispose() -> void +FixedWidthParser.Infrastructure.RecordEnumerator.MoveNext() -> bool +FixedWidthParser.Infrastructure.RecordEnumerator.RecordEnumerator() -> void +FixedWidthParser.Infrastructure.RecordEnumerator.Reset() -> void +FixedWidthParser.OverflowBehavior +FixedWidthParser.OverflowBehavior.Default = 0 -> FixedWidthParser.OverflowBehavior +FixedWidthParser.OverflowBehavior.Throw = 2 -> FixedWidthParser.OverflowBehavior +FixedWidthParser.OverflowBehavior.Truncate = 1 -> FixedWidthParser.OverflowBehavior +FixedWidthParser.TrimMode +FixedWidthParser.TrimMode.Both = 2 -> FixedWidthParser.TrimMode +FixedWidthParser.TrimMode.Leading = 1 -> FixedWidthParser.TrimMode +FixedWidthParser.TrimMode.Trailing = 0 -> FixedWidthParser.TrimMode +static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.Read(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFile(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.FixedWidth.ReadFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null) -> FixedWidthParser.Infrastructure.FixedWidthAsyncRecordEnumerable>! +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatConvert(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, TConverter converter, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatString(string! value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatUtf8Value(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +static FixedWidthParser.Infrastructure.FixedWidthRuntime.FormatValue(TValue value, System.Span slice, in FixedWidthParser.FixedWidthContext context, FixedWidthParser.Infrastructure.ColumnFormatOptions options, string! columnName) -> void +FixedWidthParser.IFixedWidthConverter.TryFormat(T value, System.Span destination, in FixedWidthParser.FixedWidthContext context, out int bytesWritten) -> bool +FixedWidthParser.IFixedWidthConverter.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, out T value) -> bool +FixedWidthParser.FixedWidthReaderOptions.DegreeOfParallelism.get -> int +FixedWidthParser.FixedWidthReaderOptions.DegreeOfParallelism.init -> void +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, FixedWidthParser.FixedWidthLayout! layout, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateAsync(System.IO.Stream! stream, bool leaveOpen = false, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthLayout! layout, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +static FixedWidthParser.FixedWidth.AggregateFileAsync(string! path, FixedWidthParser.FixedWidthReaderOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.Task! +FixedWidthParser.Infrastructure.ColumnFormatOptions.ColumnFormatOptions(FixedWidthParser.Alignment alignment, char padding, string? format, FixedWidthParser.OverflowBehavior overflow) -> void +FixedWidthParser.Infrastructure.ColumnParseOptions +FixedWidthParser.Infrastructure.ColumnParseOptions.ColumnParseOptions() -> void +FixedWidthParser.Infrastructure.ColumnParseOptions.ColumnParseOptions(char trimChar, FixedWidthParser.TrimMode trimMode, string? format) -> void +FixedWidthParser.Infrastructure.ColumnParseOptions.Format.get -> string? +FixedWidthParser.Infrastructure.ColumnParseOptions.TrimChar.get -> char +FixedWidthParser.Infrastructure.ColumnParseOptions.TrimMode.get -> FixedWidthParser.TrimMode +static FixedWidthParser.Infrastructure.FixedWidthRuntime.IsBlank(System.ReadOnlySpan column, in FixedWidthParser.Infrastructure.ColumnParseOptions options) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.String(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options) -> string! +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryConvert(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, TConverter converter, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateOnlyExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.DateOnly value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.DateTime value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDateTimeOffsetExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.DateTimeOffset value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryDouble(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out double value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryFloat(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out float value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParse(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryParseDecoded(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TValue value) -> bool +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out System.TimeOnly value) -> bool +FixedWidthParser.FixedWidthLayoutBuilder.EnumColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +FixedWidthParser.FixedWidthLayoutBuilder.NullableEnumColumn(string! name, int start, int length, FixedWidthParser.ColumnGetter? get, FixedWidthParser.ColumnSetter? set, FixedWidthParser.FixedWidthColumnOptions? options = null) -> FixedWidthParser.FixedWidthLayoutBuilder! +static FixedWidthParser.Infrastructure.FixedWidthRuntime.TryEnum(System.ReadOnlySpan column, in FixedWidthParser.FixedWidthContext context, in FixedWidthParser.Infrastructure.ColumnParseOptions options, out TEnum value) -> bool diff --git a/src/FixedWidthParser/Readers/AsyncRecordEnumeratorCore.cs b/src/FixedWidthParser/Readers/AsyncRecordEnumeratorCore.cs deleted file mode 100644 index 911e34d..0000000 --- a/src/FixedWidthParser/Readers/AsyncRecordEnumeratorCore.cs +++ /dev/null @@ -1,126 +0,0 @@ -using System.Runtime.CompilerServices; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// Shared asynchronous enumerator logic for character and UTF-8 record readers. The span scan - /// remains synchronous; only the source refill awaits a . - /// - public sealed class AsyncRecordEnumeratorCore : IAsyncEnumerator - where T : unmanaged, IEquatable - where TFormat : struct, ILineFormat - where TParser : struct, IRecordLineParser - where TSource : struct, ISource - { - private readonly TParser _strategy; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly CancellationToken _cancellationToken; - private TSource _source; - private LineBufferState _lines; - private TModel _current; - private bool _disposed; - - internal AsyncRecordEnumeratorCore( - TParser strategy, - TSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize, - CancellationToken cancellationToken) - { - _strategy = strategy; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _cancellationToken = cancellationToken; - _lines = default; - _lines.Rent(bufferSize); - _current = default!; - _disposed = false; - } - - /// - public TModel Current => _current; - - /// - public ValueTask MoveNextAsync() - { - ObjectDisposedException.ThrowIf(_disposed, nameof(AsyncRecordEnumeratorCore)); - _cancellationToken.ThrowIfCancellationRequested(); - var result = TryReadFromBuffer(); - if (result == LineStatus.Line) - { - return new ValueTask(true); - } - if (result == LineStatus.End) - { - return new ValueTask(false); - } - - return MoveNextSlowAsync(); - } - - private async ValueTask MoveNextSlowAsync() - { - while (true) - { - PrepareRefill(); - _lines.Advance(await _source - .ReadAsync(_lines.Buffer.AsMemory(_lines.End, _lines.Buffer.Length - _lines.End), _cancellationToken) - .ConfigureAwait(false)); - _cancellationToken.ThrowIfCancellationRequested(); - - var result = TryReadFromBuffer(); - if (result == LineStatus.Line) - { - return true; - } - if (result == LineStatus.End) - { - return false; - } - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private LineStatus TryReadFromBuffer() - { - var status = _lines.TryGetLine(out var line); - if (status == LineStatus.Line) - { - Parse(line); - } - return status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Parse(ReadOnlySpan line) - { - if (!_strategy.TryParse(line, _formatProvider, _stringPool, out _current)) - { - throw new FormatException( - $"Line {_lines.LineNumber} could not be parsed into {typeof(TModel).Name}: \"{TFormat.FormatForException(line)}\"."); - } - } - - private void PrepareRefill() - { - _lines.Compact(); - _lines.GrowIfFull(); - } - - /// - public ValueTask DisposeAsync() - { - if (!_disposed) - { - _lines.Return(); - _source.Dispose(); - _disposed = true; - } - return ValueTask.CompletedTask; - } - } -} diff --git a/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs deleted file mode 100644 index 520b003..0000000 --- a/src/FixedWidthParser/Readers/FixedWidthAsyncRecordEnumerable.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System.Buffers; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// Asynchronous version of : reads from a - /// via await foreach, also without allocating a string per - /// line ( buffer + slices). The - /// enumerator logic lives in the shared , - /// specialized by . - /// - public sealed class FixedWidthAsyncRecordEnumerable : IAsyncEnumerable - where TParser : struct, IRecordLineParser - { - private readonly TParser _parser; - private readonly TextReaderSource _source; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - internal FixedWidthAsyncRecordEnumerable( - TParser parser, - TextReaderSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public AsyncRecordEnumeratorCore GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - { - return new(_parser, _source.Create(_bufferSize), _formatProvider, _stringPool, _bufferSize, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/FixedWidthByteReader.cs b/src/FixedWidthParser/Readers/FixedWidthByteReader.cs deleted file mode 100644 index c13caa7..0000000 --- a/src/FixedWidthParser/Readers/FixedWidthByteReader.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// UTF-8 / byte counterpart of . Parses fixed-width records - /// straight from raw spans and streams, avoiding the UTF-8 → UTF-16 transcode - /// that the char reader pays through a . Configured once with an - /// optional culture, an optional and a buffer size; reusable across reads. - /// - /// Column offsets are measured in bytes (see ), - /// which is exact for the single-byte/ASCII payloads typical of flat files. String columns are - /// decoded with ; when a is - /// supplied, repeated values are interned (decoded once, same instance reused). A leading UTF-8 - /// byte-order mark on the stream is skipped. - /// - /// - public sealed class FixedWidthByteReader where TModel : new() - { - private readonly Utf8FixedWidthParser _parser = new(); - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - public FixedWidthByteReader(IFormatProvider? formatProvider = null, StringPool? stringPool = null, int bufferSize = 65536) - { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// - /// Parses a single UTF-8 fixed-width line (a slice of raw bytes, without the line terminator) - /// into using the configured culture and string pool. Returns - /// when the line is too short for the layout or a non-string column - /// fails to parse. - /// - public bool TryParse(ReadOnlySpan line, out TModel model) - { - return _parser.TryParse(line, _formatProvider, _stringPool, out model); - } - - /// - /// Reads from a (single pass) as raw bytes. - /// controls whether the stream is disposed when iteration completes. - /// - public Utf8FixedWidthRecordEnumerable> Read(Stream stream, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - return new Utf8FixedWidthRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), stream, ownsStream: !leaveOpen, _formatProvider, _stringPool, _bufferSize); - } - - /// Reads from a file as raw bytes. Re-enumerable: each iteration opens the file again. - public Utf8FixedWidthRecordEnumerable> ReadFile(string path) - { - ArgumentException.ThrowIfNullOrEmpty(path); - return new Utf8FixedWidthRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), path, _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a via await foreach (single pass) as raw bytes. - /// controls whether the stream is disposed when iteration completes. - /// - public Utf8FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - return new Utf8FixedWidthAsyncRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), stream, ownsStream: !leaveOpen, _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a file via await foreach, with true asynchronous I/O (FileStream useAsync). - /// Re-enumerable: each iteration reopens the file. - /// - public Utf8FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path) - { - ArgumentException.ThrowIfNullOrEmpty(path); - return new Utf8FixedWidthAsyncRecordEnumerable>( - new ReflectionUtf8LineParser(_parser), path, _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a via await foreach (single pass) as raw bytes, - /// letting the pipe own buffering and read-ahead. Use this when the source is already a pipe - /// (e.g. a Kestrel request body, a socket, or an upstream System.IO.Pipelines stage); - /// the configured buffer size does not apply since the pipe manages its own segments. - /// controls whether the reader is completed when iteration ends. - /// - public Utf8FixedWidthPipeRecordEnumerable ReadAsync(PipeReader reader, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(reader); - return new Utf8FixedWidthPipeRecordEnumerable( - _parser, reader, completeReader: !leaveOpen, _formatProvider, _stringPool); - } - } -} diff --git a/src/FixedWidthParser/Readers/FixedWidthReader.cs b/src/FixedWidthParser/Readers/FixedWidthReader.cs deleted file mode 100644 index 9012486..0000000 --- a/src/FixedWidthParser/Readers/FixedWidthReader.cs +++ /dev/null @@ -1,88 +0,0 @@ -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// Reads models from text sources (TextReader, Stream or file) lazily and with low - /// allocation. Configured once with culture, an optional and a - /// buffer size; reusable across multiple reads. - /// - public sealed class FixedWidthReader where TModel : new() - { - private readonly FixedWidthParser _parser = new(); - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - public FixedWidthReader(IFormatProvider? formatProvider = null, StringPool? stringPool = null, int bufferSize = 65536) - { - ArgumentOutOfRangeException.ThrowIfLessThan(bufferSize, 1); - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Reads from an existing (single pass; does not dispose it). - public FixedWidthRecordEnumerable> Read(TextReader reader) - { - ArgumentNullException.ThrowIfNull(reader); - return new FixedWidthRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromReader(reader), _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a (single pass). Disposes the internally created - /// ; controls closing the stream. - /// - public FixedWidthRecordEnumerable> Read(Stream stream, Encoding? encoding = null, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromStream(stream, enc, leaveOpen), _formatProvider, _stringPool, _bufferSize); - } - - /// Reads from a file. Re-enumerable: each iteration opens the file again. - public FixedWidthRecordEnumerable> ReadFile(string path, Encoding? encoding = null) - { - ArgumentException.ThrowIfNullOrEmpty(path); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromFile(path, enc, useAsync: false), _formatProvider, _stringPool, _bufferSize); - } - - /// Reads from an existing via await foreach (does not dispose it). - public FixedWidthAsyncRecordEnumerable> ReadAsync(TextReader reader) - { - ArgumentNullException.ThrowIfNull(reader); - return new FixedWidthAsyncRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromReader(reader), _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a via await foreach (single pass). Disposes the created - /// ; controls the stream. - /// - public FixedWidthAsyncRecordEnumerable> ReadAsync(Stream stream, Encoding? encoding = null, bool leaveOpen = false) - { - ArgumentNullException.ThrowIfNull(stream); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromStream(stream, enc, leaveOpen), _formatProvider, _stringPool, _bufferSize); - } - - /// - /// Reads from a file via await foreach, with true asynchronous I/O (FileStream useAsync). - /// Re-enumerable: each iteration reopens the file. - /// - public FixedWidthAsyncRecordEnumerable> ReadFileAsync(string path, Encoding? encoding = null) - { - ArgumentException.ThrowIfNullOrEmpty(path); - var enc = encoding ?? Encoding.UTF8; - return new FixedWidthAsyncRecordEnumerable>( - new ReflectionLineParser(_parser), TextReaderSource.FromFile(path, enc, useAsync: true), _formatProvider, _stringPool, _bufferSize); - } - } -} diff --git a/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs deleted file mode 100644 index f8be5b3..0000000 --- a/src/FixedWidthParser/Readers/FixedWidthRecordEnumerable.cs +++ /dev/null @@ -1,56 +0,0 @@ -using System.Buffers; -using System.Collections; -using System.Diagnostics.CodeAnalysis; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// A lazily-read sequence of models from a , without allocating a - /// string per line: lines are read in blocks into a character buffer rented from the - /// and sliced as straight into the - /// parser. Exposes a enumerator for allocation-free iteration in - /// foreach, and implements for LINQ interop. - /// - public sealed class FixedWidthRecordEnumerable : IEnumerable - where TParser : struct, IRecordLineParser - { - private readonly TParser _parser; - private readonly TextReaderSource _source; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - internal FixedWidthRecordEnumerable( - TParser parser, - TextReaderSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Struct enumerator: foreach iteration without heap allocation. - public RecordEnumeratorCore GetEnumerator() - { - return new(_parser, _source.Create(_bufferSize), _formatProvider, _stringPool, _bufferSize); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - [ExcludeFromCodeCoverage] - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - } -} diff --git a/src/FixedWidthParser/Readers/GeneratedUtf8FixedWidthPipeRecordEnumerable.cs b/src/FixedWidthParser/Readers/GeneratedUtf8FixedWidthPipeRecordEnumerable.cs deleted file mode 100644 index 3c18063..0000000 --- a/src/FixedWidthParser/Readers/GeneratedUtf8FixedWidthPipeRecordEnumerable.cs +++ /dev/null @@ -1,45 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// source for the source-generated UTF-8 path: parses fixed-width records - /// straight off a pipe via await foreach through - /// (no reflection, no delegates). The enumerator logic lives in - /// , specialized with the - /// strategy. Single-pass. - /// - public sealed class GeneratedUtf8FixedWidthPipeRecordEnumerable : IAsyncEnumerable - where TModel : IUtf8FixedWidthModel - { - private readonly PipeReader _reader; - private readonly bool _completeReader; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - - internal GeneratedUtf8FixedWidthPipeRecordEnumerable( - PipeReader reader, - bool completeReader, - IFormatProvider? formatProvider, - StringPool? stringPool) - { - _reader = reader; - _completeReader = completeReader; - _formatProvider = formatProvider; - _stringPool = stringPool; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public Utf8PipeRecordEnumeratorCore> GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - { - return new(default, _reader, _completeReader, _formatProvider, _stringPool, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/ILineParser.cs b/src/FixedWidthParser/Readers/ILineParser.cs deleted file mode 100644 index 66c6e9c..0000000 --- a/src/FixedWidthParser/Readers/ILineParser.cs +++ /dev/null @@ -1,54 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// Element-type-generic parser strategy used by the shared record enumerator cores. - /// - public interface IRecordLineParser where T : unmanaged - { - /// Parses a buffered line into a model. - bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model); - } - - /// - /// Strategy that parses a single fixed-width line into a . - /// Implemented by types so the shared enumerator cores can be - /// specialized per strategy: a struct type argument lets the JIT monomorphize each - /// (TModel, TParser) pair and devirtualize into a direct call — - /// no delegate, no boxing, no virtual dispatch on the hot path. - /// - public interface ILineParser - { - /// - bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model); - } - - // Strategy structs are plumbing for the generic enumerator cores; they are never compared, so - // value equality is meaningless for them. -#pragma warning disable CA1815 // Override equals and operator equals on value types - /// - /// Reflection-based parse strategy: forwards to a runtime . - /// - public readonly struct ReflectionLineParser(FixedWidthParser parser) : ILineParser, IRecordLineParser where TModel : new() - { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - return parser.TryParse(line, formatProvider, stringPool, out model); - } - } - - /// - /// Source-generated parse strategy: forwards to the model's static - /// , avoiding reflection and delegates. - /// - public readonly struct GeneratedLineParser : ILineParser, IRecordLineParser where TModel : IFixedWidthModel - { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - return TModel.TryParse(line, formatProvider, stringPool, out model); - } - } -#pragma warning restore CA1815 // Override equals and operator equals on value types -} diff --git a/src/FixedWidthParser/Readers/ISource.cs b/src/FixedWidthParser/Readers/ISource.cs deleted file mode 100644 index 2955fa0..0000000 --- a/src/FixedWidthParser/Readers/ISource.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace FixedWidthParser.Readers -{ - /// - /// Synchronous and asynchronous source for buffered record reading. Implemented by value types - /// so the reader cores specialize the calls without interface dispatch or allocations. - /// - public interface ISource where T : unmanaged - { - /// Reads buffered elements into . - int Read(Span buffer); - - /// Asynchronously reads buffered elements into . - ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken); - - /// Disposes the source when it owns its underlying resource. - void Dispose(); - } -} diff --git a/src/FixedWidthParser/Readers/IUtf8LineParser.cs b/src/FixedWidthParser/Readers/IUtf8LineParser.cs deleted file mode 100644 index 2e8f3f6..0000000 --- a/src/FixedWidthParser/Readers/IUtf8LineParser.cs +++ /dev/null @@ -1,47 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// UTF-8 / byte counterpart of : a strategy that parses a single - /// fixed-width line of raw s into a . Implemented by - /// types so the shared UTF-8 enumerator cores can be specialized per - /// strategy — a struct type argument lets the JIT monomorphize each (TModel, TParser) - /// pair and devirtualize into a direct call (no delegate, no boxing, no - /// virtual dispatch on the hot path). - /// - public interface IUtf8LineParser - { - /// - bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model); - } - - // Strategy structs are plumbing for the generic enumerator cores; they are never compared, so - // value equality is meaningless for them. -#pragma warning disable CA1815 // Override equals and operator equals on value types - /// - /// Reflection-based parse strategy: forwards to a runtime . - /// - public readonly struct ReflectionUtf8LineParser(Utf8FixedWidthParser parser) : IUtf8LineParser, IRecordLineParser where TModel : new() - { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - return parser.TryParse(line, formatProvider, stringPool, out model); - } - } - - /// - /// Source-generated parse strategy: forwards to the model's static - /// , avoiding reflection and delegates. - /// - public readonly struct GeneratedUtf8LineParser : IUtf8LineParser, IRecordLineParser where TModel : IUtf8FixedWidthModel - { - public bool TryParse(ReadOnlySpan line, IFormatProvider? formatProvider, StringPool? stringPool, out TModel model) - { - return TModel.TryParse(line, formatProvider, stringPool, out model); - } - - } -#pragma warning restore CA1815 // Override equals and operator equals on value types -} diff --git a/src/FixedWidthParser/Readers/LineBufferState.cs b/src/FixedWidthParser/Readers/LineBufferState.cs index b4d685a..7a396f7 100644 --- a/src/FixedWidthParser/Readers/LineBufferState.cs +++ b/src/FixedWidthParser/Readers/LineBufferState.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Text; namespace FixedWidthParser.Readers { @@ -11,111 +10,82 @@ internal enum LineStatus } /// - /// Element-type specifics for : the line-break markers - /// and an optional preamble (BOM) to skip before the first line. Implemented by - /// s so the calls devirtualize inside the generic buffer state, exactly - /// like the TParser strategy on the enumerator cores. + /// Buffers bytes read in blocks from a stream and splits them into lines as + /// slices, with no per-line allocation. Splits on \n, dropping + /// a preceding \r, skips a UTF-8 byte order mark when asked, and refuses a line longer than + /// the maximum, which bounds how far the buffer grows on a source without line breaks. /// - public interface ILineFormat where T : unmanaged + internal struct LineBufferState { - static abstract T Cr { get; } - static abstract T Lf { get; } - - /// - /// Decides how many leading elements to skip before the first line (e.g. a UTF-8 BOM). - /// Returns when more buffered data is needed to decide. - /// - static abstract bool TrySkipPreamble(ReadOnlySpan data, bool eof, out int skip); + internal byte[] Buffer; + internal int Start; + internal int End; + internal int ScanFrom; + internal bool Eof; + internal long LineNumber; + // Offset, from the first byte read, of the line TryGetLine last returned: the byte after the + // previous line break, so a skipped byte order mark belongs to the first line. + internal long LineStart; + private long _nextLineStart; + private long _compacted; + private int _lineOffset; + private int _lineLength; + private int _maxLineLength; + private bool _preambleChecked; - /// Formats a line for a parsing exception. - static abstract string FormatForException(ReadOnlySpan line); - } + /// Offset, from the first byte read, of the first byte no line has consumed yet. + internal readonly long Consumed => _compacted + Start; - /// Char line format: no preamble — the already handled the BOM. - public readonly struct CharLineFormat : ILineFormat - { - public static char Cr => '\r'; - public static char Lf => '\n'; + /// The line last returned, valid until the next . + internal readonly ReadOnlySpan CurrentLine => Buffer.AsSpan(_lineOffset, _lineLength); - public static bool TrySkipPreamble(ReadOnlySpan data, bool eof, out int skip) + internal void Rent(FixedWidthReaderOptions options) { - skip = 0; - return true; + Rent(options, options.SkipsByteOrderMark); } - public static string FormatForException(ReadOnlySpan line) + internal void Rent(FixedWidthReaderOptions options, bool skipByteOrderMark) { - return line.ToString(); + Init(ArrayPool.Shared.Rent(options.BufferSize), 0, eof: false, linesBefore: 0, options, skipByteOrderMark); } - } - /// UTF-8 byte line format: skips a leading byte-order mark (EF BB BF). - public readonly struct Utf8LineFormat : ILineFormat - { - public static byte Cr => (byte)'\r'; - public static byte Lf => (byte)'\n'; - - // Needs at least 3 buffered bytes to decide; otherwise wait for more data (unless the - // stream already ended). - public static bool TrySkipPreamble(ReadOnlySpan data, bool eof, out int skip) - { - if (data.Length >= 3) - { - skip = data is [0xEF, 0xBB, 0xBF, ..] ? 3 : 0; - return true; - } - skip = 0; - return eof; - } - - public static string FormatForException(ReadOnlySpan line) + /// + /// Takes ownership of a pooled already holding the whole input: its + /// first bytes, numbered from the line after . + /// + internal void Wrap(byte[] buffer, int length, long linesBefore, FixedWidthReaderOptions options, bool skipByteOrderMark) { - return Encoding.UTF8.GetString(line); + Init(buffer, length, eof: true, linesBefore, options, skipByteOrderMark); } - } - /// - /// Buffers elements read in blocks from the underlying source and splits them into lines as - /// , with no string (or per-line buffer) allocated. Splits on - /// , dropping a preceding (CRLF), - /// and skips the format's preamble before the first line. Used as - /// / by the -based cores and - /// as / by the -based UTF-8 cores. - /// - internal struct LineBufferState - where T : unmanaged, IEquatable - where TFormat : struct, ILineFormat - { - internal T[] Buffer; - internal int Start; - internal int End; - internal int ScanFrom; - internal bool Eof; - internal int LineNumber; - private bool _preambleChecked; - - internal void Rent(int bufferSize) + private void Init(byte[] buffer, int end, bool eof, long linesBefore, FixedWidthReaderOptions options, bool skipByteOrderMark) { - Buffer = ArrayPool.Shared.Rent(bufferSize); + Buffer = buffer; Start = 0; - End = 0; + End = end; ScanFrom = 0; - Eof = false; - LineNumber = 0; - _preambleChecked = false; + Eof = eof; + LineNumber = linesBefore; + LineStart = 0; + _nextLineStart = 0; + _compacted = 0; + _maxLineLength = options.MaxLineLength; + _preambleChecked = !skipByteOrderMark; } - internal LineStatus TryGetLine(out ReadOnlySpan line) + internal LineStatus TryGetLine(out ReadOnlySpan line) { if (!_preambleChecked) { - if (!TFormat.TrySkipPreamble(Buffer.AsSpan(Start, End - Start), Eof, out int skip)) + if (End - Start < 3 && !Eof) { line = default; return LineStatus.NeedData; } - - Start += skip; + if (Buffer.AsSpan(Start, End - Start) is [0xEF, 0xBB, 0xBF, ..]) + { + Start += 3; + } ScanFrom = Start; _preambleChecked = true; } @@ -124,11 +94,10 @@ internal LineStatus TryGetLine(out ReadOnlySpan line) { if (ScanFrom < End) { - int rel = Buffer.AsSpan(ScanFrom, End - ScanFrom).IndexOf(TFormat.Lf); + int rel = Buffer.AsSpan(ScanFrom, End - ScanFrom).IndexOf((byte)'\n'); if (rel >= 0) { int nlIndex = ScanFrom + rel; - // Content ends at the '\n' (trimming a preceding '\r'); resume past the '\n'. if (TrySliceLine(rawEnd: nlIndex, nextStart: nlIndex + 1, out line)) { return LineStatus.Line; @@ -158,26 +127,42 @@ internal LineStatus TryGetLine(out ReadOnlySpan line) } /// - /// Slices the line ending at (dropping a trailing - /// ), advances / to - /// , and counts the line. Returns for an - /// empty line so the caller skips it while still having consumed it. + /// Slices the line ending at (dropping a trailing \r), advances to + /// and counts the line. Returns for an empty + /// line so the caller skips it while still having consumed it. /// - private bool TrySliceLine(int rawEnd, int nextStart, out ReadOnlySpan line) + private bool TrySliceLine(int rawEnd, int nextStart, out ReadOnlySpan line) { int contentEnd = rawEnd; - if (contentEnd > Start && Buffer[contentEnd - 1].Equals(TFormat.Cr)) + if (contentEnd > Start && Buffer[contentEnd - 1] == (byte)'\r') { contentEnd--; } - line = Buffer.AsSpan(Start, contentEnd - Start); + if (contentEnd - Start > _maxLineLength) + { + throw ParseErrors.LineTooLong(LineNumber + 1L, _maxLineLength); + } + + _lineOffset = Start; + _lineLength = contentEnd - Start; + line = Buffer.AsSpan(_lineOffset, _lineLength); + LineStart = _nextLineStart; + _nextLineStart = _compacted + nextStart; Start = nextStart; ScanFrom = nextStart; LineNumber++; return !line.IsEmpty; } + /// Makes room in the buffer and reads the next block of into it. + internal void Fill(Stream stream) + { + Compact(); + GrowIfFull(); + Advance(stream.Read(Buffer.AsSpan(End))); + } + internal void Compact() { if (Start <= 0) @@ -193,6 +178,7 @@ internal void Compact() End = len; ScanFrom -= Start; + _compacted += Start; Start = 0; } @@ -203,9 +189,16 @@ internal void GrowIfFull() return; } - var bigger = ArrayPool.Shared.Rent(Buffer.Length * 2); + // A full buffer after Compact holds one unterminated line; +1 leaves room for its '\r'. + if (End - Start > _maxLineLength + 1L) + { + throw ParseErrors.LineTooLong(LineNumber + 1L, _maxLineLength); + } + + int size = (int)Math.Min(Buffer.Length * 2L, _maxLineLength + 2L); + var bigger = ArrayPool.Shared.Rent(size); Array.Copy(Buffer, 0, bigger, 0, End); - ArrayPool.Shared.Return(Buffer); + ArrayPool.Shared.Return(Buffer); Buffer = bigger; } @@ -225,7 +218,7 @@ internal void Return() { if (Buffer is not null) { - ArrayPool.Shared.Return(Buffer); + ArrayPool.Shared.Return(Buffer); Buffer = null!; } } diff --git a/src/FixedWidthParser/Readers/LineSinks.cs b/src/FixedWidthParser/Readers/LineSinks.cs new file mode 100644 index 0000000..d7b3b82 --- /dev/null +++ b/src/FixedWidthParser/Readers/LineSinks.cs @@ -0,0 +1,18 @@ +namespace FixedWidthParser.Readers +{ + /// + /// Where the parallel reader puts each line of a partition. A , so the reader is + /// specialized per sink and the per-line call devirtualizes. + /// + internal interface ILineSink where TSelf : struct, ILineSink + { + /// Takes one line; when it does not parse, leaving the sink as it was. + bool TryAdd(ReadOnlySpan line, in FixedWidthContext context); + + /// The exception for a line rejected. + FixedWidthParseException Diagnose(ReadOnlySpan line, long lineNumber, in FixedWidthContext context); + + /// Folds the sink of the partition that follows this one into it. + void Merge(TSelf following); + } +} diff --git a/src/FixedWidthParser/Readers/ParallelAggregator.cs b/src/FixedWidthParser/Readers/ParallelAggregator.cs new file mode 100644 index 0000000..2ad9148 --- /dev/null +++ b/src/FixedWidthParser/Readers/ParallelAggregator.cs @@ -0,0 +1,314 @@ +using System.Buffers; +using System.Runtime.ExceptionServices; +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Readers +{ + /// + /// Reads a file in byte-range partitions on several threads, folds each partition into its own + /// accumulator, and merges them in file order. A line belongs to the partition its first byte falls + /// in: a worker starts one byte before its range, skips the line it lands inside, and stops once the + /// next line would start past its range. Fixed-width lines have no quoting, so that boundary is exact + /// and every line is parsed exactly once. One partition is the sequential read. + /// + internal static class ParallelAggregator + { + private const int ChunksPerWorker = 4; + private const long MinChunkBytes = 1024 * 1024; + private const long MaxChunkBytes = 64 * 1024 * 1024; + private const int LinesPerCancellationCheck = 4096; + + internal static async Task RunAsync( + TParser parser, string path, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + using var handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read); + var source = new FileSource(handle); + return await RunPartitionsAsync(source, 0, options, + (plan, index) => AggregateChunk(parser, source, plan, index, options, ct), + static (merged, following) => + { + merged.Merge(following); + return merged; + }, ct).ConfigureAwait(false); + } + + /// + /// Reads [dataStart, source.Length) in partitions, one each, merged in + /// order. must start a line; line numbers count from the start of the source. + /// gets where the bytes of the partition its sink will read start and end; a line + /// belongs to the partition its first byte falls in. + /// + internal static Task RunAsync( + IRandomAccessSource source, long dataStart, Func create, FixedWidthReaderOptions options, CancellationToken ct) + where TSink : struct, ILineSink + { + return RunPartitionsAsync(source, dataStart, options, + (plan, index) => SinkChunk(source, plan, index, create(plan.Start(index), plan.End(index)), options, ct), + static (merged, following) => + { + merged.Merge(following); + return merged; + }, ct); + } + + private static async Task RunPartitionsAsync( + IRandomAccessSource source, long dataStart, FixedWidthReaderOptions options, + Func aggregate, Func merge, CancellationToken ct) + { + var plan = new ChunkPlan(dataStart, source.Length, options); + var results = new TResult[plan.Count]; + var failures = new ExceptionDispatchInfo?[plan.Count]; + var workers = new Task[Math.Min(options.Workers, plan.Count)]; + foreach (ref var worker in workers.AsSpan()) + { + worker = Task.Run(() => Drain(plan, aggregate, results, failures, ct), ct); + } + await Task.WhenAll(workers).ConfigureAwait(false); + + failures[0]?.Throw(); + var merged = results[0]; + for (int i = 1; i < plan.Count; i++) + { + failures[i]?.Throw(); + merged = merge(merged, results[i]); + } + return merged; + } + + private static void Drain( + ChunkPlan plan, Func aggregate, TResult[] results, ExceptionDispatchInfo?[] failures, CancellationToken ct) + { + while (plan.TryTakeNext(out int index)) + { + ct.ThrowIfCancellationRequested(); + try + { + results[index] = aggregate(plan, index); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + failures[index] = ExceptionDispatchInfo.Capture(ex); + } + } + } + + // ponytail: the model and sink loops are twins on purpose. Folding the model path into a ModelSink + // made one-call reads ~40% slower: dynamic PGO stopped devirtualizing IRecordAccumulator.Add through + // the shared generic sink. Keep them in step. + private static TAccumulator AggregateChunk( + TParser parser, IRandomAccessSource source, ChunkPlan plan, int index, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + long start = plan.Start(index); + long end = plan.End(index); + var accumulator = new TAccumulator(); + var context = options.Context; + var reader = new ChunkReader(source, index == 0 ? start : start - 1, options.MaxLineLength); + var lines = default(LineBufferState); + lines.Rent(options, start == 0 && options.SkipsByteOrderMark); + long delivered = 0; + try + { + while (reader.Origin + lines.Consumed < end) + { + if (reader.NextLine(ref lines, out var line) == LineStatus.End) + { + break; + } + long lineStart = reader.Origin + lines.LineStart; + if (lineStart < start || lineStart >= end) + { + continue; + } + if (++delivered % LinesPerCancellationCheck == 0) + { + ct.ThrowIfCancellationRequested(); + } + if (!TryAdd(parser, line, in context, ref accumulator)) + { + ParseErrors.HandleFailure(parser, line, reader.LineNumberAt(lineStart), in context, options.OnParseError); + } + } + } + finally + { + lines.Return(); + } + return accumulator; + } + + private static TSink SinkChunk( + IRandomAccessSource source, ChunkPlan plan, int index, TSink sink, FixedWidthReaderOptions options, CancellationToken ct) + where TSink : struct, ILineSink + { + long start = plan.Start(index); + long end = plan.End(index); + var context = options.Context; + var reader = new ChunkReader(source, index == 0 ? start : start - 1, options.MaxLineLength); + var lines = default(LineBufferState); + lines.Rent(options, start == 0 && options.SkipsByteOrderMark); + long delivered = 0; + try + { + while (reader.Origin + lines.Consumed < end) + { + if (reader.NextLine(ref lines, out var line) == LineStatus.End) + { + break; + } + long lineStart = reader.Origin + lines.LineStart; + if (lineStart < start || lineStart >= end) + { + continue; + } + if (++delivered % LinesPerCancellationCheck == 0) + { + ct.ThrowIfCancellationRequested(); + } + if (!sink.TryAdd(line, in context)) + { + if (options.OnSkip is { } skip) + { + skip(); + continue; + } + ParseErrors.Reject(sink.Diagnose(line, reader.LineNumberAt(lineStart), in context), options.OnParseError); + } + } + } + finally + { + lines.Return(); + } + return sink; + } + + /// Parses one line and folds it in; when it does not parse. + internal static bool TryAdd(TParser parser, ReadOnlySpan line, in FixedWidthContext context, ref TAccumulator accumulator) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator + { + if (!parser.TryParse(line, in context, out var model)) + { + return false; + } + accumulator.Add(in model); + return true; + } + + private sealed class ChunkPlan + { + private readonly long _dataStart; + private readonly long _length; + private readonly long _chunkSize; + private int _next = -1; + + internal ChunkPlan(long dataStart, long length, FixedWidthReaderOptions options) + { + _dataStart = dataStart; + _length = length; + long data = Math.Max(0, length - dataStart); + _chunkSize = ChunkSize(data, options); + Count = (int)Math.Max(1, (data + _chunkSize - 1) / _chunkSize); + } + + internal int Count { get; } + + internal long Start(int index) + { + return _dataStart + (index * _chunkSize); + } + + internal long End(int index) + { + return Math.Min(Start(index) + _chunkSize, _length); + } + + internal bool TryTakeNext(out int index) + { + index = Interlocked.Increment(ref _next); + return index < Count; + } + + private static long ChunkSize(long length, FixedWidthReaderOptions options) + { + if (options.ChunkSize > 0) + { + return options.ChunkSize; + } + if (options.Workers == 1) + { + return Math.Max(1, length); + } + return Math.Clamp(length / ((long)options.Workers * ChunksPerWorker), MinChunkBytes, MaxChunkBytes); + } + } + + /// Feeds one partition's bytes to a and numbers its lines on demand. + private sealed class ChunkReader(IRandomAccessSource source, long origin, int maxLineLength) + { + private long _readOffset = origin; + private long _countedTo; + private long _breaksCounted; + + internal long Origin { get; } = origin; + + internal LineStatus NextLine(ref LineBufferState lines, out ReadOnlySpan line) + { + try + { + while (true) + { + var status = lines.TryGetLine(out line); + if (status != LineStatus.NeedData) + { + return status; + } + lines.Compact(); + lines.GrowIfFull(); + int read = source.Read(lines.Buffer.AsSpan(lines.End), _readOffset); + _readOffset += read; + lines.Advance(read); + } + } + catch (InvalidDataException) + { + // LineBufferState numbers lines from the partition start; the file numbers them from its own. + throw ParseErrors.LineTooLong(LineNumberAt(Origin + lines.Consumed), maxLineLength); + } + } + + /// The 1-based line number of the line starting at , as a sequential read counts it. + // ponytail: counts line breaks from the start of the file on the first failure in a partition, + // then from the last count; only failing lines pay for it. + internal long LineNumberAt(long offset) + { + byte[] buffer = ArrayPool.Shared.Rent(64 * 1024); + try + { + while (_countedTo < offset) + { + int read = source.Read(buffer.AsSpan(0, (int)Math.Min(buffer.Length, offset - _countedTo)), _countedTo); + if (read == 0) + { + break; + } + _breaksCounted += buffer.AsSpan(0, read).Count((byte)'\n'); + _countedTo += read; + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + return _breaksCounted + 1; + } + } + } +} diff --git a/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs new file mode 100644 index 0000000..7cf1fe6 --- /dev/null +++ b/src/FixedWidthParser/Readers/ParallelStreamAggregator.cs @@ -0,0 +1,317 @@ +using System.Buffers; +using System.Runtime.ExceptionServices; +using System.Runtime.InteropServices; +using System.Threading.Channels; +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Readers +{ + /// + /// Parallel aggregation over a sequential source. One producer reads the source into pooled chunks, + /// cuts each after its last line break, carries the rest into the next chunk, and counts the line + /// breaks it hands over, so every chunk knows the number of its first line. Workers fold the chunks + /// from a bounded queue into one accumulator each, merged in chunk order as they complete. Every chunk + /// holds one of a few slots per worker from the moment it is produced until it is merged, which bounds + /// memory, chunks and unmerged accumulators alike, whatever the source's size. + /// + internal static class ParallelStreamAggregator + { + private const int DefaultChunkSize = 256 * 1024; + private const int LinesPerCancellationCheck = 4096; + private const int SlotsPerWorker = 4; + + [StructLayout(LayoutKind.Auto)] + private readonly record struct Chunk(int Index, byte[] Buffer, int Length, long LinesBefore); + + [StructLayout(LayoutKind.Auto)] + private readonly record struct Outcome(int Index, TAccumulator? Accumulator, ExceptionDispatchInfo? Failure); + + /// Folds , disposing it afterwards unless . + internal static async Task RunAsync( + TParser parser, Stream source, bool leaveOpen, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + try + { + if (options.Workers == 1) + { + return await SequentialAsync(parser, source, options, ct).ConfigureAwait(false); + } + return await PartitionedAsync(parser, source, options, ct).ConfigureAwait(false); + } + finally + { + if (!leaveOpen) + { + await source.DisposeAsync().ConfigureAwait(false); + } + } + } + + private static async Task SequentialAsync( + TParser parser, Stream source, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + var accumulator = new TAccumulator(); + var records = new FixedWidthAsyncRecordEnumerable(parser, source, ownsStream: false, options); + await foreach (var model in records.WithCancellation(ct).ConfigureAwait(false)) + { + accumulator.Add(in model); + } + return accumulator; + } + + private static async Task PartitionedAsync( + TParser parser, Stream source, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + int workerCount = options.Workers; + // Two chunks queued and one folding per worker leave one slot per worker for results waiting on an earlier chunk. + using var slots = new SemaphoreSlim(workerCount * SlotsPerWorker); + var queue = Channel.CreateBounded(new BoundedChannelOptions(workerCount * 2) { SingleWriter = true }); + var merge = new OrderedMerge(slots); + var workers = new Task[workerCount]; + foreach (ref var worker in workers.AsSpan()) + { +#pragma warning disable IDISP013 // Every worker is awaited below, before the slots are disposed. + worker = Task.Run(() => ConsumeAsync(parser, queue.Reader, merge, options, ct), ct); +#pragma warning restore IDISP013 + } + + ExceptionDispatchInfo? producerFailure = null; + try + { + await ProduceAsync(source, queue.Writer, slots, options, options.ChunkSize > 0 ? options.ChunkSize : DefaultChunkSize, ct).ConfigureAwait(false); + } + // Cancellation too: the workers must finish before the chunks left in the queue are returned. +#pragma warning disable CA1031 // Rethrown by merge.Result once the workers have finished. + catch (Exception ex) +#pragma warning restore CA1031 + { + producerFailure = ExceptionDispatchInfo.Capture(ex); + } + finally + { + queue.Writer.TryComplete(); + } + + try + { + await Task.WhenAll(workers).ConfigureAwait(false); + return merge.Result(producerFailure); + } + finally + { + while (queue.Reader.TryRead(out var abandoned)) + { + ArrayPool.Shared.Return(abandoned.Buffer); + } + } + } + + private static async Task ProduceAsync( + Stream source, ChannelWriter queue, SemaphoreSlim slots, FixedWidthReaderOptions options, int chunkSize, CancellationToken ct) + { + byte[] buffer = ArrayPool.Shared.Rent(chunkSize); + int filled = 0; + int index = 0; + long linesBefore = 0; + try + { + while (true) + { + if (filled == buffer.Length) + { + buffer = GrowUnterminated(buffer, linesBefore, options.MaxLineLength); + } + int read = await source.ReadAsync(buffer.AsMemory(filled), ct).ConfigureAwait(false); + filled += read; + bool eof = read == 0; + if (eof && filled == 0) + { + return; + } + if (!eof && filled < buffer.Length) + { + continue; + } + + int cut = eof ? filled : buffer.AsSpan(0, filled).LastIndexOf((byte)'\n') + 1; + if (cut == 0) + { + continue; + } + int tail = filled - cut; + byte[] next = ArrayPool.Shared.Rent(Math.Max(chunkSize, tail * 2)); + buffer.AsSpan(cut, tail).CopyTo(next); + long lines = buffer.AsSpan(0, cut).Count((byte)'\n'); + try + { + await slots.WaitAsync(ct).ConfigureAwait(false); + await queue.WriteAsync(new Chunk(index++, buffer, cut, linesBefore), ct).ConfigureAwait(false); + } + catch + { + ArrayPool.Shared.Return(next); + throw; + } + buffer = next; + filled = tail; + linesBefore += lines; + if (eof) + { + return; + } + } + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + // A full chunk with no line break holds the start of one line; grow it as LineBufferState does. + private static byte[] GrowUnterminated(byte[] buffer, long linesBefore, int maxLineLength) + { + if (buffer.Length > maxLineLength + 1L) + { + throw ParseErrors.LineTooLong(linesBefore + 1, maxLineLength); + } + byte[] bigger = ArrayPool.Shared.Rent((int)Math.Min(buffer.Length * 2L, maxLineLength + 2L)); + buffer.CopyTo(bigger, 0); + ArrayPool.Shared.Return(buffer); + return bigger; + } + + private static async Task ConsumeAsync( + TParser parser, ChannelReader queue, OrderedMerge merge, FixedWidthReaderOptions options, CancellationToken ct) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + var context = options.Context; + await foreach (var chunk in queue.ReadAllAsync(ct).ConfigureAwait(false)) + { + Outcome outcome; + try + { + outcome = new(chunk.Index, Fold(parser, chunk, options, in context, ct), null); + } + // Anything but the caller's cancellation, which stops everything, becomes this chunk's result: a chunk + // that never completes would hold its slot, and the producer would wait for it forever. + catch (Exception ex) when (ex is not OperationCanceledException || !ct.IsCancellationRequested) + { + outcome = new(chunk.Index, default, ExceptionDispatchInfo.Capture(ex)); + } + merge.Complete(outcome); + } + } + + private static TAccumulator Fold( + TParser parser, Chunk chunk, FixedWidthReaderOptions options, in FixedWidthContext context, CancellationToken ct) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + where TAccumulator : IRecordAccumulator, new() + { + var accumulator = new TAccumulator(); + var lines = default(LineBufferState); + lines.Wrap(chunk.Buffer, chunk.Length, chunk.LinesBefore, options, chunk.Index == 0 && options.SkipsByteOrderMark); + long delivered = 0; + try + { + while (lines.TryGetLine(out var line) == LineStatus.Line) + { + if (++delivered % LinesPerCancellationCheck == 0) + { + ct.ThrowIfCancellationRequested(); + } + if (!ParallelAggregator.TryAdd(parser, line, in context, ref accumulator)) + { + ParseErrors.HandleFailure(parser, line, lines.LineNumber, in context, options.OnParseError); + } + } + } + finally + { + lines.Return(); + } + return accumulator; + } + + /// + /// Merges the workers' accumulators left to right as soon as every earlier chunk has been merged, and + /// frees each chunk's slot then: a slow early chunk stalls the producer instead of letting later + /// results pile up. + /// + private sealed class OrderedMerge(SemaphoreSlim slots) + where TModel : allows ref struct + where TAccumulator : IRecordAccumulator, new() + { + private readonly Lock _gate = new(); + private readonly Dictionary> _pending = []; + private int _next; + private bool _any; + private TAccumulator? _merged; + private ExceptionDispatchInfo? _failure; + + public void Complete(Outcome outcome) + { + int released = 0; + lock (_gate) + { + _pending.Add(outcome.Index, outcome); + while (_pending.Remove(_next, out var ready)) + { + _next++; + released++; + // Past the first failure the results are dropped; only their slots still matter. + _failure ??= ready.Failure; + if (_failure is null) + { + Accept(ready.Accumulator!); + } + } + } + if (released > 0) + { + slots.Release(released); + } + } + + // Every chunk precedes the point where the producer failed, so a worker's failure comes first in the source. + public TAccumulator Result(ExceptionDispatchInfo? producerFailure) + { + _failure?.Throw(); + producerFailure?.Throw(); + return _any ? _merged! : new TAccumulator(); + } + + private void Accept(TAccumulator accumulator) + { + if (!_any) + { + _merged = accumulator; + _any = true; + return; + } + try + { + _merged!.Merge(accumulator); + } + // Thrown by Result instead, so the slots keep flowing and the producer is never left waiting. +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + _failure = ExceptionDispatchInfo.Capture(ex); + } + } + } + } +} diff --git a/src/FixedWidthParser/Readers/ParseFailure.cs b/src/FixedWidthParser/Readers/ParseFailure.cs new file mode 100644 index 0000000..0683343 --- /dev/null +++ b/src/FixedWidthParser/Readers/ParseFailure.cs @@ -0,0 +1,123 @@ +using System.Runtime.CompilerServices; +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Readers +{ + /// What a parser reported about the line it rejected, filled during a reader's diagnosis re-parse. + internal sealed class ParseFailure + { + internal string? ColumnName { get; private set; } + internal int Start { get; private set; } = -1; + internal int Length { get; private set; } + internal int RequiredLength { get; private set; } + + internal void SetColumn(string columnName, int start, int length) + { + ColumnName = columnName; + Start = start; + Length = length; + } + + internal void SetShortLine(int requiredLength) + { + RequiredLength = requiredLength; + } + } + + /// + /// The slow paths shared by every reader. A line that fails to parse is parsed again with a failure + /// sink, so the hot path pays nothing to learn which column rejected it. + /// + internal static class ParseErrors + { + /// + /// Whether an enumerator should yield . Without + /// every line is yielded and a bad one throws when + /// reads it; with it, the line is parsed here so a bad one can be skipped. + /// + internal static bool Accepts( + TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context, Func? onParseError) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + if (onParseError is null || parser.TryParse(line, in context, out _)) + { + return true; + } + HandleFailure(parser, line, lineNumber, in context, onParseError); + return false; + } + + /// Parses the line an enumerator sits on, throwing as a reader does when it does not parse. + internal static TModel Parse(TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + if (!parser.TryParse(line, in context, out var model)) + { + HandleFailure(parser, line, lineNumber, in context, onParseError: null); + } + return model; + } + + /// Returns when asks to skip the line; throws otherwise. + [MethodImpl(MethodImplOptions.NoInlining)] + internal static void HandleFailure( + TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context, Func? onParseError) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + Reject(Diagnose(parser, line, lineNumber, in context), onParseError); + } + + /// Returns when asks to skip the line describes; throws it otherwise. + internal static void Reject(FixedWidthParseException exception, Func? onParseError) + { + if (onParseError is null || !onParseError(exception)) + { + throw exception; + } + } + + /// Parses a rejected line again with a failure sink and builds the exception that names its column. + [MethodImpl(MethodImplOptions.NoInlining)] + internal static FixedWidthParseException Diagnose(TParser parser, ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + where TModel : allows ref struct + where TParser : struct, IRecordLineParser + { + var failure = new ParseFailure(); + var diagnosis = context with { Failure = failure }; + _ = parser.TryParse(line, in diagnosis, out _); + return Create(line, lineNumber, typeof(TModel).Name, failure, in context); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + internal static InvalidDataException LineTooLong(long lineNumber, int maxLineLength) + { + return new InvalidDataException( + $"Line {lineNumber} is longer than {maxLineLength} bytes (FixedWidthReaderOptions.MaxLineLength)."); + } + + private static FixedWidthParseException Create(ReadOnlySpan line, long lineNumber, string modelName, ParseFailure failure, in FixedWidthContext context) + { + if (failure.RequiredLength > line.Length) + { + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed into {modelName}: it is {line.Length} bytes long and the layout needs {failure.RequiredLength}.", + lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + + if (failure.ColumnName is { } columnName && failure.Start >= 0 && failure.Start + failure.Length <= line.Length) + { + string rawValue = context.Encoding.GetString(line.Slice(failure.Start, failure.Length)); + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed into {modelName}: column '{columnName}' (start {failure.Start}, length {failure.Length}) holds \"{rawValue}\".", + lineNumber, columnName, failure.Start, failure.Length, rawValue); + } + + return new FixedWidthParseException( + $"Line {lineNumber} could not be parsed into {modelName}: \"{context.Encoding.GetString(line)}\".", + lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + } +} diff --git a/src/FixedWidthParser/Readers/RandomAccessSources.cs b/src/FixedWidthParser/Readers/RandomAccessSources.cs new file mode 100644 index 0000000..b7ab714 --- /dev/null +++ b/src/FixedWidthParser/Readers/RandomAccessSources.cs @@ -0,0 +1,23 @@ +using Microsoft.Win32.SafeHandles; + +namespace FixedWidthParser.Readers +{ + /// Bytes read at an offset, from any thread: what the parallel reader partitions. + internal interface IRandomAccessSource + { + long Length { get; } + + /// Copies the bytes at into ; 0 at the end. + int Read(Span destination, long offset); + } + + internal sealed class FileSource(SafeFileHandle handle) : IRandomAccessSource + { + public long Length { get; } = RandomAccess.GetLength(handle); + + public int Read(Span destination, long offset) + { + return RandomAccess.Read(handle, destination, offset); + } + } +} diff --git a/src/FixedWidthParser/Readers/RecordEnumeratorCore.cs b/src/FixedWidthParser/Readers/RecordEnumeratorCore.cs deleted file mode 100644 index ae04bc2..0000000 --- a/src/FixedWidthParser/Readers/RecordEnumeratorCore.cs +++ /dev/null @@ -1,107 +0,0 @@ -using System.Collections; -using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// Shared synchronous enumerator logic for character and UTF-8 record readers. The element, - /// line format, parser, and source strategies are structs, so the JIT specializes every hot-path - /// call without allocations or interface dispatch. - /// - public struct RecordEnumeratorCore : IEnumerator - where T : unmanaged, IEquatable - where TFormat : struct, ILineFormat - where TParser : struct, IRecordLineParser - where TSource : struct, ISource - { - private readonly TParser _strategy; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private TSource _source; - private LineBufferState _lines; - private TModel _current; - private bool _disposed; - - internal RecordEnumeratorCore( - TParser strategy, - TSource source, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _strategy = strategy; - _source = source; - _formatProvider = formatProvider; - _stringPool = stringPool; - _lines = default; - _lines.Rent(bufferSize); - _current = default!; - _disposed = false; - } - - /// - public readonly TModel Current => _current; - [ExcludeFromCodeCoverage] - readonly object IEnumerator.Current => _current!; - - /// - public bool MoveNext() - { - ObjectDisposedException.ThrowIf(_disposed, nameof(RecordEnumeratorCore)); - while (true) - { - var status = _lines.TryGetLine(out var line); - if (status == LineStatus.Line) - { - Parse(line); - return true; - } - if (status == LineStatus.End) - { - return false; - } - - Refill(); - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void Parse(ReadOnlySpan line) - { - if (!_strategy.TryParse(line, _formatProvider, _stringPool, out _current)) - { - throw new FormatException( - $"Line {_lines.LineNumber} could not be parsed into {typeof(TModel).Name}: \"{TFormat.FormatForException(line)}\"."); - } - } - - private void Refill() - { - _lines.Compact(); - _lines.GrowIfFull(); - _lines.Advance(_source.Read(_lines.Buffer.AsSpan(_lines.End, _lines.Buffer.Length - _lines.End))); - } - - /// - public void Dispose() - { - if (_disposed) - { - return; - } - - _lines.Return(); - _source.Dispose(); - _disposed = true; - } - - [ExcludeFromCodeCoverage] - /// - public readonly void Reset() - { - throw new NotSupportedException("Reading is single-pass."); - } - } -} diff --git a/src/FixedWidthParser/Readers/StreamSource.cs b/src/FixedWidthParser/Readers/StreamSource.cs deleted file mode 100644 index 9443acc..0000000 --- a/src/FixedWidthParser/Readers/StreamSource.cs +++ /dev/null @@ -1,29 +0,0 @@ -namespace FixedWidthParser.Readers -{ - /// Stream-backed byte source for the shared record enumerator cores. - public readonly struct StreamSource(Stream stream, bool ownsStream) : ISource - { - /// - public int Read(Span buffer) - { - return stream.Read(buffer); - } - - /// - public ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) - { - return stream.ReadAsync(buffer, cancellationToken); - } - - /// - public void Dispose() - { -#pragma warning disable IDISP007 // Don't dispose injected - if (ownsStream) - { - stream.Dispose(); - } -#pragma warning restore IDISP007 // Don't dispose injected - } - } -} diff --git a/src/FixedWidthParser/Readers/TextReaderSource.cs b/src/FixedWidthParser/Readers/TextReaderSource.cs deleted file mode 100644 index 290c2b3..0000000 --- a/src/FixedWidthParser/Readers/TextReaderSource.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.Text; - -namespace FixedWidthParser.Readers -{ - /// - /// A closure-free description of where a char reader gets its from: - /// an existing reader (single-pass, not owned), a wrapped in a - /// (single-pass, owned), or a file path reopened per enumeration - /// (owned). Stored by value on the record enumerables, so a Read call allocates no - /// Func<TextReader> delegate or capture — only the enumerable itself. - /// - public readonly struct TextReaderSource : ISource - { - private readonly TextReader? _reader; - private readonly Stream? _stream; - private readonly string? _path; - private readonly Encoding? _encoding; - private readonly bool _leaveOpen; - private readonly bool _useAsync; - - private TextReaderSource( - TextReader? reader, Stream? stream, string? path, Encoding? encoding, bool leaveOpen, bool useAsync, bool ownsReader) - { - _reader = reader; - _stream = stream; - _path = path; - _encoding = encoding; - _leaveOpen = leaveOpen; - _useAsync = useAsync; - OwnsReader = ownsReader; - } - - /// Whether the enumerator should dispose the reader (true except for an injected reader). - public bool OwnsReader { get; } - - /// Existing reader: single pass, not disposed by the enumerator. - public static TextReaderSource FromReader(TextReader reader) - { - return new(reader, null, null, null, leaveOpen: false, useAsync: false, ownsReader: false); - } - - /// Stream wrapped in a : single pass; controls closing the stream. - public static TextReaderSource FromStream(Stream stream, Encoding encoding, bool leaveOpen) - { - return new(null, stream, null, encoding, leaveOpen, useAsync: false, ownsReader: true); - } - - /// File path reopened per enumeration; opts into asynchronous file I/O. - public static TextReaderSource FromFile(string path, Encoding encoding, bool useAsync) - { - return new(null, null, path, encoding, leaveOpen: false, useAsync, ownsReader: true); - } - - /// - /// Materializes a read source. Returns the injected reader as-is, or allocates a - /// (and, for an async file, a ) owned by the - /// returned source. BOM detection mirrors the original reader behavior. - /// - public TextReaderSource Create(int bufferSize) - { - if (_reader is not null) - { - return new(_reader, null, null, null, leaveOpen: false, useAsync: false, ownsReader: OwnsReader); - } - if (_stream is not null) - { - return new(new StreamReader(_stream, _encoding, detectEncodingFromByteOrderMarks: true, bufferSize: bufferSize, leaveOpen: _leaveOpen), null, null, null, leaveOpen: false, useAsync: false, ownsReader: true); - } - if (_useAsync) - { - return new(new StreamReader( - new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.Asynchronous | FileOptions.SequentialScan), - _encoding, detectEncodingFromByteOrderMarks: true, bufferSize: bufferSize), null, null, null, leaveOpen: false, useAsync: false, ownsReader: true); - } - return new(new StreamReader(_path!, _encoding, detectEncodingFromByteOrderMarks: true, bufferSize: bufferSize), null, null, null, leaveOpen: false, useAsync: false, ownsReader: true); - } - - /// - public int Read(Span buffer) - { - return _reader!.Read(buffer); - } - - /// - public ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) - { - return _reader!.ReadAsync(buffer, cancellationToken); - } - - /// - public void Dispose() - { -#pragma warning disable IDISP007 // Don't dispose injected - if (OwnsReader) - { - _reader?.Dispose(); - } -#pragma warning restore IDISP007 // Don't dispose injected - } - } -} diff --git a/src/FixedWidthParser/Readers/Utf8FixedWidthAsyncRecordEnumerable.cs b/src/FixedWidthParser/Readers/Utf8FixedWidthAsyncRecordEnumerable.cs deleted file mode 100644 index e2e0261..0000000 --- a/src/FixedWidthParser/Readers/Utf8FixedWidthAsyncRecordEnumerable.cs +++ /dev/null @@ -1,73 +0,0 @@ -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// Asynchronous version of : reads from a - /// via await foreach, also straight from raw bytes (no - /// , no transcode, no string per line). The enumerator logic lives in - /// . - /// - /// The source is stored as either a fixed (single-pass) or a file path - /// (reopened per enumeration with useAsync) — never a captured delegate, so a - /// ReadAsync call allocates only the enumerable itself, no closure. - /// - /// - public sealed class Utf8FixedWidthAsyncRecordEnumerable : IAsyncEnumerable - where TParser : struct, IRecordLineParser - { - private readonly TParser _parser; - private readonly Stream? _stream; - private readonly string? _path; - private readonly bool _ownsStream; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - /// Single-pass source: a fixed stream. - internal Utf8FixedWidthAsyncRecordEnumerable( - TParser parser, - Stream stream, - bool ownsStream, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _stream = stream; - _ownsStream = ownsStream; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Re-enumerable source: a file path reopened (with useAsync) on each enumeration. - internal Utf8FixedWidthAsyncRecordEnumerable( - TParser parser, - string path, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _path = path; - _ownsStream = true; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public AsyncRecordEnumeratorCore GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - { - var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.Asynchronous | FileOptions.SequentialScan); - return new(_parser, new StreamSource(stream, _ownsStream), _formatProvider, _stringPool, _bufferSize, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/Utf8FixedWidthPipeRecordEnumerable.cs b/src/FixedWidthParser/Readers/Utf8FixedWidthPipeRecordEnumerable.cs deleted file mode 100644 index 4b63269..0000000 --- a/src/FixedWidthParser/Readers/Utf8FixedWidthPipeRecordEnumerable.cs +++ /dev/null @@ -1,48 +0,0 @@ -using System.IO.Pipelines; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Readers -{ - /// - /// source for the reflection-based UTF-8 path: parses fixed-width records - /// straight off a pipe via await foreach, letting the pipe own buffering/read-ahead. The - /// enumerator logic lives in , specialized - /// with the strategy. Single-pass (a pipe is consumed - /// as it is read). - /// - public sealed class Utf8FixedWidthPipeRecordEnumerable : IAsyncEnumerable where TModel : new() - { - private readonly Utf8FixedWidthParser _parser; - private readonly PipeReader _reader; - private readonly bool _completeReader; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - - internal Utf8FixedWidthPipeRecordEnumerable( - Utf8FixedWidthParser parser, - PipeReader reader, - bool completeReader, - IFormatProvider? formatProvider, - StringPool? stringPool) - { - _parser = parser; - _reader = reader; - _completeReader = completeReader; - _formatProvider = formatProvider; - _stringPool = stringPool; - } - -#pragma warning disable HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - public Utf8PipeRecordEnumeratorCore> GetAsyncEnumerator(CancellationToken cancellationToken = default) -#pragma warning restore HLQ006 // GetEnumerator() or GetAsyncEnumerator() should return a value type - { - return new(new ReflectionUtf8LineParser(_parser), _reader, _completeReader, _formatProvider, _stringPool, cancellationToken); - } - - IAsyncEnumerator IAsyncEnumerable.GetAsyncEnumerator(CancellationToken cancellationToken) - { - return GetAsyncEnumerator(cancellationToken); - } - } -} diff --git a/src/FixedWidthParser/Readers/Utf8FixedWidthRecordEnumerable.cs b/src/FixedWidthParser/Readers/Utf8FixedWidthRecordEnumerable.cs deleted file mode 100644 index 3f02d80..0000000 --- a/src/FixedWidthParser/Readers/Utf8FixedWidthRecordEnumerable.cs +++ /dev/null @@ -1,82 +0,0 @@ -using System.Collections; -using System.Diagnostics.CodeAnalysis; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// UTF-8 / byte counterpart of : a lazily-read - /// sequence of models read straight from a as raw bytes (no - /// , no transcode, no string per line). Exposes a - /// enumerator for allocation-free foreach and implements for - /// LINQ interop. - /// - /// The source is stored as either a fixed (single-pass) or a file path - /// (reopened per enumeration) — never a captured delegate, so a Read call allocates only the - /// enumerable itself, no closure. - /// - /// - public sealed class Utf8FixedWidthRecordEnumerable : IEnumerable - where TParser : struct, IRecordLineParser - { - private readonly TParser _parser; - private readonly Stream? _stream; - private readonly string? _path; - private readonly bool _ownsStream; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly int _bufferSize; - - /// Single-pass source: a fixed stream. - internal Utf8FixedWidthRecordEnumerable( - TParser parser, - Stream stream, - bool ownsStream, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _stream = stream; - _ownsStream = ownsStream; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Re-enumerable source: a file path reopened on each enumeration. - internal Utf8FixedWidthRecordEnumerable( - TParser parser, - string path, - IFormatProvider? formatProvider, - StringPool? stringPool, - int bufferSize) - { - _parser = parser; - _path = path; - _ownsStream = true; - _formatProvider = formatProvider; - _stringPool = stringPool; - _bufferSize = bufferSize; - } - - /// Struct enumerator: foreach iteration without heap allocation. - public RecordEnumeratorCore GetEnumerator() - { - var stream = _stream ?? new FileStream(_path!, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 1, FileOptions.SequentialScan); - return new(_parser, new StreamSource(stream, _ownsStream), _formatProvider, _stringPool, _bufferSize); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - [ExcludeFromCodeCoverage] - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - } -} diff --git a/src/FixedWidthParser/Readers/Utf8PipeRecordEnumeratorCore.cs b/src/FixedWidthParser/Readers/Utf8PipeRecordEnumeratorCore.cs deleted file mode 100644 index 2d1758b..0000000 --- a/src/FixedWidthParser/Readers/Utf8PipeRecordEnumeratorCore.cs +++ /dev/null @@ -1,281 +0,0 @@ -using System.Buffers; -using System.IO.Pipelines; -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; - -namespace FixedWidthParser.Readers -{ - /// - /// -based counterpart of - /// : parses fixed-width records from a - /// , letting the pipe own buffering and read-ahead instead of the manual - /// pooled the stream cores use. Lines are split out of the - /// with a ; a line is parsed in place - /// from its when it sits in a single segment, and copied - /// into a pooled scratch buffer only when it straddles segment boundaries (the parser needs a - /// contiguous span). Specialized by a strategy - /// so parsing devirtualizes, exactly like the stream cores. - /// - /// Line semantics match the stream path: a leading UTF-8 BOM is skipped once, lines split on - /// \n (a preceding \r is dropped), empty lines are counted but not yielded, and a final - /// line without a trailing newline is yielded. - /// - /// - public sealed class Utf8PipeRecordEnumeratorCore : IAsyncEnumerator - where TParser : struct, IUtf8LineParser - { - private const byte Cr = (byte)'\r'; - private const byte Lf = (byte)'\n'; - - private readonly TParser _strategy; - private readonly bool _completeReader; - private readonly IFormatProvider? _formatProvider; - private readonly StringPool? _stringPool; - private readonly CancellationToken _cancellationToken; - private PipeReader? _reader; - private byte[]? _scratch; - private long _lineNumber; - private bool _preambleChecked; - private TModel _current; - - // Stateful PipeReader tracking (PERF-B) - private ReadResult _currentResult; - private bool _hasResult; - private SequencePosition _consumed; - private SequencePosition _examined; - private long _scannedBytes; - - internal Utf8PipeRecordEnumeratorCore( - TParser strategy, - PipeReader reader, - bool completeReader, - IFormatProvider? formatProvider, - StringPool? stringPool, - CancellationToken cancellationToken) - { - _strategy = strategy; - _reader = reader; - _completeReader = completeReader; - _formatProvider = formatProvider; - _stringPool = stringPool; - _cancellationToken = cancellationToken; - _current = default!; - } - - public TModel Current => _current; - - public async ValueTask MoveNextAsync() - { - ObjectDisposedException.ThrowIf(_reader is null, nameof(Utf8PipeRecordEnumeratorCore<,>)); - var reader = _reader; - - try - { - while (true) - { - _cancellationToken.ThrowIfCancellationRequested(); - - if (!_hasResult) - { - _currentResult = await reader.ReadAsync(_cancellationToken).ConfigureAwait(false); - if (_currentResult.IsCanceled) - { - throw new OperationCanceledException(_cancellationToken); - } - _hasResult = true; - - // Reset positions to point to the start of the new buffer (which corresponds to previous _consumed) - _consumed = _currentResult.Buffer.Start; - _examined = _currentResult.Buffer.Start; - } - - var status = TryReadFromResult(out _consumed, out _examined); - - if (status == LineStatus.Line) - { - return true; - } - - // Before reading more data or returning false at the end, advance the reader - reader.AdvanceTo(_consumed, _examined); - _hasResult = false; - - if (status == LineStatus.End || _currentResult.IsCompleted) - { - return false; - } - } - } - catch (Exception) - { - if (_hasResult) - { - reader.AdvanceTo(_consumed, _examined); - _hasResult = false; - } - throw; - } - } - - private LineStatus TryReadFromResult(out SequencePosition consumed, out SequencePosition examined) - { - ReadOnlySequence buffer = _currentResult.Buffer; - bool isCompleted = _currentResult.IsCompleted; - - if (!_preambleChecked) - { - if (buffer.Length >= 3) - { - Span head = stackalloc byte[3]; - buffer.Slice(0, 3).CopyTo(head); - int skip = head is [0xEF, 0xBB, 0xBF] ? 3 : 0; - if (skip > 0) - { - _consumed = buffer.GetPosition(skip); - } - _preambleChecked = true; - } - else if (isCompleted) - { - _preambleChecked = true; // too short to hold a BOM; no preamble - } - else - { - consumed = _consumed; - examined = buffer.End; - return LineStatus.NeedData; - } - } - - while (true) - { - ReadOnlySequence remaining = buffer.Slice(_consumed); - var scanStart = remaining.GetPosition(_scannedBytes); - var seqReader = new SequenceReader(remaining.Slice(scanStart)); - - if (seqReader.TryReadTo(out ReadOnlySpan _, Lf, advancePastDelimiter: true)) - { - _lineNumber++; - var positionOfLf = remaining.GetPosition(seqReader.Consumed - 1, scanStart); - var scopeLine = TrimTrailingCr(remaining.Slice(0, positionOfLf)); - - var nextPosition = remaining.GetPosition(seqReader.Consumed, scanStart); - _consumed = nextPosition; - _examined = nextPosition; - _scannedBytes = 0; - - if (!scopeLine.IsEmpty) - { - Parse(in scopeLine); - consumed = _consumed; - examined = _consumed; - return LineStatus.Line; - } - continue; - } - - if (!isCompleted) - { - _scannedBytes = remaining.Length; - consumed = _consumed; - examined = buffer.End; - return LineStatus.NeedData; - } - - consumed = buffer.End; - examined = buffer.End; - if (remaining.Length <= 0) - { - _consumed = buffer.End; - _examined = buffer.End; - _scannedBytes = 0; - return LineStatus.End; - } - _lineNumber++; - var line = TrimTrailingCr(in remaining); - if (!line.IsEmpty) - { - _consumed = buffer.End; - _examined = buffer.End; - _scannedBytes = 0; - Parse(in line); - return LineStatus.Line; - } - _consumed = buffer.End; - _examined = buffer.End; - _scannedBytes = 0; - return LineStatus.End; - } - } - - private void Parse(in ReadOnlySequence line) - { - if (line.IsSingleSegment) - { - ParseSpan(line.FirstSpan); - return; - } - - int length = (int)line.Length; - if (_scratch is null || _scratch.Length < length) - { - if (_scratch is not null) - { - var oldScratch = _scratch; - _scratch = null; - ArrayPool.Shared.Return(oldScratch); - } - _scratch = ArrayPool.Shared.Rent(length); - } - line.CopyTo(_scratch); - ParseSpan(_scratch.AsSpan(0, length)); - } - - private void ParseSpan(ReadOnlySpan line) - { - if (!_strategy.TryParse(line, _formatProvider, _stringPool, out _current)) - { - throw new FormatException( - $"Line {_lineNumber} could not be parsed into {typeof(TModel).Name}: \"{Encoding.UTF8.GetString(line)}\"."); - } - } - - private static ReadOnlySequence TrimTrailingCr(in ReadOnlySequence line) - { - long length = line.Length; - if (length == 0) - { - return line; - } - byte last = line.Slice(length - 1).FirstSpan[0]; - return last == Cr ? line.Slice(0, length - 1) : line; - } - - public async ValueTask DisposeAsync() - { - if (_scratch is not null) - { - var oldScratch = _scratch; - _scratch = null; - ArrayPool.Shared.Return(oldScratch); - } - - var reader = _reader; - _reader = null; - if (reader is null) - { - return; - } - if (_hasResult) - { - reader.AdvanceTo(_consumed, _examined); - _hasResult = false; - } - - if (_completeReader) - { - await reader.CompleteAsync().ConfigureAwait(false); - } - } - } -} diff --git a/src/FixedWidthParser/Utf8FixedWidthRuntime.cs b/src/FixedWidthParser/Utf8FixedWidthRuntime.cs deleted file mode 100644 index 1302cae..0000000 --- a/src/FixedWidthParser/Utf8FixedWidthRuntime.cs +++ /dev/null @@ -1,127 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Globalization; -using System.Runtime.CompilerServices; -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Processors; - -namespace FixedWidthParser -{ - /// - /// UTF-8 / byte counterpart of : low-level helpers called by the - /// source-generated implementations. They - /// centralize the column slicing, UTF-8 string decoding/pooling and numeric parsing so the generated - /// code stays small and matches the runtime byte parser's semantics exactly. - /// Public because the generated code lives in the consumer assembly; not intended for direct use. - /// - public static class Utf8FixedWidthRuntime - { - /// - /// Converts a FixedColumnAttribute.TrimChar (or any other single character configured for - /// the char path) to the single UTF-8 byte it maps to. Only ASCII (<= 0x7F) characters are a - /// single byte in UTF-8; anything else throws rather than silently trimming the wrong byte. - /// - /// The character is not a single ASCII byte. - public static byte ToAsciiByte(char value, string columnName) - { - if (value > '\x7F') - { - throw new NotSupportedException( - $"The trim character '{value}' (U+{(int)value:X4}) for column \"{columnName}\" is not a single ASCII " + - "byte. The UTF-8 byte parser only supports ASCII trim characters; use the char-based parser for this column."); - } - return (byte)value; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static ReadOnlySpan TrimColumn(ReadOnlySpan column, byte trimChar, Attributes.TrimMode trimMode) - { - return trimMode switch - { - Attributes.TrimMode.Leading => column.TrimStart(trimChar), - Attributes.TrimMode.Both => column.Trim(trimChar), - _ => column.TrimEnd(trimChar) - }; - } - - /// Materializes a string column: trims spaces and UTF-8 decodes, interning via the pool when supplied. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static string String(ReadOnlySpan column, StringPool? stringPool, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - return stringPool is null ? Encoding.UTF8.GetString(trimmed) : stringPool.GetOrAdd(trimmed, Encoding.UTF8); - } - - /// Parses a column, honoring the provider's decimal separator (see ). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDouble(ReadOnlySpan column, IFormatProvider? formatProvider, out double value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return CultureHelpers.TryParseDouble(column, formatProvider, out value, trimChar, trimMode); - } - - /// Parses a column, honoring the provider's decimal separator (see ). - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryFloat(ReadOnlySpan column, IFormatProvider? formatProvider, out float value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - return CultureHelpers.TryParseFloat(column, formatProvider, out value, trimChar, trimMode); - } - - /// - /// Parses any column (int, decimal, DateTime, …), trimming - /// spaces. A means - /// — see - /// for why (same contract on both the char and UTF-8 paths). - /// - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryParse(ReadOnlySpan column, IFormatProvider? formatProvider, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - where TValue : IUtf8SpanParsable - { - return TValue.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider ?? CultureInfo.InvariantCulture, out value); - } - - /// Parses a column via a FixedColumnAttribute.Converter instance, trimming spaces. - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryConvert( - ReadOnlySpan column, IFormatProvider? formatProvider, TConverter converter, [MaybeNullWhen(false)] out TValue value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - where TConverter : IUtf8FixedWidthConverter - { - return converter.TryParse(TrimColumn(column, trimChar, trimMode), formatProvider, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTime value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return DateTime.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateOnly value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return DateOnly.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryTimeOnlyExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out TimeOnly value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return TimeOnly.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool TryDateTimeOffsetExact(ReadOnlySpan column, string format, IFormatProvider? formatProvider, out DateTimeOffset value, byte trimChar = (byte)' ', Attributes.TrimMode trimMode = Attributes.TrimMode.Trailing) - { - var trimmed = TrimColumn(column, trimChar, trimMode); - Span chars = trimmed.Length <= 128 ? stackalloc char[128] : new char[trimmed.Length]; - int written = Encoding.UTF8.GetChars(trimmed, chars); - return DateTimeOffset.TryParseExact(chars[..written], format, formatProvider ?? CultureInfo.InvariantCulture, DateTimeStyles.None, out value); - } - } -} diff --git a/src/FixedWidthParser/Writers/FixedWidthWriter.cs b/src/FixedWidthParser/Writers/FixedWidthWriter.cs deleted file mode 100644 index 9a10b3d..0000000 --- a/src/FixedWidthParser/Writers/FixedWidthWriter.cs +++ /dev/null @@ -1,313 +0,0 @@ -using System.Buffers; -using System.Linq.Expressions; -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; -using FixedWidthParser.Attributes; -using FixedWidthParser.Formatters; -using FixedWidthParser.Processors; - -namespace FixedWidthParser.Writers -{ - public sealed class FixedWidthWriter - { - private static readonly IColumnFormatter[] _cachedFormatters; - private static readonly int _cachedLineLength; - private static readonly ExceptionDispatchInfo? _buildError; - - static FixedWidthWriter() - { - try - { - var formatters = new List>(); - int maxLen = 0; - - ModelColumns.ForEachColumn(typeof(TModel), (member, attribute) => - { - maxLen = Math.Max(maxLen, attribute.Start + attribute.Length); - formatters.Add(CreateFormatter(member, attribute)); - }); - - _cachedFormatters = formatters.ToArray(); - _cachedLineLength = maxLen; - _buildError = null; - } - catch (Exception ex) - { - _cachedFormatters = []; - _cachedLineLength = 0; - _buildError = ExceptionDispatchInfo.Capture(ex); - } - } - - private readonly IColumnFormatter[] _formatters; - private readonly int _lineLength; - - public FixedWidthWriter() - { - _buildError?.Throw(); - _formatters = _cachedFormatters; - _lineLength = _cachedLineLength; - } - - private static OverflowBehavior DetermineOverflowBehavior(FixedColumnAttribute attribute, bool isString) - { - if (attribute.Overflow != OverflowBehavior.Default) - { - return attribute.Overflow; - } - - if (isString) - { - return OverflowBehavior.Truncate; - } - - return OverflowBehavior.Throw; - } - - private static IColumnFormatter CreateFormatter(MemberInfo member, FixedColumnAttribute attribute) - { - var (memberType, model, access) = ModelColumns.MemberAccess(typeof(TModel), member); - - if (attribute.Overflow == OverflowBehavior.Truncate && memberType != typeof(string)) - { - throw new InvalidOperationException( - $"OverflowBehavior.Truncate is not supported for non-string column '{member.Name}' of type '{memberType.Name}'."); - } - - var delegateType = typeof(RefGetter<,>).MakeGenericType(typeof(TModel), memberType); - var getter = Expression.Lambda(delegateType, access, model).Compile(); - // Resolve Default overflow per type: string truncates, numeric (incl. T?) throws. - var overflow = DetermineOverflowBehavior(attribute, memberType == typeof(string)); - var options = new ColumnFormatOptions(attribute.Alignment, attribute.Padding, attribute.Format, overflow); - - var underlyingType = Nullable.GetUnderlyingType(memberType); - if (underlyingType is null) - { - return BuildValueFormatter(member, attribute, memberType, options, getter); - } - - // A null value writes as a blank (padding-filled) column instead of formatting - // otherwise the underlying T formats exactly as a non-nullable column would. - var underlyingGetter = BuildUnderlyingGetter(underlyingType, getter); - var inner = BuildValueFormatter(member, attribute, underlyingType, options, underlyingGetter); - var nullableFormatterType = typeof(NullableColumnFormatter<,>).MakeGenericType(typeof(TModel), underlyingType); - return (IColumnFormatter)Activator.CreateInstance( - nullableFormatterType, attribute.Start, attribute.Length, attribute.TrimChar, getter, inner)!; - } - - // Adapts a RefGetter into a RefGetter (reads .Value - // only ever invoked by NullableColumnFormatter after it has confirmed HasValue). - private static Delegate BuildUnderlyingGetter(Type underlyingType, Delegate nullableGetter) - { - var modelParam = Expression.Parameter(typeof(TModel).MakeByRefType(), "model"); - var invoke = Expression.Invoke(Expression.Constant(nullableGetter), modelParam); - var value = Expression.Property(invoke, "Value"); - var adapterType = typeof(RefGetter<,>).MakeGenericType(typeof(TModel), underlyingType); - return Expression.Lambda(adapterType, value, modelParam).Compile(); - } - - private static IColumnFormatter BuildValueFormatter( - MemberInfo member, FixedColumnAttribute attribute, Type valueType, ColumnFormatOptions options, Delegate getter) - { - if (attribute.Converter is { } converterType) - { - var requiredInterface = typeof(IFixedWidthConverter<>).MakeGenericType(valueType); - if (!requiredInterface.IsAssignableFrom(converterType)) - { - throw new InvalidOperationException( - $"Converter '{converterType}' for column '{member.Name}' must implement '{requiredInterface}'."); - } - var converterInstance = Activator.CreateInstance(converterType) - ?? throw new InvalidOperationException($"Could not instantiate converter '{converterType}' for column '{member.Name}'."); - var converterFormatterType = typeof(ConverterColumnFormatter<,,>).MakeGenericType(typeof(TModel), valueType, converterType); - return (IColumnFormatter)Activator.CreateInstance( - converterFormatterType, attribute.Start, attribute.Length, options, member.Name, getter, converterInstance)!; - } - - var formatterType = valueType == typeof(string) - ? typeof(StringColumnFormatter<>).MakeGenericType(typeof(TModel)) - : typeof(SpanFormattableColumnFormatter<,>).MakeGenericType(typeof(TModel), valueType); - return (IColumnFormatter)Activator.CreateInstance( - formatterType, attribute.Start, attribute.Length, options, member.Name, getter)!; - } - - /// - /// Formats a single model line into the given buffer: fills it with spaces and applies - /// each column formatter. Shared core used by every write overload. - /// - private void FormatLine(in TModel model, Span lineBuffer, IFormatProvider? formatProvider) - { - lineBuffer.Fill(' '); - foreach (ref readonly var formatter in _formatters.AsSpan()) - { - formatter.Format(in model, lineBuffer, formatProvider); - } - } - - public async Task WriteAsync(StreamWriter writer, TModel model, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) - { - var lineBuffer = ArrayPool.Shared.Rent(_lineLength); - try - { - FormatLine(in model, lineBuffer.AsSpan(0, _lineLength), formatProvider); - await writer.WriteLineAsync(lineBuffer.AsMemory(0, _lineLength), cancellationToken).ConfigureAwait(false); - } - finally - { - ArrayPool.Shared.Return(lineBuffer); - } - } - public async Task WriteAsync(Stream stream, TModel model, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) - { - StreamWriter writer = new(stream, leaveOpen: true); - await using (writer.ConfigureAwait(false)) - { - await WriteAsync(writer, model, formatProvider, cancellationToken).ConfigureAwait(false); - } - } - public async Task WriteManyAsync(Stream stream, IEnumerable models, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) - { - StreamWriter writer = new(stream, leaveOpen: true); - await using (writer.ConfigureAwait(false)) - { - await WriteManyAsync(writer, models, formatProvider, cancellationToken).ConfigureAwait(false); - } - } - public async Task WriteManyAsync(StreamWriter writer, IEnumerable models, IFormatProvider? formatProvider = null, CancellationToken cancellationToken = default) - { - var lineBuffer = ArrayPool.Shared.Rent(_lineLength); - try - { - foreach (var model in models) - { - FormatLine(in model, lineBuffer.AsSpan(0, _lineLength), formatProvider); - await writer.WriteLineAsync(lineBuffer.AsMemory(0, _lineLength), cancellationToken).ConfigureAwait(false); - } - } - finally - { - ArrayPool.Shared.Return(lineBuffer); - } - } - [SkipLocalsInit] - public void Write(StreamWriter writer, in TModel model, IFormatProvider? formatProvider = null) - { - if (_lineLength <= 1024) - { - Span lineBuffer = stackalloc char[_lineLength]; - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - return; - } - - var rented = ArrayPool.Shared.Rent(_lineLength); - try - { - var lineBuffer = rented.AsSpan(0, _lineLength); - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - - /// - /// Writes a single model directly to a Stream. - /// - public void Write(Stream stream, in TModel model, IFormatProvider? formatProvider = null) - { - using var writer = new StreamWriter(stream, leaveOpen: true); - Write(writer, in model, formatProvider); - } - - /// - /// Writes a collection of models to a Stream continuously. - /// - public void WriteMany(Stream stream, IEnumerable models, IFormatProvider? formatProvider = null) - { - using var writer = new StreamWriter(stream, leaveOpen: true); - WriteMany(writer, models, formatProvider); - } - - /// - /// Writes a collection of models reusing an existing StreamWriter - /// (avoids allocating/disposing a StreamWriter per call). - /// - [SkipLocalsInit] - public void WriteMany(StreamWriter writer, IEnumerable models, IFormatProvider? formatProvider = null) - { - if (_lineLength <= 1024) - { - Span lineBuffer = stackalloc char[_lineLength]; - foreach (var model in models) - { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } - return; - } - - var rented = ArrayPool.Shared.Rent(_lineLength); - try - { - var lineBuffer = rented.AsSpan(0, _lineLength); - foreach (var model in models) - { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - - /// - /// Writes a contiguous collection of models to a Stream without allocating an enumerator - /// (iterates by ref readonly, avoiding a copy of each struct). - /// - public void WriteMany(Stream stream, ReadOnlySpan models, IFormatProvider? formatProvider = null) - { - using var writer = new StreamWriter(stream, leaveOpen: true); - WriteMany(writer, models, formatProvider); - } - - /// - /// Writes a contiguous collection of models reusing an existing StreamWriter, without - /// allocating an enumerator (iterates by ref readonly, avoiding a copy of each struct). - /// - [SkipLocalsInit] - public void WriteMany(StreamWriter writer, ReadOnlySpan models, IFormatProvider? formatProvider = null) - { - if (_lineLength <= 1024) - { - Span lineBuffer = stackalloc char[_lineLength]; - foreach (ref readonly var model in models) - { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } - return; - } - - var rented = ArrayPool.Shared.Rent(_lineLength); - try - { - var lineBuffer = rented.AsSpan(0, _lineLength); - foreach (ref readonly var model in models) - { - FormatLine(in model, lineBuffer, formatProvider); - writer.WriteLine(lineBuffer); - } - } - finally - { - ArrayPool.Shared.Return(rented); - } - } - } -} diff --git a/src/FixedWidthParser/Writers/RecordFormatters.cs b/src/FixedWidthParser/Writers/RecordFormatters.cs new file mode 100644 index 0000000..50ab1a4 --- /dev/null +++ b/src/FixedWidthParser/Writers/RecordFormatters.cs @@ -0,0 +1,42 @@ +using FixedWidthParser.Infrastructure; + +namespace FixedWidthParser.Writers +{ + /// + /// Format strategy is specialized by, the write-side twin of + /// : implemented by types so each + /// (TModel, TFormatter) pair is compiled once and the call devirtualizes. + /// + internal interface IRecordFormatter + where TModel : allows ref struct + { + /// Formats one line, without its line break. Returns only when is shorter than the line. + bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten); + } + + internal readonly struct LayoutRecordFormatter(FixedWidthLayout layout) : IRecordFormatter + where TModel : allows ref struct + { + public bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten) + { + int lineLength = layout.LineLength; + if (destination.Length < lineLength) + { + bytesWritten = 0; + return false; + } + layout.FormatLine(in model, destination[..lineLength], in context); + bytesWritten = lineLength; + return true; + } + } + + internal readonly struct GeneratedRecordFormatter : IRecordFormatter + where TModel : IFixedWidthModel + { + public bool TryFormat(in TModel model, Span destination, in FixedWidthContext context, out int bytesWritten) + { + return TModel.TryFormat(in model, destination, in context, out bytesWritten); + } + } +} diff --git a/src/FixedWidthParser/Writers/RecordWriter.cs b/src/FixedWidthParser/Writers/RecordWriter.cs new file mode 100644 index 0000000..ff739a1 --- /dev/null +++ b/src/FixedWidthParser/Writers/RecordWriter.cs @@ -0,0 +1,167 @@ +using System.Buffers; +using System.Runtime.CompilerServices; + +namespace FixedWidthParser.Writers +{ + /// + /// Writes records, each followed by a line break, batched into one pooled buffer. The line length is + /// not known up front (a generated model formats itself), so a record that does not fit the rest of the + /// batch flushes it, and one that does not fit an empty batch grows it. + /// + internal static class RecordWriter + { + private const int BatchSize = 64 * 1024; + private const int StackRecordSize = 1024; + + [SkipLocalsInit] + public static void Write(Stream stream, in TModel model, TFormatter formatter, FixedWidthWriterOptions options) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + Span record = stackalloc byte[StackRecordSize]; + int written = TryFormatRecord(in model, formatter, record, in context, options.NewLineBytes); + if (written > 0) + { + stream.Write(record[..written]); + return; + } + + byte[] buffer = ArrayPool.Shared.Rent(StackRecordSize * 2); + try + { + written = FormatGrowing(in model, formatter, ref buffer, in context, options.NewLineBytes); + stream.Write(buffer, 0, written); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static void WriteMany(Stream stream, IEnumerable models, TFormatter formatter, FixedWidthWriterOptions options) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(BatchSize); + try + { + int used = 0; + foreach (var model in models) + { + used = Append(stream, in model, formatter, ref buffer, used, in context, options.NewLineBytes); + } + stream.Write(buffer, 0, used); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static void WriteMany(Stream stream, ReadOnlySpan models, TFormatter formatter, FixedWidthWriterOptions options) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(BatchSize); + try + { + int used = 0; + foreach (ref readonly var model in models) + { + used = Append(stream, in model, formatter, ref buffer, used, in context, options.NewLineBytes); + } + stream.Write(buffer, 0, used); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static async Task WriteAsync(Stream stream, TModel model, TFormatter formatter, FixedWidthWriterOptions options, CancellationToken cancellationToken) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(StackRecordSize); + try + { + int written = FormatGrowing(in model, formatter, ref buffer, in context, options.NewLineBytes); + await stream.WriteAsync(buffer.AsMemory(0, written), cancellationToken).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static async Task WriteManyAsync(Stream stream, IEnumerable models, TFormatter formatter, FixedWidthWriterOptions options, CancellationToken cancellationToken) + where TFormatter : struct, IRecordFormatter + { + var context = options.Context; + byte[] buffer = ArrayPool.Shared.Rent(BatchSize); + try + { + int used = 0; + foreach (var model in models) + { + int written = TryFormatRecord(in model, formatter, buffer.AsSpan(used), in context, options.NewLineBytes); + if (written == 0) + { + await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); + used = 0; + written = FormatGrowing(in model, formatter, ref buffer, in context, options.NewLineBytes); + } + used += written; + } + await stream.WriteAsync(buffer.AsMemory(0, used), cancellationToken).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private static int Append( + Stream stream, in TModel model, TFormatter formatter, ref byte[] buffer, int used, in FixedWidthContext context, byte[] newLine) + where TFormatter : struct, IRecordFormatter + { + int written = TryFormatRecord(in model, formatter, buffer.AsSpan(used), in context, newLine); + if (written > 0) + { + return used + written; + } + stream.Write(buffer, 0, used); + return FormatGrowing(in model, formatter, ref buffer, in context, newLine); + } + + // Formats into the whole of buffer, replacing it with one twice the size until the record fits. + // The new buffer is rented before the old one is returned, so the caller's finally never returns a buffer twice. + private static int FormatGrowing(in TModel model, TFormatter formatter, ref byte[] buffer, in FixedWidthContext context, byte[] newLine) + where TFormatter : struct, IRecordFormatter + { + while (true) + { + int written = TryFormatRecord(in model, formatter, buffer, in context, newLine); + if (written > 0) + { + return written; + } + byte[] larger = ArrayPool.Shared.Rent(checked(buffer.Length * 2)); + ArrayPool.Shared.Return(buffer); + buffer = larger; + } + } + + /// Formats a record and its line break into ; returns 0 when they do not fit. + private static int TryFormatRecord(in TModel model, TFormatter formatter, Span destination, in FixedWidthContext context, byte[] newLine) + where TFormatter : struct, IRecordFormatter + { + if (!formatter.TryFormat(in model, destination, in context, out int written) || destination.Length - written < newLine.Length) + { + return 0; + } + newLine.CopyTo(destination[written..]); + return written + newLine.Length; + } + } +} diff --git a/tests/AotSmoke/AotSmoke.csproj b/tests/AotSmoke/AotSmoke.csproj new file mode 100644 index 0000000..9322536 --- /dev/null +++ b/tests/AotSmoke/AotSmoke.csproj @@ -0,0 +1,24 @@ + + + + + Exe + net10.0 + enable + enable + true + true + false + true + false + + $(NoWarn);CA2007;MA0004 + + + + + + + diff --git a/tests/AotSmoke/Order.cs b/tests/AotSmoke/Order.cs new file mode 100644 index 0000000..7f6f719 --- /dev/null +++ b/tests/AotSmoke/Order.cs @@ -0,0 +1,10 @@ +namespace AotSmoke +{ + internal sealed record class Order + { + public int Id { get; set; } + public string Name { get; set; } = ""; + public decimal Amount { get; set; } + public DateOnly Date { get; set; } + } +} diff --git a/tests/AotSmoke/Program.cs b/tests/AotSmoke/Program.cs new file mode 100644 index 0000000..fcb08fb --- /dev/null +++ b/tests/AotSmoke/Program.cs @@ -0,0 +1,28 @@ +using AotSmoke; +using FixedWidthParser; + +var layout = FixedWidthLayout.Build(b => b + .Factory(() => new Order()) + .Column("Id", 0, 6, (in Order o) => o.Id, (ref Order o, int v) => o.Id = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) + .Column("Name", 6, 20, (in Order o) => o.Name, (ref Order o, string v) => o.Name = v) + .Column("Amount", 26, 12, (in Order o) => o.Amount, (ref Order o, decimal v) => o.Amount = v) + .DecodedColumn("Date", 38, 8, (in Order o) => o.Date, (ref Order o, DateOnly v) => o.Date = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" })); + +var orders = Enumerable.Range(1, 1000) + .Select(i => new Order { Id = i, Name = "Order " + i, Amount = i * 1.25m, Date = new DateOnly(2026, 1, 1).AddDays(i) }) + .ToList(); + +await using var stream = new MemoryStream(); +await FixedWidth.WriteManyAsync(stream, orders, layout); +stream.Position = 0; +var read = await FixedWidth.ReadAsync(stream, layout, leaveOpen: true).ToListAsync(); + +if (!orders.SequenceEqual(read)) +{ + await Console.Error.WriteLineAsync("Round trip mismatch."); + return 1; +} +await Console.Out.WriteLineAsync($"Round-tripped {read.Count} orders."); +return 0; diff --git a/tests/Benchmarks/Benchmarks.csproj b/tests/Benchmarks/Benchmarks.csproj index 4f246c0..3b3140c 100644 --- a/tests/Benchmarks/Benchmarks.csproj +++ b/tests/Benchmarks/Benchmarks.csproj @@ -6,6 +6,7 @@ enable enable true + $(NoWarn);CA1707 diff --git a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs index f89aa8f..8f8c08d 100644 --- a/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/AsyncReaderBenchmarks.cs @@ -1,28 +1,21 @@ -using System.Globalization; using System.Text; using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Parsers; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { /// - /// Asynchronous reading: compares the span-based await foreach (FixedWidthReader.ReadAsync) - /// with the naive baseline ReadLineAsync()+TryParse. In-memory source (StringReader, whose - /// ReadAsync completes synchronously) to isolate the async state-machine overhead + buffering. + /// Asynchronous reading from an in-memory stream: await foreach over the readers against a + /// baseline of plus parsing the re-encoded line. /// [Config(typeof(RegressionConfig))] public class AsyncReaderBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthParser _parser = new(); - private readonly FixedWidthReader _reader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthReader _pooledReader = new(CultureInfo.InvariantCulture, new StringPool()); - private readonly StringPool _generatedStringPool = new(); - private string _text = string.Empty; + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private readonly FixedWidthReaderOptions _pooledReaderOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + private readonly FixedWidthReaderOptions _generatedPooled = new() { StringPool = new StringPool() }; + private byte[] _bytes = []; [Params(100, 1000)] public int Count { get; set; } @@ -30,27 +23,18 @@ public class AsyncReaderBenchmarks [GlobalSetup] public void Setup() { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - _text = Encoding.UTF8.GetString(ms.ToArray()); + _bytes = SampleData.Bytes(Count); } - /// ReadLineAsync() allocates a string per line; then TryParse. Baseline. [Benchmark(Baseline = true)] public async Task Naive_ReadLineAsync() { - using var reader = new StringReader(_text); + using var reader = new StreamReader(new MemoryStream(_bytes, writable: false), Encoding.UTF8); int sum = 0; string? line; while ((line = await reader.ReadLineAsync().ConfigureAwait(false)) is not null) { - if (_parser.TryParse(line, Culture, null, out var model)) + if (FixedWidth.TryParse(Encoding.UTF8.GetBytes(line), _layout, default, out var model)) { sum += model.Age; } @@ -58,59 +42,47 @@ public async Task Naive_ReadLineAsync() return sum; } - /// Span-based await foreach: slices each line from the buffer, no string per line. [Benchmark] - public async Task SpanReader_ReadAsync() + public async Task Reader_ReadAsync() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in _reader.ReadAsync(reader).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false), _layout).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// Span-based await foreach + StringPool: also interns the string columns. [Benchmark] - public async Task SpanReader_ReadAsync_Pooled() + public async Task Reader_ReadAsync_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in _pooledReader.ReadAsync(reader).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false), _layout, options: _pooledReaderOptions).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// Source-generated async reader: same scanner, static generated TryParse, no reflection. [Benchmark] public async Task GeneratedReader_ReadAsync() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in FixedWidth.ReadAsync(reader, formatProvider: Culture).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false)).ConfigureAwait(false)) { sum += model.Age; } - return sum; } - /// Source-generated async reader + StringPool: static TryParse plus interned string columns. [Benchmark] public async Task GeneratedReader_ReadAsync_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - await foreach (var model in FixedWidth.ReadAsync(reader, formatProvider: Culture, stringPool: _generatedStringPool).ConfigureAwait(false)) + await foreach (var model in FixedWidth.ReadAsync(new MemoryStream(_bytes, writable: false), options: _generatedPooled).ConfigureAwait(false)) { sum += model.Age; } - return sum; } } diff --git a/tests/Benchmarks/Perf/ByteReaderBenchmarks.cs b/tests/Benchmarks/Perf/ByteReaderBenchmarks.cs deleted file mode 100644 index c180b46..0000000 --- a/tests/Benchmarks/Perf/ByteReaderBenchmarks.cs +++ /dev/null @@ -1,108 +0,0 @@ -using System.Globalization; -using System.Text; -using BenchmarkDotNet.Attributes; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; - -namespace Benchmarks.Perf -{ - /// - /// UTF-8 / byte parsing vs the char path. Isolates the per-line parse from I/O: lines are - /// pre-split in into char[] and UTF-8 byte[] forms (ASCII fixtures, so byte - /// offsets equal char offsets). Three lanes: - /// - /// baseline: parse lines that are already char (no transcode at all); - /// char-after-decode: UTF-8 → string per line, then parse — the cost a StreamReader-backed - /// char reader effectively pays on a UTF-8 source; - /// byte: parse straight from UTF-8 bytes with — - /// the transcode is skipped entirely. - /// - /// This measures the parse core only; covers the - /// end-to-end stream read (StreamReader-backed char reader vs the raw-byte reader). - /// - [Config(typeof(RegressionConfig))] - public class ByteReaderBenchmarks - { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthParser _charParser = new(); - private readonly Utf8FixedWidthParser _byteParser = new(); - - private string[] _charLines = []; - private byte[][] _byteLines = []; - - [Params(100, 1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - - var text = Encoding.UTF8.GetString(ms.ToArray()); - _charLines = text.Split('\n', StringSplitOptions.RemoveEmptyEntries); - foreach (ref var v in _charLines.AsSpan()) - { - v = v.TrimEnd('\r'); - } - _byteLines = new byte[_charLines.Length][]; - for (int i = 0; i < _charLines.Length; i++) - { - _byteLines[i] = Encoding.UTF8.GetBytes(_charLines[i]); - } - } - - /// Char parse over lines that are already UTF-16: no transcode. Baseline. - [Benchmark(Baseline = true)] - public int CharParser_Parse() - { - int sum = 0; - foreach (var line in _charLines) - { - if (_charParser.TryParse(line, Culture, null, out var model)) - { - sum += model.Age; - } - } - return sum; - } - - /// UTF-8 → string per line, then char parse: the transcode a char reader pays on a UTF-8 source. - [Benchmark] - public int CharParser_Parse_AfterUtf8Decode() - { - int sum = 0; - foreach (var bytes in _byteLines) - { - var line = Encoding.UTF8.GetString(bytes); - if (_charParser.TryParse(line, Culture, null, out var model)) - { - sum += model.Age; - } - } - return sum; - } - - /// Byte parse straight from UTF-8: no transcode, no string per line. - [Benchmark] - public int ByteParser_Parse() - { - int sum = 0; - foreach (var bytes in _byteLines) - { - if (_byteParser.TryParse(bytes, Culture, null, out var model)) - { - sum += model.Age; - } - } - return sum; - } - } -} diff --git a/tests/Benchmarks/Perf/ByteReaderPoolingBenchmarks.cs b/tests/Benchmarks/Perf/ByteReaderPoolingBenchmarks.cs deleted file mode 100644 index f357bac..0000000 --- a/tests/Benchmarks/Perf/ByteReaderPoolingBenchmarks.cs +++ /dev/null @@ -1,101 +0,0 @@ -using System.Globalization; -using BenchmarkDotNet.Attributes; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; - -namespace Benchmarks.Perf -{ - /// - /// Where the byte reader's per-line allocation actually comes from — the string column — and the - /// two ways to remove it: a (intern repeated values) or a model with no - /// string at all. End-to-end stream read over an in-memory . The pool is - /// reused across invocations, so after warmup the repeated names ("Name0".."Name99") are already - /// interned and the pooled lane allocates ~nothing per line. - /// - /// WithString_NoPool: , a fresh string decoded per line (baseline); - /// WithString_Pooled: same model + StringPool — the string is interned, not re-decoded; - /// NoString: — no string column, nothing to decode or pool. - /// - /// - [Config(typeof(RegressionConfig))] - public class ByteReaderPoolingBenchmarks - { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthByteReader _reader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthByteReader _pooledReader = new(CultureInfo.InvariantCulture, new StringPool()); - private readonly FixedWidthByteReader _numericReader = new(CultureInfo.InvariantCulture); - private MemoryStream _stream = new(); - private MemoryStream _numericStream = new(); - - [Params(100, 1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - var models = new SampleModel[Count]; - var numeric = new NumericModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - numeric[i] = new NumericModel { Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - _stream = new MemoryStream(); - new FixedWidthWriter().WriteMany(_stream, models.AsSpan(), Culture); - - _numericStream = new MemoryStream(); - new FixedWidthWriter().WriteMany(_numericStream, numeric.AsSpan(), Culture); - } - - [GlobalCleanup] - public void Cleanup() - { - _stream.Dispose(); - _numericStream.Dispose(); - } - - /// Byte reader, string model, no pool: a fresh string decoded per line. Baseline. - [Benchmark(Baseline = true)] - public int WithString_NoPool() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _reader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - - /// Byte reader, string model, StringPool: repeated values interned, not re-decoded. - [Benchmark] - public int WithString_Pooled() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _pooledReader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - - /// Byte reader, numeric-only model: no string column, nothing to decode or pool. - [Benchmark] - public int NoString() - { - _numericStream.Position = 0; - int sum = 0; - foreach (var model in _numericReader.Read(_numericStream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - } -} diff --git a/tests/Benchmarks/Perf/ByteReaderStreamBenchmarks.cs b/tests/Benchmarks/Perf/ByteReaderStreamBenchmarks.cs deleted file mode 100644 index b597ff3..0000000 --- a/tests/Benchmarks/Perf/ByteReaderStreamBenchmarks.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System.Globalization; -using BenchmarkDotNet.Attributes; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; - -namespace Benchmarks.Perf -{ - /// - /// End-to-end streaming read: the char (which decodes the - /// stream UTF-8 → UTF-16 through a ) versus the raw-byte - /// (which slices lines straight from the byte buffer, - /// no transcode). The source is an in-memory so the comparison isolates - /// buffering+decoding+parsing from disk I/O. Both readers are rewound per invocation. - /// - [Config(typeof(RegressionConfig))] - public class ByteReaderStreamBenchmarks - { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthReader _charReader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthByteReader _byteReader = new(CultureInfo.InvariantCulture); - private MemoryStream _stream = new(); - - [Params(100, 1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - var writer = new FixedWidthWriter(); - _stream = new MemoryStream(); - writer.WriteMany(_stream, models.AsSpan(), Culture); - } - - [GlobalCleanup] - public void Cleanup() - { - _stream.Dispose(); - } - - /// Char reader: StreamReader decodes UTF-8 → UTF-16, then span slicing. Baseline. - [Benchmark(Baseline = true)] - public int CharReader_Read() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _charReader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - - /// Byte reader: slices lines straight from the byte buffer, no transcode, no string per line. - [Benchmark] - public int ByteReader_Read() - { - _stream.Position = 0; - int sum = 0; - foreach (var model in _byteReader.Read(_stream, leaveOpen: true)) - { - sum += model.Age; - } - - return sum; - } - } -} diff --git a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs index f551241..39b10e7 100644 --- a/tests/Benchmarks/Perf/ComparisonBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonBenchmarks.cs @@ -1,11 +1,8 @@ -using System.Globalization; using System.Text; using BenchmarkDotNet.Attributes; using FileHelpers; -using FlatFiles; using FixedWidthParser; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; +using FlatFiles; using RecordParser.Builders.Reader; using RecordParser.Extensions; @@ -16,30 +13,31 @@ namespace Benchmarks.Perf public sealed class FileHelpersRecord { [FieldFixedLength(10)] - [FieldTrim(TrimMode.Both)] + [FieldTrim(FileHelpers.TrimMode.Both)] public string? Name; [FieldFixedLength(5)] - [FieldTrim(TrimMode.Both)] + [FieldTrim(FileHelpers.TrimMode.Both)] public int Age; [FieldFixedLength(10)] - [FieldTrim(TrimMode.Both)] + [FieldTrim(FileHelpers.TrimMode.Both)] public double Salary; } #pragma warning restore CA1051 /// - /// Cross-library comparison: FileHelpers and FlatFiles against FixedWidthParser on the same - /// in-memory fixed-width data (25 chars/line: 10 string + 5 int + 10 double). + /// Cross-library comparison on the same in-memory fixed-width data (25 bytes/line: 10 string + 5 + /// int + 10 double). The other libraries read text, so they get the decoded string; FixedWidthParser + /// reads the file's bytes, which is what a caller holding the file would hand each of them. /// [Config(typeof(RegressionConfig))] public class ComparisonBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthReader _reader = new(CultureInfo.InvariantCulture); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private readonly FileHelperEngine _engine = new(); private readonly FixedLengthSchema _schema = CreateSchema(); private readonly FixedLengthReaderOptions _options = CreateReader(); private string _text = string.Empty; + private byte[] _bytes = []; [Params(100, 1000)] public int Count { get; set; } @@ -60,21 +58,14 @@ private static FixedLengthReaderOptions CreateReader() .Map(m => m.Age, 10, 5) .Map(m => m.Salary, 15, 10) .Build(); - return new (){ Parser = parser.Parse, ParallelismOptions = new() { Enabled = false } }; + return new() { Parser = parser.Parse, ParallelismOptions = new() { Enabled = false } }; } [GlobalSetup] public void Setup() { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - _text = Encoding.UTF8.GetString(ms.ToArray()); + _bytes = SampleData.Bytes(Count); + _text = Encoding.UTF8.GetString(_bytes); } [Benchmark(Baseline = true)] @@ -119,9 +110,8 @@ public int RecordParser_Read() [Benchmark] public int FixedWidthParser_Read() { - using var stringReader = new StringReader(_text); int sum = 0; - foreach (var model in _reader.Read(stringReader)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout)) { sum += model.Age; } @@ -132,9 +122,8 @@ public int FixedWidthParser_Read() [Benchmark] public int FixedWidthParser_Generated() { - using var stringReader = new StringReader(_text); int sum = 0; - foreach (var model in FixedWidth.Read(stringReader, formatProvider: Culture)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false))) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs b/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs index 2705e5d..5905fe3 100644 --- a/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs +++ b/tests/Benchmarks/Perf/ComparisonWriterBenchmarks.cs @@ -1,9 +1,8 @@ -using System.Globalization; +using FixedWidthParser; using BenchmarkDotNet.Attributes; using FileHelpers; using FlatFiles; using FlatFiles.TypeMapping; -using FixedWidthParser.Writers; using RecordParser.Parsers; using RecordParser.Builders.Writer; using RecordParser.Extensions; @@ -19,14 +18,14 @@ public sealed class FlatFilesWriteModel /// /// Cross-library comparison of fixed-width writing: FileHelpers and FlatFiles against - /// FixedWidthParser. All writes target Stream.Null to isolate formatting from I/O. + /// FixedWidthParser. All writes target Stream.Null to isolate formatting from I/O; the others go + /// through a StreamWriter because they write text, FixedWidthParser writes bytes straight to the stream. /// [Config(typeof(RegressionConfig))] public class ComparisonWriterBenchmarks : IDisposable { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; private static readonly FixedLengthOptions FlatFilesOptions = new(); - private readonly FixedWidthWriter _writer = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private readonly FileHelperEngine _engine = new(); private readonly IFixedLengthWriter _fixedWidthWriter = CreateWriter(); private StreamWriter _sink = null!; @@ -101,7 +100,7 @@ public void RecordParser_Write() [Benchmark] public void FixedWidthParser_Write() { - _writer.WriteMany(_sink, _models.AsSpan(), Culture); + FixedWidth.WriteMany(Stream.Null, _models.AsSpan(), _layout); } public virtual void Dispose() diff --git a/tests/Benchmarks/Perf/ParserBenchmarks.cs b/tests/Benchmarks/Perf/ParserBenchmarks.cs index c560d9d..e967fbe 100644 --- a/tests/Benchmarks/Perf/ParserBenchmarks.cs +++ b/tests/Benchmarks/Perf/ParserBenchmarks.cs @@ -1,7 +1,7 @@ -using System.Globalization; +using System.Text; using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; +using FixedWidthParser; namespace Benchmarks.Perf { @@ -12,9 +12,9 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class ParserBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthParser _parser = new(); - private StringPool _stringPool = null!; + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private FixedWidthContext _pooled; + private byte[] _line = []; [Params( "John Doe 30 60000.00 ", @@ -24,20 +24,21 @@ public class ParserBenchmarks [GlobalSetup] public void Setup() { - _stringPool = new StringPool(); + _line = Encoding.UTF8.GetBytes(Line); + _pooled = new FixedWidthContext { StringPool = new StringPool() }; } [Benchmark(Baseline = true)] public SampleModel Parse_NoPool() { - _parser.TryParse(Line, Culture, null, out var model); + FixedWidth.TryParse(_line, _layout, default, out var model); return model; } [Benchmark] public SampleModel Parse_WithStringPool() { - _parser.TryParse(Line, Culture, _stringPool, out var model); + FixedWidth.TryParse(_line, _layout, _pooled, out var model); return model; } } diff --git a/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs b/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs deleted file mode 100644 index f074ca7..0000000 --- a/tests/Benchmarks/Perf/PipeReaderBenchmarks.cs +++ /dev/null @@ -1,91 +0,0 @@ -using System.Globalization; -using System.IO.Pipelines; -using BenchmarkDotNet.Attributes; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; - -namespace Benchmarks.Perf -{ - /// - /// Async byte reading from a (the existing pooled LineBufferState core) - /// versus a (the pipe owns buffering/read-ahead). The source is an in-memory - /// , rewound per invocation, so the comparison isolates buffering + line - /// splitting + parsing from disk I/O. forces a tiny pipe - /// buffer so most records straddle segment boundaries, exercising the pooled scratch-copy path. - /// - [Config(typeof(RegressionConfig))] - public class PipeReaderBenchmarks - { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthByteReader _byteReader = new(CultureInfo.InvariantCulture); - private MemoryStream _stream = new(); - - [Params(100, 1000)] - public int Count { get; set; } - - [GlobalSetup] - public void Setup() - { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - - var writer = new FixedWidthWriter(); - _stream = new MemoryStream(); - writer.WriteMany(_stream, models.AsSpan(), Culture); - } - - [GlobalCleanup] - public void Cleanup() - { - _stream.Dispose(); - } - - /// Stream async path: manual pooled buffer with compaction/growth. Baseline. - [Benchmark(Baseline = true)] - public async Task Stream_ReadAsync() - { - _stream.Position = 0; - int sum = 0; - await foreach (var model in _byteReader.ReadAsync(_stream, leaveOpen: true).ConfigureAwait(false)) - { - sum += model.Age; - } - - return sum; - } - - /// PipeReader path with default segment sizing: lines are typically contiguous (parsed in place). - [Benchmark] - public async Task Pipe_ReadAsync() - { - _stream.Position = 0; - // leaveOpen on the pipe options keeps the reused MemoryStream alive; the reader completes the pipe. - var pipe = PipeReader.Create(_stream, new StreamPipeReaderOptions(leaveOpen: true)); - int sum = 0; - await foreach (var model in _byteReader.ReadAsync(pipe).ConfigureAwait(false)) - { - sum += model.Age; - } - - return sum; - } - - /// PipeReader path with a tiny buffer: most lines span segments → pooled scratch copy. - [Benchmark] - public async Task Pipe_ReadAsync_SmallSegments() - { - _stream.Position = 0; - var pipe = PipeReader.Create(_stream, new StreamPipeReaderOptions(bufferSize: 16, leaveOpen: true)); - int sum = 0; - await foreach (var model in _byteReader.ReadAsync(pipe).ConfigureAwait(false)) - { - sum += model.Age; - } - - return sum; - } - } -} diff --git a/tests/Benchmarks/Perf/ReaderBenchmarks.cs b/tests/Benchmarks/Perf/ReaderBenchmarks.cs index d215c46..141cc28 100644 --- a/tests/Benchmarks/Perf/ReaderBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderBenchmarks.cs @@ -1,28 +1,22 @@ -using System.Globalization; using System.Text; using BenchmarkDotNet.Attributes; using CommunityToolkit.HighPerformance.Buffers; using FixedWidthParser; -using FixedWidthParser.Parsers; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { /// - /// Synchronous reading: compares the span-based path (FixedWidthReader, no string per line) - /// with the naive baseline ReadLine()+TryParse (one string allocated per line). The source is - /// an in-memory string (StringReader) to isolate parsing+buffering from the cost of disk I/O. + /// Synchronous reading from an in-memory stream, to isolate parsing and buffering from disk I/O. + /// The baseline reads a string per line with a and encodes it back to + /// bytes before parsing: what a caller would do by hand. /// [Config(typeof(RegressionConfig))] public class ReaderBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthParser _parser = new(); - private readonly FixedWidthReader _reader = new(CultureInfo.InvariantCulture); - private readonly FixedWidthReader _pooledReader = new(CultureInfo.InvariantCulture, new StringPool()); - private readonly StringPool _generatedStringPool = new(); - private string _text = string.Empty; + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private readonly FixedWidthReaderOptions _pooledReaderOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + private readonly FixedWidthReaderOptions _generatedPooled = new() { StringPool = new StringPool() }; + private byte[] _bytes = []; [Params(100, 1000)] public int Count { get; set; } @@ -30,27 +24,18 @@ public class ReaderBenchmarks [GlobalSetup] public void Setup() { - var models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; - } - var writer = new FixedWidthWriter(); - using var ms = new MemoryStream(); - writer.WriteMany(ms, models.AsSpan(), Culture); - _text = Encoding.UTF8.GetString(ms.ToArray()); + _bytes = SampleData.Bytes(Count); } - /// ReadLine() allocates a string per line; then TryParse over it. Baseline. [Benchmark(Baseline = true)] public int Naive_ReadLine() { - using var reader = new StringReader(_text); + using var reader = new StreamReader(new MemoryStream(_bytes, writable: false), Encoding.UTF8); int sum = 0; string? line; while ((line = reader.ReadLine()) is not null) { - if (_parser.TryParse(line, Culture, null, out var model)) + if (FixedWidth.TryParse(Encoding.UTF8.GetBytes(line), _layout, default, out var model)) { sum += model.Age; } @@ -58,59 +43,47 @@ public int Naive_ReadLine() return sum; } - /// Span-based reading: slices each line from the buffer, no string per line. [Benchmark] - public int SpanReader_Read() + public int Reader_Read() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in _reader.Read(reader)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout)) { sum += model.Age; } - return sum; } - /// Span-based + StringPool: also interns the string columns (tends to zero-alloc). [Benchmark] - public int SpanReader_Read_Pooled() + public int Reader_Read_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in _pooledReader.Read(reader)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout, options: _pooledReaderOptions)) { sum += model.Age; } - return sum; } - /// Source-generated reader: same scanner, static generated TryParse, no reflection. [Benchmark] public int GeneratedReader_Read() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in FixedWidth.Read(reader, formatProvider: Culture)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false))) { sum += model.Age; } - return sum; } - /// Source-generated reader + StringPool: static TryParse plus interned string columns. [Benchmark] public int GeneratedReader_Read_Pooled() { - using var reader = new StringReader(_text); int sum = 0; - foreach (var model in FixedWidth.Read(reader, formatProvider: Culture, stringPool: _generatedStringPool)) + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), options: _generatedPooled)) { sum += model.Age; } - return sum; } } diff --git a/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs new file mode 100644 index 0000000..54de4b3 --- /dev/null +++ b/tests/Benchmarks/Perf/ReaderPoolingBenchmarks.cs @@ -0,0 +1,70 @@ +using BenchmarkDotNet.Attributes; +using CommunityToolkit.HighPerformance.Buffers; +using FixedWidthParser; + +namespace Benchmarks.Perf +{ + /// + /// Where the reader's per-line allocation comes from — the string column — and what a + /// saves. The pool is reused across invocations, so after warmup the + /// repeated names ("Name0".."Name99") are already interned. + /// + [Config(typeof(RegressionConfig))] + public class ReaderPoolingBenchmarks + { + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private readonly FixedWidthReaderOptions _pooledReaderOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + private static readonly FixedWidthLayout _numericLayout = FixedWidthLayout.FromAttributes(); + private byte[] _bytes = []; + private byte[] _numericBytes = []; + + [Params(1000)] + public int Count { get; set; } + + [GlobalSetup] + public void Setup() + { + _bytes = SampleData.Bytes(Count); + var numeric = SampleData.Models(Count).Select(m => new NumericModel { Age = m.Age, Salary = m.Salary }).ToArray(); + using var ms = new MemoryStream(); + FixedWidth.WriteMany(ms, numeric.AsSpan(), FixedWidthLayout.FromAttributes()); + _numericBytes = ms.ToArray(); + } + + /// String model, no pool: a fresh string decoded per line. Baseline. + [Benchmark(Baseline = true)] + public int WithString_NoPool() + { + int sum = 0; + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout)) + { + sum += model.Age; + } + return sum; + } + + /// String model, StringPool: repeated values interned, not re-decoded. + [Benchmark] + public int WithString_Pooled() + { + int sum = 0; + foreach (var model in FixedWidth.Read(new MemoryStream(_bytes, writable: false), _layout, options: _pooledReaderOptions)) + { + sum += model.Age; + } + return sum; + } + + /// Numeric-only model: no string column, nothing to decode or pool. + [Benchmark] + public int NoString() + { + int sum = 0; + foreach (var model in FixedWidth.Read(new MemoryStream(_numericBytes, writable: false), _numericLayout)) + { + sum += model.Age; + } + return sum; + } + } +} diff --git a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs index b059311..2bc8a41 100644 --- a/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs +++ b/tests/Benchmarks/Perf/ReaderStartupBenchmarks.cs @@ -1,7 +1,5 @@ -using System.Globalization; using BenchmarkDotNet.Attributes; using FixedWidthParser; -using FixedWidthParser.Readers; namespace Benchmarks.Perf { @@ -13,16 +11,13 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class ReaderStartupBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private const string Text = "John Doe 30 60000.00 \n"; + private static readonly byte[] Text = "John Doe 30 60000.00 \n"u8.ToArray(); [Benchmark(Baseline = true)] public int ReflectionReader_ConstructAndReadOne() { - var fixedWidthReader = new FixedWidthReader(Culture); - using var reader = new StringReader(Text); int sum = 0; - foreach (var model in fixedWidthReader.Read(reader)) + foreach (var model in FixedWidth.Read(new MemoryStream(Text, writable: false), FixedWidthLayout.FromAttributes())) { sum += model.Age; } @@ -32,9 +27,8 @@ public int ReflectionReader_ConstructAndReadOne() [Benchmark] public int GeneratedReader_ReadOne() { - using var reader = new StringReader(Text); int sum = 0; - foreach (var model in FixedWidth.Read(reader, formatProvider: Culture)) + foreach (var model in FixedWidth.Read(new MemoryStream(Text, writable: false))) { sum += model.Age; } diff --git a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs index 9916e07..18c86f5 100644 --- a/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs +++ b/tests/Benchmarks/Perf/SourceGenParserBenchmarks.cs @@ -1,8 +1,6 @@ -using System.Globalization; +using System.Text; using BenchmarkDotNet.Attributes; using FixedWidthParser; -using FixedWidthParser.Attributes; -using FixedWidthParser.Parsers; namespace Benchmarks.Perf { @@ -21,23 +19,29 @@ namespace Benchmarks.Perf [Config(typeof(RegressionConfig))] public class SourceGenParserBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthParser _reflection = new(); + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); + private byte[] _line = []; [Params("John Doe 30 6000000.00")] public string Line { get; set; } = string.Empty; + [GlobalSetup] + public void Setup() + { + _line = Encoding.UTF8.GetBytes(Line); + } + [Benchmark(Baseline = true)] public int Reflection() { - _reflection.TryParse(Line, Culture, null, out var model); + FixedWidth.TryParse(_line, _layout, default, out var model); return model.Age; } [Benchmark] public int Generated() { - FixedWidth.TryParse(Line, Culture, null, out var model); + FixedWidth.TryParse(_line, default, out var model); return model.Age; } } diff --git a/tests/Benchmarks/Perf/WriterBenchmarks.cs b/tests/Benchmarks/Perf/WriterBenchmarks.cs index d9dedb3..a84e834 100644 --- a/tests/Benchmarks/Perf/WriterBenchmarks.cs +++ b/tests/Benchmarks/Perf/WriterBenchmarks.cs @@ -1,21 +1,17 @@ -using System.Globalization; +using FixedWidthParser; using BenchmarkDotNet.Attributes; -using FixedWidthParser.Writers; namespace Benchmarks.Perf { /// - /// Measures the hot path of fixed-width writing. Writes to Stream.Null to isolate formatting + - /// encoding from the cost of disk I/O. WriteMany_NewStream is the baseline. + /// Measures the hot path of fixed-width writing. Writes to Stream.Null to isolate formatting from + /// the cost of disk I/O. WriteMany_Enumerable is the baseline. /// [Config(typeof(RegressionConfig))] - public class WriterBenchmarks : IDisposable + public class WriterBenchmarks { - private static readonly CultureInfo Culture = CultureInfo.InvariantCulture; - private readonly FixedWidthWriter _writer = new(); - private StreamWriter _sink = null!; + private static readonly FixedWidthLayout _layout = FixedWidthLayout.FromAttributes(); private SampleModel[] _models = []; - private bool _disposed; [Params(1, 100, 1000)] public int Count { get; set; } @@ -23,76 +19,27 @@ public class WriterBenchmarks : IDisposable [GlobalSetup] public void Setup() { - _sink?.Dispose(); - _sink = new StreamWriter(Stream.Null); - _models = new SampleModel[Count]; - for (int i = 0; i < Count; i++) - { - _models[i] = new SampleModel - { - Name = "Name" + i, - Age = 20 + (i % 50), - Salary = 1000.0 + i - }; - } + _models = SampleData.Models(Count); } - [GlobalCleanup] - public void Cleanup() - { - _sink.Dispose(); - } - - /// Creates/disposes a StreamWriter per call (over the Stream). Baseline. [Benchmark(Baseline = true)] - public void WriteMany_NewStream() - { - _writer.WriteMany(Stream.Null, _models, Culture); - } - - /// Reuses an existing StreamWriter (new overload). Should eliminate the fixed allocation. - [Benchmark] - public void WriteMany_ReuseWriter() + public void WriteMany_Enumerable() { - _writer.WriteMany(_sink, _models, Culture); + FixedWidth.WriteMany(Stream.Null, (IEnumerable)_models, _layout); } - /// Writer reuse + ReadOnlySpan: no StreamWriter per call and no enumerator. Should be zero-alloc. + /// ReadOnlySpan: no enumerator, each record read by reference. Should be zero-alloc. [Benchmark] - public void WriteMany_ReuseWriterSpan() + public void WriteMany_Span() { - _writer.WriteMany(_sink, _models.AsSpan(), Culture); + FixedWidth.WriteMany(Stream.Null, _models.AsSpan(), _layout); } - /// - /// Async creating a StreamWriter per call (Stream overload). Mixes the cost of the new - /// StreamWriter with the async state-machine overhead — comparable to NewStream. - /// + /// The async overload: the difference from WriteMany_Enumerable is the async state machine. [Benchmark] - public Task WriteMany_AsyncNewStream() - { - return _writer.WriteManyAsync(Stream.Null, _models, Culture); - } - - /// - /// Async reusing the StreamWriter (StreamWriter overload). Isolates the pure async - /// overhead: compared to WriteMany_ReuseWriter, the difference is only the async. - /// - [Benchmark] - public Task WriteMany_AsyncReuseWriter() - { - return _writer.WriteManyAsync(_sink, _models, Culture); - } - - public virtual void Dispose() + public Task WriteManyAsync() { - if (_disposed) - { - return; - } - _disposed = true; - _sink.Dispose(); - GC.SuppressFinalize(this); + return FixedWidth.WriteManyAsync(Stream.Null, _models, _layout); } } } diff --git a/tests/Benchmarks/SampleData.cs b/tests/Benchmarks/SampleData.cs new file mode 100644 index 0000000..61a1fc2 --- /dev/null +++ b/tests/Benchmarks/SampleData.cs @@ -0,0 +1,25 @@ +using FixedWidthParser; + +namespace Benchmarks +{ + /// The fixed-width input every read benchmark parses: lines. + internal static class SampleData + { + public static SampleModel[] Models(int count) + { + var models = new SampleModel[count]; + for (int i = 0; i < count; i++) + { + models[i] = new SampleModel { Name = "Name" + (i % 100), Age = 20 + (i % 50), Salary = 1000.0 + i }; + } + return models; + } + + public static byte[] Bytes(int count) + { + using var ms = new MemoryStream(); + FixedWidth.WriteMany(ms, Models(count).AsSpan(), FixedWidthLayout.FromAttributes()); + return ms.ToArray(); + } + } +} diff --git a/tests/Benchmarks/SampleModel.cs b/tests/Benchmarks/SampleModel.cs index 0198732..5dbcc17 100644 --- a/tests/Benchmarks/SampleModel.cs +++ b/tests/Benchmarks/SampleModel.cs @@ -1,4 +1,4 @@ -using FixedWidthParser.Attributes; +using FixedWidthParser; namespace Benchmarks { diff --git a/tests/Benchmarks/bench_delta.py b/tests/Benchmarks/bench_delta.py new file mode 100644 index 0000000..8bd2301 --- /dev/null +++ b/tests/Benchmarks/bench_delta.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Renders the PR benchmark comment as a delta against the series published to GitHub Pages. + +The PR job measures only the pull request's own build, so its absolute numbers carry no verdict on +their own: runner variance on a shared machine swamps most real changes. benchmark-pages.yml already +publishes a per-commit series for master, so a baseline costs one file read rather than a second +benchmark run. + +Usage: bench_delta.py --results [--baseline ] [--threshold 10] +""" + +from __future__ import annotations + +import argparse +import glob +import json +import os + + +def load_baseline(path: str | None) -> tuple[str | None, dict[str, float]]: + """The newest run's measurements, as (commit sha, {benchmark name: mean ns}).""" + if not path or not os.path.exists(path): + return None, {} + with open(path, encoding="utf-8") as handle: + raw = handle.read() + data = json.loads(raw[raw.index("{"):]) + runs = data.get("entries", {}).get("Benchmark", []) + if not runs: + return None, {} + newest = runs[-1] + measurements = {bench["name"]: bench["value"] for bench in newest.get("benches", [])} + return newest.get("commit", {}).get("id"), measurements + + +def load_results(directory: str) -> tuple[dict[str, float], list[str]]: + """(measured means, benchmarks that ran but produced no measurement). FullName matches the + baseline's `name` verbatim. BenchmarkDotNet writes null Statistics when the benchmark process + crashed or threw.""" + results: dict[str, float] = {} + failed: list[str] = [] + for path in sorted(glob.glob(os.path.join(directory, "*-report-full.json"))): + with open(path, encoding="utf-8") as handle: + for bench in json.load(handle).get("Benchmarks", []): + statistics = bench.get("Statistics") + if statistics is None: + failed.append(bench["FullName"]) + else: + results[bench["FullName"]] = statistics["Mean"] + return results, failed + + +def humanize(nanoseconds: float) -> str: + for unit, scale in (("s", 1e9), ("ms", 1e6), ("µs", 1e3)): + if nanoseconds >= scale: + return f"{nanoseconds / scale:.2f} {unit}" + return f"{nanoseconds:.0f} ns" + + +def render(baseline_sha, baseline, results, threshold, failed=()) -> str: + lines = ["## Benchmark Results", ""] + if failed: + lines += ["### Failed to run", "", "These produced no measurement; check the job log for the crash or exception.", ""] + lines += [f"- `{name}`" for name in failed] + lines.append("") + if not baseline: + lines += [ + ("_No published baseline was found, so these are absolute numbers only. They come from a " + "shared `ubuntu-latest` runner and cannot be compared against another machine's._"), + "", + ] + return "\n".join(lines) + + lines += [ + (f"_Baseline: master [`{baseline_sha[:8]}`](../commit/{baseline_sha}), from the series " + "published by `benchmark-pages.yml`. Both sides ran on a shared `ubuntu-latest` runner, so " + f"treat anything under ±{threshold:g}% as noise._"), + "", + ] + + moved, steady, unmatched = [], 0, 0 + for name, current in results.items(): + before = baseline.get(name) + if before is None: + unmatched += 1 + continue + if before == 0: + continue + percent = (current - before) / before * 100 + if abs(percent) >= threshold: + moved.append((percent, name, before, current)) + else: + steady += 1 + + if moved: + moved.sort(reverse=True) + lines += [ + f"### Moved by at least {threshold:g}%", + "", + "| Benchmark | master | This PR | Delta |", + "|---|---:|---:|---:|", + ] + lines += [ + f"| `{name}` | {humanize(before)} | {humanize(current)} | {percent:+.1f}% |" + for percent, name, before, current in moved + ] + lines.append("") + else: + lines += [f"No benchmark moved by {threshold:g}% or more.", ""] + + summary = [f"{steady} within ±{threshold:g}%"] + if unmatched: + summary.append(f"{unmatched} with no baseline entry (new, renamed, or reparameterized)") + lines += ["_" + ", ".join(summary) + "._", ""] + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--results", required=True, help="Directory holding *-report-full.json") + parser.add_argument("--baseline", help="gh-pages dev/bench/data.js; absent means no comparison") + parser.add_argument("--threshold", type=float, default=10.0, help="Percent change worth reporting") + args = parser.parse_args() + + baseline_sha, baseline = load_baseline(args.baseline) + results, failed = load_results(args.results) + print(render(baseline_sha, baseline, results, args.threshold, failed)) + + +if __name__ == "__main__": + main() diff --git a/tests/Benchmarks/benchmark-groups.json b/tests/Benchmarks/benchmark-groups.json new file mode 100644 index 0000000..46ebe46 --- /dev/null +++ b/tests/Benchmarks/benchmark-groups.json @@ -0,0 +1,32 @@ +[ + { + "name": "parser", + "filter": "Benchmarks.Perf.ParserBenchmarks.* Benchmarks.Perf.SourceGenParserBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "reader", + "filter": "Benchmarks.Perf.ReaderBenchmarks.* Benchmarks.Perf.ReaderStartupBenchmarks.* Benchmarks.Perf.AsyncReaderBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "byte", + "filter": "Benchmarks.Perf.ReaderPoolingBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "writer", + "filter": "Benchmarks.Perf.WriterBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + }, + { + "name": "comparison", + "filter": "Benchmarks.Perf.ComparisonWriterBenchmarks.* Benchmarks.Perf.ComparisonBenchmarks.*", + "pagesArgs": "--warmupCount 3 --iterationCount 10", + "prArgs": "--warmupCount 1 --iterationCount 5" + } +] diff --git a/tests/Benchmarks/check-benchmark-groups.sh b/tests/Benchmarks/check-benchmark-groups.sh new file mode 100644 index 0000000..f77d769 --- /dev/null +++ b/tests/Benchmarks/check-benchmark-groups.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +groups_file="$here/benchmark-groups.json" +project="$here/Benchmarks.csproj" + +filters="$(python3 -c ' +import json, sys +with open(sys.argv[1], encoding="utf-8") as f: + for group in json.load(f): + for pattern in group["filter"].split(): + print(pattern) +' "$groups_file" | tr -d '\r')" + +ids="$(dotnet run --project "$project" -c Release --no-build -- --list flat | + tr -d '\r' | grep '^Benchmarks\.')" + +regexes=() +while IFS= read -r filter; do + escaped=${filter//./\\.} + regexes+=("^${escaped//\*/.*}$") +done <<<"$filters" + +missing=() +while IFS= read -r id; do + matched=0 + for regex in "${regexes[@]}"; do + if [[ $id =~ $regex ]]; then + matched=1 + break + fi + done + ((matched)) || missing+=("$id") +done <<<"$ids" + +if ((${#missing[@]})); then + echo "Benchmarks matched by no group in benchmark-groups.json:" >&2 + printf ' %s\n' "${missing[@]}" >&2 + echo "Add them to a group, or add a new group." >&2 + exit 1 +fi + +echo "Every benchmark is covered by a group." diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs index 4f72cd1..6ceb25b 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthModelGeneratorTests.cs @@ -1,7 +1,7 @@ using System.Collections.Immutable; using CommunityToolkit.HighPerformance.Buffers; -using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis; namespace FixedWidthParser.Generator.Tests { @@ -21,18 +21,16 @@ public void ValidModel_GeneratesTryParse() """); Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("public static bool TryParse", generated, StringComparison.Ordinal); - Assert.Contains("if (line.Length < 18) { model = default!; return false; }", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryParse(line.Slice(5, 3), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryDouble(line.Slice(8, 5), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.TryFloat(line.Slice(13, 5), formatProvider", generated, StringComparison.Ordinal); - Assert.DoesNotContain("global::System.Math.Min", generated, StringComparison.Ordinal); - Assert.DoesNotContain("FixedWidthRuntime.Column", generated, StringComparison.Ordinal); + Assert.Contains("TryParse(global::System.ReadOnlySpan line, in global::FixedWidthParser.FixedWidthContext context", generated, StringComparison.Ordinal); + Assert.Contains("if (line.Length < 18) { context.ReportShortLine(18); model = default!; return false; }", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.String(line.Slice(0, 5), context, in __parse0)", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryParse(line.Slice(5, 3), context, in __parse1, out int __v1)) { context.ReportFailedColumn(\"Quantity\", 5, 3); model = default!; return false; }", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryDouble(line.Slice(8, 5), context, in __parse2", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryFloat(line.Slice(13, 5), context, in __parse3", generated, StringComparison.Ordinal); + Assert.DoesNotContain("ReadOnlySpan", generated, StringComparison.Ordinal); Assert.Contains("Code = __v0", generated, StringComparison.Ordinal); - Assert.Contains("Quantity = __v1", generated, StringComparison.Ordinal); - Assert.Contains("Price = __v2", generated, StringComparison.Ordinal); Assert.Contains("Discount = __v3", generated, StringComparison.Ordinal); } @@ -48,134 +46,127 @@ public void ValidModel_GeneratesTryFormat() """); Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("public static bool TryFormat", generated, StringComparison.Ordinal); - Assert.Contains("if (destination.Length < 8) { charsWritten = 0; return false; }", generated, StringComparison.Ordinal); + Assert.Contains("global::System.Span destination, in global::FixedWidthParser.FixedWidthContext context, out int bytesWritten", generated, StringComparison.Ordinal); + Assert.Contains("if (destination.Length < 8) { bytesWritten = 0; return false; }", generated, StringComparison.Ordinal); Assert.Contains("FixedWidthRuntime.FormatString(model.Code,", generated, StringComparison.Ordinal); - Assert.Contains("FixedWidthRuntime.FormatValue(model.Quantity,", generated, StringComparison.Ordinal); - Assert.Contains("ColumnFormatOptions __options0", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.FormatUtf8Value(model.Quantity,", generated, StringComparison.Ordinal); Assert.Contains("ColumnFormatOptions __options1", generated, StringComparison.Ordinal); } [Fact] - public void CharModel_ColumnNotFormattable_EmitsParseOnly_AndReportsFwp010() + public void EveryColumnKind_CompilesCleanly() { - // A column type that is ISpanParsable but NOT ISpanFormattable: the read side is unaffected - // (FWP010 only blocks the write method), so TryParse is still emitted but TryFormat is not. var result = Run(""" - public readonly struct ParsableNotFormattable : System.ISpanParsable + public readonly record struct Cents(long Value); + + public sealed class CentsConverter : FixedWidthParser.IFixedWidthConverter { - public static ParsableNotFormattable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; - public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } - public static ParsableNotFormattable Parse(string s, System.IFormatProvider? p) => default; - public static bool TryParse(string? s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } + public bool TryParse(System.ReadOnlySpan column, in FixedWidthContext context, out Cents value) { value = default; return true; } + public bool TryFormat(Cents value, System.Span destination, in FixedWidthContext context, out int bytesWritten) { bytesWritten = 0; return true; } } - public readonly partial record struct WriteBadModel : IFixedWidthModel + public readonly partial record struct AllKinds : IFixedWidthModel { - [FixedColumn(0, 5)] public ParsableNotFormattable Value { get; init; } + [FixedColumn(0, 5)] public string Text { get; init; } + [FixedColumn(5, 5)] public int? Number { get; init; } + [FixedColumn(10, 8, Format = "yyyyMMdd")] public System.DateOnly Day { get; init; } + [FixedColumn(18, 8, Converter = typeof(CentsConverter))] public Cents? Amount { get; init; } + [FixedColumn(26, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public decimal Rate { get; init; } + [FixedColumn(31, 36)] public System.Guid Id { get; init; } } """); - Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP010", StringComparison.Ordinal)); + Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("public static bool TryParse", generated, StringComparison.Ordinal); - Assert.DoesNotContain("public static bool TryFormat", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.IsBlank(line.Slice(5, 5), in __parse1)", generated, StringComparison.Ordinal); + Assert.Contains("TryDateOnlyExact(line.Slice(10, 8), context, in __parse", generated, StringComparison.Ordinal); + Assert.Contains("global::FixedWidthParser.TrimMode.Trailing, \"yyyyMMdd\");", generated, StringComparison.Ordinal); + Assert.Contains("= new('0', global::FixedWidthParser.TrimMode.Leading", generated, StringComparison.Ordinal); } [Fact] - public void DualMarkerModel_GeneratesBothCharAndByteTryParse() + public void CharOnlyParsableColumn_ParsesThroughDecodedText() { var result = Run(""" - public readonly partial record struct DualModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly struct OnlyCharParsable : System.ISpanParsable, System.ISpanFormattable { - [FixedColumn(0, 5)] public string Code { get; init; } - [FixedColumn(5, 3)] public int Quantity { get; init; } - [FixedColumn(8, 5)] public double Price { get; init; } - [FixedColumn(13, 5)] public float Discount { get; init; } + public static OnlyCharParsable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; + public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public static OnlyCharParsable Parse(string s, System.IFormatProvider? p) => default; + public static bool TryParse(string? s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public bool TryFormat(System.Span d, out int w, System.ReadOnlySpan f, System.IFormatProvider? p) { w = 0; return true; } + public string ToString(string? f, System.IFormatProvider? p) => ""; } - """); - - Assert.Empty(result.GeneratorDiagnostics); - // Both overloads live in one generated file. - var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - // Char method still routes through FixedWidthRuntime. - Assert.Contains("FixedWidthParser.FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); - // Byte method routes through Utf8FixedWidthRuntime. - Assert.Contains("Utf8FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.TryParse(line.Slice(5, 3), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.TryDouble(line.Slice(8, 5), formatProvider", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.TryFloat(line.Slice(13, 5), formatProvider", generated, StringComparison.Ordinal); - } - [Fact] - public void Utf8OnlyMarkerModel_GeneratesByteTryParseOnly() - { - var result = Run(""" - public readonly partial record struct Utf8OnlyModel : IUtf8FixedWidthModel + public readonly partial record struct DecodedModel : IFixedWidthModel { - [FixedColumn(0, 5)] public string Code { get; init; } - [FixedColumn(5, 3)] public int Quantity { get; init; } + [FixedColumn(0, 5)] public OnlyCharParsable Value { get; init; } } """); Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.DoesNotContain("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.Contains("Utf8FixedWidthRuntime.String(line.Slice(0, 5), stringPool)", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.TryParseDecoded", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.FormatValue(model.Value,", generated, StringComparison.Ordinal); } [Fact] - public void Utf8Model_ColumnOnlyCharParsable_ReportsFwp007() + public void ColumnNotFormattable_ReportsFwp010_AndEmitsNothing() { - // A column type that is ISpanParsable but NOT IUtf8SpanParsable cannot be handled by the - // generated byte parser, so a UTF-8-marked model must report FWP007 and emit no source. + // The interface needs TryFormat too, so a column the writer can't handle blocks the whole emission. var result = Run(""" - public readonly struct OnlyCharParsable : System.ISpanParsable + public readonly struct ParsableNotFormattable : System.ISpanParsable { - public static OnlyCharParsable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; - public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } - public static OnlyCharParsable Parse(string s, System.IFormatProvider? p) => default; - public static bool TryParse(string? s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public static ParsableNotFormattable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; + public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } + public static ParsableNotFormattable Parse(string s, System.IFormatProvider? p) => default; + public static bool TryParse(string? s, System.IFormatProvider? p, out ParsableNotFormattable r) { r = default; return true; } } - public readonly partial record struct Utf8BadModel : IUtf8FixedWidthModel + public readonly partial record struct WriteBadModel : IFixedWidthModel { - [FixedColumn(0, 5)] public OnlyCharParsable Value { get; init; } + [FixedColumn(0, 5)] public ParsableNotFormattable Value { get; init; } } """); - Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP007", StringComparison.Ordinal)); + Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP010", StringComparison.Ordinal)); Assert.Empty(result.GeneratedSources); } [Fact] - public void DualMarker_ColumnOnlyCharParsable_EmitsCharMethodOnly_AndReportsFwp007() + public void ConverterForAnotherType_ReportsFwp008() { - // A dual-marked model whose column is ISpanParsable but not IUtf8SpanParsable: FWP007 blocks - // ONLY the byte path, so the char TryParse is still emitted (partial emit). var result = Run(""" - public readonly struct OnlyCharParsable : System.ISpanParsable + public sealed class IntConverter : FixedWidthParser.IFixedWidthConverter { - public static OnlyCharParsable Parse(System.ReadOnlySpan s, System.IFormatProvider? p) => default; - public static bool TryParse(System.ReadOnlySpan s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } - public static OnlyCharParsable Parse(string s, System.IFormatProvider? p) => default; - public static bool TryParse(string? s, System.IFormatProvider? p, out OnlyCharParsable r) { r = default; return true; } + public bool TryParse(System.ReadOnlySpan column, in FixedWidthContext context, out int value) { value = 0; return true; } + public bool TryFormat(int value, System.Span destination, in FixedWidthContext context, out int bytesWritten) { bytesWritten = 0; return true; } } - public readonly partial record struct DualPartialModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct MismatchModel : IFixedWidthModel { - [FixedColumn(0, 5)] public OnlyCharParsable Value { get; init; } + [FixedColumn(0, 5, Converter = typeof(IntConverter))] public string Value { get; init; } } """); - Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP007", StringComparison.Ordinal)); - var generated = Assert.Single(result.GeneratedSources); - Assert.Contains("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); - Assert.DoesNotContain("global::System.ReadOnlySpan line", generated, StringComparison.Ordinal); + Assert.Contains(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP008", StringComparison.Ordinal)); + Assert.Empty(result.GeneratedSources); + } + + [Theory] + [InlineData("[FixedColumn(0, 5, Alignment = Alignment.Right, Padding = '\u00e9')] public int Value { get; init; }", "Padding")] + [InlineData("[FixedColumn(0, 5, TrimChar = '\u00a0')] public int Value { get; init; }", "TrimChar")] + public void NonAsciiPaddingOrTrimChar_ReportsFwp012(string column, string setting) + { + var result = Run("public readonly partial record struct NonAsciiModel : IFixedWidthModel { " + column + " }"); + + var diagnostic = Assert.Single(result.GeneratorDiagnostics, d => string.Equals(d.Id, "FWP012", StringComparison.Ordinal)); + Assert.Contains(setting, diagnostic.GetMessage(System.Globalization.CultureInfo.InvariantCulture), StringComparison.Ordinal); + Assert.Empty(result.GeneratedSources); } [Fact] @@ -199,6 +190,30 @@ public partial class IgnoringModel : IFixedWidthModel Assert.DoesNotContain("Ignored2", generated, StringComparison.Ordinal); } + [Fact] + public void Model_IncludesInheritedColumns() + { + var result = Run(""" + public class BaseModel + { + [FixedColumn(0, 5)] public int Id { get; set; } + [FixedColumn(5, 5)] public string Code; + } + + public partial class DerivedModel : BaseModel, IFixedWidthModel + { + [FixedColumn(10, 5)] public string Name { get; set; } + } + """); + + Assert.Empty(result.GeneratorDiagnostics); + Assert.Empty(result.CompilationErrors); + var generated = Assert.Single(result.GeneratedSources); + Assert.Contains("Id = __v", generated, StringComparison.Ordinal); + Assert.Contains("Code = __v", generated, StringComparison.Ordinal); + Assert.Contains("FixedWidthRuntime.FormatString(model.Code,", generated, StringComparison.Ordinal); + } + [Theory] [InlineData(""" public readonly record struct BadModel : IFixedWidthModel @@ -240,6 +255,17 @@ public partial class Outer [FixedColumn(2, 5)] public string Right { get; init; } } """, "FWP006")] + [InlineData(""" + public class BadBase + { + [FixedColumn(0, 5)] public string Left { get; set; } + } + + public partial class BadModel : BadBase, IFixedWidthModel + { + [FixedColumn(2, 5)] public string Right { get; set; } + } + """, "FWP006")] public void InvalidModel_ReportsExpectedDiagnostic(string modelSource, string diagnosticId) { var result = Run(modelSource); @@ -420,15 +446,14 @@ public void IncrementalCaching_ColumnChange_RegeneratesOutput() var second = driver.GetRunResult().Results[0].GeneratedSources.Single().SourceText.ToString(); Assert.NotEqual(first, second); - Assert.Contains("line.Length < 8", second, StringComparison.Ordinal); + Assert.Contains("line.Length < 8)", second, StringComparison.Ordinal); } private static CSharpCompilation CreateCompilation(string modelSource, CSharpParseOptions parseOptions) { string source = """ using FixedWidthParser; - using FixedWidthParser.Attributes; - + namespace GeneratorSmoke; """ + modelSource; @@ -445,8 +470,7 @@ private static GeneratorRunResult Run(string modelSource) { string source = """ using FixedWidthParser; - using FixedWidthParser.Attributes; - + namespace GeneratorSmoke; """ + modelSource; @@ -464,7 +488,7 @@ namespace GeneratorSmoke; GeneratorDriver driver = CSharpGeneratorDriver.Create([generator], parseOptions: parseOptions); driver = driver.RunGeneratorsAndUpdateCompilation( compilation, - out _, + out var updated, out var generatorDiagnostics, TestContext.Current.CancellationToken); @@ -472,8 +496,11 @@ namespace GeneratorSmoke; var generatedSources = runResult.GeneratedTrees .Select(t => t.GetText(TestContext.Current.CancellationToken).ToString()) .ToImmutableArray(); + var compilationErrors = updated.GetDiagnostics(TestContext.Current.CancellationToken) + .Where(d => d.Severity == DiagnosticSeverity.Error) + .ToImmutableArray(); - return new GeneratorRunResult(generatorDiagnostics, generatedSources); + return new GeneratorRunResult(generatorDiagnostics, generatedSources, compilationErrors); } private static readonly Lazy> References = new(CreateReferences); @@ -508,10 +535,12 @@ void Add(string path) private sealed class GeneratorRunResult( ImmutableArray generatorDiagnostics, - ImmutableArray generatedSources) + ImmutableArray generatedSources, + ImmutableArray compilationErrors) { public ImmutableArray GeneratorDiagnostics { get; } = generatorDiagnostics; public ImmutableArray GeneratedSources { get; } = generatedSources; + public ImmutableArray CompilationErrors { get; } = compilationErrors; } } } diff --git a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj index 5a19001..29760e0 100644 --- a/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj +++ b/tests/FixedWidthParser.Generator.Tests/FixedWidthParser.Generator.Tests.csproj @@ -3,7 +3,7 @@ true true - net8.0;net10.0 + net10.0 enable enable false @@ -13,13 +13,13 @@ - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + diff --git a/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt b/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt new file mode 100644 index 0000000..1a74195 --- /dev/null +++ b/tests/FixedWidthParser.NativeSmoke/CMakeLists.txt @@ -0,0 +1,40 @@ +# Builds and runs smoke.c against the published FixedWidthParser.Native shared library. The library is +# loaded dynamically (see smoke.c), so nothing links against it at build time. +# +# Usage, from the repository root, after `python python/scripts/build_native.py`: +# cmake -S tests/FixedWidthParser.NativeSmoke -B build/nativesmoke +# cmake --build build/nativesmoke --config Release +# ctest --test-dir build/nativesmoke --output-on-failure +cmake_minimum_required(VERSION 3.16) +project(FixedWidthParserNativeSmoke C) + +set(FIXEDWIDTH_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/../..") + +if(WIN32) + set(_lib_name "FixedWidthParser.Native.dll") +elseif(APPLE) + set(_lib_name "FixedWidthParser.Native.dylib") +else() + set(_lib_name "FixedWidthParser.Native.so") +endif() + +# build_native.py publishes the library and copies it to this OS-normalized path for the Python package. +set(FIXEDWIDTH_LIB_PATH "${FIXEDWIDTH_ROOT}/python/src/fixedwidth/_lib/${_lib_name}" + CACHE FILEPATH "Path to the published FixedWidthParser.Native shared library") + +add_executable(fw_smoke smoke.c) +target_include_directories(fw_smoke PRIVATE "${FIXEDWIDTH_ROOT}/src/FixedWidthParser.Native/include") +target_compile_definitions(fw_smoke PRIVATE FIXEDWIDTH_LIB_PATH_DEFAULT="${FIXEDWIDTH_LIB_PATH}") +if(MSVC) + # C4152: the smoke test casts GetProcAddress results to function pointers on purpose. + target_compile_options(fw_smoke PRIVATE /W4 /WX /wd4152) +else() + target_compile_options(fw_smoke PRIVATE -Wall -Wextra -Werror) +endif() +if(NOT WIN32) + target_link_libraries(fw_smoke PRIVATE ${CMAKE_DL_LIBS}) +endif() + +enable_testing() +add_test(NAME fw_smoke COMMAND fw_smoke "${FIXEDWIDTH_LIB_PATH}" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") +set_tests_properties(fw_smoke PROPERTIES PASS_REGULAR_EXPRESSION "smoke ok") diff --git a/tests/FixedWidthParser.NativeSmoke/README.md b/tests/FixedWidthParser.NativeSmoke/README.md new file mode 100644 index 0000000..fd045f8 --- /dev/null +++ b/tests/FixedWidthParser.NativeSmoke/README.md @@ -0,0 +1,15 @@ +# FixedWidthParser.NativeSmoke + +The reference C consumer of `FixedWidthParser.Native`. `smoke.c` loads the published library dynamically +(`LoadLibraryA`/`dlopen`: NativeAOT ships no import library on Windows), pins the offset and size of every +ABI struct with static asserts, and calls every export: layouts, whole and parallel reads, batches, Arrow +arrays and streams, writes, parse errors, and `NULL`/`struct_size`/freed-handle misuse. + + dotnet publish src/FixedWidthParser.Native -c Release -r + cmake -S tests/FixedWidthParser.NativeSmoke -B build/nativesmoke -DFIXEDWIDTH_LIB_PATH= + cmake --build build/nativesmoke --config Release + ctest --test-dir build/nativesmoke --output-on-failure + +Without `-DFIXEDWIDTH_LIB_PATH`, it uses the copy `python/scripts/build_native.py` puts in +`python/src/fixedwidth/_lib/`. A failing static assert means `include/fixedwidth.h` and the managed structs +disagree. diff --git a/tests/FixedWidthParser.NativeSmoke/smoke.c b/tests/FixedWidthParser.NativeSmoke/smoke.c new file mode 100644 index 0000000..aab6319 --- /dev/null +++ b/tests/FixedWidthParser.NativeSmoke/smoke.c @@ -0,0 +1,375 @@ +/* Reference C consumer of the FixedWidthParser native library: loads it dynamically (NativeAOT ships + * no import library on Windows), pins every ABI struct's layout, and exercises every export. + * + * fw_smoke [path to FixedWidthParser.Native.{dll,so,dylib}] + * + * Prints "smoke ok" and exits 0 when everything matches. */ +#include "fixedwidth.h" +#include "fixedwidth_arrow.h" + +#include +#include +#include +#include + +#ifdef _WIN32 +#include +typedef HMODULE fw_lib; +#define FW_OPEN(path) LoadLibraryA(path) +#define FW_SYM(lib, name) ((void*)GetProcAddress(lib, name)) +#else +#include +typedef void* fw_lib; +#define FW_OPEN(path) dlopen(path, RTLD_NOW) +#define FW_SYM(lib, name) dlsym(lib, name) +#endif + +#ifndef FIXEDWIDTH_LIB_PATH_DEFAULT +#define FIXEDWIDTH_LIB_PATH_DEFAULT "FixedWidthParser.Native.dll" +#endif + +#define FW_STATIC_ASSERT(cond, name) typedef char fw_static_assert_##name[(cond) ? 1 : -1] + +/* Every published target is 64-bit with natural alignment; the offsets below follow from that. */ +FW_STATIC_ASSERT(sizeof(void*) == 8, pointer_is_64_bit); +FW_STATIC_ASSERT(offsetof(fw_column_spec, name) == 8, spec_name); +FW_STATIC_ASSERT(offsetof(fw_column_spec, start) == 20, spec_start); +FW_STATIC_ASSERT(offsetof(fw_column_spec, scale) == 36, spec_scale); +FW_STATIC_ASSERT(offsetof(fw_column_spec, format) == 40, spec_format); +FW_STATIC_ASSERT(offsetof(fw_column_spec, overflow) == 68, spec_overflow); +FW_STATIC_ASSERT(offsetof(fw_column_spec, dictionary) == 72, spec_dictionary); +FW_STATIC_ASSERT(sizeof(fw_column_spec) == 80, spec_size); +FW_STATIC_ASSERT(offsetof(fw_stream, user_data) == 8, stream_user_data); +FW_STATIC_ASSERT(offsetof(fw_stream, read) == 16, stream_read); +FW_STATIC_ASSERT(sizeof(fw_stream) == 32, stream_size); +FW_STATIC_ASSERT(offsetof(fw_sink, write) == 16, sink_write); +FW_STATIC_ASSERT(sizeof(fw_sink) == 24, sink_size); +FW_STATIC_ASSERT(sizeof(fw_layout_options) == 12, layout_options_size); +FW_STATIC_ASSERT(offsetof(fw_read_options, header_lines) == 8, read_header_lines); +FW_STATIC_ASSERT(offsetof(fw_read_options, skip_byte_order_mark) == 32, read_bom); +FW_STATIC_ASSERT(sizeof(fw_read_options) == 40, read_options_size); +FW_STATIC_ASSERT(offsetof(fw_column, length) == 8, column_length); +FW_STATIC_ASSERT(offsetof(fw_column, values) == 16, column_values); +FW_STATIC_ASSERT(offsetof(fw_column, data_len) == 40, column_data_len); +FW_STATIC_ASSERT(sizeof(fw_column) == 48, column_size); +FW_STATIC_ASSERT(offsetof(fw_table, row_count) == 8, table_row_count); +FW_STATIC_ASSERT(sizeof(fw_table) == 24, table_size); +FW_STATIC_ASSERT(sizeof(fw_read_result) == 24, read_result_size); +FW_STATIC_ASSERT(sizeof(fw_parse_error) == 24, parse_error_size); +FW_STATIC_ASSERT(offsetof(fw_reader_options, read) == 8, reader_read); +FW_STATIC_ASSERT(offsetof(fw_reader_options, batch_rows) == 48, reader_batch_rows); +FW_STATIC_ASSERT(sizeof(fw_reader_options) == 56, reader_options_size); +FW_STATIC_ASSERT(offsetof(fw_write_options, newline) == 8, write_newline); +FW_STATIC_ASSERT(offsetof(fw_write_options, batch_rows) == 24, write_batch_rows); +FW_STATIC_ASSERT(sizeof(fw_write_options) == 32, write_options_size); +FW_STATIC_ASSERT(sizeof(fw_buffer) == 16, buffer_size); + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + fprintf(stderr, "%s:%d: check failed: %s (%s)\n", __FILE__, __LINE__, #cond, \ + last_error()); \ + exit(1); \ + } \ + } while (0) + +static struct { + int32_t (*abi_version)(void); + int32_t (*last_error)(uint8_t*, int32_t, int32_t*); + int32_t (*last_parse_error)(fw_parse_error*); + int32_t (*layout_create)(const fw_column_spec*, int32_t, const fw_layout_options*, fw_layout**); + void (*layout_free)(fw_layout*); + int32_t (*read_file)(const uint8_t*, int32_t, const fw_layout*, const fw_read_options*, fw_table*, fw_read_result*); + int32_t (*read_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_read_options*, fw_table*, fw_read_result*); + void (*free_table)(fw_table*); + int32_t (*reader_open_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_reader_options*, fw_reader**); + int32_t (*reader_next)(fw_reader*, fw_table*); + void (*reader_close)(fw_reader*); + int32_t (*read_arrow_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_read_options*, + struct ArrowArray*, struct ArrowSchema*, fw_read_result*); + int32_t (*read_arrow_stream_memory)(const uint8_t*, int64_t, const fw_layout*, const fw_reader_options*, + struct ArrowArrayStream*); + int32_t (*write_file)(const uint8_t*, int32_t, const fw_table*, const fw_layout*, const fw_write_options*); + int32_t (*write_memory)(const fw_table*, const fw_layout*, const fw_write_options*, fw_buffer*); + void (*free_buffer)(fw_buffer*); +} fw; + +static char error_text[1024]; + +static const char* last_error(void) +{ + int32_t length = 0; + error_text[0] = 0; + if (fw.last_error != NULL && fw.last_error((uint8_t*)error_text, sizeof(error_text) - 1, &length) == FW_OK) { + error_text[length] = 0; + } + return error_text; +} + +static void* symbol(fw_lib lib, const char* name) +{ + void* address = FW_SYM(lib, name); + if (address == NULL) { + fprintf(stderr, "missing export %s\n", name); + exit(1); + } + return address; +} + +static void load(const char* path) +{ + fw_lib lib = FW_OPEN(path); + if (lib == NULL) { + fprintf(stderr, "cannot load %s\n", path); + exit(1); + } + *(void**)&fw.abi_version = symbol(lib, "fw_abi_version"); + *(void**)&fw.last_error = symbol(lib, "fw_last_error"); + *(void**)&fw.last_parse_error = symbol(lib, "fw_last_parse_error"); + *(void**)&fw.layout_create = symbol(lib, "fw_layout_create"); + *(void**)&fw.layout_free = symbol(lib, "fw_layout_free"); + *(void**)&fw.read_file = symbol(lib, "fw_read_file"); + *(void**)&fw.read_memory = symbol(lib, "fw_read_memory"); + *(void**)&fw.free_table = symbol(lib, "fw_free_table"); + *(void**)&fw.reader_open_memory = symbol(lib, "fw_reader_open_memory"); + *(void**)&fw.reader_next = symbol(lib, "fw_reader_next"); + *(void**)&fw.reader_close = symbol(lib, "fw_reader_close"); + *(void**)&fw.read_arrow_memory = symbol(lib, "fw_read_arrow_memory"); + *(void**)&fw.read_arrow_stream_memory = symbol(lib, "fw_read_arrow_stream_memory"); + *(void**)&fw.write_file = symbol(lib, "fw_write_file"); + *(void**)&fw.write_memory = symbol(lib, "fw_write_memory"); + *(void**)&fw.free_buffer = symbol(lib, "fw_free_buffer"); + symbol(lib, "fw_reader_open_file"); + symbol(lib, "fw_read_arrow_file"); + symbol(lib, "fw_read_arrow_stream_file"); + /* Exercised by the C++ tests through std::istream and std::ostream. */ + symbol(lib, "fw_reader_open_stream"); + symbol(lib, "fw_read_arrow_stream_stream"); + symbol(lib, "fw_set_source_error"); + symbol(lib, "fw_write_stream"); +} + +/* A COTAHIST-like file: a header, quotes, an invalid line and a trailer. */ +static const char CONTENT[] = + "00COTAHIST.2023\n" + "01PETR4 0000000123456000000000000001000\n" + "01VALE3 0000000067890 \n" + "01ITUB4 0000000004321000000000000000050\n" + "99TRAILER\n"; + +static fw_layout* cotahist_layout(void) +{ + fw_column_spec columns[4]; + memset(columns, 0, sizeof(columns)); + const char* names[4] = { "tipreg", "codneg", "preabe", "quatot" }; + const int32_t starts[4] = { 0, 2, 14, 27 }; + const int32_t lengths[4] = { 2, 12, 13, 18 }; + const int32_t types[4] = { FW_T_STRING, FW_T_STRING, FW_T_FLOAT64, FW_T_INT64 }; + for (int i = 0; i < 4; i++) { + columns[i].struct_size = sizeof(fw_column_spec); + columns[i].name = (const uint8_t*)names[i]; + columns[i].name_len = (int32_t)strlen(names[i]); + columns[i].start = starts[i]; + columns[i].length = lengths[i]; + columns[i].type = types[i]; + } + columns[2].scale = 2; + columns[2].padding = '0'; + columns[3].nullable = 1; + columns[3].padding = '0'; + fw_layout* layout = NULL; + CHECK(fw.layout_create(columns, 4, NULL, &layout) == FW_OK); + CHECK(layout != NULL); + return layout; +} + +static fw_read_options read_options(int32_t degree) +{ + fw_read_options options; + memset(&options, 0, sizeof(options)); + options.struct_size = sizeof(options); + options.header_lines = 1; + options.skip_invalid = 1; + options.degree_of_parallelism = degree; + return options; +} + +static void check_table(const fw_table* table) +{ + CHECK(table->column_count == 4); + CHECK(table->row_count == 3); + const int32_t* offsets = (const int32_t*)table->columns[1].values; + CHECK(offsets[1] - offsets[0] == 5 && memcmp(table->columns[1].data, "PETR4", 5) == 0); + const double* prices = (const double*)table->columns[2].values; + CHECK(prices[0] == 1234.56 && prices[2] == 43.21); + const int64_t* quantities = (const int64_t*)table->columns[3].values; + CHECK(quantities[0] == 1000 && quantities[2] == 50); + CHECK(table->columns[3].validity != NULL && (table->columns[3].validity[0] & 2) == 0); +} + +static void reads(fw_layout* layout, const char* path) +{ + fw_read_options options = read_options(1); + fw_table table; + fw_read_result result = { sizeof(fw_read_result), 0, 0 }; + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &table, &result) == FW_OK); + check_table(&table); + CHECK(result.lines_read == 3 && result.lines_skipped == 2); + fw.free_table(&table); + CHECK(table.columns == NULL); + + options = read_options(4); + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &table, NULL) == FW_OK); + check_table(&table); + fw.free_table(&table); + + FILE* file = fopen(path, "wb"); + CHECK(file != NULL); + fwrite(CONTENT, 1, sizeof(CONTENT) - 1, file); + fclose(file); + options = read_options(1); + CHECK(fw.read_file((const uint8_t*)path, (int32_t)strlen(path), layout, &options, &table, NULL) == FW_OK); + check_table(&table); + fw.free_table(&table); +} + +static void parse_error(fw_layout* layout) +{ + fw_read_options options = read_options(1); + options.skip_invalid = 0; + fw_table table; + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &table, NULL) == FW_PARSE_ERROR); + fw_parse_error error = { sizeof(fw_parse_error), 0, 0 }; + CHECK(fw.last_parse_error(&error) == FW_OK); + CHECK(error.line_number == 5 && error.column == -1); + CHECK(strstr(last_error(), "Line 5") != NULL); +} + +static void batches(fw_layout* layout) +{ + fw_reader_options options; + memset(&options, 0, sizeof(options)); + options.struct_size = sizeof(options); + options.read = read_options(0); + options.batch_rows = 2; + fw_reader* reader = NULL; + CHECK(fw.reader_open_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &reader) == FW_OK); + fw_table table; + int64_t rows = 0; + int32_t status; + while ((status = fw.reader_next(reader, &table)) == FW_OK) { + rows += table.row_count; + fw.free_table(&table); + } + CHECK(status == FW_EOF && rows == 3); + fw.reader_close(reader); + CHECK(fw.reader_next(reader, &table) == FW_INVALID_HANDLE); +} + +static void arrow(fw_layout* layout) +{ + fw_read_options options = read_options(1); + struct ArrowArray array; + struct ArrowSchema schema; + CHECK(fw.read_arrow_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &options, &array, &schema, NULL) == FW_OK); + CHECK(strcmp(schema.format, "+s") == 0 && schema.n_children == 4); + CHECK(strcmp(schema.children[2]->format, "g") == 0 && strcmp(schema.children[2]->name, "preabe") == 0); + CHECK(array.length == 3 && array.n_children == 4 && array.children[3]->null_count == 1); + CHECK(((const double*)array.children[2]->buffers[1])[0] == 1234.56); + array.release(&array); + schema.release(&schema); + CHECK(array.release == NULL && schema.release == NULL); + + fw_reader_options reader = { sizeof(fw_reader_options), read_options(1), 2 }; + struct ArrowArrayStream stream; + CHECK(fw.read_arrow_stream_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &reader, &stream) == FW_OK); + CHECK(stream.get_schema(&stream, &schema) == 0); + schema.release(&schema); + int64_t rows = 0; + while (stream.get_next(&stream, &array) == 0 && array.release != NULL) { + rows += array.length; + array.release(&array); + } + CHECK(rows == 3); + stream.release(&stream); +} + +static void writes(fw_layout* layout, const char* path) +{ + fw_read_options read = read_options(1); + fw_table table; + CHECK(fw.read_memory((const uint8_t*)CONTENT, sizeof(CONTENT) - 1, layout, &read, &table, NULL) == FW_OK); + fw_write_options options; + memset(&options, 0, sizeof(options)); + options.struct_size = sizeof(options); + options.degree_of_parallelism = 4; + options.batch_rows = 1; + fw_buffer buffer; + CHECK(fw.write_memory(&table, layout, &options, &buffer) == FW_OK); + const char* first = "01PETR4 0000000123456000000000000001000\n"; + CHECK(buffer.length == 3 * 46 && memcmp(buffer.data, first, 46) == 0); + CHECK(fw.write_file((const uint8_t*)path, (int32_t)strlen(path), &table, layout, &options) == FW_OK); + FILE* file = fopen(path, "rb"); + CHECK(file != NULL); + char written[3 * 46]; + CHECK(fread(written, 1, sizeof(written), file) == sizeof(written)); + fclose(file); + CHECK(memcmp(written, buffer.data, sizeof(written)) == 0); + fw.free_buffer(&buffer); + CHECK(buffer.data == NULL); + fw.free_table(&table); +} + +static void null_arguments(fw_layout* layout) +{ + fw_table table; + fw_layout* other = NULL; + CHECK(fw.layout_create(NULL, 1, NULL, &other) == FW_INVALID_ARGUMENT); + CHECK(fw.read_memory((const uint8_t*)CONTENT, 4, layout, NULL, NULL, NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.read_memory(NULL, 4, layout, NULL, &table, NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.last_error(NULL, 0, NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.last_parse_error(NULL) == FW_INVALID_ARGUMENT); + CHECK(fw.write_memory(NULL, layout, NULL, NULL) == FW_INVALID_ARGUMENT); + + fw_read_options bad = read_options(1); + bad.struct_size = 12; + CHECK(fw.read_memory((const uint8_t*)CONTENT, 4, layout, &bad, &table, NULL) == FW_INVALID_ARGUMENT); + CHECK(strstr(last_error(), "struct_size") != NULL); + + fw_column_spec overlap[2]; + memset(overlap, 0, sizeof(overlap)); + for (int i = 0; i < 2; i++) { + overlap[i].struct_size = sizeof(fw_column_spec); + overlap[i].name = (const uint8_t*)(i == 0 ? "a" : "b"); + overlap[i].name_len = 1; + overlap[i].start = i * 2; + overlap[i].length = 4; + } + CHECK(fw.layout_create(overlap, 2, NULL, &other) == FW_INVALID_ARGUMENT && other == NULL); + CHECK(strstr(last_error(), "Overlapping") != NULL); + + fw.layout_free(layout); + CHECK(fw.read_memory((const uint8_t*)CONTENT, 4, layout, NULL, &table, NULL) == FW_INVALID_HANDLE); + fw.layout_free(layout); + fw.free_table(NULL); + fw.free_buffer(NULL); +} + +int main(int argc, char** argv) +{ + load(argc > 1 ? argv[1] : FIXEDWIDTH_LIB_PATH_DEFAULT); + CHECK(fw.abi_version() == FW_ABI_VERSION); + const char* path = "fw_smoke.txt"; + + fw_layout* layout = cotahist_layout(); + reads(layout, path); + parse_error(layout); + batches(layout); + arrow(layout); + writes(layout, path); + null_arguments(layout); + remove(path); + + printf("smoke ok\n"); + return 0; +} diff --git a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs index 18dcf92..22edd22 100644 --- a/tests/FixedWidthParser.Tests/AllocationGuardTests.cs +++ b/tests/FixedWidthParser.Tests/AllocationGuardTests.cs @@ -1,187 +1,118 @@ -using System.Globalization; using System.Text; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; namespace FixedWidthParser.Tests { /// /// Fast allocation-regression guard (runs in milliseconds, unlike the BenchmarkDotNet suite). - /// Proves the pooled read hot path allocates nothing per line: with a - /// and a repeated value, the only would-be per-line allocation (the string column) is interned - /// away, and the struct enumerator adds no heap per iteration. We read two different line counts - /// and assert the marginal per-line cost is ~0 — subtracting the two cancels the fixed - /// per-call overhead (enumerable, closure, buffer rent) and JIT/warmup noise, so the test is - /// robust without pinning exact byte counts. + /// Proves the pooled read hot path allocates nothing per line: with a and a + /// repeated value, the string column is interned away and the struct enumerator adds no heap per + /// iteration. Reading two line counts and subtracting cancels the fixed per-call overhead. /// public class AllocationGuardTests { + private const string Line = "ABC"; // CodeModel/GenCodeModel: [FixedColumn(0, 3)] Code + private const int Small = 2_000; + private const int Large = 6_000; + /// - /// stores the stream - /// directly instead of a Func<Stream>, so a call allocates only the enumerable - /// object — no captured-variable display class and no delegate. Measured at 64 B/call; a - /// regressed closure would add a delegate (~56 B+) and push it well past the threshold. + /// A read call stores the stream directly instead of a Func<Stream>, so it allocates + /// only the enumerable — no display class and no delegate (~56 B more if one sneaks back in). /// [Fact] - public void ByteReaderRead_AllocatesEnumerableOnly_NoClosure() + public void Read_AllocatesEnumerableOnly_NoClosure() { - var reader = new FixedWidthByteReader(); + var layout = FixedWidthLayout.FromAttributes(); using var stream = new MemoryStream(); - _ = reader.Read(stream); // warmup/JIT - - const int n = 10_000; - long before = GC.GetAllocatedBytesForCurrentThread(); - for (int i = 0; i < n; i++) - { - _ = reader.Read(stream); - } + _ = FixedWidth.Read(stream, layout); + _ = FixedWidth.Read(stream); - double perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; - - Assert.True(perCall <= 80, $"Read allocated {perCall} B/call (expected ~64 B: enumerable only, no closure)."); + Assert.True(PerCall(() => FixedWidth.Read(stream, layout)) <= 72, "Reflection Read allocated more than the enumerable."); + Assert.True(PerCall(() => FixedWidth.Read(stream)) <= 72, "Generated Read allocated more than the enumerable."); } - /// - /// stores the source in a - /// by-value TextReaderSource struct instead of a Func<TextReader>, so a call - /// allocates only the enumerable object — no captured-variable display class and no delegate. - /// Measured at 88 B/call; a regressed closure would add a delegate (~56 B+) past the threshold. - /// [Fact] - public void CharReaderRead_AllocatesEnumerableOnly_NoClosure() + public void ReflectionPooledReader_AllocatesNothingPerLine() { - var reader = new FixedWidthReader(); - using var tr = new StringReader(""); - _ = reader.Read(tr); // warmup/JIT + var options = new FixedWidthReaderOptions { StringPool = new StringPool() }; + var layout = FixedWidthLayout.FromAttributes(); + byte[] small = Repeat(Small), large = Repeat(Large); + SumReflection(layout, options, small); + SumReflection(layout, options, large); - const int n = 10_000; - long before = GC.GetAllocatedBytesForCurrentThread(); - for (int i = 0; i < n; i++) - { - _ = reader.Read(tr); - } - - double perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; + double perLine = MarginalPerLine(() => SumReflection(layout, options, small), () => SumReflection(layout, options, large)); - Assert.True(perCall <= 104, $"Read allocated {perCall} B/call (expected ~88 B: enumerable only, no closure)."); + Assert.True(perLine <= 1, $"Reflection pooled reader regressed to {perLine} B/line (expected ~0)."); } - /// - /// - /// stores the stream directly (like the reflection byte reader), so a call allocates only the - /// enumerable object — no captured-variable display class and no delegate. - /// [Fact] - public void GeneratedUtf8Read_AllocatesEnumerableOnly_NoClosure() + public void GeneratedPooledReader_AllocatesNothingPerLine() { - using var stream = new MemoryStream(); - _ = FixedWidthUtf8.Read(stream); // warmup/JIT + var options = new FixedWidthReaderOptions { StringPool = new StringPool() }; + byte[] small = Repeat(Small), large = Repeat(Large); + SumGenerated(options, small); + SumGenerated(options, large); + + double perLine = MarginalPerLine(() => SumGenerated(options, small), () => SumGenerated(options, large)); + Assert.True(perLine <= 1, $"Generated pooled reader regressed to {perLine} B/line (expected ~0)."); + } + + private static double PerCall(Func read) + { const int n = 10_000; long before = GC.GetAllocatedBytesForCurrentThread(); for (int i = 0; i < n; i++) { - _ = FixedWidthUtf8.Read(stream); + _ = read(); } - - double perCall = (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; - - Assert.True(perCall <= 80, $"Read allocated {perCall} B/call (expected ~64 B: enumerable only, no closure)."); + return (GC.GetAllocatedBytesForCurrentThread() - before) / (double)n; } - private const string Line = "ABC"; // CodeModel/GenCodeModel: [FixedColumn(0, 3)] Code - private const int Small = 2_000; - private const int Large = 6_000; - - private static string Repeat(int lines) + private static byte[] Repeat(int lines) { var sb = new StringBuilder((Line.Length + 1) * lines); for (int i = 0; i < lines; i++) { sb.Append(Line).Append('\n'); } - - return sb.ToString(); - } - - [Fact] - public void ReflectionPooledReader_AllocatesNothingPerLine() - { - var reader = new FixedWidthReader(CultureInfo.InvariantCulture, new StringPool()); - string small = Repeat(Small), large = Repeat(Large); - - // Warmup: JIT every path and intern "ABC" into the pool. - SumReflection(reader, small); - SumReflection(reader, large); - - double perLine = MarginalPerLine(() => SumReflection(reader, small), () => SumReflection(reader, large)); - Assert.True(perLine <= 1, $"Reflection pooled reader regressed to {perLine} B/line (expected ~0)."); - } - - [Fact] - public void GeneratedPooledReader_AllocatesNothingPerLine() - { - var pool = new StringPool(); - string small = Repeat(Small), large = Repeat(Large); - - SumGenerated(pool, small); - SumGenerated(pool, large); - - double perLine = MarginalPerLine(() => SumGenerated(pool, small), () => SumGenerated(pool, large)); - Assert.True(perLine <= 1, $"Generated pooled reader regressed to {perLine} B/line (expected ~0)."); - } - - [Fact] - public void GeneratedUtf8PooledReader_AllocatesNothingPerLine() - { - var pool = new StringPool(); - byte[] small = Encoding.UTF8.GetBytes(Repeat(Small)), large = Encoding.UTF8.GetBytes(Repeat(Large)); - - SumGeneratedUtf8(pool, small); - SumGeneratedUtf8(pool, large); - - double perLine = MarginalPerLine(() => SumGeneratedUtf8(pool, small), () => SumGeneratedUtf8(pool, large)); - Assert.True(perLine <= 1, $"Generated UTF-8 pooled reader regressed to {perLine} B/line (expected ~0)."); + return Encoding.UTF8.GetBytes(sb.ToString()); } - private static int SumReflection(FixedWidthReader reader, string text) + private static int SumReflection(FixedWidthLayout layout, FixedWidthReaderOptions options, byte[] bytes) { int total = 0; - using var sr = new StringReader(text); - foreach (var m in reader.Read(sr)) + foreach (var m in FixedWidth.Read(new MemoryStream(bytes, writable: false), layout, options: options)) { total += m.Code.Length; } - return total; } - private static int SumGenerated(StringPool pool, string text) + private static int SumGenerated(FixedWidthReaderOptions options, byte[] bytes) { int total = 0; - using var sr = new StringReader(text); - foreach (var m in FixedWidth.Read(sr, CultureInfo.InvariantCulture, pool)) + foreach (var m in FixedWidth.Read(new MemoryStream(bytes, writable: false), options: options)) { total += m.Code.Length; } - return total; } - private static int SumGeneratedUtf8(StringPool pool, byte[] bytes) + // Bytes allocated per extra line, isolating per-line cost from fixed per-call overhead. The best of + // three attempts: while other test classes run, the runtime can make a one-off allocation on this + // thread during one read, which a per-line regression would not survive in every attempt. + private static double MarginalPerLine(Action readSmall, Action readLarge) { - int total = 0; - using var ms = new MemoryStream(bytes, writable: false); - foreach (var m in FixedWidthUtf8.Read(ms, formatProvider: CultureInfo.InvariantCulture, stringPool: pool)) + double best = double.MaxValue; + for (int attempt = 0; attempt < 3; attempt++) { - total += m.Code.Length; + best = Math.Min(best, MarginalPerLineOnce(readSmall, readLarge)); } - - return total; + return best; } - // Bytes allocated per extra line, isolating per-line cost from fixed per-call overhead. - private static double MarginalPerLine(Action readSmall, Action readLarge) + private static double MarginalPerLineOnce(Action readSmall, Action readLarge) { long b1 = GC.GetAllocatedBytesForCurrentThread(); readSmall(); diff --git a/tests/FixedWidthParser.Tests/ByteReaderStreamTests.cs b/tests/FixedWidthParser.Tests/ByteReaderStreamTests.cs deleted file mode 100644 index c9d184c..0000000 --- a/tests/FixedWidthParser.Tests/ByteReaderStreamTests.cs +++ /dev/null @@ -1,342 +0,0 @@ -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the streaming Stream/file overloads of (sync - /// and async): raw-byte reading with no transcode, line-ending handling, BOM skipping, buffer - /// growth, error reporting, leaveOpen behavior, and byte/char parity. - /// - public class ByteReaderStreamTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Synchronous ----------------------------- - - [Fact] - public void Read_Stream_ParsesAll() - { - var reader = new FixedWidthByteReader(Inv); - using var stream = Utf8(TwoPeople); - - var people = reader.Read(stream).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line - public void Read_HandlesLineEndingsAndTrailingNewline(string text) - { - var reader = new FixedWidthByteReader(); - using var stream = Utf8(text); - - var codes = reader.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsEmptyLines() - { - var reader = new FixedWidthByteReader(); - using var stream = Utf8("ABC\n\n\nDEF\n"); - - var codes = reader.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsLeadingUtf8Bom() - { - var reader = new FixedWidthByteReader(); - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - using var stream = new MemoryStream(bytes); - - var codes = reader.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_LineLongerThanBuffer_StillParses() - { - // A tiny bufferSize forces buffer compaction and growth (25-byte lines). - var reader = new FixedWidthByteReader(Inv, bufferSize: 4); - using var stream = Utf8(TwoPeople); - - var people = reader.Read(stream).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_InvalidLine_ThrowsWithLineNumber() - { - var reader = new FixedWidthByteReader(Inv); - using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = Assert.Throws(() => reader.Read(stream).ToList()); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public void Read_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthByteReader(); - using var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream, leaveOpen: true).ToList(); - - Assert.True(stream.CanRead); - } - - [Fact] - public void Read_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthByteReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); - } - - [Fact] - public void Read_NullStream_Throws() - { - var reader = new FixedWidthByteReader(); - - Assert.Throws(() => reader.Read(null!)); - } - - [Fact] - public void GetEnumerator_IsStruct_ForAllocationFreeForeach() - { - Assert.True(typeof(RecordEnumeratorCore, StreamSource>).IsValueType); - - var reader = new FixedWidthByteReader(); - using var stream = Utf8("ABC\nDEF\n"); - int count = 0; - foreach (var _ in reader.Read(stream)) - { - count++; - } - - Assert.Equal(2, count); - } - - [Fact] - public void ReadFile_IsReEnumerable() - { - string path = Path.GetTempFileName(); - try - { - File.WriteAllText(path, "ABC\nDEF\n"); - var reader = new FixedWidthByteReader(); - - var first = reader.ReadFile(path).Select(m => m.Code).ToList(); - var second = reader.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], first); - Assert.Equal(first, second); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- Asynchronous ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - var reader = new FixedWidthByteReader(Inv); - await using var stream = Utf8(TwoPeople); - - var people = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_LineLongerThanBuffer_StillParses() - { - var reader = new FixedWidthByteReader(Inv, bufferSize: 4); - await using var stream = Utf8(TwoPeople); - - var people = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_SkipsLeadingUtf8Bom() - { - var reader = new FixedWidthByteReader(); - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - await using var stream = new MemoryStream(bytes); - - var codes = await reader.ReadAsync(stream).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_InvalidLine_ThrowsWithLineNumber() - { - var reader = new FixedWidthByteReader(Inv); - await using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = await Assert.ThrowsAsync(async () => - { - await foreach (var _ in reader.ReadAsync(stream).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task ReadAsync_CancelledToken_Throws() - { - var reader = new FixedWidthByteReader(); - await using var stream = Utf8("ABC\nDEF\n"); - using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); - - await Assert.ThrowsAnyAsync(async () => - { - await foreach (var _ in reader.ReadAsync(stream).WithCancellation(cts.Token).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthByteReader(); - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthByteReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_NullStream_Throws() - { - var reader = new FixedWidthByteReader(); - - Assert.Throws(() => reader.ReadAsync((Stream)null!)); - } - - [Fact] - public async Task ReadFileAsync_MatchesSyncReadFile() - { - string path = Path.GetTempFileName(); - try - { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); - var reader = new FixedWidthByteReader(); - - var asyncCodes = await reader.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - var syncCodes = reader.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); - Assert.Equal(syncCodes, asyncCodes); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- StringPool ----------------------------- - - [Fact] - public void Read_Stream_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var reader = new FixedWidthByteReader(stringPool: pool); - using var stream = Utf8("ABC\nABC\n"); - - var codes = reader.Read(stream).ToList(); - - Assert.Equal("ABC", codes[0].Code); - Assert.Same(codes[0].Code, codes[1].Code); - } - - [Fact] - public async Task ReadAsync_Stream_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var reader = new FixedWidthByteReader(stringPool: pool); - await using var stream = Utf8("ABC\nABC\n"); - - var codes = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal("ABC", codes[0].Code); - Assert.Same(codes[0].Code, codes[1].Code); - } - - // ----------------------------- char/byte parity ----------------------------- - - [Fact] - public void ByteReader_MatchesCharReader_ForAsciiStream() - { - var charReader = new FixedWidthReader(Inv); - var byteReader = new FixedWidthByteReader(Inv); - - var fromChars = charReader.Read(new StringReader(TwoPeople)).ToList(); - using var stream = Utf8(TwoPeople); - var fromBytes = byteReader.Read(stream).ToList(); - - Assert.Equal(fromChars.Count, fromBytes.Count); - for (int i = 0; i < fromChars.Count; i++) - { - Assert.Equal(fromChars[i].Name, fromBytes[i].Name); - Assert.Equal(fromChars[i].Age, fromBytes[i].Age); - Assert.Equal(fromChars[i].Salary, fromBytes[i].Salary, 2); - } - } - } -} diff --git a/tests/FixedWidthParser.Tests/ByteReaderTests.cs b/tests/FixedWidthParser.Tests/ByteReaderTests.cs deleted file mode 100644 index 13d77d4..0000000 --- a/tests/FixedWidthParser.Tests/ByteReaderTests.cs +++ /dev/null @@ -1,373 +0,0 @@ -using System.Globalization; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the UTF-8 / byte parse path: and the - /// facade. Column offsets are measured in bytes; - /// all fixtures here are ASCII, so byte offsets equal character offsets. Mirrors the assertions - /// in / to prove byte/char parity. - /// - public class ByteReaderTests - { - private static readonly CultureInfo DeDe = CultureInfo.GetCultureInfo("de-DE"); - - // ----------------------- Core parsing ----------------------- - - [Fact] - public void Parser_ParsesAllColumns() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("John Doe", model.Name); - Assert.Equal(30, model.Age); - Assert.Equal(60000.00, model.Salary, 2); - } - - [Fact] - public void Parser_StringColumn_TrimsTrailingSpacesAndDecodes() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("ABC"u8, null, null, out var model); - - Assert.True(ok); - Assert.Equal("ABC", model.Code); - } - - [Fact] - public void Parser_DecimalColumn_UsesUtf8SpanParsableFallback() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("1234.56 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void Parser_FloatColumn_UsesCsFastFloat() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("3.14 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_FieldBasedModel_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("ABCDE0042"u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("ABCDE", model.Code); - Assert.Equal(42, model.Quantity); - } - -#if NET9_0_OR_GREATER - [Fact] - public void Parser_SupportsRefStructModel() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("John Doe 30 60000.00 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("John Doe", model.Name); - Assert.Equal(30, model.Age); - Assert.Equal(60000.00, model.Salary, 2); - } -#endif - - // ----------------------- Rejection paths ----------------------- - - [Fact] - public void Parser_LineShorterThanLayout_ReturnsFalse() - { - var parser = new Utf8FixedWidthParser(); - - // Layout requires 25 bytes; supply fewer. - bool ok = parser.TryParse("John Doe 30"u8, Inv, null, out _); - - Assert.False(ok); - } - - [Fact] - public void Parser_InvalidNumericColumn_ReturnsFalse() - { - var parser = new Utf8FixedWidthParser(); - - // "XX" in the Age column [10,5) is not an integer. - bool ok = parser.TryParse("John Doe XX 60000.00 "u8, Inv, null, out _); - - Assert.False(ok); - } - - // ----------------------- Culture handling ----------------------- - - [Fact] - public void Parser_DoubleColumn_HonorsCommaCulture() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234,50 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - [Fact] - public void Parser_DoubleColumn_NullProvider_DefaultsToDot() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234.50 "u8, null, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - [Fact] - public void Parser_DecimalColumn_HonorsCommaCulture() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("1234,56 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void Parser_FloatColumn_HonorsCommaCulture() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("3,14 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() - { - // de-DE: '.' groups thousands, ',' is the decimal separator — "1.234,50" means 1234.50. - // Regression coverage for a bug where csFastFloat silently truncated at the '.' it wasn't - // told about and returned 1.0 instead of failing or parsing the full value. - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1.234,50 "u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - [Fact] - public void Parser_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1,234.50 "u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); - } - - // ----- Null provider means invariant for every column type on the byte path too (see - // CultureTests.Parse_DecimalColumn_NullProvider_DefaultsToInvariant for the char-path version and - // rationale). ----- - - [Fact] - public void Parser_DecimalColumn_NullProvider_DefaultsToInvariant() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("1234.56 "u8, null, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void Parser_DoubleColumn_TrailingGarbageAfterNumber_Fails() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 12x "u8, Inv, null, out _); - - Assert.False(ok); - } - - // ----- Right-aligned (leading-space-padded) numeric columns: same regression as - // CultureTests' char-path equivalents — csFastFloat's characters_consumed did not reliably - // include skipped leading whitespace across its double/float overloads. ----- - - [Fact] - public void Parser_DoubleColumn_RightAligned_DotCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123.45"u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(123.45, model.Salary, 2); - } - - [Fact] - public void Parser_FloatColumn_RightAligned_DotCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse(" 3.14"u8, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_DoubleColumn_RightAligned_CommaCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123,45"u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(123.45, model.Salary, 2); - } - - [Fact] - public void Parser_FloatColumn_RightAligned_CommaCulture_Parses() - { - var parser = new Utf8FixedWidthParser(); - - bool ok = parser.TryParse(" 3,14"u8, DeDe, null, out var model); - - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); - } - - [Fact] - public void Parser_DoubleColumn_NonAsciiSeparatorCulture_Throws() - { - // The byte parser matches the decimal separator against raw UTF-8 bytes, so a non-ASCII - // separator (here U+066B ARABIC DECIMAL SEPARATOR) cannot be represented as a single byte. - // It must be rejected with a clear error rather than silently truncated to the wrong byte. - var culture = (CultureInfo)CultureInfo.InvariantCulture.Clone(); - culture.NumberFormat.NumberDecimalSeparator = "٫"; - - var parser = new Utf8FixedWidthParser(); - - var ex = Assert.Throws( - () => parser.TryParse("Bob 30 1234.50 "u8, culture, null, out _)); - Assert.Contains("ASCII", ex.Message, StringComparison.Ordinal); - } - - // ----------------------- Facade ----------------------- - - [Fact] - public void ByteReader_TryParse_UsesConfiguredCulture() - { - var reader = new FixedWidthByteReader(Inv); - - bool ok = reader.TryParse("Jane 28 55000.00 "u8, out var model); - - Assert.True(ok); - Assert.Equal("Jane", model.Name); - Assert.Equal(28, model.Age); - Assert.Equal(55000.00, model.Salary, 2); - } - - [Fact] - public void ByteReader_TryParse_CommaCulture() - { - var reader = new FixedWidthByteReader(DeDe); - - bool ok = reader.TryParse("1234,56 "u8, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); - } - - [Fact] - public void ByteReader_TryParse_TooShort_ReturnsFalse() - { - var reader = new FixedWidthByteReader(Inv); - - bool ok = reader.TryParse("short"u8, out _); - - Assert.False(ok); - } - - // ----------------------- char/byte parity ----------------------- - - [Fact] - public void ByteParser_MatchesCharParser_ForAsciiInput() - { - const string line = "John Doe 30 60000.00 "; - var charParser = new FixedWidthParser(); - var byteParser = new Utf8FixedWidthParser(); - - bool charOk = charParser.TryParse(line, Inv, null, out var fromChars); - bool byteOk = byteParser.TryParse("John Doe 30 60000.00 "u8, Inv, null, out var fromBytes); - - Assert.True(charOk); - Assert.True(byteOk); - Assert.Equal(fromChars.Name, fromBytes.Name); - Assert.Equal(fromChars.Age, fromBytes.Age); - Assert.Equal(fromChars.Salary, fromBytes.Salary, 2); - } - - // ----------------------- StringPool interning ----------------------- - - [Fact] - public void Parser_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var parser = new Utf8FixedWidthParser(); - - Assert.True(parser.TryParse("ABC"u8, null, pool, out var first)); - Assert.True(parser.TryParse("ABC"u8, null, pool, out var second)); - - Assert.Equal("ABC", first.Code); - Assert.Same(first.Code, second.Code); - } - - [Fact] - public void Parser_WithStringPool_MatchesNonPooledDecode() - { - var pool = new StringPool(); - var parser = new Utf8FixedWidthParser(); - - Assert.True(parser.TryParse("ABC"u8, null, null, out var nonPooled)); - Assert.True(parser.TryParse("ABC"u8, null, pool, out var pooled)); - - Assert.Equal(nonPooled.Code, pooled.Code); - } - - [Fact] - public void ByteReader_TryParse_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - var reader = new FixedWidthByteReader(stringPool: pool); - - Assert.True(reader.TryParse("ABC"u8, out var first)); - Assert.True(reader.TryParse("ABC"u8, out var second)); - - Assert.Same(first.Code, second.Code); - } - - } -} diff --git a/tests/FixedWidthParser.Tests/ColumnValidationTests.cs b/tests/FixedWidthParser.Tests/ColumnValidationTests.cs index 32808e2..1620f2a 100644 --- a/tests/FixedWidthParser.Tests/ColumnValidationTests.cs +++ b/tests/FixedWidthParser.Tests/ColumnValidationTests.cs @@ -1,5 +1,4 @@ -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; +using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { @@ -12,49 +11,63 @@ public class ColumnValidationTests [Fact] public void Parser_OverlappingColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); } [Fact] public void Writer_OverlappingColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthWriter()); + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); } [Fact] public void Parser_NegativeStart_Throws() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("Start", ex.Message, StringComparison.Ordinal); } [Fact] public void Writer_ZeroLength_Throws() { - var ex = Assert.Throws(() => new FixedWidthWriter()); + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); Assert.Contains("Length", ex.Message, StringComparison.Ordinal); } [Fact] public void Parser_NoColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthParser()); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); Assert.Contains("no [FixedColumn]", ex.Message, StringComparison.Ordinal); } [Fact] - public void Utf8Parser_NoColumns_ThrowsOnConstruction() + public void Parser_NonAsciiTrimChar_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new Utf8FixedWidthParser()); - Assert.Contains("no [FixedColumn]", ex.Message, StringComparison.Ordinal); + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.Contains("TrimChar", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Writer_NonAsciiPadding_ThrowsOnConstruction() + { + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); + Assert.Contains("Padding", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Parser_UnsupportedColumnType_ThrowsOnConstruction() + { + var ex = Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.Contains("Converter", ex.Message, StringComparison.Ordinal); } [Fact] public void Writer_NoColumns_ThrowsOnConstruction() { - var ex = Assert.Throws(() => new FixedWidthWriter()); + var ex = Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); Assert.Contains("no [FixedColumn]", ex.Message, StringComparison.Ordinal); } @@ -62,8 +75,8 @@ public void Writer_NoColumns_ThrowsOnConstruction() public void AdjacentColumns_AreValid() { // Touching columns (end of one == start of the next) do not overlap. - var parser = new FixedWidthParser(); - var writer = new FixedWidthWriter(); + var parser = FixedWidthLayout.FromAttributes(); + var writer = FixedWidthLayout.FromAttributes(); Assert.NotNull(parser); Assert.NotNull(writer); diff --git a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs index b3a8026..d521278 100644 --- a/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs +++ b/tests/FixedWidthParser.Tests/ConverterAttributeTests.cs @@ -1,89 +1,53 @@ -using System.Text; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Covers : a per-property converter that - /// takes priority over the built-in ISpanParsable fallback, across all four combinations of - /// reflection/generated and char/UTF-8, plus the write side and the build-time mismatch check. + /// : a per-property converter that takes + /// priority over the built-in parsers, on the reflection and generated paths, for parse and write, + /// plus the build-time mismatch check. /// public class ConverterAttributeTests { [Fact] - public void Reflection_Char_RoundTrips() + public void Reflection_RoundTrips() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); var original = new CentsConverterModel { Amount = new CentsValue(12345) }; - string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + string line = WriteOne(FixedWidthLayout.FromAttributes(), original); - Assert.True(ok); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var parsed)); Assert.Equal(original.Amount, parsed.Amount); } [Fact] - public void Reflection_Utf8_Parses() + public void Generated_MatchesReflection() { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes("12345 "); + bool okR = FixedWidth.TryParse("12345 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse("12345 "u8, Ctx, out var g); - bool ok = parser.TryParse(bytes, Inv, null, out var parsed); - - Assert.True(ok); - Assert.Equal(new CentsValue(12345), parsed.Amount); - } - - [Fact] - public void Generated_Char_MatchesReflection() - { - var reflection = new FixedWidthParser(); - const string line = "12345 "; - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Amount, g.Amount); - } - - [Fact] - public void Generated_Utf8_MatchesReflection() - { - byte[] bytes = Encoding.UTF8.GetBytes("12345 "); - var reflection = new Utf8FixedWidthParser(); - - bool okR = reflection.TryParse(bytes, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(bytes, Inv, null, out var g); - - Assert.Equal(okR, okG); + Assert.True(okR); + Assert.True(okG); Assert.Equal(r.Amount, g.Amount); } [Fact] public void Converter_RejectingInput_ReturnsFalse() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("notanum ", Inv, null, out _); - - Assert.False(ok); + Assert.False(FixedWidth.TryParse("notanum "u8, FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.False(FixedWidth.TryParse("notanum "u8, Ctx, out _)); } [Fact] public void MismatchedConverter_ParserConstruction_Throws() { - Assert.Throws(() => new FixedWidthParser()); + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); } [Fact] public void MismatchedConverter_WriterConstruction_Throws() { - Assert.Throws(() => new FixedWidthWriter()); + Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); } } } diff --git a/tests/FixedWidthParser.Tests/CorrectnessTests.cs b/tests/FixedWidthParser.Tests/CorrectnessTests.cs index e8a4d24..e06ad70 100644 --- a/tests/FixedWidthParser.Tests/CorrectnessTests.cs +++ b/tests/FixedWidthParser.Tests/CorrectnessTests.cs @@ -1,6 +1,3 @@ -using System.Text; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -8,162 +5,83 @@ namespace FixedWidthParser.Tests public class CorrectnessTests { [Fact] - public void DateTimeExact_RoundTrip_AllPaths() + public void DateTimeExact_RoundTrip_BothPaths() { var date = new DateTime(2026, 7, 16, 0, 0, 0, DateTimeKind.Unspecified); var dateOnly = new DateOnly(2026, 7, 16); const string input = "2026071620260716"; - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var model1)); - Assert.Equal(date, model1.Date); - Assert.Equal(dateOnly, model1.DateOnlyVal); + var layout = FixedWidthLayout.FromAttributes(); + Assert.True(FixedWidth.TryParse(Bytes(input), layout, Ctx, out var reflected)); + Assert.Equal(date, reflected.Date); + Assert.Equal(dateOnly, reflected.DateOnlyVal); + Assert.Equal(input, WriteOne(FixedWidthLayout.FromAttributes(), reflected)); - var wChar = new FixedWidthWriter(); - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - wChar.Write(sw, in model1, Inv); - sw.Flush(); - Assert.Equal(input + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); - } - - // 2. Reflection Utf8 - var pUtf8 = new Utf8FixedWidthParser(); - var bytes = "2026071620260716"u8; - Assert.True(pUtf8.TryParse(bytes, Inv, null, out var model2)); - Assert.Equal(date, model2.Date); - Assert.Equal(dateOnly, model2.DateOnlyVal); - - // 3. Generated Char - Assert.True(FixedWidth.TryParse(input, Inv, null, out var model3)); - Assert.Equal(date, model3.Date); - Assert.Equal(dateOnly, model3.DateOnlyVal); - - Span charDest = stackalloc char[16]; - Assert.True(FixedWidth.TryFormat(model3, charDest, Inv, out int cw)); - Assert.Equal(16, cw); - Assert.Equal(input, charDest.ToString()); - - // 4. Generated Utf8 - Assert.True(FixedWidthUtf8.TryParse(bytes, Inv, null, out var model4)); - Assert.Equal(date, model4.Date); - Assert.Equal(dateOnly, model4.DateOnlyVal); + Assert.True(FixedWidth.TryParse(Bytes(input), Ctx, out var generated)); + Assert.Equal(date, generated.Date); + Assert.Equal(dateOnly, generated.DateOnlyVal); + Assert.Equal(input, Format(generated)); } [Fact] public void ZeroPadded_Leading_TrimMode() { - const string input = "0004203.14"; - var bytes = "0004203.14"u8; - - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var model1)); - Assert.Equal(42, model1.Value); - Assert.Equal(3.14, model1.DoubleValue); - - // 2. Reflection Utf8 - var pUtf8 = new Utf8FixedWidthParser(); - Assert.True(pUtf8.TryParse(bytes, Inv, null, out var model2)); - Assert.Equal(42, model2.Value); - Assert.Equal(3.14, model2.DoubleValue); - - // 3. Generated Char - Assert.True(FixedWidth.TryParse(input, Inv, null, out var model3)); - Assert.Equal(42, model3.Value); - Assert.Equal(3.14, model3.DoubleValue); - - // 4. Generated Utf8 - Assert.True(FixedWidthUtf8.TryParse(bytes, Inv, null, out var model4)); - Assert.Equal(42, model4.Value); - Assert.Equal(3.14, model4.DoubleValue); + var layout = FixedWidthLayout.FromAttributes(); + Assert.True(FixedWidth.TryParse("0004203.14"u8, layout, Ctx, out var reflected)); + Assert.Equal(42, reflected.Value); + Assert.Equal(3.14, reflected.DoubleValue); + + Assert.True(FixedWidth.TryParse("0004203.14"u8, Ctx, out var generated)); + Assert.Equal(42, generated.Value); + Assert.Equal(3.14, generated.DoubleValue); } [Fact] public void ZeroPadded_Both_TrimMode() { - const string input = "00420"; - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var model1)); - Assert.Equal(42, model1.Value); + Assert.True(FixedWidth.TryParse("00420"u8, FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); + Assert.Equal(42, reflected.Value); - // 2. Generated Char - Assert.True(FixedWidth.TryParse(input, Inv, null, out var model3)); - Assert.Equal(42, model3.Value); + Assert.True(FixedWidth.TryParse("00420"u8, Ctx, out var generated)); + Assert.Equal(42, generated.Value); } [Fact] public void Nullable_BlankDetection_WithCustomTrimChar() { - // Nullable model with custom TrimChar: should detect spaces or custom TrimChar as null - const string inputSpace = " "; - const string inputZeroAsterisk = "00000*****"; - - // 1. Reflection Char - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(inputSpace, Inv, null, out var model1)); - Assert.Null(model1.Value); - Assert.Null(model1.DoubleValue); - - Assert.True(pChar.TryParse(inputZeroAsterisk, Inv, null, out var model2)); - Assert.Null(model2.Value); - Assert.Null(model2.DoubleValue); - - var wChar = new FixedWidthWriter(); - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - wChar.Write(sw, in model2, Inv); - sw.Flush(); - Assert.Equal(inputZeroAsterisk + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); - } - - // 2. Generated Char - Assert.True(FixedWidth.TryParse(inputSpace, Inv, null, out var model3)); - Assert.Null(model3.Value); - Assert.Null(model3.DoubleValue); - - Assert.True(FixedWidth.TryParse(inputZeroAsterisk, Inv, null, out var model4)); - Assert.Null(model4.Value); - Assert.Null(model4.DoubleValue); - - Span charDest = stackalloc char[10]; - Assert.True(FixedWidth.TryFormat(model4, charDest, Inv, out int cw)); - Assert.Equal(10, cw); - Assert.Equal(inputZeroAsterisk, charDest.ToString()); + const string spaces = " "; + const string padded = "00000*****"; + + var layout = FixedWidthLayout.FromAttributes(); + Assert.True(FixedWidth.TryParse(Bytes(spaces), layout, Ctx, out var fromSpaces)); + Assert.Null(fromSpaces.Value); + Assert.Null(fromSpaces.DoubleValue); + Assert.True(FixedWidth.TryParse(Bytes(padded), layout, Ctx, out var fromPadding)); + Assert.Null(fromPadding.Value); + Assert.Null(fromPadding.DoubleValue); + Assert.Equal(padded, WriteOne(FixedWidthLayout.FromAttributes(), fromPadding)); + + Assert.True(FixedWidth.TryParse(Bytes(spaces), Ctx, out var generatedSpaces)); + Assert.Null(generatedSpaces.Value); + Assert.Null(generatedSpaces.DoubleValue); + Assert.True(FixedWidth.TryParse(Bytes(padded), Ctx, out var generatedPadding)); + Assert.Null(generatedPadding.Value); + Assert.Equal(padded, Format(generatedPadding)); } [Fact] public void SignAware_ZeroPadding() { - var modelNegative = new SignPaddedModel { Value = -42 }; - var modelPositive = new SignPaddedModel { Value = 42 }; - - var writer = new FixedWidthWriter(); - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - writer.Write(sw, in modelNegative, Inv); - sw.Flush(); - Assert.Equal("-00042" + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); - } + var layout = FixedWidthLayout.FromAttributes(); - using (var ms = new MemoryStream()) - using (var sw = new StreamWriter(ms, new UTF8Encoding(false), leaveOpen: true)) - { - writer.Write(sw, in modelPositive, Inv); - sw.Flush(); - Assert.Equal("000042" + Environment.NewLine, Encoding.UTF8.GetString(ms.ToArray())); // Default int doesn't format with +, so just zero pad - } + Assert.Equal("-00042", WriteOne(layout, new SignPaddedModel { Value = -42 })); + Assert.Equal("000042", WriteOne(layout, new SignPaddedModel { Value = 42 })); } [Fact] public void Writer_RejectsTruncateOnNonString() { - Assert.Throws(() => new FixedWidthWriter()); + Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); } } } diff --git a/tests/FixedWidthParser.Tests/CoverageTests.cs b/tests/FixedWidthParser.Tests/CoverageTests.cs index 6b26749..4a1b96a 100644 --- a/tests/FixedWidthParser.Tests/CoverageTests.cs +++ b/tests/FixedWidthParser.Tests/CoverageTests.cs @@ -1,119 +1,99 @@ +using System.Buffers; using System.Globalization; using System.Text; -using FixedWidthParser; -using FixedWidthParser.Attributes; -using FixedWidthParser.Formatters; -using FixedWidthParser.Parsers; +using FixedWidthParser.Infrastructure; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Fills coverage gaps left by the type-focused suites: the TimeOnly/DateTimeOffset exact-parser - /// branches (reflection factories and generated runtime helpers), the UTF-8 culture parser's - /// wide-field ArrayPool transcode branch, and the converter formatter's ArrayPool growth/exhaustion. + /// Branches the type-focused suites leave out: the TimeOnly/DateTimeOffset exact parsers, the + /// decode fallback for wide numeric fields, and the converter formatter's ArrayPool growth/exhaustion. /// public class CoverageTests { - private static readonly CultureInfo DeDe = CultureInfo.GetCultureInfo("de-DE"); - - // ----- TimeOnly + DateTimeOffset exact parsing (both factories + both generated runtimes) ----- - [Fact] - public void TimeExact_RoundTrip_AllPaths() + public void TimeExact_RoundTrip_BothPaths() { var time = new TimeOnly(13, 45, 30); var timestamp = new DateTimeOffset(2026, 7, 16, 13, 45, 30, TimeSpan.FromHours(3)); - const string input = "13453020260716134530+03:00"; - var bytes = Encoding.UTF8.GetBytes(input); - - // Reflection char (ColumnParserFactory.BuildExact: TimeOnly + DateTimeOffset branches) - var pChar = new FixedWidthParser(); - Assert.True(pChar.TryParse(input, Inv, null, out var m1)); - Assert.Equal(time, m1.Time); - Assert.Equal(timestamp, m1.Timestamp); - - // Reflection UTF-8 (Utf8ColumnParserFactory.BuildExact) - var pUtf8 = new Utf8FixedWidthParser(); - Assert.True(pUtf8.TryParse(bytes, Inv, null, out var m2)); - Assert.Equal(time, m2.Time); - Assert.Equal(timestamp, m2.Timestamp); - - // Generated char (FixedWidthRuntime.TryTimeOnlyExact / TryDateTimeOffsetExact) - Assert.True(FixedWidth.TryParse(input, Inv, null, out var m3)); - Assert.Equal(time, m3.Time); - Assert.Equal(timestamp, m3.Timestamp); - - // Generated UTF-8 (Utf8FixedWidthRuntime.TryTimeOnlyExact / TryDateTimeOffsetExact) - Assert.True(FixedWidthUtf8.TryParse(bytes, Inv, null, out var m4)); - Assert.Equal(time, m4.Time); - Assert.Equal(timestamp, m4.Timestamp); + byte[] input = Bytes("13453020260716134530+03:00"); + + Assert.True(FixedWidth.TryParse(input, FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); + Assert.Equal(time, reflected.Time); + Assert.Equal(timestamp, reflected.Timestamp); + + Assert.True(FixedWidth.TryParse(input, Ctx, out var generated)); + Assert.Equal(time, generated.Time); + Assert.Equal(timestamp, generated.Timestamp); } [Fact] public void TimeExact_InvalidField_ReturnsFalse() { - // A non-time first column makes the exact parser return false (the failure branch in both factories). - const string bad = "xxxxxx20260716134530+03:00"; - var badBytes = Encoding.UTF8.GetBytes(bad); + byte[] bad = Bytes("xxxxxx20260716134530+03:00"); - Assert.False(new FixedWidthParser().TryParse(bad, Inv, null, out _)); - Assert.False(new Utf8FixedWidthParser().TryParse(badBytes, Inv, null, out _)); + Assert.False(FixedWidth.TryParse(bad, FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.False(FixedWidth.TryParse(bad, Ctx, out _)); } [Fact] - public void Utf8_ConverterRejectsField_ReturnsFalse() + public void StrictDecoding_InvalidWideColumn_ReturnsTheRentedBuffer() { - // A non-numeric field makes CentsConverter.TryParse (byte) fail — the converter-failure - // branch in Utf8ColumnParserFactory.BuildConverter. - Assert.False(new Utf8FixedWidthParser().TryParse("notanum!"u8, Inv, null, out _)); + // Wider than the 256-char stack buffer, so decoding rents; the trailing 0xFF makes the strict decoder throw. + var strict = new UTF8Encoding(false, true); + byte[] column = new byte[300]; + column.AsSpan().Fill((byte)'1'); + column[^1] = 0xFF; + var options = new ColumnParseOptions(' ', TrimMode.Trailing, "yyyyMMdd"); + int size = strict.GetMaxCharCount(column.Length); + + // The shared pool hands a thread back the array it last returned, so a lost rental shows as a different array. + char[] cached = ArrayPool.Shared.Rent(size); + ArrayPool.Shared.Return(cached); + + Assert.Throws(() => FixedWidthRuntime.TryDateOnlyExact(column, CtxWith(encoding: strict), in options, out _)); + + char[] next = ArrayPool.Shared.Rent(size); + ArrayPool.Shared.Return(next); + Assert.Same(cached, next); } - // ----- UTF-8 culture parser: wide field forces the ArrayPool transcode branch ----- - [Fact] - public void Utf8_WideNumericField_CommaCulture_UsesTranscodeArrayPoolBranch() + public void WideNumericField_CommaCulture_DecodesThroughArrayPool() { - // Fields longer than the 128-char stack buffer, and a non-'.' separator so the parser takes - // the transcode path (TryParseDoubleViaTranscode / TryParseFloatViaTranscode) — its long branch. + // Wider than the 128-char stack buffer, with a ',' separator so the BCL fallback runs. string field = ("0," + new string('1', 150)).PadRight(200); - var bytes = Encoding.UTF8.GetBytes(field + field); - bool ok = new Utf8FixedWidthParser().TryParse(bytes, DeDe, null, out var m); + bool ok = FixedWidth.TryParse(Bytes(field + field), FixedWidthLayout.FromAttributes(), CtxWith(CultureInfo.GetCultureInfo("de-DE")), out var m); Assert.True(ok); Assert.InRange(m.D, 0.11, 0.12); Assert.InRange(m.F, 0.11f, 0.12f); } - // ----- Converter formatter: ArrayPool growth loop and exhaustion ----- - [Fact] public void FormatConvert_OutputLargerThanStackBuffer_UsesArrayPoolGrowth() { var options = new ColumnFormatOptions(Alignment.Left, ' ', null, OverflowBehavior.Truncate); - Span slice = stackalloc char[200]; + byte[] slice = new byte[400]; - // 100 chars > the 64-char stack buffer → the ArrayPool growth loop writes it instead. - FixedWidthRuntime.FormatConvert(new BigValue(100), slice, Inv, new BigConverter(), options, "Col"); + // 300 bytes > the 256-byte stack buffer, so the ArrayPool growth loop writes it instead. + FixedWidthRuntime.FormatConvert(new BigValue(300), slice, Ctx, new BigConverter(), options, "Col"); - Assert.Equal(new string('x', 100) + new string(' ', 100), slice.ToString()); + Assert.Equal(new string('x', 300) + new string(' ', 100), Encoding.ASCII.GetString(slice)); } [Fact] public void FormatConvert_OutputNeverFits_Throws() - { - var ex = Record.Exception(FormatConvertThatNeverFits); - var iop = Assert.IsType(ex); - Assert.Contains("Col", iop.Message, StringComparison.Ordinal); - } - - // Separate method: a Span can't be captured by the Record.Exception lambda. - private static void FormatConvertThatNeverFits() { var options = new ColumnFormatOptions(Alignment.Left, ' ', null, OverflowBehavior.Truncate); - Span slice = stackalloc char[8]; // maxSize = max(1024, 8*16) = 1024 - FixedWidthRuntime.FormatConvert(new BigValue(2000), slice, Inv, new BigConverter(), options, "Col"); + byte[] slice = new byte[8]; + + var ex = Assert.Throws( + () => FixedWidthRuntime.FormatConvert(new BigValue(10_000), slice, Ctx, new BigConverter(), options, "Col")); + + Assert.Contains("Col", ex.Message, StringComparison.Ordinal); } } } diff --git a/tests/FixedWidthParser.Tests/CultureTests.cs b/tests/FixedWidthParser.Tests/CultureTests.cs index 35bf73b..f6ca75a 100644 --- a/tests/FixedWidthParser.Tests/CultureTests.cs +++ b/tests/FixedWidthParser.Tests/CultureTests.cs @@ -1,279 +1,366 @@ using System.Globalization; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; +using System.Text; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Covers culture handling. The generic path (ISpanParsable, e.g. decimal) and writing via - /// ISpanFormattable honor the IFormatProvider. The double/float processors derive the decimal - /// separator from the culture and pass it to csFastFloat, so they also honor the - /// IFormatProvider (with '.' as the default when null). + /// Culture handling. The format provider comes from ; the default + /// context means the invariant culture for every column type, on parse and on write. /// public class CultureTests { private static readonly CultureInfo DeDe = CultureInfo.GetCultureInfo("de-DE"); - - // ----- Generic path (decimal): honors the culture ----- + private static readonly FixedWidthContext German = CtxWith(DeDe); [Fact] public void Parse_DecimalColumn_HonorsCommaCulture() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("1234,56 ", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse("1234,56 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(1234.56m, model.Amount); } [Fact] - public void Parse_DecimalColumn_HonorsDotCulture() + public void Parse_DecimalColumn_DefaultContext_IsInvariant() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("1234.56 ", CultureInfo.InvariantCulture, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse("1234.56 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(1234.56m, model.Amount); } - // ----- Double path: decimal separator derived from the culture ----- - [Fact] public void Parse_DoubleColumn_HonorsCommaCulture() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234,50 ", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse("Bob 30 1234,50 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(1234.50, model.Salary, 2); } [Fact] - public void Parse_DoubleColumn_HonorsDotCulture() + public void Parse_DoubleColumn_DefaultContext_UsesDot() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234.50 ", CultureInfo.InvariantCulture, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse("Bob 30 1234.50 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(1234.50, model.Salary, 2); } [Fact] - public void Parse_DoubleColumn_NullProvider_DefaultsToDot() + public void Parse_FloatColumn_HonorsCommaCulture() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1234.50 ", null, null, out var model); + Assert.True(FixedWidth.TryParse("3,14 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); + Assert.Equal(3.14f, model.Value, 2); + } - Assert.True(ok); + // The fast path refuses a thousands separator; the BCL fallback accepts the value and still + // rejects real garbage. + [Fact] + public void Parse_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() + { + Assert.True(FixedWidth.TryParse("Bob 30 1,234.50 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(1234.50, model.Salary, 2); } + // Regression: "1.234,50" under de-DE used to come back as 1.0 from a prefix-only fast parse. [Fact] - public void Parse_FloatColumn_HonorsCommaCulture() + public void Parse_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() { - var parser = new FixedWidthParser(); + Assert.True(FixedWidth.TryParse("Bob 30 1.234,50 "u8, FixedWidthLayout.FromAttributes(), German, out var model)); + Assert.Equal(1234.50, model.Salary, 2); + } - bool ok = parser.TryParse("3,14 ", DeDe, null, out var model); + // The fast path takes any one-ASCII-character separator, so every accepted value must match the BCL's. + [Theory] + [InlineData("de-DE")] + [InlineData("pt-BR")] + [InlineData("fr-FR")] + public void Parse_Double_CommaCulture_MatchesTheBcl(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format: null); + string[] edges = ["-0,5", "+1,5", "1,5e3", "1,5E-3", "1.234,5", "0,1", ",5", "5,", "007,10", "1,7976931348623157E+308", "4,9E-324", "1,2,3", "--1", "1e", "1,5x", "1 000,5"]; + var random = new Random(42); + var values = edges.Concat(Enumerable.Range(0, 2000).Select(_ => (random.NextDouble() * Math.Pow(10, random.Next(-8, 12)) * (random.Next(2) * 2 - 1)).ToString("R", culture))); + + foreach (string text in values) + { + bool expected = double.TryParse(text, NumberStyles.Float | NumberStyles.AllowThousands, culture, out double bcl); + bool actual = Infrastructure.FixedWidthRuntime.TryDouble(Bytes(text), in context, in options, out double parsed); - Assert.True(ok); - Assert.Equal(3.14f, model.Value, 2); + Assert.True(expected == actual, $"'{text}': BCL {expected}, parser {actual}"); + Assert.True(!expected || bcl.Equals(parsed), $"'{text}': BCL {bcl:R}, parser {parsed:R}"); + } } - // ----- Null provider means invariant for EVERY column type, not just double/float: before this - // fix, ISpanParsable columns (decimal, int, DateTime, …) fell through to the BCL's own default - // of CurrentCulture on a null provider, while double/float were hard-coded to invariant — so the - // same line could parse two different ways for two different column types under one null-provider - // call. Pinning both to invariant also makes this test's outcome independent of whatever culture - // happens to be ambient when the suite runs. ----- - - [Fact] - public void Parse_DecimalColumn_NullProvider_DefaultsToInvariant() + // The digit-only fast path for exact formats must agree with TryParseExact, including where it must + // step aside: a Buddhist calendar (th-TH) and "/" or ":" standing for other separators (de-DE). + [Theory] + [InlineData("")] + [InlineData("pt-BR")] + [InlineData("de-DE")] + [InlineData("th-TH")] + public void Parse_ExactDates_MatchTheBcl(string cultureName) { - var parser = new FixedWidthParser(); + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + string[] formats = ["yyyyMMdd", "yyyy-MM-dd", "dd/MM/yyyy", "yyyyMMddHHmmss", "yyyy-MM-ddTHH:mm:ss", "HHmmss", "HH:mm:ss", "HH", "MMdd", "yyyyMM", "yyyy.MM.dd"]; + string[] edges = ["00000101", "20230230", "20240229", "20230229", "20231301", "99991231", "2023010", "2023o105", "-0230105", "24:00:00", "23:59:60", "235959", "000000", "2023-01-05", "05/01/2023", "05.01.2023"]; + var random = new Random(42); - bool ok = parser.TryParse("1234.56 ", null, null, out var model); - - Assert.True(ok); - Assert.Equal(1234.56m, model.Amount); + foreach (string format in formats) + { + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format); + var samples = Enumerable.Range(0, 300) + .Select(_ => new DateTime(1, 1, 1).AddSeconds(random.NextInt64(0, (long)(DateTime.MaxValue - DateTime.MinValue).TotalSeconds))) + .SelectMany(d => new[] { d.ToString(format, CultureInfo.InvariantCulture), d.ToString(format, culture) }); + foreach (string text in edges.Concat(samples)) + { + byte[] bytes = Bytes(text); + bool expected = DateTime.TryParseExact(text, format, culture, DateTimeStyles.None, out var bclDateTime); + Assert.True(expected == Infrastructure.FixedWidthRuntime.TryDateTimeExact(bytes, in context, in options, out var dateTime) && (!expected || bclDateTime == dateTime && bclDateTime.Kind == dateTime.Kind), + $"DateTime '{text}' as {format}: BCL {expected} {bclDateTime:O}, parser {dateTime:O}"); + expected = DateOnly.TryParseExact(text, format, culture, DateTimeStyles.None, out var bclDate); + Assert.True(expected == Infrastructure.FixedWidthRuntime.TryDateOnlyExact(bytes, in context, in options, out var date) && (!expected || bclDate == date), + $"DateOnly '{text}' as {format}: BCL {expected} {bclDate:O}, parser {date:O}"); + expected = TimeOnly.TryParseExact(text, format, culture, DateTimeStyles.None, out var bclTime); + Assert.True(expected == Infrastructure.FixedWidthRuntime.TryTimeOnlyExact(bytes, in context, in options, out var time) && (!expected || bclTime == time), + $"TimeOnly '{text}' as {format}: BCL {expected} {bclTime:O}, parser {time:O}"); + } + } } - // ----- Round-trip under the default (null) provider must preserve the value regardless of the - // machine's ambient culture — before this fix, write (CurrentCulture via ISpanFormattable.TryFormat) - // and parse (invariant via CultureHelpers) disagreed on what null means, so a comma-decimal machine - // would write "1234,5" and then fail to parse its own output back with the same null provider. ----- - - [Fact] - public void RoundTrip_DoubleColumn_NullProvider_PreservesValue() + // sv-SE's negative sign is U+2212, so a leading "-" must not take the integer fast path there. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("sv-SE")] + public void Parse_Integers_MatchTheBcl(string cultureName) { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); - var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Trailing, format: null); + string[] edges = ["-5", "+5", " 12", "12 ", "0007", "-0", "-", "", "1,000", "1.000", "999999999", "2147483647", "2147483648", "-2147483648", "-2147483649", + "999999999999999999", "9223372036854775807", "9223372036854775808", "-9223372036854775808", "−5", "1e3", "12x"]; + var random = new Random(42); + var values = Enumerable.Range(0, 2000).Select(_ => random.NextInt64(long.MinValue, long.MaxValue) >> random.Next(0, 63)).Select(v => v.ToString(CultureInfo.InvariantCulture)); + + foreach (string text in edges.Concat(values)) + { + byte[] bytes = Bytes(text); + bool expected = long.TryParse(bytes, culture, out long bclLong); + bool actual = Infrastructure.FixedWidthRuntime.TryParse(bytes, in context, in options, out long parsedLong); + Assert.True(expected == actual && (!expected || bclLong == parsedLong), $"long '{text}': BCL {expected} {bclLong}, parser {actual} {parsedLong}"); + expected = int.TryParse(bytes, culture, out int bclInt); + actual = Infrastructure.FixedWidthRuntime.TryParse(bytes, in context, in options, out int parsedInt); + Assert.True(expected == actual && (!expected || bclInt == parsedInt), $"int '{text}': BCL {expected} {bclInt}, parser {actual} {parsedInt}"); + } + } - string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, null, null, out var parsed); + // The integer and date fast paths for writing must lay out the same bytes as the BCL's char path. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("th-TH")] + [InlineData("sv-SE")] + public void Format_IntegersAndDates_MatchTheBcl(string cultureName) + { + var context = CtxWith(CultureInfo.GetCultureInfo(cultureName)); + var random = new Random(42); + long[] edges = [0, 1, -1, 9, -9, 10, 123, -171, int.MaxValue, int.MinValue, long.MaxValue, long.MinValue]; + var longs = edges.Concat(Enumerable.Range(0, 1000).Select(_ => random.NextInt64(long.MinValue, long.MaxValue) >> random.Next(0, 63))); + var dates = Enumerable.Range(0, 300).Select(_ => new DateTime(1, 1, 1).AddSeconds(random.NextInt64(0, (long)(DateTime.MaxValue - DateTime.MinValue).TotalSeconds))); + string[] formats = ["yyyyMMdd", "yyyy-MM-dd", "dd/MM/yyyy", "yyyy-MM-ddTHH:mm:ss", "HHmmss", "HH:mm:ss"]; + + foreach (var alignment in new[] { Alignment.Left, Alignment.Right }) + { + foreach (char padding in new[] { ' ', '0' }) + { + var plain = new Infrastructure.ColumnFormatOptions(alignment, padding, null, OverflowBehavior.Throw); + foreach (long value in longs) + { + // Room for sv-SE's three-byte minus sign too. + Same(value, plain, 23); + Same((int)value, plain, 14); + } + foreach (string format in formats) + { + var options = new Infrastructure.ColumnFormatOptions(alignment, padding, format, OverflowBehavior.Throw); + foreach (var date in dates) + { + Same(date, options, 24); + if (!format.Contains('H', StringComparison.Ordinal)) + { + Same(DateOnly.FromDateTime(date), options, 24); + } + if (!format.Contains('y', StringComparison.Ordinal)) + { + Same(TimeOnly.FromDateTime(date), options, 24); + } + } + } + } + } - Assert.True(ok); - Assert.Equal(original.Salary, parsed.Salary, 2); + void Same(T value, Infrastructure.ColumnFormatOptions options, int width) + where T : IUtf8SpanFormattable, ISpanFormattable + { + byte[] fast = new byte[width]; + byte[] bcl = new byte[width]; + Infrastructure.FixedWidthRuntime.FormatUtf8Value(value, fast, in context, options, "c"); + Infrastructure.FixedWidthRuntime.FormatValue(value, bcl, in context, options, "c"); + Assert.True(fast.AsSpan().SequenceEqual(bcl), $"{typeof(T).Name} {value} ({options.Format}, {options.Alignment}, '{options.Padding}'): fast \"{Encoding.UTF8.GetString(fast)}\", BCL \"{Encoding.UTF8.GetString(bcl)}\""); + } } - // ----- Thousands separators under the invariant '.' fast path: csFastFloat's single - // decimal_separator override can't express a thousands separator, so it used to fail the - // full-consumption check and reject the whole line even though the value is valid; the BCL - // fallback now retried on that failure accepts it (and still rejects real garbage, see - // Parse_DoubleColumn_TrailingGarbageAfterNumber_Fails above). ----- - - [Fact] - public void Parse_DoubleColumn_DotCultureThousandsSeparator_ParsesFullValue() + // The float fast path rounds once in binary32; going through a double could land one ulp off. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("fr-FR")] + public void Parse_Float_MatchesTheBcl(string cultureName) { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 1,234.50 ", Inv, null, out var model); + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format: null); + string separator = culture.NumberFormat.NumberDecimalSeparator; + string[] edges = ["-0", "16777216", "16777217", $"1{separator}5e3", $"0{separator}1", $"3{separator}4028235E+38", "1e", "--1"]; + var random = new Random(42); + var shortDecimals = Enumerable.Range(0, 2000).Select(_ => (random.Next(-20_000_000, 20_000_000) / Math.Pow(10, random.Next(0, 11))).ToString("0.##########", culture)); + var roundTrips = Enumerable.Range(0, 2000).Select(_ => ((float)(random.NextDouble() * Math.Pow(10, random.Next(-8, 12)))).ToString("R", culture)); + + foreach (string text in edges.Concat(shortDecimals).Concat(roundTrips)) + { + bool expected = float.TryParse(text, NumberStyles.Float | NumberStyles.AllowThousands, culture, out float bcl); + bool actual = Infrastructure.FixedWidthRuntime.TryFloat(Bytes(text), in context, in options, out float parsed); - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); + Assert.True(expected == actual, $"'{text}': BCL {expected}, parser {actual}"); + Assert.True(!expected || bcl.Equals(parsed), $"'{text}': BCL {bcl:R}, parser {parsed:R}"); + } } - // ----- Thousands separators: csFastFloat's decimal_separator override silently truncates at - // the first character it doesn't recognize (e.g. a thousands separator) instead of failing - // only real NumberFormatInfo-aware parsing (used whenever the separator isn't '.') gets this - // right. Regression coverage for a bug where "1.234,50" under de-DE silently became 1.0. ----- - - [Fact] - public void Parse_DoubleColumn_DeDeThousandsSeparator_ParsesFullValue() + // The decimal fast path keeps the scale as written, as the BCL does: 1,50 stays 1,50. + [Theory] + [InlineData("")] + [InlineData("de-DE")] + [InlineData("fr-FR")] + public void Parse_Decimal_MatchesTheBcl(string cultureName) { - var parser = new FixedWidthParser(); - - // de-DE: '.' groups thousands, ',' is the decimal separator — "1.234,50" means 1234.50. - bool ok = parser.TryParse("Bob 30 1.234,50 ", DeDe, null, out var model); + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new Infrastructure.ColumnParseOptions(' ', TrimMode.Both, format: null); + string separator = culture.NumberFormat.NumberDecimalSeparator; + string[] edges = ["-0", $"1{separator}50", $"{separator}5", $"5{separator}", "9999999999999999999", "99999999999999999999", $"0{separator}0000000000000000001", "1e3", "--1", "+7"]; + var random = new Random(42); + var values = Enumerable.Range(0, 2000).Select(_ => (random.NextInt64(-1_000_000_000_000, 1_000_000_000_000) / (decimal)Math.Pow(10, random.Next(0, 13))).ToString(culture)); + + foreach (string text in edges.Concat(values)) + { + bool expected = decimal.TryParse(text, NumberStyles.Number, culture, out decimal bcl); + bool actual = Infrastructure.FixedWidthRuntime.TryParse(Bytes(text), in context, in options, out decimal parsed); - Assert.True(ok); - Assert.Equal(1234.50, model.Salary, 2); + Assert.True(expected == actual, $"'{text}': BCL {expected}, parser {actual}"); + Assert.True(!expected || bcl.ToString(CultureInfo.InvariantCulture) == parsed.ToString(CultureInfo.InvariantCulture), $"'{text}': BCL {bcl}, parser {parsed}"); + } } - // ----- Trailing garbage after a dot-separated number: the fast path must reject it instead of - // silently truncating to the leading digits and reporting success. ----- - [Fact] public void Parse_DoubleColumn_TrailingGarbageAfterNumber_Fails() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 12x ", Inv, null, out _); - - Assert.False(ok); + Assert.False(FixedWidth.TryParse("Bob 30 12x "u8, FixedWidthLayout.FromAttributes(), Ctx, out _)); } [Fact] public void Parse_FloatColumn_TrailingGarbageAfterNumber_Fails() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("12x ", Inv, null, out _); - - Assert.False(ok); + Assert.False(FixedWidth.TryParse("12x "u8, FixedWidthLayout.FromAttributes(), Ctx, out _)); } - // ----- Right-aligned (leading-space-padded) numeric columns: csFastFloat's characters_consumed - // does not consistently include skipped leading whitespace across its double/float overloads - // (FastDoubleParser counted it, FastFloatParser didn't, and vice versa on the UTF-8 overloads), - // so a right-aligned column could fail the full-consumption check even under the invariant '.' - // fast path — the exact opposite of a version that should always accept it. ----- - + // The fast path takes no whitespace, so right-aligned columns must be trimmed before it. [Fact] public void Parse_DoubleColumn_RightAligned_DotCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123.45", Inv, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse("Bob 30 123.45"u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(123.45, model.Salary, 2); } [Fact] public void Parse_FloatColumn_RightAligned_DotCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(" 3.14", Inv, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse(" 3.14"u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(3.14f, model.Value, 2); } [Fact] public void Parse_DoubleColumn_RightAligned_CommaCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("Bob 30 123,45", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse("Bob 30 123,45"u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(123.45, model.Salary, 2); } [Fact] public void Parse_FloatColumn_RightAligned_CommaCulture_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(" 3,14", DeDe, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse(" 3,14"u8, FixedWidthLayout.FromAttributes(), German, out var model)); Assert.Equal(3.14f, model.Value, 2); } - // ----- Regression guard for the old single-entry decimal-separator memo: alternating between - // two providers must not thrash into wrong results for either one. ----- + // A separator that is not one byte (U+066B in UTF-8) used to be refused; the value is now + // decoded and parsed by the BCL with the real culture. + [Fact] + public void Parse_DoubleColumn_NonAsciiSeparatorCulture_Parses() + { + var culture = (CultureInfo)CultureInfo.InvariantCulture.Clone(); + culture.NumberFormat.NumberDecimalSeparator = "٫"; + Assert.True(FixedWidth.TryParse(Bytes("Bob 30 1234٫50 "), FixedWidthLayout.FromAttributes(), CtxWith(culture), out var model)); + Assert.Equal(1234.50, model.Salary, 2); + } + + // Regression guard for the old single-entry decimal-separator memo. [Fact] public void Parse_DoubleColumn_AlternatingProviders_BothStayCorrect() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); for (int i = 0; i < 5; i++) { - Assert.True(parser.TryParse("Bob 30 1234.50 ", Inv, null, out var invModel)); - Assert.Equal(1234.50, invModel.Salary, 2); + Assert.True(FixedWidth.TryParse("Bob 30 1234.50 "u8, layout, Ctx, out var invariant)); + Assert.Equal(1234.50, invariant.Salary, 2); - Assert.True(parser.TryParse("Bob 30 1234,50 ", DeDe, null, out var deModel)); - Assert.Equal(1234.50, deModel.Salary, 2); + Assert.True(FixedWidth.TryParse("Bob 30 1234,50 "u8, layout, German, out var german)); + Assert.Equal(1234.50, german.Salary, 2); } } - // ----- Writing: honors the culture (ISpanFormattable) ----- - [Fact] public void Write_DoubleColumn_UsesCultureDecimalSeparator() { - var writer = new FixedWidthWriter(); + var writerOptions = new FixedWidthWriterOptions { FormatProvider = DeDe }; + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, DeDe); + string line = WriteOne(layout, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, writerOptions); - // Salary column at [15,25): formatted with a comma under de-DE. Assert.Equal("1234,5", line[15..21]); } - // ----- Round-trip under a comma culture now preserves the value ----- + [Fact] + public void RoundTrip_DoubleColumn_DefaultOptions_PreservesValue() + { + var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; + + string line = WriteOne(FixedWidthLayout.FromAttributes(), original); + + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var parsed)); + Assert.Equal(original.Salary, parsed.Salary, 2); + } [Fact] public void RoundTrip_DoubleColumn_UnderCommaCulture_PreservesValue() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writerOptions = new FixedWidthWriterOptions { FormatProvider = DeDe }; + var layout = FixedWidthLayout.FromAttributes(); var original = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; - // Writing produces "1234,5"; reading uses the same culture separator → same value. - string line = WriteOne(writer, original, DeDe); - bool ok = parser.TryParse(line, DeDe, null, out var parsed); + string line = WriteOne(layout, original, writerOptions); - Assert.True(ok); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), German, out var parsed)); Assert.Equal(original.Salary, parsed.Salary, 2); } } diff --git a/tests/FixedWidthParser.Tests/DoubleFormatTests.cs b/tests/FixedWidthParser.Tests/DoubleFormatTests.cs new file mode 100644 index 0000000..82aaca6 --- /dev/null +++ b/tests/FixedWidthParser.Tests/DoubleFormatTests.cs @@ -0,0 +1,45 @@ +using System.Diagnostics.CodeAnalysis; +using System.Globalization; +using System.Text; +using FixedWidthParser.Infrastructure; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The writer's short-cut for plain doubles must match byte for byte. + public class DoubleFormatTests + { + [Theory] + [InlineData("")] + [InlineData("pt-BR")] + [SuppressMessage("Security", "CA5394:Do not use insecure randomness", + Justification = "Needs a reproducible seeded PRNG for deterministic test content, not cryptographic randomness.")] + public void FormattedBytesMatchDoubleTryFormat(string cultureName) + { + var culture = CultureInfo.GetCultureInfo(cultureName); + var context = CtxWith(culture); + var options = new ColumnFormatOptions(Alignment.Right, ' ', null, OverflowBehavior.Throw); + Random rng = new(42); + List values = [0, -0.0, 1e9, 999_999_999.9999, -999_999_999.9999, 0.0001, 0.00001, 0.00015, 0.1 + 0.2, 1.0 / 3, 1e15, 2.5, -2.5, double.Epsilon, double.MaxValue, double.NaN, double.NegativeInfinity]; + foreach (double edge in values.ToArray()) + { + values.Add(Math.BitIncrement(edge)); + values.Add(Math.BitDecrement(edge)); + } + for (int i = 0; i < 100_000; i++) + { + long digits = rng.NextInt64(-2_000_000_000_000, 2_000_000_000_000) / (long)Math.Pow(10, rng.Next(0, 12)); + values.Add(digits / Math.Pow(10, rng.Next(0, 7))); + values.Add(rng.NextDouble() * Math.Pow(10, rng.Next(-8, 12))); + values.Add(BitConverter.Int64BitsToDouble(rng.NextInt64())); + } + byte[] actual = new byte[32]; + foreach (double value in values) + { + FixedWidthRuntime.FormatUtf8Value(value, actual, in context, options, "Value"); + string text = value.ToString(culture); + Assert.Equal(text.PadLeft(32 - Encoding.UTF8.GetByteCount(text) + text.Length), Encoding.UTF8.GetString(actual)); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/EnumColumnTests.cs b/tests/FixedWidthParser.Tests/EnumColumnTests.cs new file mode 100644 index 0000000..9b3b984 --- /dev/null +++ b/tests/FixedWidthParser.Tests/EnumColumnTests.cs @@ -0,0 +1,146 @@ +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + public enum Side { Buy = 1, Sell = 2 } + + [Flags] + public enum Access { None = 0, Read = 1, Write = 2 } + + public sealed class EnumModel + { + [FixedColumn(0, 10)] public Side Side { get; set; } + [FixedColumn(10, 5)] public Side? MaybeSide { get; set; } + [FixedColumn(15, 10)] public Access Access { get; set; } + } + + public partial class GenEnumModel : IFixedWidthModel + { + [FixedColumn(0, 10)] public Side Side { get; set; } + [FixedColumn(10, 5)] public Side? MaybeSide { get; set; } + [FixedColumn(15, 10)] public Access Access { get; set; } + } + + public enum Kind { Ação = 1, Fundo = 2 } + + public sealed class KindModel + { + [FixedColumn(0, 6)] public Kind Kind { get; set; } + } + + public sealed class EnumCodeModel + { + [FixedColumn(0, 2, Format = "D", Padding = '0', Alignment = Alignment.Right)] public Side Side { get; set; } + } + + public partial class GenEnumCodeModel : IFixedWidthModel + { + [FixedColumn(0, 2, Format = "D", Padding = '0', Alignment = Alignment.Right)] public Side Side { get; set; } + } + + /// + /// Enum columns, as ExcelReader reads them: a member's name in any case or its number, nothing the enum does not + /// define. Attribute, generated and fluent layouts agree. + /// + public class EnumColumnTests + { + private static byte[] Line(string side, string maybeSide = "", string access = "Read") + { + return Bytes(side.PadRight(10) + maybeSide.PadRight(5) + access.PadRight(10)); + } + + private static FixedWidthLayout Fluent() + { + return FixedWidthLayout.Build(b => b + .Factory(() => new EnumModel()) + .EnumColumn("Side", 0, 10, (in EnumModel m) => m.Side, (ref EnumModel m, Side v) => m.Side = v) + .NullableEnumColumn("MaybeSide", 10, 5, (in EnumModel m) => m.MaybeSide, (ref EnumModel m, Side? v) => m.MaybeSide = v) + .EnumColumn("Access", 15, 10, (in EnumModel m) => m.Access, (ref EnumModel m, Access v) => m.Access = v)); + } + + [Theory] + [InlineData("Buy", Side.Buy)] + [InlineData("sell", Side.Sell)] + [InlineData("SELL", Side.Sell)] + [InlineData("1", Side.Buy)] + [InlineData("02", Side.Sell)] + [InlineData(" Sell", Side.Sell)] + public void NameOrNumber_Parses(string text, Side expected) + { + Assert.True(FixedWidth.TryParse(Line(text), FixedWidthLayout.FromAttributes(), Ctx, out var attributes)); + Assert.True(FixedWidth.TryParse(Line(text), Ctx, out var generated)); + Assert.True(FixedWidth.TryParse(Line(text), Fluent(), Ctx, out var fluent)); + Assert.Equal(expected, attributes.Side); + Assert.Equal(expected, generated.Side); + Assert.Equal(expected, fluent.Side); + } + + [Theory] + [InlineData("Hold")] + [InlineData("3")] + [InlineData("0")] + [InlineData("")] + [InlineData("Buy,Sell")] + [InlineData("0Buy")] + [InlineData("00")] + [InlineData("-1")] + public void UndefinedValue_DoesNotParse(string text) + { + Assert.False(FixedWidth.TryParse(Line(text), FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.False(FixedWidth.TryParse(Line(text), Ctx, out _)); + Assert.False(FixedWidth.TryParse(Line(text), Fluent(), Ctx, out _)); + } + + [Fact] + public void FlagsCombination_DoesNotParse() + { + Assert.True(FixedWidth.TryParse(Line("Buy", access: "Write"), FixedWidthLayout.FromAttributes(), Ctx, out var single)); + Assert.Equal(Access.Write, single.Access); + Assert.True(FixedWidth.TryParse(Line("Buy", access: "00"), Ctx, out var zero)); + Assert.Equal(Access.None, zero.Access); + Assert.False(FixedWidth.TryParse(Line("Buy", access: "Read,Write"), FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.False(FixedWidth.TryParse(Line("Buy", access: "3"), Ctx, out _)); + } + + [Fact] + public void Nullable_BlankIsNull() + { + Assert.True(FixedWidth.TryParse(Line("Buy"), FixedWidthLayout.FromAttributes(), Ctx, out var blank)); + Assert.Null(blank.MaybeSide); + Assert.True(FixedWidth.TryParse(Line("Buy", "sell"), Ctx, out var present)); + Assert.Equal(Side.Sell, present.MaybeSide); + Assert.True(FixedWidth.TryParse(Line("Buy", "2"), Fluent(), Ctx, out var fluent)); + Assert.Equal(Side.Sell, fluent.MaybeSide); + } + + [Fact] + public void Write_WritesTheName() + { + const string expected = "Sell None "; + var model = new EnumModel { Side = Side.Sell, MaybeSide = null, Access = Access.None }; + + Assert.Equal(expected, WriteOne(FixedWidthLayout.FromAttributes(), model)); + Assert.Equal(expected, WriteOne(Fluent(), model)); + Assert.Equal(expected, Format(new GenEnumModel { Side = Side.Sell, MaybeSide = null, Access = Access.None })); + Assert.Equal("Buy Sell Read ", Format(new GenEnumModel { Side = Side.Buy, MaybeSide = Side.Sell, Access = Access.Read })); + } + + [Fact] + public void NonAsciiName_IsDecodedWithTheEncoding() + { + var latin1 = CtxWith(encoding: System.Text.Encoding.Latin1); + + Assert.True(FixedWidth.TryParse(System.Text.Encoding.Latin1.GetBytes("AÇÃO "), FixedWidthLayout.FromAttributes(), latin1, out var model)); + Assert.Equal(Kind.Ação, model.Kind); + } + + [Fact] + public void FormatD_WritesTheNumber_AndReadsItBack() + { + Assert.Equal("02", WriteOne(FixedWidthLayout.FromAttributes(), new EnumCodeModel { Side = Side.Sell })); + Assert.Equal("01", Format(new GenEnumCodeModel { Side = Side.Buy })); + Assert.True(FixedWidth.TryParse("02"u8, Ctx, out var parsed)); + Assert.Equal(Side.Sell, parsed.Side); + } + } +} diff --git a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj index d090f90..160ccbd 100644 --- a/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj +++ b/tests/FixedWidthParser.Tests/FixedWidthParser.Tests.csproj @@ -1,17 +1,18 @@ - + true true - net8.0;net10.0 + net10.0 latest enable enable false + true Exe true - $(NoWarn);HLQ005;CA1707;CA1815;S3241;CA1051 + $(NoWarn);HLQ005;CA1707;CA1815;S3241;CA1051;S6640 @@ -19,15 +20,15 @@ + - - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + @@ -36,6 +37,7 @@ + diff --git a/tests/FixedWidthParser.Tests/FluentLayoutTests.cs b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs new file mode 100644 index 0000000..f254099 --- /dev/null +++ b/tests/FixedWidthParser.Tests/FluentLayoutTests.cs @@ -0,0 +1,301 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// A layout from reads and writes exactly like the same layout declared with attributes. + public sealed class FluentLayoutTests : IDisposable + { + private static readonly FixedWidthWriterOptions Lf = new() { NewLine = "\n" }; + + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + private static FixedWidthLayout Person() + { + return FixedWidthLayout.Build(b => b + .Factory(() => new MutablePersonModel()) + .Column("Name", 0, 10, (in MutablePersonModel p) => p.Name, (ref MutablePersonModel p, string v) => p.Name = v) + .Column("Age", 10, 5, (in MutablePersonModel p) => p.Age, (ref MutablePersonModel p, int v) => p.Age = v) + .Column("Salary", 15, 10, (in MutablePersonModel p) => p.Salary, (ref MutablePersonModel p, double v) => p.Salary = v)); + } + + private static FixedWidthLayout Options() + { + return FixedWidthLayout.Build(b => b + .Factory(() => new MutableOptionsModel()) + .Column("ZeroPadded", 0, 5, (in MutableOptionsModel m) => m.ZeroPadded, (ref MutableOptionsModel m, int v) => m.ZeroPadded = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Padding = '0' }) + .Column("Fixed2", 5, 8, (in MutableOptionsModel m) => m.Fixed2, (ref MutableOptionsModel m, double v) => m.Fixed2 = v, + new FixedWidthColumnOptions { Format = "F2" }) + .Column("Truncated", 13, 4, (in MutableOptionsModel m) => m.Truncated, (ref MutableOptionsModel m, string v) => m.Truncated = v, + new FixedWidthColumnOptions { Alignment = Alignment.Right, Overflow = OverflowBehavior.Truncate }) + .Column("Starred", 17, 5, (in MutableOptionsModel m) => m.Starred, (ref MutableOptionsModel m, int v) => m.Starred = v, + new FixedWidthColumnOptions { TrimChar = '*' }) + .NullableColumn("NullableAge", 22, 5, (in MutableOptionsModel m) => m.NullableAge, (ref MutableOptionsModel m, int? v) => m.NullableAge = v) + .NullableColumn("NullableAmount", 27, 10, (in MutableOptionsModel m) => m.NullableAmount, (ref MutableOptionsModel m, decimal? v) => m.NullableAmount = v) + .ConvertedColumn("Cents", 37, 8, (in MutableOptionsModel m) => m.Cents, (ref MutableOptionsModel m, CentsValue v) => m.Cents = v, new CentsConverter()) + .DecodedColumn("Date", 45, 8, (in MutableOptionsModel m) => m.Date, (ref MutableOptionsModel m, DateOnly v) => m.Date = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" }) + .NullableDecodedColumn("NullableDate", 53, 8, (in MutableOptionsModel m) => m.NullableDate, (ref MutableOptionsModel m, DateOnly? v) => m.NullableDate = v, + new FixedWidthColumnOptions { Format = "yyyyMMdd" }) + .NullableConvertedColumn("NullableCents", 61, 8, (in MutableOptionsModel m) => m.NullableCents, (ref MutableOptionsModel m, CentsValue? v) => m.NullableCents = v, + new CentsConverter())); + } + + [Theory] + [InlineData("John Doe 30 6000000.00")] + [InlineData("Jane 28 55000.00 ")] + [InlineData("John")] + [InlineData("")] + [InlineData("Ann abc 10.5 ")] + public void Fluent_TryParse_MatchesAttributes(string line) + { + bool okA = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var a); + bool okF = FixedWidth.TryParse(Bytes(line), Person(), Ctx, out var f); + + Assert.Equal(okA, okF); + if (okA) + { + Assert.Equal(a, f); + } + } + + [Fact] + public void Fluent_WriteMany_MatchesAttributes() + { + MutablePersonModel[] people = [new() { Name = "John Doe", Age = 30, Salary = 6000000 }, new() { Name = "VeryLongName", Age = 1 }, new()]; + + Assert.Equal(Write(people, FixedWidthLayout.FromAttributes()), Write(people, Person())); + } + + [Theory] + [InlineData(42, 3.14159, "ABCDEF", 7, 30, "12.5")] + [InlineData(-3, -0.5, "X", 0, null, null)] + public void Fluent_Options_MatchAttributes(int zero, double fixed2, string truncated, int starred, int? age, string? amount) + { + var model = new MutableOptionsModel + { + ZeroPadded = zero, + Fixed2 = fixed2, + Truncated = truncated, + Starred = starred, + NullableAge = age, + NullableAmount = amount is null ? null : decimal.Parse(amount, Inv), + Cents = new CentsValue(1234), + Date = new DateOnly(2026, 10, 23), + NullableDate = age is null ? null : new DateOnly(2027, 1, 2), + NullableCents = age is null ? null : new CentsValue(99), + }; + var attributes = FixedWidthLayout.FromAttributes(); + + string attributeLine = Write([model], attributes); + Assert.Equal(attributeLine, Write([model], Options())); + + byte[] line = Bytes(attributeLine.TrimEnd('\n')); + Assert.True(FixedWidth.TryParse(line, attributes, Ctx, out var a)); + Assert.True(FixedWidth.TryParse(line, Options(), Ctx, out var f)); + Assert.Equal(a, f); + } + + [Fact] + public void Fluent_StarTrim_ParsesLikeAttributes() + { + byte[] line = Bytes("00042" + "3.14 " + "ABCD" + "42***" + new string(' ', 15) + "1234 " + "20261023" + new string(' ', 16)); + + Assert.True(FixedWidth.TryParse(line, FixedWidthLayout.FromAttributes(), Ctx, out var a)); + Assert.True(FixedWidth.TryParse(line, Options(), Ctx, out var f)); + Assert.Equal(42, f.Starred); + Assert.Equal(a, f); + } + + [Fact] + public void Fluent_ParseError_SameExceptionAsAttributes() + { + File.WriteAllText(_path, "John Doe 30 6000000.00\nJane abc 55000.00 \n"); + + var a = Assert.Throws(() => FixedWidth.ReadFile(_path, FixedWidthLayout.FromAttributes()).ToList()); + var f = Assert.Throws(() => FixedWidth.ReadFile(_path, Person()).ToList()); + + Assert.Equal((a.LineNumber, a.ColumnName, a.ColumnStart, a.ColumnLength, a.RawValue), (f.LineNumber, f.ColumnName, f.ColumnStart, f.ColumnLength, f.RawValue)); + Assert.Equal(2, f.LineNumber); + Assert.Equal("Age", f.ColumnName); + } + + [Fact] + public void Fluent_ShortLine_ReportsRequiredLength() + { + File.WriteAllText(_path, "John\n"); + + var ex = Assert.Throws(() => FixedWidth.ReadFile(_path, Person()).ToList()); + + Assert.Contains("the layout needs 25", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_StructModel_SetterMutatesResult() + { + var layout = FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in MutablePersonStruct p) => p.Name, (ref MutablePersonStruct p, string v) => p.Name = v) + .Column("Age", 10, 5, (in MutablePersonStruct p) => p.Age, (ref MutablePersonStruct p, int v) => p.Age = v) + .Column("Salary", 15, 10, (in MutablePersonStruct p) => p.Salary, (ref MutablePersonStruct p, double v) => p.Salary = v)); + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, layout, Ctx, out var p)); + + Assert.Equal(new MutablePersonStruct { Name = "John Doe", Age = 30, Salary = 6000000 }, p); + } + + [Fact] + public void Fluent_RefStruct_TryParseAndTryFormat() + { + var layout = FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in RefPersonModel p) => p.Name, (ref RefPersonModel p, string v) => p.Name = v) + .Column("Age", 10, 5, (in RefPersonModel p) => p.Age, (ref RefPersonModel p, int v) => p.Age = v) + .Column("Salary", 15, 10, (in RefPersonModel p) => p.Salary, (ref RefPersonModel p, double v) => p.Salary = v)); + Span buffer = stackalloc byte[25]; + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000 "u8, layout, Ctx, out var p)); + Assert.True(FixedWidth.TryFormat(in p, layout, buffer, Ctx, out int written)); + + Assert.Equal("John Doe 30 6000000 ", Encoding.UTF8.GetString(buffer[..written])); + } + + [Fact] + public void Fluent_WriteOnlyColumn_ReadThrowsAtCall() + { + var layout = FixedWidthLayout.Build(b => b + .Factory(() => new MutablePersonModel()) + .Column("Name", 0, 10, (in MutablePersonModel p) => p.Name, null)); + File.WriteAllText(_path, "John Doe \n"); + + var ex = Assert.Throws(() => FixedWidth.ReadFile(_path, layout)); + + Assert.Contains("without a setter", ex.Message, StringComparison.Ordinal); + Assert.Equal("John Doe \n", Write([new MutablePersonModel { Name = "John Doe" }], layout)); + } + + [Fact] + public void Fluent_ReadOnlyColumn_WriteThrows() + { + var layout = FixedWidthLayout.Build(b => b + .Factory(() => new MutablePersonModel()) + .Column("Name", 0, 10, null, (ref MutablePersonModel p, string v) => p.Name = v)); + using var stream = new MemoryStream(); + + var ex = Assert.Throws(() => FixedWidth.WriteMany(stream, [new MutablePersonModel()], layout)); + + Assert.Contains("without a getter", ex.Message, StringComparison.Ordinal); + Assert.True(FixedWidth.TryParse("John Doe "u8, layout, Ctx, out var p)); + Assert.Equal("John Doe", p.Name); + } + + [Fact] + public void Fluent_Overlap_NamesColumns() + { + var attributes = Assert.Throws(() => FixedWidthLayout.FromAttributes()); + + var fluent = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Left", 0, 5, (in OverlapReadModel m) => m.Left, null) + .Column("Right", 2, 5, (in OverlapReadModel m) => m.Right, null))); + + Assert.Equal(attributes.Message, fluent.Message); + } + + [Fact] + public void Fluent_NoColumns_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(_ => { })); + + Assert.Contains("has no columns", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_NegativeStart_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", -1, 10, (in MutablePersonStruct p) => p.Name, null))); + + Assert.Contains("negative Start", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_ZeroLength_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", 0, 0, (in MutablePersonStruct p) => p.Name, null))); + + Assert.Contains("invalid Length", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_NonAsciiPadding_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in MutablePersonStruct p) => p.Name, null, new FixedWidthColumnOptions { Padding = 'é' }))); + + Assert.Contains("Padding", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_TruncateOnNonString_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Age", 0, 5, (in MutablePersonStruct p) => p.Age, null, new FixedWidthColumnOptions { Overflow = OverflowBehavior.Truncate }))); + + Assert.Contains("Truncate", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Fluent_ClassWithoutFactory_Throws() + { + var ex = Assert.Throws(() => FixedWidthLayout.Build(b => b + .Column("Name", 0, 10, (in MutablePersonModel p) => p.Name, (ref MutablePersonModel p, string v) => p.Name = v))); + + Assert.Contains("Factory", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public async Task Fluent_AggregateFileAsync_MatchesSequential() + { + var text = new StringBuilder(); + for (int i = 0; i < 100_000; i++) + { + text.Append(("N" + (i % 10)).PadRight(10)).Append((i % 1000).ToString(Inv).PadRight(5)).Append(i.ToString(Inv).PadRight(10)).Append('\n'); + } + File.WriteAllText(_path, text.ToString()); + var layout = Person(); + + var parallel = await FixedWidth.AggregateFileAsync( + _path, layout, new FixedWidthReaderOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(FixedWidth.ReadFile(_path, layout).Select(p => p.Age), parallel.Values); + Assert.Equal(100_000, parallel.Values.Count); + } + + public sealed class Ages : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in MutablePersonModel model) + { + Values.Add(model.Age); + } + + public void Merge(Ages following) + { + Values.AddRange(following.Values); + } + } + + private static string Write(T[] models, FixedWidthLayout layout) + { + using var stream = new MemoryStream(); + FixedWidth.WriteMany(stream, models, layout, Lf); + return Encoding.UTF8.GetString(stream.ToArray()); + } + } +} diff --git a/tests/FixedWidthParser.Tests/GeneratedModels.cs b/tests/FixedWidthParser.Tests/GeneratedModels.cs index 080f67a..1c94964 100644 --- a/tests/FixedWidthParser.Tests/GeneratedModels.cs +++ b/tests/FixedWidthParser.Tests/GeneratedModels.cs @@ -1,15 +1,11 @@ -using FixedWidthParser.Attributes; - namespace FixedWidthParser.Tests { - // Models that opt into the source-generated parser. Each declares BOTH IFixedWidthModel - // (char) and IUtf8FixedWidthModel (byte), so the generator emits both TryParse overloads - // into the same partial — exercising marker coexistence. Each mirrors a reflection-based - // counterpart in TestModels.cs so the parity tests can compare the generated TryParse against the - // runtime FixedWidthParser / Utf8FixedWidthParser over the same lines. + // Models that opt into the source-generated parser and writer. Each mirrors a reflection-based + // counterpart in TestModels.cs so the parity tests can compare the generated TryParse/TryFormat + // against the same type's attribute layout (FixedWidthLayout.FromAttributes) over the same lines. /// Property-based model (string + int + double) — mirrors . - public readonly partial record struct GenPersonModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenPersonModel : IFixedWidthModel { [FixedColumn(0, 10)] public string Name { get; init; } [FixedColumn(10, 5)] public int Age { get; init; } @@ -17,141 +13,184 @@ namespace FixedWidthParser.Tests } /// Public-field model (exercises the field path) — mirrors . - public partial struct GenProductModel : IFixedWidthModel, IUtf8FixedWidthModel + public partial struct GenProductModel : IFixedWidthModel { [FixedColumn(0, 5)] public string Code; [FixedColumn(5, 4)] public int Quantity; } /// Float column — mirrors . - public readonly partial record struct GenMeasurementModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenMeasurementModel : IFixedWidthModel { [FixedColumn(0, 8)] public float Value { get; init; } } /// Trailing string column — mirrors . - public readonly partial record struct GenTrailingStringModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenTrailingStringModel : IFixedWidthModel { [FixedColumn(0, 5)] public int Id { get; init; } [FixedColumn(5, 10)] public string Note { get; init; } } - /// Decimal column (ISpanParsable / IUtf8SpanParsable path) — mirrors . - public readonly partial record struct GenDecimalModel : IFixedWidthModel, IUtf8FixedWidthModel + /// Decimal column (IUtf8SpanParsable path) — mirrors . + public readonly partial record struct GenDecimalModel : IFixedWidthModel { [FixedColumn(0, 12)] public decimal Amount { get; init; } } /// Single-column generated model — mirrors for reader tests. - public readonly partial record struct GenCodeModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenCodeModel : IFixedWidthModel { [FixedColumn(0, 3)] public string Code { get; init; } } /// Custom-converter column () — mirrors . - public readonly partial record struct GenCentsConverterModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenCentsConverterModel : IFixedWidthModel { [FixedColumn(0, 8, Converter = typeof(CentsConverter))] public CentsValue Amount { get; init; } } /// Nullable value-type columns (T?) — mirrors . - public readonly partial record struct GenNullableModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNullableModel : IFixedWidthModel { [FixedColumn(0, 5)] public int? Age { get; init; } [FixedColumn(5, 10)] public decimal? Amount { get; init; } } /// Nullable value-type column + converter — mirrors . - public readonly partial record struct GenNullableConverterModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNullableConverterModel : IFixedWidthModel { [FixedColumn(0, 8, Converter = typeof(CentsConverter))] public CentsValue? Amount { get; init; } } - /// Right-aligned integer — mirrors (write formatting options). - public readonly partial record struct GenRightAlignedModel : IFixedWidthModel, IUtf8FixedWidthModel + /// Right-aligned integer — mirrors . + public readonly partial record struct GenRightAlignedModel : IFixedWidthModel { [FixedColumn(0, 5, Alignment = Alignment.Right)] public int Value { get; init; } } /// Zero-padded right-aligned integer — mirrors . - public readonly partial record struct GenZeroPaddedModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenZeroPaddedModel : IFixedWidthModel { [FixedColumn(0, 5, Alignment = Alignment.Right, Padding = '0')] public int Value { get; init; } } /// Double with a fixed format string — mirrors . - public readonly partial record struct GenFormattedModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenFormattedModel : IFixedWidthModel { [FixedColumn(0, 8, Format = "F2")] public double Amount { get; init; } } /// Narrow numeric column; default overflow (throws) — mirrors . - public readonly partial record struct GenNarrowModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNarrowModel : IFixedWidthModel { [FixedColumn(0, 3)] public int Value { get; init; } } /// '*'-padded integer, read with a matching TrimChar — mirrors . - public readonly partial record struct GenAsteriskTrimIntModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenAsteriskTrimIntModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '*')] public int Value { get; init; } } /// '#'-padded string, read with a matching TrimChar — mirrors . - public readonly partial record struct GenHashTrimStringModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenHashTrimStringModel : IFixedWidthModel { [FixedColumn(0, 8, TrimChar = '#')] public string Code { get; init; } } - /// Non-ASCII TrimChar (U+00A0) — mirrors ; UTF-8 must throw. - public readonly partial record struct GenNonAsciiTrimModel : IFixedWidthModel, IUtf8FixedWidthModel - { - [FixedColumn(0, 5, TrimChar = ' ')] public int Value { get; init; } - } - -#if NET9_0_OR_GREATER /// Ref struct model — proves generated parsing works under `allows ref struct`. Mirrors . - public ref partial struct GenRefPersonModel : IFixedWidthModel, IUtf8FixedWidthModel + public ref partial struct GenRefPersonModel : IFixedWidthModel { [FixedColumn(0, 10)] public string Name { get; set; } [FixedColumn(10, 5)] public int Age { get; set; } [FixedColumn(15, 10)] public double Salary { get; set; } } -#endif - public readonly partial record struct GenDateTimeExactModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenDateTimeExactModel : IFixedWidthModel { [FixedColumn(0, 8, Format = "yyyyMMdd")] public DateTime Date { get; init; } [FixedColumn(8, 8, Format = "yyyyMMdd")] public DateOnly DateOnlyVal { get; init; } } - /// TimeOnly + DateTimeOffset exact columns — mirrors ; drives - /// the generated code's TryTimeOnlyExact / TryDateTimeOffsetExact runtime helpers (char and UTF-8). - public readonly partial record struct GenTimeExactModel : IFixedWidthModel, IUtf8FixedWidthModel + /// TimeOnly + DateTimeOffset exact columns — mirrors . + public readonly partial record struct GenTimeExactModel : IFixedWidthModel { [FixedColumn(0, 6, Format = "HHmmss")] public TimeOnly Time { get; init; } [FixedColumn(6, 20, Format = "yyyyMMddHHmmsszzz")] public DateTimeOffset Timestamp { get; init; } } - public readonly partial record struct GenZeroPaddedLeadingModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenZeroPaddedLeadingModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public int Value { get; init; } [FixedColumn(5, 5, TrimChar = '0', TrimMode = TrimMode.Leading)] public double DoubleValue { get; init; } } - public readonly partial record struct GenZeroPaddedBothModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenZeroPaddedBothModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '0', TrimMode = TrimMode.Both)] public int Value { get; init; } } - public readonly partial record struct GenSignPaddedModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenSignPaddedModel : IFixedWidthModel { [FixedColumn(0, 6, Alignment = Alignment.Right, Padding = '0')] public int Value { get; init; } } - public readonly partial record struct GenNullablePaddingModel : IFixedWidthModel, IUtf8FixedWidthModel + public readonly partial record struct GenNullablePaddingModel : IFixedWidthModel { [FixedColumn(0, 5, TrimChar = '0')] public int? Value { get; init; } [FixedColumn(5, 5, TrimChar = '*')] public double? DoubleValue { get; init; } } + + /// Character-only parsable column — mirrors . + public readonly partial record struct GenShoutModel : IFixedWidthModel + { + [FixedColumn(0, 6)] public Shout Word { get; init; } + } + + /// Latin-1 name and number — mirrors . + public readonly partial record struct GenNameAgeModel : IFixedWidthModel + { + [FixedColumn(0, 6)] public string Name { get; init; } + [FixedColumn(6, 3)] public int Age { get; init; } + } + + /// Currency column — mirrors . + public readonly partial record struct GenPriceModel : IFixedWidthModel + { + [FixedColumn(0, 8, Alignment = Alignment.Right, Format = "C")] public decimal Price { get; init; } + } + + /// A line longer than the single-record stack buffer. + public readonly partial record struct GenLongNameModel : IFixedWidthModel + { + [FixedColumn(0, 1200)] public string Name { get; init; } + } + + /// A line longer than a whole write batch. + public readonly partial record struct GenHugeNameModel : IFixedWidthModel + { + [FixedColumn(0, 70_000)] public string Name { get; init; } + } + + /// Base class of : a mapped property, a field, a virtual and a hidden column. + public class GenBaseModel + { + [FixedColumn(0, 4)] public int Id { get; set; } + [FixedColumn(4, 3)] public string Code = ""; + [FixedColumn(7, 5)] public virtual string Tag { get; set; } = ""; + [FixedColumn(12, 5)] public string Label { get; set; } = ""; + } + + /// Generated model whose columns are partly inherited; its own attribute layout is the reflection counterpart. + public sealed partial class GenDerivedModel : GenBaseModel, IFixedWidthModel + { + [FixedColumn(17, 6)] public string Name { get; set; } = ""; + + // Inherits [FixedColumn(7, 5)] from the property it overrides. + public override string Tag { get; set; } = ""; + + // Hides the base column, so only this narrower one is mapped. + [FixedColumn(12, 2)] public new string Label { get; set; } = ""; + } } diff --git a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs index ff012bf..6bd316a 100644 --- a/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs +++ b/tests/FixedWidthParser.Tests/GeneratedReaderTests.cs @@ -1,227 +1,140 @@ using System.Text; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; +using FixedWidthParser.Infrastructure; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { + /// + /// The read methods for generated models, sync and async, and their + /// parity with reading through . + /// public class GeneratedReaderTests { private const string TwoPeople = "John Doe 30 60000.00 \n" + "Jane 28 55000.00 "; - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Stream overloads – sync ----------------------------- - [Fact] public void Read_Stream_ParsesAll() { - using var stream = Utf8(TwoPeople); - - var people = FixedWidth.Read(stream, formatProvider: Inv).ToList(); + var people = FixedWidth.Read(StreamOf(TwoPeople)).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); + Assert.Equal(30, people[0].Age); Assert.Equal("Jane", people[1].Name); Assert.Equal(55000.00, people[1].Salary, 2); } [Fact] - public void Read_Stream_CustomEncoding_IsHonored() + public void Read_StructEnumerator_IteratesManually() { - using var stream = new MemoryStream(Encoding.Latin1.GetBytes("Áb \nCDé\n")); - - var codes = FixedWidth.Read(stream, encoding: Encoding.Latin1) - .Select(m => m.Code) - .ToList(); - - Assert.Equal(["Áb", "CDé"], codes); - } - - [Fact] - public void Read_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - using var stream = Utf8("ABC\nDEF\n"); - - _ = FixedWidth.Read(stream, leaveOpen: true).ToList(); + using var enumerator = FixedWidth.Read(StreamOf(TwoPeople)).GetEnumerator(); + var names = new List(); + while (enumerator.MoveNext()) + { + names.Add(enumerator.Current.Name); + } - Assert.True(stream.CanRead); + Assert.Equal(["John Doe", "Jane"], names); } - [Fact] - public void Read_Stream_LeaveOpenFalse_DisposesStream() + [Theory] + [InlineData("ABC\nDEF\n")] + [InlineData("ABC\r\nDEF\r\n")] + [InlineData("ABC\nDEF")] + public void Read_HandlesLineEndingsAndTrailingNewline(string text) { - var stream = Utf8("ABC\nDEF\n"); + var codes = FixedWidth.Read(StreamOf(text)).Select(m => m.Code).ToList(); - _ = FixedWidth.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); + Assert.Equal(["ABC", "DEF"], codes); } [Fact] - public void Read_Stream_NullStream_Throws() + public void Read_SkipsEmptyLinesAndBom() { - Assert.Throws(() => FixedWidth.Read((Stream)null!)); - } + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\n\n\nDEF\n"u8]; - // ----------------------------- Stream overloads – async ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - await using var stream = Utf8(TwoPeople); + var codes = FixedWidth.Read(new MemoryStream(bytes)).Select(m => m.Code).ToList(); - var people = await FixedWidth.ReadAsync(stream, formatProvider: Inv).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); + Assert.Equal(["ABC", "DEF"], codes); } [Fact] - public async Task ReadAsync_Stream_CustomEncoding_IsHonored() + public void Read_Latin1_DecodesStringsAndKeepsBytePositions() { - await using var stream = new MemoryStream(Encoding.Latin1.GetBytes("Áb \nCDé\n")); + // 'Á' and 'é' are one byte each in Latin-1, so the 3-byte column holds three characters. + var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; - var codes = await FixedWidth.ReadAsync(stream, encoding: Encoding.Latin1) + var codes = FixedWidth.Read(new MemoryStream(Encoding.Latin1.GetBytes("Áb \nCDé\n")), options: options) .Select(m => m.Code) - .ToListAsync().ConfigureAwait(true); + .ToList(); Assert.Equal(["Áb", "CDé"], codes); } [Fact] - public async Task ReadAsync_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidth.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_Stream_LeaveOpenFalse_DisposesStream() - { - var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidth.ReadAsync(stream).ToListAsync().ConfigureAwait(true); // default leaveOpen: false - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_Stream_NullStream_Throws() - { - Assert.Throws(() => FixedWidth.ReadAsync((Stream)null!)); - } - - // ----------------------------- TextReader overloads ----------------------------- - - [Fact] - public void Read_MultipleLines_MatchesReflectionReader() + public void Read_LineLongerThanBuffer_MatchesReflectionReader() { - var reflection = new FixedWidthReader(Inv); + var options = new FixedWidthReaderOptions { BufferSize = 4 }; - var expected = reflection.Read(new StringReader(TwoPeople)).ToList(); - var actual = FixedWidth.Read( - new StringReader(TwoPeople), - formatProvider: Inv).ToList(); + var expected = FixedWidth.Read(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes(), options: options).ToList(); + var actual = FixedWidth.Read(StreamOf(TwoPeople), options: options).ToList(); - Assert.Equal(expected.Count, actual.Count); Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[0].Age, actual[0].Age); - Assert.Equal(expected[0].Salary, actual[0].Salary, 2); Assert.Equal(expected[1].Name, actual[1].Name); - Assert.Equal(expected[1].Age, actual[1].Age); Assert.Equal(expected[1].Salary, actual[1].Salary, 2); } - [Theory] - [InlineData("ABC\nDEF\n")] - [InlineData("ABC\r\nDEF\r\n")] - [InlineData("ABC\nDEF")] - public void Read_HandlesLineEndingsAndTrailingNewline(string text) - { - var codes = FixedWidth.Read(new StringReader(text)) - .Select(m => m.Code) - .ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - [Fact] - public void Read_SkipsEmptyLines() + public void Read_InvalidLine_ThrowsSameErrorAsReflectionReader() { - var codes = FixedWidth.Read(new StringReader("ABC\n\n\nDEF\n")) - .Select(m => m.Code) - .ToList(); + const string text = "John Doe 30 60000.00 \nJane XX 55000.00 "; - Assert.Equal(["ABC", "DEF"], codes); + var expected = Assert.Throws(() => FixedWidth.Read(StreamOf(text), FixedWidthLayout.FromAttributes()).ToList()); + var actual = Assert.Throws(() => FixedWidth.Read(StreamOf(text)).ToList()); + + Assert.Equal(expected.LineNumber, actual.LineNumber); + Assert.Equal(expected.ColumnName, actual.ColumnName); + Assert.Equal(expected.RawValue, actual.RawValue); } [Fact] - public void Read_LineLongerThanBuffer_MatchesReflectionReader() + public void Read_LeaveOpen_ControlsDisposal() { - var reflection = new FixedWidthReader(Inv, bufferSize: 4); + using var kept = StreamOf("ABC\n"); + var owned = StreamOf("ABC\n"); - var expected = reflection.Read(new StringReader(TwoPeople)).ToList(); - var actual = FixedWidth.Read( - new StringReader(TwoPeople), - formatProvider: Inv, - bufferSize: 4).ToList(); + _ = FixedWidth.Read(kept, leaveOpen: true).ToList(); + _ = FixedWidth.Read(owned).ToList(); - Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[1].Name, actual[1].Name); - Assert.Equal(expected[1].Salary, actual[1].Salary, 2); + Assert.True(kept.CanRead); + Assert.False(owned.CanRead); } [Fact] - public void Read_InvalidLine_ThrowsWithSameLineNumber() + public void Read_NullArguments_Throw() { - var reflection = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; - - var expected = Assert.Throws( - () => reflection.Read(new StringReader(text)).ToList()); - var actual = Assert.Throws( - () => FixedWidth.Read(new StringReader(text), formatProvider: Inv).ToList()); - - Assert.Contains("Line 2", expected.Message, StringComparison.Ordinal); - Assert.Contains("Line 2", actual.Message, StringComparison.Ordinal); + Assert.Throws(() => FixedWidth.Read(null!)); + Assert.Throws(() => FixedWidth.ReadAsync((Stream)null!)); + Assert.Throws(() => FixedWidth.ReadFile("")); } [Fact] public void Read_WithStringPool_InternsRepeatedValues() { - var pool = new StringPool(); + var options = new FixedWidthReaderOptions { StringPool = new StringPool() }; - var codes = FixedWidth.Read( - new StringReader("ABC\nABC\n"), - stringPool: pool).ToList(); + var codes = FixedWidth.Read(StreamOf("ABC\nABC\n"), options: options).ToList(); - Assert.Equal("ABC", codes[0].Code); Assert.Same(codes[0].Code, codes[1].Code); } [Fact] public void GetEnumerator_IsStruct_ForAllocationFreeForeach() { - Assert.True(typeof(RecordEnumeratorCore, TextReaderSource>).IsValueType); - - int count = 0; - foreach (var _ in FixedWidth.Read(new StringReader("ABC\nDEF\n"))) - { - count++; - } - - Assert.Equal(2, count); + Assert.True(typeof(RecordEnumerator>).IsValueType); } [Fact] @@ -245,54 +158,41 @@ public void ReadFile_IsReEnumerable() } [Fact] - public async Task ReadAsync_MultipleLines_MatchesReflectionReader() + public async Task ReadAsync_MatchesReflectionReader() { - var reflection = new FixedWidthReader(Inv); - var expected = await reflection.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); - var actual = await FixedWidth.ReadAsync( - new StringReader(TwoPeople), - formatProvider: Inv).ToListAsync().ConfigureAwait(true); + var expected = await FixedWidth.ReadAsync(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes()).ToListAsync(TestContext.Current.CancellationToken); + var actual = await FixedWidth.ReadAsync(StreamOf(TwoPeople)).ToListAsync(TestContext.Current.CancellationToken); Assert.Equal(expected.Count, actual.Count); - Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[0].Age, actual[0].Age); Assert.Equal(expected[1].Name, actual[1].Name); Assert.Equal(expected[1].Salary, actual[1].Salary, 2); } [Fact] - public async Task ReadAsync_LineLongerThanBuffer_MatchesReflectionReader() + public async Task ReadAsync_LeaveOpen_ControlsDisposal() { - var reflection = new FixedWidthReader(Inv, bufferSize: 4); - var expected = await reflection.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); - var actual = await FixedWidth.ReadAsync( - new StringReader(TwoPeople), - formatProvider: Inv, - bufferSize: 4).ToListAsync().ConfigureAwait(true); + using var kept = StreamOf("ABC\n"); + var owned = StreamOf("ABC\n"); - Assert.Equal(expected[0].Name, actual[0].Name); - Assert.Equal(expected[1].Salary, actual[1].Salary, 2); + _ = await FixedWidth.ReadAsync(kept, leaveOpen: true).ToListAsync(TestContext.Current.CancellationToken); + _ = await FixedWidth.ReadAsync(owned).ToListAsync(TestContext.Current.CancellationToken); + + Assert.True(kept.CanRead); + Assert.False(owned.CanRead); } [Fact] - public async Task ReadAsync_InvalidLine_ThrowsWithSameLineNumber() + public async Task ReadAsync_CancelledToken_Throws() { - var reflection = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); - var expected = await Assert.ThrowsAsync(async () => + await Assert.ThrowsAnyAsync(async () => { - using StringReader reader = new(text); - await foreach (var _ in reflection.ReadAsync(reader).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - var actual = await Assert.ThrowsAsync(async () => - { - using StringReader reader = new(text); - await foreach (var _ in FixedWidth.ReadAsync(reader, formatProvider: Inv).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - - Assert.Contains("Line 2", expected.Message, StringComparison.Ordinal); - Assert.Contains("Line 2", actual.Message, StringComparison.Ordinal); + await foreach (var _ in FixedWidth.ReadAsync(StreamOf("ABC\nDEF\n")).WithCancellation(cts.Token)) + { + } + }); } [Fact] @@ -301,9 +201,9 @@ public async Task ReadFileAsync_MatchesSyncReadFile() string path = Path.GetTempFileName(); try { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); + await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n", TestContext.Current.CancellationToken); - var asyncCodes = await FixedWidth.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); + var asyncCodes = await FixedWidth.ReadFileAsync(path).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); var syncCodes = FixedWidth.ReadFile(path).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); diff --git a/tests/FixedWidthParser.Tests/GeneratedUtf8ReaderTests.cs b/tests/FixedWidthParser.Tests/GeneratedUtf8ReaderTests.cs deleted file mode 100644 index 28ae2fe..0000000 --- a/tests/FixedWidthParser.Tests/GeneratedUtf8ReaderTests.cs +++ /dev/null @@ -1,328 +0,0 @@ -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the source-generated UTF-8 streaming path via (sync and - /// async): raw-byte reading with no transcode, line-ending handling, BOM skipping, buffer growth, - /// error reporting, leaveOpen behavior, struct-enumerator allocation freedom and cancellation. - /// Mirrors but exercises the generated TryParse strategy - /// instead of the reflection parser. - /// - public class GeneratedUtf8ReaderTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Synchronous ----------------------------- - - - [Fact] - public void Read_Stream_ParsesAll() - { - using var stream = Utf8(TwoPeople); - - var people = FixedWidthUtf8.Read(stream, formatProvider: Inv).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_StructEnumerator_IteratesManually() - { - // Exercises the public struct core returned by the generated UTF-8 enumerable directly - // (GetEnumerator / MoveNext / Current / Dispose) rather than through foreach. - using var stream = Utf8(TwoPeople); - var enumerable = FixedWidthUtf8.Read(stream, formatProvider: Inv); - - using var enumerator = enumerable.GetEnumerator(); - var names = new List(); - while (enumerator.MoveNext()) - { - names.Add(enumerator.Current.Name); - } - - Assert.Equal(["John Doe", "Jane"], names); - } - - [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line - public void Read_HandlesLineEndingsAndTrailingNewline(string text) - { - using var stream = Utf8(text); - - var codes = FixedWidthUtf8.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsEmptyLines() - { - using var stream = Utf8("ABC\n\n\nDEF\n"); - - var codes = FixedWidthUtf8.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_SkipsLeadingUtf8Bom() - { - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - using var stream = new MemoryStream(bytes); - - var codes = FixedWidthUtf8.Read(stream).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public void Read_LineLongerThanBuffer_StillParses() - { - // A tiny bufferSize forces buffer compaction and growth (25-byte lines). - using var stream = Utf8(TwoPeople); - - var people = FixedWidthUtf8.Read(stream, formatProvider: Inv, bufferSize: 4).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_InvalidLine_ThrowsWithLineNumber() - { - using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = Assert.Throws(() => FixedWidthUtf8.Read(stream, formatProvider: Inv).ToList()); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public void Read_LeaveOpenTrue_KeepsStreamOpen() - { - using var stream = Utf8("ABC\nDEF\n"); - - _ = FixedWidthUtf8.Read(stream, leaveOpen: true).ToList(); - - Assert.True(stream.CanRead); - } - - [Fact] - public void Read_LeaveOpenFalse_DisposesStream() - { - var stream = Utf8("ABC\nDEF\n"); - - _ = FixedWidthUtf8.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); - } - - [Fact] - public void Read_NullStream_Throws() - { - Assert.Throws(() => FixedWidthUtf8.Read(null!)); - } - - [Fact] - public void GetEnumerator_IsStruct_ForAllocationFreeForeach() - { - Assert.True(typeof(RecordEnumeratorCore, StreamSource>).IsValueType); - - using var stream = Utf8("ABC\nDEF\n"); - int count = 0; - foreach (var _ in FixedWidthUtf8.Read(stream)) - { - count++; - } - - - Assert.Equal(2, count); - } - - [Fact] - public void ReadFile_IsReEnumerable() - { - string path = Path.GetTempFileName(); - try - { - File.WriteAllText(path, "ABC\nDEF\n"); - - var first = FixedWidthUtf8.ReadFile(path).Select(m => m.Code).ToList(); - var second = FixedWidthUtf8.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], first); - Assert.Equal(first, second); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- Asynchronous ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - await using var stream = Utf8(TwoPeople); - - var people = await FixedWidthUtf8.ReadAsync(stream, formatProvider: Inv).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_LineLongerThanBuffer_StillParses() - { - await using var stream = Utf8(TwoPeople); - - var people = await FixedWidthUtf8.ReadAsync(stream, formatProvider: Inv, bufferSize: 4).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_SkipsLeadingUtf8Bom() - { - byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; - await using var stream = new MemoryStream(bytes); - - var codes = await FixedWidthUtf8.ReadAsync(stream).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_InvalidLine_ThrowsWithLineNumber() - { - await using var stream = Utf8("John Doe 30 60000.00 \nJane XX 55000.00 "); - - var ex = await Assert.ThrowsAsync(async () => - { - await foreach (var _ in FixedWidthUtf8.ReadAsync(stream, formatProvider: Inv).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); - } - - [Fact] - public async Task ReadAsync_CancelledToken_Throws() - { - await using var stream = Utf8("ABC\nDEF\n"); - using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); - - await Assert.ThrowsAnyAsync(async () => - { - await foreach (var _ in FixedWidthUtf8.ReadAsync(stream).WithCancellation(cts.Token).ConfigureAwait(false)) { } - }).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_LeaveOpenTrue_KeepsStreamOpen() - { - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidthUtf8.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_LeaveOpenFalse_DisposesStream() - { - var stream = Utf8("ABC\nDEF\n"); - - _ = await FixedWidthUtf8.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_NullStream_Throws() - { - Assert.Throws(() => FixedWidthUtf8.ReadAsync((Stream)null!)); - } - - [Fact] - public async Task ReadFileAsync_MatchesSyncReadFile() - { - string path = Path.GetTempFileName(); - try - { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); - - var asyncCodes = await FixedWidthUtf8.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - var syncCodes = FixedWidthUtf8.ReadFile(path).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); - Assert.Equal(syncCodes, asyncCodes); - } - finally - { - File.Delete(path); - } - } - - // ----------------------------- StringPool ----------------------------- - - [Fact] - public void Read_Stream_WithStringPool_InternsRepeatedValues() - { - var pool = new StringPool(); - using var stream = Utf8("ABC\nABC\n"); - - var codes = FixedWidthUtf8.Read(stream, stringPool: pool).ToList(); - - Assert.Equal("ABC", codes[0].Code); - Assert.Same(codes[0].Code, codes[1].Code); - } - - // ----------------------------- generated/reflection parity ----------------------------- - - [Fact] - public void GeneratedReader_MatchesReflectionByteReader_ForAsciiStream() - { - var reflection = new FixedWidthByteReader(Inv); - - using var reflectionStream = Utf8(TwoPeople); - var fromReflection = reflection.Read(reflectionStream).ToList(); - using var generatedStream = Utf8(TwoPeople); - var fromGenerated = FixedWidthUtf8.Read(generatedStream, formatProvider: Inv).ToList(); - - Assert.Equal(fromReflection.Count, fromGenerated.Count); - for (int i = 0; i < fromReflection.Count; i++) - { - Assert.Equal(fromReflection[i].Name, fromGenerated[i].Name); - Assert.Equal(fromReflection[i].Age, fromGenerated[i].Age); - Assert.Equal(fromReflection[i].Salary, fromGenerated[i].Salary, 2); - } - } - } -} diff --git a/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs b/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs index 1bf877f..9d2b826 100644 --- a/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs +++ b/tests/FixedWidthParser.Tests/GeneratedWriterTests.cs @@ -1,23 +1,19 @@ -using FixedWidthParser.Writers; +using System.Globalization; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Covers the source-generated TryFormat (via ): - /// parity against the reflection-based , round-tripping - /// through the generated TryParse, formatting options (alignment/padding/format/overflow), - /// nullable columns, custom converters, culture, and the destination-too-small case. + /// The source-generated TryFormat (via ): parity + /// with the attribute layout (), round-tripping through the generated TryParse, + /// formatting options, nullable columns, converters, culture, and a destination that is too small. /// public class GeneratedWriterTests { private static string WriteGenerated(in TModel model, IFormatProvider? formatProvider = null) where TModel : IFixedWidthModel { - Span buffer = stackalloc char[256]; - bool ok = FixedWidth.TryFormat(in model, buffer, formatProvider ?? Inv, out int written); - Assert.True(ok); - return new string(buffer[..written]); + return Format(in model, CtxWith(formatProvider)); } [Theory] @@ -26,14 +22,11 @@ private static string WriteGenerated(in TModel model, IFormatProvider? f [InlineData("Max", -5, -1234.56)] public void Person_MatchesReflection(string name, int age, double salary) { - var writer = new FixedWidthWriter(); - var reflectionModel = new PersonModel { Name = name, Age = age, Salary = salary }; - var generatedModel = new GenPersonModel { Name = name, Age = age, Salary = salary }; + var layout = FixedWidthLayout.FromAttributes(); - string reflectionLine = WriteOne(writer, reflectionModel); - string generatedLine = WriteGenerated(generatedModel); - - Assert.Equal(reflectionLine, generatedLine); + Assert.Equal( + WriteOne(layout, new PersonModel { Name = name, Age = age, Salary = salary }), + WriteGenerated(new GenPersonModel { Name = name, Age = age, Salary = salary })); } [Fact] @@ -42,9 +35,8 @@ public void Person_RoundTripsThroughGeneratedParse() var original = new GenPersonModel { Name = "Bob", Age = 42, Salary = 999.5 }; string line = WriteGenerated(original); - bool ok = FixedWidth.TryParse(line, Inv, null, out var parsed); - Assert.True(ok); + Assert.True(FixedWidth.TryParse(Bytes(line), Ctx, out var parsed)); Assert.Equal(original.Name, parsed.Name); Assert.Equal(original.Age, parsed.Age); Assert.Equal(original.Salary, parsed.Salary, 2); @@ -53,152 +45,128 @@ public void Person_RoundTripsThroughGeneratedParse() [Fact] public void FieldModel_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new ProductModel { Code = "ABCDE", Quantity = 1234 }; - var generatedModel = new GenProductModel { Code = "ABCDE", Quantity = 1234 }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(FixedWidthLayout.FromAttributes(), new ProductModel { Code = "ABCDE", Quantity = 1234 }), + WriteGenerated(new GenProductModel { Code = "ABCDE", Quantity = 1234 })); } [Fact] public void FloatColumn_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new MeasurementModel { Value = 3.14f }; - var generatedModel = new GenMeasurementModel { Value = 3.14f }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(FixedWidthLayout.FromAttributes(), new MeasurementModel { Value = 3.14f }), + WriteGenerated(new GenMeasurementModel { Value = 3.14f })); } [Fact] public void DecimalColumn_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new DecimalModel { Amount = 1234.56m }; - var generatedModel = new GenDecimalModel { Amount = 1234.56m }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(FixedWidthLayout.FromAttributes(), new DecimalModel { Amount = 1234.56m }), + WriteGenerated(new GenDecimalModel { Amount = 1234.56m })); } [Fact] public void RightAligned_MatchesReflection() { - var writer = new FixedWidthWriter(); - Assert.Equal( - WriteOne(writer, new RightAlignedModel { Value = 30 }), + WriteOne(FixedWidthLayout.FromAttributes(), new RightAlignedModel { Value = 30 }), WriteGenerated(new GenRightAlignedModel { Value = 30 })); } [Fact] public void ZeroPadded_MatchesReflection() { - var writer = new FixedWidthWriter(); - Assert.Equal( - WriteOne(writer, new ZeroPaddedModel { Value = 30 }), + WriteOne(FixedWidthLayout.FromAttributes(), new ZeroPaddedModel { Value = 30 }), WriteGenerated(new GenZeroPaddedModel { Value = 30 })); } [Fact] public void FormatString_MatchesReflection() { - var writer = new FixedWidthWriter(); - Assert.Equal( - WriteOne(writer, new FormattedModel { Amount = 1234.5 }), + WriteOne(FixedWidthLayout.FromAttributes(), new FormattedModel { Amount = 1234.5 }), WriteGenerated(new GenFormattedModel { Amount = 1234.5 })); } [Fact] public void Overflow_Throws_MatchesReflection() { - var writer = new FixedWidthWriter(); - - Assert.Throws(() => WriteOne(writer, new NarrowModel { Value = 12345 })); + Assert.Throws(() => WriteOne(FixedWidthLayout.FromAttributes(), new NarrowModel { Value = 12345 })); - var buffer = new char[16]; + var buffer = new byte[16]; var generatedModel = new GenNarrowModel { Value = 12345 }; - Assert.Throws(() => FixedWidth.TryFormat(in generatedModel, buffer, Inv, out _)); + Assert.Throws(() => FixedWidth.TryFormat(in generatedModel, buffer, Ctx, out _)); } [Fact] - public void DestinationTooSmall_ReturnsFalseWithZeroCharsWritten() + public void DestinationTooSmall_ReturnsFalseWithZeroBytesWritten() { var model = new GenPersonModel { Name = "Bob", Age = 1, Salary = 1 }; - Span tooSmall = stackalloc char[10]; + Span tooSmall = stackalloc byte[10]; - bool ok = FixedWidth.TryFormat(in model, tooSmall, Inv, out int charsWritten); + bool ok = FixedWidth.TryFormat(in model, tooSmall, Ctx, out int bytesWritten); Assert.False(ok); - Assert.Equal(0, charsWritten); + Assert.Equal(0, bytesWritten); } - // ----- Nullable columns ----- - [Fact] public void Nullable_Null_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new NullableModel { Age = null, Amount = null }; - var generatedModel = new GenNullableModel { Age = null, Amount = null }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(FixedWidthLayout.FromAttributes(), new NullableModel { Age = null, Amount = null }), + WriteGenerated(new GenNullableModel { Age = null, Amount = null })); } [Fact] public void Nullable_Value_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new NullableModel { Age = 42, Amount = 123.45m }; - var generatedModel = new GenNullableModel { Age = 42, Amount = 123.45m }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(FixedWidthLayout.FromAttributes(), new NullableModel { Age = 42, Amount = 123.45m }), + WriteGenerated(new GenNullableModel { Age = 42, Amount = 123.45m })); } - // ----- Custom converter ----- - [Fact] public void Converter_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new CentsConverterModel { Amount = new CentsValue(12345) }; - var generatedModel = new GenCentsConverterModel { Amount = new CentsValue(12345) }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(FixedWidthLayout.FromAttributes(), new CentsConverterModel { Amount = new CentsValue(12345) }), + WriteGenerated(new GenCentsConverterModel { Amount = new CentsValue(12345) })); } [Fact] - public void NullableConverter_Null_MatchesReflection() + public void NullableConverter_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new NullableConverterModel { Amount = null }; - var generatedModel = new GenNullableConverterModel { Amount = null }; + var layout = FixedWidthLayout.FromAttributes(); - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(layout, new NullableConverterModel { Amount = null }), + WriteGenerated(new GenNullableConverterModel { Amount = null })); + Assert.Equal( + WriteOne(layout, new NullableConverterModel { Amount = new CentsValue(999) }), + WriteGenerated(new GenNullableConverterModel { Amount = new CentsValue(999) })); } [Fact] - public void NullableConverter_Value_MatchesReflection() + public void DecodedParsable_MatchesReflection() { - var writer = new FixedWidthWriter(); - var reflectionModel = new NullableConverterModel { Amount = new CentsValue(999) }; - var generatedModel = new GenNullableConverterModel { Amount = new CentsValue(999) }; - - Assert.Equal(WriteOne(writer, reflectionModel), WriteGenerated(generatedModel)); + Assert.Equal( + WriteOne(FixedWidthLayout.FromAttributes(), new ShoutModel { Word = new Shout("HEY") }), + WriteGenerated(new GenShoutModel { Word = new Shout("HEY") })); } - // ----- Culture ----- - [Fact] public void Culture_HonorsDecimalSeparator_MatchesReflection() { - var deDe = System.Globalization.CultureInfo.GetCultureInfo("de-DE"); - var writer = new FixedWidthWriter(); - var reflectionModel = new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; - var generatedModel = new GenPersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }; + var deDe = CultureInfo.GetCultureInfo("de-DE"); + var writerOptions = new FixedWidthWriterOptions { FormatProvider = deDe }; + var layout = FixedWidthLayout.FromAttributes(); - Assert.Equal(WriteOne(writer, reflectionModel, deDe), WriteGenerated(generatedModel, deDe)); + Assert.Equal( + WriteOne(layout, new PersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, writerOptions), + WriteGenerated(new GenPersonModel { Name = "Bob", Age = 30, Salary = 1234.5 }, deDe)); } } } diff --git a/tests/FixedWidthParser.Tests/LayoutReadTests.cs b/tests/FixedWidthParser.Tests/LayoutReadTests.cs new file mode 100644 index 0000000..ac60137 --- /dev/null +++ b/tests/FixedWidthParser.Tests/LayoutReadTests.cs @@ -0,0 +1,90 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The read and aggregate overloads that take a layout. + public sealed class LayoutReadTests : IDisposable + { + private const string People = "John Doe 30 6000000.00\nJane 28 55000.00 \nAnn 7 1.5 \n"; + + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + [Fact] + public void ReadFile_WithLayout_YieldsSameRecordsAsReader() + { + File.WriteAllText(_path, People); + + var records = FixedWidth.ReadFile(_path, FixedWidthLayout.FromAttributes()) + .Select(p => (p.Name, p.Age, p.Salary)) + .ToList(); + + Assert.Equal([("John Doe", 30, 6000000.00), ("Jane", 28, 55000.00), ("Ann", 7, 1.5)], records); + } + + [Fact] + public async Task ReadAsync_WithLayout_ParseError_ReportsLineAndColumn() + { + using var stream = StreamOf("John Doe 30 6000000.00\nJane abc 55000.00 \n"); + + var ex = await Assert.ThrowsAsync(async () => + { + await foreach (var _ in FixedWidth.ReadAsync(stream, FixedWidthLayout.FromAttributes())) + { + } + }); + + Assert.Equal(2, ex.LineNumber); + Assert.Equal("Age", ex.ColumnName); + Assert.Equal(10, ex.ColumnStart); + Assert.Equal(5, ex.ColumnLength); + } + + [Fact] + public async Task AggregateFileAsync_WithLayout_MatchesSequentialFold() + { + File.WriteAllText(_path, NameAges(200_000)); + var layout = FixedWidthLayout.FromAttributes(); + + var parallel = await FixedWidth.AggregateFileAsync( + _path, layout, new FixedWidthReaderOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(SequentialAges(layout), parallel.Values); + } + + [Fact] + public async Task AggregateAsync_WithLayout_MatchesSequentialFold() + { + File.WriteAllText(_path, NameAges(200_000)); + var layout = FixedWidthLayout.FromAttributes(); + using var stream = new MemoryStream(File.ReadAllBytes(_path)); + + var parallel = await FixedWidth.AggregateAsync( + stream, layout, options: new FixedWidthReaderOptions { DegreeOfParallelism = 4 }, cancellationToken: TestContext.Current.CancellationToken); + + Assert.Equal(SequentialAges(layout), parallel.Values); + } + + private List SequentialAges(FixedWidthLayout layout) + { + var ages = FixedWidth.ReadFile(_path, layout).Select(m => m.Age).ToList(); + Assert.Equal(200_000, ages.Count); + return ages; + } + + private static string NameAges(int lines) + { + var text = new StringBuilder(); + for (int i = 1; i <= lines; i++) + { + text.Append("Name").Append(i % 10).Append(' ').Append((i % 1000).ToString("D3", Inv)).Append('\n'); + } + return text.ToString(); + } + } +} diff --git a/tests/FixedWidthParser.Tests/LayoutTests.cs b/tests/FixedWidthParser.Tests/LayoutTests.cs new file mode 100644 index 0000000..64038a0 --- /dev/null +++ b/tests/FixedWidthParser.Tests/LayoutTests.cs @@ -0,0 +1,55 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// A layout built from attributes parses and formats through the overloads. + public class LayoutTests + { + [Fact] + public void FromAttributes_TryParse_MatchesParser() + { + var layout = FixedWidthLayout.FromAttributes(); + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, layout, Ctx, out var p)); + Assert.Equal(("John Doe", 30, 6000000.00), (p.Name, p.Age, p.Salary)); + } + + [Fact] + public void FromAttributes_TryFormat_WritesLine() + { + var layout = FixedWidthLayout.FromAttributes(); + Span buffer = stackalloc byte[25]; + + Assert.True(FixedWidth.TryFormat(new PersonModel { Name = "Ann", Age = 7, Salary = 1.5 }, layout, buffer, Ctx, out int written)); + Assert.Equal("Ann 7 1.5 ", Encoding.UTF8.GetString(buffer[..written])); + Assert.False(FixedWidth.TryFormat(new PersonModel(), layout, stackalloc byte[24], Ctx, out _)); + } + + [Fact] + public void FromAttributes_IsCachedPerType() + { + Assert.Same(FixedWidthLayout.FromAttributes(), FixedWidthLayout.FromAttributes()); + } + + [Fact] + public void FromAttributes_InvalidLayout_ThrowsEveryCall() + { + for (int i = 0; i < 2; i++) + { + var ex = Assert.Throws(() => FixedWidthLayout.FromAttributes()); + Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public void FromAttributes_RefStruct_ParsesButCannotFormat() + { + var layout = FixedWidthLayout.FromAttributes(); + + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, layout, Ctx, out var r)); + Assert.Equal(30, r.Age); + Assert.Throws(() => FixedWidth.TryFormat(new RefPersonModel(), layout, new byte[25], Ctx, out _)); + } + } +} diff --git a/tests/FixedWidthParser.Tests/LayoutWriteTests.cs b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs new file mode 100644 index 0000000..73d85a3 --- /dev/null +++ b/tests/FixedWidthParser.Tests/LayoutWriteTests.cs @@ -0,0 +1,156 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The write overloads, for generated models and for layouts. + public class LayoutWriteTests + { + private static readonly FixedWidthWriterOptions Lf = new() { NewLine = "\n" }; + + private static readonly GenPersonModel[] GenPeople = + [ + new() { Name = "John Doe", Age = 30, Salary = 6000000 }, + new() { Name = "Jane", Age = 28, Salary = 55000 }, + new() { Name = "Ann", Age = 7, Salary = 1.5 }, + ]; + + private static readonly PersonModel[] People = + [ + new() { Name = "John Doe", Age = 30, Salary = 6000000 }, + new() { Name = "Jane", Age = 28, Salary = 55000 }, + new() { Name = "Ann", Age = 7, Salary = 1.5 }, + ]; + + [Fact] + public void Generated_WriteMany_MatchesTryFormatPlusNewLine() + { + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, GenPeople, Lf); + + Assert.Equal(string.Concat(GenPeople.Select(p => Format(p) + "\n")), Decode(stream)); + } + + [Fact] + public void Generated_WriteMany_CrossesBatchBoundary() + { + int count = (3 * 64 * 1024 / 26) + 17; + var models = Enumerable.Range(0, count).Select(i => new GenPersonModel { Name = "N" + i, Age = i % 1000, Salary = i }).ToList(); + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, models, Lf); + + byte[] bytes = stream.ToArray(); + Assert.Equal(count * 26, bytes.Length); + for (int i = 0; i < count; i++) + { + Assert.True(FixedWidth.TryParse(bytes.AsSpan(i * 26, 25), Ctx, out var parsed)); + Assert.Equal(models[i], parsed); + } + } + + [Fact] + public void Generated_Write_RecordLongerThanStackBuffer() + { + using var stream = new MemoryStream(); + + FixedWidth.Write(stream, new GenLongNameModel { Name = "Wide" }, Lf); + + Assert.Equal("Wide".PadRight(1200) + "\n", Decode(stream)); + } + + [Fact] + public void Generated_WriteMany_RecordLongerThanBatch() + { + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, [new GenHugeNameModel { Name = "A" }, new GenHugeNameModel { Name = "B" }], Lf); + + Assert.Equal("A".PadRight(70_000) + "\n" + "B".PadRight(70_000) + "\n", Decode(stream)); + } + + [Fact] + public async Task LongRecords_AsyncTwinsMatchSync() + { + GenHugeNameModel[] huge = [new() { Name = "A" }, new() { Name = "B" }]; + var hugeLayout = FixedWidthLayout.Build(b => b.Column("Name", 0, 70_000, (in GenHugeNameModel m) => m.Name, null)); + var ct = TestContext.Current.CancellationToken; + using var genSync = new MemoryStream(); + await using var genAsync = new MemoryStream(); + using var layoutSync = new MemoryStream(); + await using var layoutAsync = new MemoryStream(); + using var oneSync = new MemoryStream(); + await using var oneAsync = new MemoryStream(); + + FixedWidth.WriteMany(genSync, huge, Lf); + await FixedWidth.WriteManyAsync(genAsync, huge, Lf, ct); + FixedWidth.WriteMany(layoutSync, huge, hugeLayout, Lf); + await FixedWidth.WriteManyAsync(layoutAsync, huge, hugeLayout, Lf, ct); + FixedWidth.Write(oneSync, new GenLongNameModel { Name = "Wide" }, Lf); + await FixedWidth.WriteAsync(oneAsync, new GenLongNameModel { Name = "Wide" }, Lf, ct); + + string expected = "A".PadRight(70_000) + "\n" + "B".PadRight(70_000) + "\n"; + Assert.Equal(expected, Decode(genSync)); + Assert.Equal(expected, Decode(genAsync)); + Assert.Equal(expected, Decode(layoutSync)); + Assert.Equal(expected, Decode(layoutAsync)); + Assert.Equal(Decode(oneSync), Decode(oneAsync)); + } + + [Fact] + public void Layout_WriteMany_MatchesWriter() + { + using var stream = new MemoryStream(); + + FixedWidth.WriteMany(stream, People, FixedWidthLayout.FromAttributes(), Lf); + + Assert.Equal(WriteMany(FixedWidthLayout.FromAttributes(), (IEnumerable)People, Lf), Decode(stream)); + } + + [Fact] + public async Task WriteManyAsync_MatchesWriteMany() + { + var layout = FixedWidthLayout.FromAttributes(); + using var sync = new MemoryStream(); + await using var async = new MemoryStream(); + using var genSync = new MemoryStream(); + await using var genAsync = new MemoryStream(); + + FixedWidth.WriteMany(sync, People, layout, Lf); + await FixedWidth.WriteManyAsync(async, People, layout, Lf, TestContext.Current.CancellationToken); + FixedWidth.WriteMany(genSync, GenPeople, Lf); + await FixedWidth.WriteManyAsync(genAsync, GenPeople, Lf, TestContext.Current.CancellationToken); + + Assert.Equal(Decode(sync), Decode(async)); + Assert.Equal(Decode(genSync), Decode(genAsync)); + Assert.Equal(Decode(sync), Decode(genSync)); + } + + [Fact] + public async Task WriteAsync_MatchesWrite() + { + using var sync = new MemoryStream(); + await using var async = new MemoryStream(); + + FixedWidth.Write(sync, GenPeople[0], Lf); + await FixedWidth.WriteAsync(async, GenPeople[0], Lf, TestContext.Current.CancellationToken); + + Assert.Equal(Format(GenPeople[0]) + "\n", Decode(sync)); + Assert.Equal(Decode(sync), Decode(async)); + } + + [Fact] + public void Write_NullNewLine_ThrowsArgumentException() + { + using var stream = new MemoryStream(); + + Assert.Throws(() => FixedWidth.Write(stream, GenPeople[0], new FixedWidthWriterOptions { NewLine = null! })); + } + + private static string Decode(MemoryStream stream) + { + return Encoding.UTF8.GetString(stream.ToArray()); + } + } +} diff --git a/tests/FixedWidthParser.Tests/LineSinkTests.cs b/tests/FixedWidthParser.Tests/LineSinkTests.cs new file mode 100644 index 0000000..a7c8639 --- /dev/null +++ b/tests/FixedWidthParser.Tests/LineSinkTests.cs @@ -0,0 +1,101 @@ +using System.Text; +using FixedWidthParser.Readers; + +namespace FixedWidthParser.Tests +{ + /// The internal parallel read over a line sink and a random-access source, used by the native layer. + public sealed class LineSinkTests : IDisposable + { + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + private struct Lines : ILineSink + { + private List? _lines; + + internal readonly List Values => _lines ?? []; + + public bool TryAdd(ReadOnlySpan line, in FixedWidthContext context) + { + if (line.SequenceEqual("XX"u8)) + { + return false; + } + (_lines ??= []).Add(Encoding.UTF8.GetString(line)); + return true; + } + + public readonly FixedWidthParseException Diagnose(ReadOnlySpan line, long lineNumber, in FixedWidthContext context) + { + return new FixedWidthParseException("rejected", lineNumber, columnName: null, columnStart: -1, columnLength: 0, rawValue: null); + } + + public void Merge(Lines following) + { + (_lines ??= []).AddRange(following.Values); + } + } + + private sealed class ArraySource(byte[] bytes) : IRandomAccessSource + { + public long Length => bytes.Length; + + public int Read(Span destination, long offset) + { + var rest = bytes.AsSpan((int)Math.Min(offset, bytes.Length)); + int count = Math.Min(rest.Length, destination.Length); + rest[..count].CopyTo(destination); + return count; + } + } + + private static FixedWidthReaderOptions Options(int chunkSize) + { + return FixedWidthReaderOptions.Validate(new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(64)] + public async Task RunAsync_FromDataStart_SkipsEarlierBytes(int chunkSize) + { + var source = new ArraySource("HDR\nA1\nA2\n"u8.ToArray()); + + var lines = await ParallelAggregator.RunAsync(source, dataStart: 4, (_, _) => new Lines(), Options(chunkSize), CancellationToken.None); + + Assert.Equal(["A1", "A2"], lines.Values); + } + + [Theory] + [InlineData(1)] + [InlineData(64)] + public async Task RunAsync_ErrorLineNumber_CountsFromSourceStart(int chunkSize) + { + var source = new ArraySource("HDR\nA1\nXX\nA3\n"u8.ToArray()); + + var ex = await Assert.ThrowsAsync(() => + ParallelAggregator.RunAsync(source, dataStart: 4, (_, _) => new Lines(), Options(chunkSize), CancellationToken.None)); + + Assert.Equal(3, ex.LineNumber); + } + + [Fact] + public async Task RunAsync_MemorySource_EqualsFileSource() + { + byte[] content = "A1\r\nA2\n\nA3"u8.ToArray(); + await File.WriteAllBytesAsync(_path, content); + using var handle = File.OpenHandle(_path); + + var fromFile = await ParallelAggregator.RunAsync(new FileSource(handle), 0, (_, _) => new Lines(), Options(3), CancellationToken.None); + var fromMemory = await ParallelAggregator.RunAsync(new ArraySource(content), 0, (_, _) => new Lines(), Options(3), CancellationToken.None); + + Assert.Equal(["A1", "A2", "A3"], fromFile.Values); + Assert.Equal(fromFile.Values, fromMemory.Values); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs b/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs new file mode 100644 index 0000000..46381bf --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeApiTests.cs @@ -0,0 +1,100 @@ +using System.Text; +using FixedWidthParser.Native; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeApiTests + { + private static string LastError() + { + Span buffer = stackalloc byte[512]; + Assert.Equal(NativeStatus.Ok, NativeApi.LastError(buffer, out int length)); + return Encoding.UTF8.GetString(buffer[..length]); + } + + [Fact] + public void LastError_IsPerThread() + { + NativeApi.SetLastError("on this thread"); + + string? other = null; + var thread = new Thread(() => other = LastError()); + thread.Start(); + thread.Join(); + + Assert.Equal("", other); + Assert.Equal("on this thread", LastError()); + } + + [Fact] + public void LastError_BufferTooSmall_ReportsLength() + { + NativeApi.SetLastError("twelve bytes"); + + Assert.Equal(NativeStatus.BufferTooSmall, NativeApi.LastError(new byte[4], out int length)); + Assert.Equal(12, length); + } + + [Fact] + public void ParseError_CarriesLineAndColumn() + { + var exception = new FixedWidthParseException("bad", 7, "Age", 10, 5, "XX"); + + NativeApi.SetParseError(exception, column: 2); + + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(7, line); + Assert.Equal(2, column); + Assert.Equal("bad", LastError()); + } + + [Fact] + public void SetLastError_ClearsTheParseError() + { + NativeApi.SetParseError(new FixedWidthParseException("bad", 7, null, -1, 0, null), column: -1); + + NativeApi.SetLastError("other"); + + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(0, line); + Assert.Equal(-1, column); + } + + [Fact] + public void HandleTable_IdIsNeverReissued() + { + nint first = NativeHandleTable.Register(new object()); + Assert.True(NativeHandleTable.TryUnregister(first, out _)); + + nint second = NativeHandleTable.Register(new object()); + + Assert.NotEqual(first, second); + Assert.Null(NativeHandleTable.Resolve(first)); + } + + [Fact] + public void DefFile_ListsEveryExport() + { + var exports = typeof(NativeApi).Assembly.GetType("FixedWidthParser.Native.Exports")! + .GetMethods(System.Reflection.BindingFlags.Public | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static) + .Select(m => m.GetCustomAttributes(typeof(System.Runtime.InteropServices.UnmanagedCallersOnlyAttribute), false).Cast().FirstOrDefault()?.EntryPoint) + .OfType() + .Order(StringComparer.Ordinal); + string def = Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", "..", "src", "FixedWidthParser.Native", "include", "fixedwidth.def"); + + var listed = File.ReadAllLines(def).Skip(1).Select(l => l.Trim()).Where(l => l.Length > 0).Order(StringComparer.Ordinal); + + Assert.Equal(exports, listed); + } + + [Fact] + public void HandleTable_ResolvesOnlyItsOwnKind() + { + nint id = NativeHandleTable.Register("a string"); + + Assert.Null(NativeHandleTable.Resolve>(id)); + Assert.Equal("a string", NativeHandleTable.Resolve(id)); + Assert.True(NativeHandleTable.TryUnregister(id, out _)); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs b/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs new file mode 100644 index 0000000..46bc56b --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeArrowTests.cs @@ -0,0 +1,279 @@ +using System.Text; +using Apache.Arrow; +using Apache.Arrow.C; +using Apache.Arrow.Ipc; +using Apache.Arrow.Types; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeArrowTests + { + private const string Content = + "Ann " + "00042" + " 12.500" + "00012345" + "20231024" + "133005" + "20231024133000" + "\n" + + " " + " " + " -0.5" + "-0000001" + "19700101" + "000000" + "19700101000000" + "\n" + + "Cid " + "00007" + " 3.250" + "00000100" + "20000229" + "235959" + "20000229235959" + "\n"; + + private static readonly int ReaderOptionsSize = System.Runtime.InteropServices.Marshal.SizeOf(); + + private static NativeLayout Layout() + { + return Build( + new Col("name", 0, 6, TString) { Nullable = true }, + new Col("qty", 6, 5, TInt64) { Nullable = true }, + new Col("price", 11, 8, TFloat64), + new Col("amount", 19, 8, TDecimal) { Scale = 2 }, + new Col("day", 27, 8, TDate) { Format = "yyyyMMdd" }, + new Col("time", 35, 6, TTime) { Format = "HHmmss" }, + new Col("stamp", 41, 14, TTimestamp) { Format = "yyyyMMddHHmmss" }); + } + + private static unsafe RecordBatch ReadBatch(string content, NativeLayout layout, NativeReadOptionsRaw? options = null) + { + byte[] bytes = Encoding.UTF8.GetBytes(content); + ArrowArray array = default; + ArrowSchema schema = default; + NativeReadOptionsRaw value = options ?? default; + fixed (byte* data = bytes) + { + int status = ArrowApi.ReadMemory(data, bytes.Length, layout, options is null ? null : &value, &array, &schema, null); + Assert.True(status == NativeStatus.Ok, LastError()); + } + var imported = CArrowSchemaImporter.ImportSchema((CArrowSchema*)&schema); + return CArrowArrayImporter.ImportRecordBatch((CArrowArray*)&array, imported); + } + + private static object?[] Values(IArrowArray column) + { + return column switch + { + StringArray s => [.. Enumerable.Range(0, s.Length).Select(i => s.IsNull(i) ? null : s.GetString(i))], + Int64Array l => [.. Enumerable.Range(0, l.Length).Select(i => (object?)l.GetValue(i))], + DoubleArray d => [.. Enumerable.Range(0, d.Length).Select(i => (object?)d.GetValue(i))], + Decimal128Array m => [.. Enumerable.Range(0, m.Length).Select(i => (object?)m.GetValue(i))], + Date32Array t => [.. Enumerable.Range(0, t.Length).Select(i => (object?)t.GetValue(i))], + Time64Array t => [.. Enumerable.Range(0, t.Length).Select(i => (object?)t.GetValue(i))], + TimestampArray t => [.. Enumerable.Range(0, t.Length).Select(i => (object?)t.GetValue(i))], + _ => throw new InvalidOperationException(column.GetType().Name), + }; + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public void Arrow_DictionaryColumn_DecodesToTheSameStrings(int degree) + { + string[] codes = ["PETR4", "VALE3", "ITUB4", "", "B3SA3"]; + string content = string.Concat(Enumerable.Range(0, 5000).Select(i => $"{codes[i * 7 % codes.Length],-6}{i % 97:D5}\n")); + var options = Options(degreeOfParallelism: degree); + using var plain = ReadBatch(content, Build(new Col("code", 0, 6, TString) { Nullable = true }, new Col("n", 6, 5, TInt64)), options); + using var encoded = ReadBatch(content, Build(new Col("code", 0, 6, TString) { Nullable = true, Dictionary = 1 }, new Col("n", 6, 5, TInt64)), options); + + var dictionary = Assert.IsType(encoded.Column(0)); + var values = (StringArray)dictionary.Dictionary; + var indices = (Int32Array)dictionary.Indices; + // First-seen order: i * 7 % 5 visits 0, 2, 4, 1, 3. A null row's index points at the empty string. + Assert.Equal(["PETR4", "ITUB4", "B3SA3", "VALE3", ""], Enumerable.Range(0, values.Length).Select(i => values.GetString(i))); + var decoded = Enumerable.Range(0, indices.Length).Select(i => dictionary.IsNull(i) ? null : values.GetString(indices.GetValue(i)!.Value)); + Assert.Equal(Values(plain.Column(0)), decoded); + Assert.Equal(1000, dictionary.NullCount); + Assert.Equal(Values(plain.Column(1)), Values(encoded.Column(1))); + } + + [Fact] + public void Dictionary_OnANonStringColumn_IsRejected() + { + Assert.Equal(NativeStatus.InvalidArgument, Create(out _, new Col("n", 0, 5, TInt64) { Dictionary = 1 })); + Assert.Contains("dictionary", LastError(), StringComparison.Ordinal); + } + + [Fact] + public async Task ArrowStream_DictionaryColumn_HasADictionaryPerBatch() + { + var layout = Build(new Col("code", 0, 6, TString) { Dictionary = 1 }); + byte[] bytes = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("AAAAAA\nBBBBBB\nAAAAAA\n", 4))); + + var decoded = new List(); + IArrowArrayStream stream = OpenStream(bytes, layout, new NativeReaderOptionsRaw { StructSize = ReaderOptionsSize, BatchRows = 5 }); + try + { + while (await stream.ReadNextRecordBatchAsync() is { } batch) + { + using (batch) + { + var dictionary = (DictionaryArray)batch.Column(0); + var values = (StringArray)dictionary.Dictionary; + var indices = (Int32Array)dictionary.Indices; + decoded.AddRange(Enumerable.Range(0, indices.Length).Select(i => values.GetString(indices.GetValue(i)!.Value))); + } + } + } + finally + { + stream.Dispose(); + } + + Assert.Equal(Enumerable.Repeat(new[] { "AAAAAA", "BBBBBB", "AAAAAA" }, 4).SelectMany(x => x), decoded); + } + + [Fact] + public void Arrow_EveryType_MatchesColumnarRead() + { + var layout = Layout(); + var columnar = FromMemory(Encoding.UTF8.GetBytes(Content), layout); + + using var batch = ReadBatch(Content, layout); + + Assert.Equal(3, batch.Length); + Assert.Equal(["name", "qty", "price", "amount", "day", "time", "stamp"], batch.Schema.FieldsList.Select(f => f.Name)); + Assert.Equal(columnar.Column(0), Values(batch.Column(0))); + Assert.Equal(columnar.Column(1), Values(batch.Column(1))); + Assert.Equal(columnar.Column(2), Values(batch.Column(2))); + Assert.Equal([123.45m, -0.01m, 1.00m], Values(batch.Column(3))); + Assert.Equal(columnar.Column(4), Values(batch.Column(4))); + Assert.Equal(columnar.Column(5), Values(batch.Column(5))); + Assert.Equal(columnar.Column(6), Values(batch.Column(6))); + } + + [Fact] + public void Arrow_Nulls_HaveValidity() + { + using var batch = ReadBatch(Content, Layout()); + + Assert.Equal(1, batch.Column(0).NullCount); + Assert.True(batch.Column(1).IsNull(1)); + Assert.Equal(0, batch.Column(2).NullCount); + Assert.True(batch.Schema.GetFieldByName("name").IsNullable); + Assert.False(batch.Schema.GetFieldByName("price").IsNullable); + } + + [Fact] + public void Arrow_Decimal_HasScale() + { + using var batch = ReadBatch(Content, Layout()); + + var type = Assert.IsType(batch.Schema.GetFieldByName("amount").DataType); + Assert.Equal(18, type.Precision); + Assert.Equal(2, type.Scale); + Assert.IsType(batch.Schema.GetFieldByName("stamp").DataType); + Assert.Equal(TimeUnit.Microsecond, ((TimestampType)batch.Schema.GetFieldByName("stamp").DataType).Unit); + } + + [Fact] + public void Arrow_Parallel_EqualsSequential() + { + var layout = Layout(); + string content = string.Concat(Enumerable.Repeat(Content, 50)); + + using var sequential = ReadBatch(content, layout, Options(degreeOfParallelism: 1)); + using var parallel = ReadBatch(content, layout, Options(degreeOfParallelism: 4)); + + for (int i = 0; i < 7; i++) + { + Assert.Equal(Values(sequential.Column(i)), Values(parallel.Column(i))); + } + } + + [Fact] + public async Task ArrowStream_BatchesConcatenate_ToTheWholeRead() + { + var layout = Layout(); + string content = string.Concat(Enumerable.Repeat(Content, 5)); + using var whole = ReadBatch(content, layout); + byte[] bytes = Encoding.UTF8.GetBytes(content); + var options = new NativeReaderOptionsRaw { StructSize = ReaderOptionsSize, BatchRows = 4 }; + + var batches = new List(); + IArrowArrayStream stream = OpenStream(bytes, layout, options); + try + { + while (await stream.ReadNextRecordBatchAsync() is { } batch) + { + batches.Add(batch); + } + } + finally + { + stream.Dispose(); + } + + Assert.Equal([4, 4, 4, 3], batches.Select(b => b.Length)); + for (int i = 0; i < 7; i++) + { + Assert.Equal(Values(whole.Column(i)), batches.SelectMany(b => Values(b.Column(i)))); + } + batches.ForEach(b => b.Dispose()); + } + + [Fact] + public async Task ArrowStream_ParseError_IsReported() + { + var layout = Build(new Col("n", 0, 2, TInt64)); + byte[] bytes = "01\nXX\n"u8.ToArray(); + + IArrowArrayStream stream = OpenStream(bytes, layout, new NativeReaderOptionsRaw { StructSize = ReaderOptionsSize, BatchRows = 1 }); + try + { + using var first = await stream.ReadNextRecordBatchAsync(); + var ex = await Assert.ThrowsAnyAsync(async () => await stream.ReadNextRecordBatchAsync()); + Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); + } + finally + { + stream.Dispose(); + } + } + + [Fact] + public unsafe void Arrow_ParseError_IsReported() + { + var layout = Build(new Col("n", 0, 2, TInt64)); + ArrowArray array = default; + ArrowSchema schema = default; + fixed (byte* data = "XX\n"u8) + { + Assert.Equal(NativeStatus.ParseError, ArrowApi.ReadMemory(data, 3, layout, null, &array, &schema, null)); + } + Assert.Equal(IntPtr.Zero, array.Release); + Assert.Equal(IntPtr.Zero, schema.Release); + } + + [Fact] + public unsafe void Arrow_Release_IsIdempotent() + { + byte[] bytes = Encoding.UTF8.GetBytes(Content); + ArrowArray array = default; + ArrowSchema schema = default; + fixed (byte* data = bytes) + { + Assert.Equal(NativeStatus.Ok, ArrowApi.ReadMemory(data, bytes.Length, Layout(), null, &array, &schema, null)); + } + + ArrowApi.ReleaseArrowArray((IntPtr)(&array)); + ArrowApi.ReleaseArrowArray((IntPtr)(&array)); + ArrowApi.ReleaseArrowSchema((IntPtr)(&schema)); + ArrowApi.ReleaseArrowSchema((IntPtr)(&schema)); + + Assert.Equal(IntPtr.Zero, array.Release); + Assert.Equal(IntPtr.Zero, schema.Release); + } + + private static unsafe IArrowArrayStream OpenStream(byte[] bytes, NativeLayout layout, NativeReaderOptionsRaw options) + { + var stream = (CArrowArrayStream*)System.Runtime.InteropServices.NativeMemory.AllocZeroed((nuint)sizeof(CArrowArrayStream)); + fixed (byte* data = bytes) + { + // The stream reads the memory lazily, so it is copied to memory that outlives this call. + byte* copy = (byte*)System.Runtime.InteropServices.NativeMemory.Alloc((nuint)bytes.Length); + new ReadOnlySpan(data, bytes.Length).CopyTo(new Span(copy, bytes.Length)); + Assert.Equal(NativeStatus.Ok, ArrowApi.OpenStreamMemory(copy, bytes.Length, layout, &options, (ArrowArrayStream*)stream)); + } + return CArrowArrayStreamImporter.ImportArrayStream(stream); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs b/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs new file mode 100644 index 0000000..b2108e1 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeLayoutTests.cs @@ -0,0 +1,153 @@ +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeLayoutTests + { + private static void AssertInvalid(string expectedFragment, params Col[] columns) + { + Assert.Equal(NativeStatus.InvalidArgument, Create(out var layout, columns)); + Assert.Null(layout); + Assert.Contains(expectedFragment, LastError(), StringComparison.Ordinal); + } + + [Fact] + public void Create_ValidCotahistColumns_Succeeds() + { + var layout = Build( + new Col("tipreg", 0, 2, TString), + new Col("data_pregao", 2, 8, TDate) { Format = "yyyyMMdd" }, + new Col("codneg", 12, 12, TString), + new Col("preabe", 56, 13, TFloat64) { Scale = 2 }, + new Col("quatot", 152, 18, TInt64), + new Col("voltot", 170, 18, TDecimal) { Scale = 2, Nullable = true }); + + Assert.Equal(6, layout.Columns.Length); + Assert.Equal("preabe", layout.Columns[3].Name); + Assert.Equal(2, layout.Columns[3].Scale); + Assert.Equal("yyyyMMdd", layout.Columns[1].Parse.Format); + Assert.True(layout.Columns[5].Nullable); + } + + [Fact] + public void Create_Defaults_FollowTheSpec() + { + var layout = Build(new Col("name", 0, 5, TString), new Col("qty", 5, 4, TInt64)); + + Assert.Equal(TrimMode.Both, layout.Columns[0].Parse.TrimMode); + Assert.Equal(' ', layout.Columns[0].Parse.TrimChar); + Assert.Equal(Alignment.Left, layout.Columns[0].FormatOptions.Alignment); + Assert.Equal(Alignment.Right, layout.Columns[1].FormatOptions.Alignment); + Assert.Equal(' ', layout.Columns[1].FormatOptions.Padding); + Assert.Equal(OverflowBehavior.Throw, layout.Columns[0].FormatOptions.Overflow); + Assert.Equal((byte)' ', layout.Filler); + } + + [Fact] + public void Create_Overlap_NamesBothColumns() + { + AssertInvalid("\"a\" [0, 5) and \"b\" [3, 6)", new Col("a", 0, 5, TString), new Col("b", 3, 3, TString)); + } + + [Fact] + public void Create_NegativeStart_Fails() + { + AssertInvalid("negative Start (-1)", new Col("a", -1, 5, TString)); + } + + [Fact] + public void Create_ZeroLength_Fails() + { + AssertInvalid("invalid Length (0)", new Col("a", 0, 0, TString)); + } + + [Fact] + public void Create_NonAsciiPadding_Fails() + { + AssertInvalid("padding", new Col("a", 0, 5, TString) { Padding = 'é' }); + } + + [Fact] + public void Create_ScaleOutOfRange_Fails() + { + AssertInvalid("scale", new Col("a", 0, 20, TFloat64) { Scale = 19 }); + } + + [Fact] + public void Create_ScaleOnInt64_Fails() + { + AssertInvalid("scale", new Col("a", 0, 5, TInt64) { Scale = 2 }); + } + + [Fact] + public void Create_FormatOnString_Fails() + { + AssertInvalid("format", new Col("a", 0, 5, TString) { Format = "x" }); + } + + [Fact] + public void Create_TruncateOnNumber_Fails() + { + AssertInvalid("truncate", new Col("a", 0, 5, TInt64) { Overflow = 1 }); + } + + [Fact] + public void Create_UnknownType_Fails() + { + AssertInvalid("type", new Col("a", 0, 5, 42)); + } + + [Fact] + public void Create_NoName_Fails() + { + AssertInvalid("name", new Col("", 0, 5, TString)); + } + + [Fact] + public void Create_NoColumns_Fails() + { + AssertInvalid("at least one column"); + } + + [Fact] + public void Create_MixedStructSizes_Fails() + { + AssertInvalid("struct_size", new Col("a", 0, 5, TString), new Col("b", 5, 5, TString) { StructSize = 8 }); + } + + [Fact] + public void Options_UnknownStructSize_IsInvalidArgument() + { + var options = LayoutOptions() with { StructSize = 4 }; + + Assert.Equal(NativeStatus.InvalidArgument, Create(out _, options, new Col("a", 0, 5, TString))); + Assert.Contains("fw_layout_options struct_size is 4", LastError(), StringComparison.Ordinal); + } + + [Fact] + public void LineLength_DefaultsToLastColumnEnd() + { + var layout = Build(new Col("a", 10, 5, TString), new Col("b", 0, 3, TString)); + + Assert.Equal(15, layout.LineLength); + } + + [Fact] + public void LineLength_ShorterThanColumns_Fails() + { + Assert.Equal(NativeStatus.InvalidArgument, Create(out _, LayoutOptions(lineLength: 4), new Col("a", 0, 5, TString))); + Assert.Contains("line_length", LastError(), StringComparison.Ordinal); + } + + [Fact] + public void LineLength_AndFiller_AreKept() + { + Assert.Equal(NativeStatus.Ok, Create(out var layout, LayoutOptions(lineLength: 245, filler: '0'), new Col("a", 0, 5, TString))); + + Assert.Equal(245, layout!.LineLength); + Assert.Equal((byte)'0', layout.Filler); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeLimitsTests.cs b/tests/FixedWidthParser.Tests/Native/NativeLimitsTests.cs new file mode 100644 index 0000000..e3c986e --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeLimitsTests.cs @@ -0,0 +1,85 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + /// The size limits of one table, exercised with small limits instead of gigabytes of input. + public sealed unsafe class NativeLimitsTests + { + private static int ReadWith(byte[] content, TableLimits limits, int degree, int chunkSize, out NativeTableRaw table) + { + var options = Options(degreeOfParallelism: degree); + NativeTableRaw result = default; + int status; + fixed (byte* data = content) + { + status = ReadApi.ReadMemory(data, content.Length, Build(new Col("s", 0, 4, TString)), &options, &result, null, chunkSize, limits); + } + table = result; + return status; + } + + [Fact] + public void PartitionedText_OverTheLimit_FailsBeforePublishing() + { + byte[] content = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("abcd\n", 10))); + + int status = ReadWith(content, new TableLimits(MaxTextBytes: 30, MaxRows: int.MaxValue), degree: 4, chunkSize: 5, out var table); + + Assert.Equal(NativeStatus.Error, status); + Assert.Contains("more than 30 bytes of text", LastError(), StringComparison.Ordinal); + Assert.True(table.Columns is null); + } + + [Fact] + public void SequentialText_OverTheLimit_Fails() + { + byte[] content = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("abcd\n", 10))); + + Assert.Equal(NativeStatus.Error, ReadWith(content, new TableLimits(MaxTextBytes: 30, MaxRows: int.MaxValue), degree: 1, chunkSize: 0, out _)); + } + + [Theory] + [InlineData(1, 0)] + [InlineData(4, 5)] + public void Rows_OverTheLimit_Fail(int degree, int chunkSize) + { + byte[] content = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("abcd\n", 10))); + + int status = ReadWith(content, new TableLimits(MaxTextBytes: int.MaxValue, MaxRows: 4), degree, chunkSize, out var table); + + Assert.Equal(NativeStatus.Error, status); + Assert.Contains("more than 4 rows", LastError(), StringComparison.Ordinal); + Assert.True(table.Columns is null); + } + + [Fact] + public void BufferGrowth_PastHalfOfInt_ClampsInsteadOfOverflowing() + { + Assert.Equal(int.MaxValue, GrowableNativeBuffer.NextCapacity(1 << 30, (1L << 30) + 1)); + Assert.Equal(512, GrowableNativeBuffer.NextCapacity(256, 257)); + Assert.Throws(() => GrowableNativeBuffer.NextCapacity(int.MaxValue, (long)int.MaxValue + 1)); + } + + [Fact] + public void DecimalBuffer_SizeIsComputedIn64Bits() + { + Assert.Equal(4_294_967_312L, ArrowApi.DecimalBufferBytes(268_435_457)); + Assert.Equal(16L, ArrowApi.DecimalBufferBytes(0)); + } + + [Fact] + public void Reader_BatchRowsOverInt_IsInvalidArgument() + { + var options = new NativeReaderOptionsRaw { StructSize = sizeof(NativeReaderOptionsRaw), BatchRows = (long)int.MaxValue + 1 }; + fixed (byte* data = "abcd\n"u8) + { + Assert.Equal(NativeStatus.InvalidArgument, ReaderApi.OpenMemory(data, 5, Build(new Col("s", 0, 4, TString)), &options, out _)); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeParallelPropertyTests.cs b/tests/FixedWidthParser.Tests/Native/NativeParallelPropertyTests.cs new file mode 100644 index 0000000..fa981c7 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeParallelPropertyTests.cs @@ -0,0 +1,109 @@ +using System.Text; +using CsCheck; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + /// + /// A partitioned read parses into slices of the final columns sized from the first line's length + /// (TableArena), then closes the gaps in place or copies a column out when a slice overflows. Random files + /// drive every path through it: invalid, empty and short lines, trailing bytes and CRLF that change the line + /// length, a byte order mark and header lines, multi-byte text, and Latin-1 text whose UTF-8 is longer than the + /// column. Whatever the chunk size and thread count, the result must be the sequential read's. + /// + public sealed class NativeParallelPropertyTests + { + private const int Latin1 = 28591; + + private sealed record Case(string[] Lines, int HeaderLines, bool Bom, bool FinalNewline, bool Latin, bool SkipInvalid, int ChunkSize, int Degree) + { + public byte[] Bytes() + { + var text = new StringBuilder(); + for (int i = 0; i < HeaderLines; i++) + { + text.Append("HEADER ").Append(i).Append('\n'); + } + text.AppendJoin("", Lines); + if (!FinalNewline && text.Length > 0 && text[^1] == '\n') + { + text.Length--; + if (text.Length > 0 && text[^1] == '\r') + { + text.Length--; + } + } + byte[] body = (Latin ? Encoding.Latin1 : Encoding.UTF8).GetBytes(text.ToString()); + return Bom && !Latin ? [0xEF, 0xBB, 0xBF, .. body] : body; + } + } + + // code[0,4) text, qty[4,8) nullable integer, price[8,14) two implied decimals. + private static NativeLayout Layout() + { + return Build( + new Col("code", 0, 4, TString), + new Col("qty", 4, 4, TInt64) { Nullable = true }, + new Col("price", 8, 6, TFloat64) { Scale = 2 }); + } + + private static Gen GenLine(bool latin) + { + // Four bytes in either encoding; as UTF-8, the Latin-1 one is eight. + var code = Gen.OneOf(Gen.Char["ABCXYZ019"].Array[4].Select(c => new string(c)), Gen.Const(latin ? "ÉÉÉÉ" : "ÉÉ")); + var qty = Gen.OneOf(Gen.Int[0, 9999].Select(q => q.ToString("D4", System.Globalization.CultureInfo.InvariantCulture)), Gen.Const(" ")); + var price = Gen.Int[0, 999_999].Select(p => p.ToString("D6", System.Globalization.CultureInfo.InvariantCulture)); + var tail = Gen.Char[" #"].Array[0, 3].Select(c => new string(c)); + var newline = Gen.OneOf(Gen.Const("\n"), Gen.Const("\r\n")); + var valid = Gen.Select(code, qty, price, tail, newline, (c, q, p, t, n) => c + q + p + t + n); + return Gen.Frequency( + (12, valid), + (1, newline.Select(n => "AB01XXXX123456" + n)), + (1, newline.Select(n => "AB" + n)), + (1, newline)); + } + + private static readonly Gen GenCase = + from latin in Gen.Bool + from lines in GenLine(latin).Array[0, 60] + from header in Gen.Int[0, 2] + from bom in Gen.Bool + from finalNewline in Gen.Bool + from skipInvalid in Gen.Bool + from chunk in Gen.Int[1, 64] + from degree in Gen.Int[2, 8] + select new Case(lines, header, bom, finalNewline, latin, skipInvalid, chunk, degree); + + [Fact] + public void ParallelRead_IsTheSequentialRead() + { + var layout = Layout(); + GenCase.Sample(c => + { + byte[] bytes = c.Bytes(); + int encoding = c.Latin ? Latin1 : 0; + + var sequential = FromMemory(bytes, layout, Options(c.HeaderLines, c.SkipInvalid, degreeOfParallelism: 1, encoding: encoding)); + NativeApi.LastParseError(out long sequentialLine, out int sequentialColumn); + var parallel = FromMemory(bytes, layout, Options(c.HeaderLines, c.SkipInvalid, degreeOfParallelism: c.Degree, encoding: encoding), c.ChunkSize); + NativeApi.LastParseError(out long parallelLine, out int parallelColumn); + + Assert.Equal(sequential.Status, parallel.Status); + if (sequential.Status != NativeStatus.Ok) + { + Assert.Equal((sequentialLine, sequentialColumn), (parallelLine, parallelColumn)); + return; + } + Assert.Equal(sequential.RowCount, parallel.RowCount); + Assert.Equal(sequential.LinesSkipped, parallel.LinesSkipped); + for (int column = 0; column < 3; column++) + { + Assert.Equal(sequential.Column(column), parallel.Column(column)); + } + }, iter: 2000); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs b/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs new file mode 100644 index 0000000..13af12a --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeParallelReadTests.cs @@ -0,0 +1,128 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed class NativeParallelReadTests + { + private static NativeLayout Layout() + { + return Build( + new Col("code", 0, 4, TString), + new Col("qty", 4, 4, TInt64) { Nullable = true }, + new Col("price", 8, 6, TFloat64) { Scale = 2 }); + } + + // A BOM, CRLF and LF breaks, empty lines, invalid lines, nulls and no final line break. + private static byte[] Content(bool withBom) + { + var text = new StringBuilder(withBom ? "" : ""); + text.Append("HEADER ONE\r\nHEADER TWO\n"); + for (int i = 1; i <= 200; i++) + { + string qty = i % 7 == 0 ? " " : (i % 10000).ToString("D4", System.Globalization.CultureInfo.InvariantCulture); + string price = i % 13 == 0 ? "XXXXXX" : (i * 3).ToString("D6", System.Globalization.CultureInfo.InvariantCulture); + text.Append(i % 3 == 0 ? "CD" : "AB").Append(System.Globalization.CultureInfo.InvariantCulture, $"{i % 100:D2}").Append(qty).Append(price); + text.Append(i % 2 == 0 ? "\r\n" : "\n"); + if (i % 17 == 0) + { + text.Append('\n'); + } + } + text.Append("LAST0001000123"); + return Encoding.UTF8.GetBytes(text.ToString()); + } + + public static TheoryData Partitions() + { + var data = new TheoryData(); + foreach (int chunkSize in new[] { 1, 7, 64, 4096 }) + { + foreach (int degree in new[] { 1, 4 }) + { + data.Add(chunkSize, degree, false); + data.Add(chunkSize, degree, true); + } + } + return data; + } + + [Theory] + [MemberData(nameof(Partitions))] + public void ParallelRead_EqualsSequential(int chunkSize, int degree, bool withBom) + { + var layout = Layout(); + byte[] content = Content(withBom); + + var sequential = FromMemory(content, layout, Options(headerLines: 2, skipInvalid: true, degreeOfParallelism: 1)); + var parallel = FromMemory(content, layout, Options(headerLines: 2, skipInvalid: true, degreeOfParallelism: degree), chunkSize); + + Assert.Equal(NativeStatus.Ok, sequential.Status); + Assert.True(parallel.Status == NativeStatus.Ok, parallel.Error); + Assert.Equal(sequential.RowCount, parallel.RowCount); + for (int column = 0; column < 3; column++) + { + Assert.Equal(sequential.Column(column), parallel.Column(column)); + } + Assert.Equal(sequential.LinesRead, parallel.LinesRead); + Assert.Equal(sequential.LinesSkipped, parallel.LinesSkipped); + Assert.Equal(200 - (200 / 13) + 1, parallel.RowCount); + } + + // Lines of one length with invalid ones among them leave gaps in the shared columns that publishing closes in + // place; a longer first line makes every partition outgrow the room it was lent. + [Theory] + [InlineData(1, false)] + [InlineData(7, false)] + [InlineData(64, false)] + [InlineData(4096, false)] + [InlineData(7, true)] + [InlineData(64, true)] + public void ParallelRead_UniformLines_EqualsSequential(int chunkSize, bool longFirstLine) + { + var layout = Layout(); + var text = new StringBuilder(longFirstLine ? "AB000001000003 trailing\n" : ""); + for (int i = 1; i <= 300; i++) + { + string price = i % 11 == 0 ? "XXXXXX" : (i * 3).ToString("D6", System.Globalization.CultureInfo.InvariantCulture); + text.Append(System.Globalization.CultureInfo.InvariantCulture, $"AB{i % 100:D2}{i % 10000:D4}{price}\n"); + } + byte[] content = Encoding.UTF8.GetBytes(text.ToString()); + + var sequential = FromMemory(content, layout, Options(skipInvalid: true, degreeOfParallelism: 1)); + var parallel = FromMemory(content, layout, Options(skipInvalid: true, degreeOfParallelism: 4), chunkSize); + + Assert.True(parallel.Status == NativeStatus.Ok, parallel.Error); + Assert.Equal(300 - (300 / 11) + (longFirstLine ? 1 : 0), parallel.RowCount); + Assert.Equal(sequential.RowCount, parallel.RowCount); + for (int column = 0; column < 3; column++) + { + Assert.Equal(sequential.Column(column), parallel.Column(column)); + } + } + + [Theory] + [InlineData(1)] + [InlineData(7)] + [InlineData(64)] + public void ParallelRead_ErrorLineNumber_EqualsSequential(int chunkSize) + { + var layout = Layout(); + byte[] content = Content(withBom: true); + + var sequential = FromMemory(content, layout, Options(headerLines: 2, degreeOfParallelism: 1)); + NativeApi.LastParseError(out long sequentialLine, out int sequentialColumn); + var parallel = FromMemory(content, layout, Options(headerLines: 2, degreeOfParallelism: 4), chunkSize); + NativeApi.LastParseError(out long parallelLine, out int parallelColumn); + + Assert.Equal(NativeStatus.ParseError, sequential.Status); + Assert.Equal(NativeStatus.ParseError, parallel.Status); + Assert.Equal(sequentialLine, parallelLine); + Assert.Equal(sequentialColumn, parallelColumn); + Assert.Equal(2, parallelColumn); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs new file mode 100644 index 0000000..4e9f565 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeReadTests.cs @@ -0,0 +1,354 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed unsafe class NativeReadTests + { + private static readonly DateOnly Epoch = new(1970, 1, 1); + + private static int Days(int year, int month, int day) + { + return new DateOnly(year, month, day).DayNumber - Epoch.DayNumber; + } + + private static ReadOutcome[] Ok(string content, NativeLayout layout, NativeReadOptionsRaw? options = null, Encoding? encoding = null) + { + var outcomes = Both(content, layout, options, encoding); + foreach (var outcome in outcomes) + { + Assert.True(outcome.Status == NativeStatus.Ok, $"status {outcome.Status}: {outcome.Error}"); + } + return outcomes; + } + + [Fact] + public void Read_EveryType_ProducesColumns() + { + var layout = Build( + new Col("name", 0, 6, TString), + new Col("qty", 6, 5, TInt64), + new Col("price", 11, 8, TFloat64), + new Col("amount", 19, 8, TDecimal) { Scale = 2 }, + new Col("day", 27, 8, TDate) { Format = "yyyyMMdd" }, + new Col("time", 35, 6, TTime) { Format = "HHmmss" }, + new Col("stamp", 41, 14, TTimestamp) { Format = "yyyyMMddHHmmss" }); + const string Content = + "Ann " + "00042" + " 12.500" + "00012345" + "20231024" + "133005" + "20231024133000" + "\n" + + "Bob " + " -7" + " -0.5" + "-0000001" + "19700101" + "000000" + "19700101000000" + "\n"; + + foreach (var outcome in Ok(Content, layout)) + { + Assert.Equal(2, outcome.RowCount); + Assert.Equal(["Ann", "Bob"], outcome.Column(0)); + Assert.Equal([42L, -7L], outcome.Column(1)); + Assert.Equal([12.5, -0.5], outcome.Column(2)); + Assert.Equal([12345L, -1L], outcome.Column(3)); + Assert.Equal([Days(2023, 10, 24), 0], outcome.Column(4)); + Assert.Equal([new TimeOnly(13, 30, 5).Ticks / 10, 0L], outcome.Column(5)); + Assert.Equal([(new DateTime(2023, 10, 24, 13, 30, 0, DateTimeKind.Utc) - DateTime.UnixEpoch).Ticks / 10, 0L], outcome.Column(6)); + Assert.Equal(2, outcome.Columns[3].Scale); + Assert.Equal(2, outcome.LinesRead); + } + } + + [Fact] + public void Read_ScaledFloat_DividesByPowerOfTen() + { + var layout = Build(new Col("preabe", 0, 13, TFloat64) { Scale = 2 }); + + foreach (var outcome in Ok("0000000123456\n0000000000000\n", layout)) + { + Assert.Equal([1234.56, 0.0], outcome.Column(0)); + } + } + + [Fact] + public void Read_ScaledNegative_KeepsSign() + { + var layout = Build(new Col("v", 0, 11, TFloat64) { Scale = 2 }); + + foreach (var outcome in Ok("-0000012345\n", layout)) + { + Assert.Equal([-123.45], outcome.Column(0)); + } + } + + [Fact] + public void Read_ScaledWithDecimalPoint_ParsesTheText() + { + var layout = Build(new Col("v", 0, 8, TFloat64) { Scale = 2 }, new Col("d", 8, 8, TDecimal) { Scale = 2 }); + + foreach (var outcome in Ok(" 123.45 67.8 \n", layout)) + { + Assert.Equal([123.45], outcome.Column(0)); + Assert.Equal([6780L], outcome.Column(1)); + } + } + + [Fact] + public void Read_DecimalWithTooManyFractionalDigits_IsParseError() + { + var layout = Build(new Col("d", 0, 8, TDecimal) { Scale = 2 }); + + Assert.All(Both(" 1.234\n", layout), o => Assert.Equal(NativeStatus.ParseError, o.Status)); + } + + [Fact] + public void Read_Decimal_KeepsUnscaled() + { + var layout = Build(new Col("voltot", 0, 18, TDecimal) { Scale = 2 }); + + foreach (var outcome in Ok("000000000000123456\n", layout)) + { + Assert.Equal([123456L], outcome.Column(0)); + } + } + + [Fact] + public void Read_Nullable_BlankIsNull() + { + var layout = Build(new Col("a", 0, 4, TInt64) { Nullable = true }, new Col("s", 4, 3, TString) { Nullable = true }); + + foreach (var outcome in Ok("0001abc\n \n0003xyz\n", layout)) + { + Assert.Equal([1L, null, 3L], outcome.Column(0)); + Assert.Equal(["abc", null, "xyz"], outcome.Column(1)); + } + } + + [Fact] + public void Read_NoNulls_HasNoValidity() + { + var layout = Build(new Col("a", 0, 4, TInt64) { Nullable = true }); + + foreach (var outcome in Ok("0001\n0002\n", layout)) + { + Assert.True(outcome.Columns[0].Validity is null); + } + } + + [Fact] + public void Read_NotNullableBlank_IsParseError() + { + var layout = Build(new Col("a", 0, 4, TInt64)); + + foreach (var outcome in Both("0001\n \n", layout)) + { + Assert.Equal(NativeStatus.ParseError, outcome.Status); + Assert.Contains("Line 2", outcome.Error, StringComparison.Ordinal); + } + } + + [Fact] + public void Read_NotNullableBlankString_IsEmpty() + { + var layout = Build(new Col("s", 0, 4, TString), new Col("n", 4, 1, TInt64)); + + foreach (var outcome in Ok(" 1\n", layout)) + { + Assert.Equal([""], outcome.Column(0)); + } + } + + [Fact] + public void Read_HeaderLines_AreNotParsed() + { + var layout = Build(new Col("n", 0, 3, TInt64)); + + foreach (var outcome in Ok("HEADER\nXX\n001\n002\n", layout, Options(headerLines: 2))) + { + Assert.Equal([1L, 2L], outcome.Column(0)); + Assert.Equal(2, outcome.LinesSkipped); + } + } + + [Fact] + public void Read_HeaderLinesBeyondTheFile_ReadNothing() + { + var layout = Build(new Col("n", 0, 3, TInt64)); + + foreach (var outcome in Ok("HDR\n", layout, Options(headerLines: 5))) + { + Assert.Equal(0, outcome.RowCount); + Assert.Equal(1, outcome.LinesSkipped); + } + } + + [Fact] + public void Read_SkipInvalid_SkipsTrailerAndCounts() + { + var layout = Build(new Col("tipreg", 0, 2, TString), new Col("price", 2, 5, TFloat64) { Scale = 2 }); + const string Content = "00COTAHIST\n0100123\n0100456\n99TRAILER\n"; + + foreach (var outcome in Ok(Content, layout, Options(skipInvalid: true))) + { + Assert.Equal(["01", "01"], outcome.Column(0)); + Assert.Equal([1.23, 4.56], outcome.Column(1)); + Assert.Equal(2, outcome.LinesSkipped); + Assert.Equal(2, outcome.LinesRead); + } + } + + [Fact] + public void Read_RejectedLineMidway_LeavesNoPartialValues() + { + var layout = Build(new Col("s", 0, 3, TString), new Col("n", 3, 2, TInt64), new Col("t", 5, 2, TString)); + + foreach (var outcome in Ok("abc01xy\ndefXXzz\nghi03uv\n", layout, Options(skipInvalid: true))) + { + Assert.Equal(["abc", "ghi"], outcome.Column(0)); + Assert.Equal([1L, 3L], outcome.Column(1)); + Assert.Equal(["xy", "uv"], outcome.Column(2)); + } + } + + [Fact] + public void Read_ParseError_ReportsLineAndColumn() + { + var layout = Build(new Col("name", 0, 3, TString), new Col("age", 3, 2, TInt64)); + + foreach (var outcome in Both("Ann30\nBobXX\n", layout)) + { + Assert.Equal(NativeStatus.ParseError, outcome.Status); + Assert.Contains("column 'age'", outcome.Error, StringComparison.Ordinal); + Assert.Contains("\"XX\"", outcome.Error, StringComparison.Ordinal); + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(2, line); + Assert.Equal(1, column); + } + } + + [Fact] + public void Read_ShortLine_IsParseErrorWithColumnMinusOne() + { + var layout = Build(new Col("a", 0, 3, TString), new Col("b", 3, 4, TString)); + + foreach (var outcome in Both("abc\n", layout)) + { + Assert.Equal(NativeStatus.ParseError, outcome.Status); + Assert.Contains("it is 3 bytes long and the layout needs 7", outcome.Error, StringComparison.Ordinal); + NativeApi.LastParseError(out _, out int column); + Assert.Equal(-1, column); + } + } + + [Fact] + public void Read_Latin1Text_IsTranscodedToUtf8() + { + var layout = Build(new Col("name", 0, 6, TString), new Col("n", 6, 2, TInt64)); + var latin1 = Encoding.Latin1; + + foreach (var outcome in Ok("AÇÃO 12\n", layout, Options(encoding: 28591), latin1)) + { + Assert.Equal(["AÇÃO"], outcome.Column(0)); + Assert.Equal([12L], outcome.Column(1)); + } + } + + [Fact] + public void Read_InvalidUtf8_IsReplacedLikeTheCore() + { + var layout = Build(new Col("name", 0, 6, TString), new Col("n", 6, 2, TInt64)); + byte[] latin1 = Encoding.Latin1.GetBytes("AÇÃO 01\n"); + + var outcome = FromMemory(latin1, layout); + + Assert.Equal(NativeStatus.Ok, outcome.Status); + Assert.Equal([Encoding.UTF8.GetString(latin1.AsSpan(0, 4))], outcome.Column(0)); + } + + [Fact] + public void Read_SplitMultiByteCharacter_StaysValidUtf8() + { + var layout = Build(new Col("a", 0, 2, TString), new Col("b", 2, 3, TString)); + byte[] bytes = Encoding.UTF8.GetBytes("AÇBC\n"); + + var outcome = FromMemory(bytes, layout); + + Assert.Equal(["A�", "�BC"], outcome.Column(0).Concat(outcome.Column(1))); + } + + [Fact] + public void Read_TimestampWithZoneOrOffset_IsUtc() + { + var layout = Build(new Col("t", 0, 25, TTimestamp) { Nullable = true }); + long midnight = (new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc) - DateTime.UnixEpoch).Ticks / 10; + + foreach (var outcome in Ok("2024-01-01T00:00:00Z \n2024-01-01T00:00:00-03:00\n2024-01-01T00:00:00 \n", layout)) + { + Assert.Equal([midnight, midnight + (3L * 3600 * 1_000_000), midnight], outcome.Column(0)); + } + } + + [Fact] + public void Read_DecimalOver18Digits_IsParseError() + { + var layout = Build(new Col("d", 0, 19, TDecimal) { Scale = 2 }); + + Assert.All(Both("1234567890123456789\n", layout), o => Assert.Equal(NativeStatus.ParseError, o.Status)); + } + + [Fact] + public void Read_ScaledFloatBeyondInt64_KeepsTheScale() + { + var layout = Build(new Col("v", 0, 20, TFloat64) { Scale = 2 }); + + foreach (var outcome in Ok("99999999999999999999\n", layout)) + { + Assert.Equal([1e18], outcome.Column(0)); + } + } + + [Fact] + public void Read_EmptyFile_ReturnsZeroRows() + { + var layout = Build(new Col("a", 0, 3, TString)); + + foreach (var outcome in Ok("", layout)) + { + Assert.Equal(0, outcome.RowCount); + Assert.Equal(1, outcome.Columns.Length); + } + } + + [Fact] + public void Read_NullOptions_UsesDefaults() + { + var layout = Build(new Col("a", 0, 3, TString)); + + foreach (var outcome in Ok("abc\r\ndef", layout)) + { + Assert.Equal(["abc", "def"], outcome.Column(0)); + } + } + + [Fact] + public void Read_BadOptionsStructSize_IsInvalidArgument() + { + var layout = Build(new Col("a", 0, 3, TString)); + + var outcome = FromMemory("abc"u8.ToArray(), layout, Options() with { StructSize = 12 }); + + Assert.Equal(NativeStatus.InvalidArgument, outcome.Status); + Assert.Contains("fw_read_options struct_size is 12", outcome.Error, StringComparison.Ordinal); + } + + [Fact] + public void Read_MissingFile_IsError() + { + var layout = Build(new Col("a", 0, 3, TString)); + byte[] path = Encoding.UTF8.GetBytes(Path.Combine(Path.GetTempPath(), "fwp-missing-" + Guid.NewGuid())); + + unsafe + { + NativeTableRaw table = default; + Assert.Equal(NativeStatus.Error, ReadApi.ReadFile(path, layout, null, &table, null)); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeReaderTests.cs b/tests/FixedWidthParser.Tests/Native/NativeReaderTests.cs new file mode 100644 index 0000000..ae1ddd6 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeReaderTests.cs @@ -0,0 +1,159 @@ +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed unsafe class NativeReaderTests + { + private static readonly byte[] Content = Encoding.UTF8.GetBytes( + "HEADER\n" + string.Concat(Enumerable.Range(1, 10).Select(i => i == 5 ? "ZZXX\r\n" : $"AB{i:D2}\n")) + "AB11"); + + private static NativeLayout Layout() + { + return Build(new Col("code", 0, 2, TString), new Col("n", 2, 2, TInt64)); + } + + private static NativeReaderOptionsRaw ReaderOptions(long batchRows, bool skipInvalid = true) + { + return new NativeReaderOptionsRaw + { + StructSize = sizeof(NativeReaderOptionsRaw), + Read = Options(headerLines: 1, skipInvalid: skipInvalid), + BatchRows = batchRows, + }; + } + + private static nint Open(NativeLayout layout, NativeReaderOptionsRaw options, byte* data) + { + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenMemory(data, Content.Length, layout, &options, out nint reader)); + return reader; + } + + private static List Drain(nint reader) + { + var batches = new List(); + while (true) + { + NativeTableRaw table = default; + int status = ReaderApi.Next(reader, &table); + if (status == NativeStatus.Eof) + { + return batches; + } + Assert.True(status == NativeStatus.Ok, LastError()); + batches.Add(Collect(status, &table, default)); + } + } + + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(1000)] + public void Reader_BatchesConcatenate_ToTheWholeRead(long batchRows) + { + var layout = Layout(); + var whole = FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)); + + fixed (byte* data = Content) + { + nint reader = Open(layout, ReaderOptions(batchRows), data); + var batches = Drain(reader); + ReaderApi.Close(reader); + + Assert.All(batches, b => Assert.InRange(b.RowCount, 1, batchRows)); + Assert.Equal(whole.Column(0), batches.SelectMany(b => b.Column(0))); + Assert.Equal(whole.Column(1), batches.SelectMany(b => b.Column(1))); + Assert.Equal(10, batches.Sum(b => b.RowCount)); + } + } + + [Fact] + public void Reader_File_EqualsMemory() + { + var layout = Layout(); + string path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, Content); + var options = ReaderOptions(4); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenFile(Encoding.UTF8.GetBytes(path), layout, &options, out nint reader)); + var batches = Drain(reader); + ReaderApi.Close(reader); + + Assert.Equal([4L, 4L, 2L], batches.Select(b => b.RowCount)); + Assert.Equal(FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)).Column(1), batches.SelectMany(b => b.Column(1))); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void Reader_AfterEof_ReturnsEof() + { + fixed (byte* data = Content) + { + nint reader = Open(Layout(), ReaderOptions(100), data); + Drain(reader); + NativeTableRaw table = default; + + Assert.Equal(NativeStatus.Eof, ReaderApi.Next(reader, &table)); + Assert.Equal(0, table.ColumnCount); + ReaderApi.Close(reader); + } + } + + [Fact] + public void Reader_ParseError_StopsWithLineNumber() + { + fixed (byte* data = Content) + { + nint reader = Open(Layout(), ReaderOptions(2, skipInvalid: false), data); + NativeTableRaw table = default; + + Assert.Equal(NativeStatus.Ok, ReaderApi.Next(reader, &table)); + NativeTable.Free(&table); + Assert.Equal(NativeStatus.Ok, ReaderApi.Next(reader, &table)); + NativeTable.Free(&table); + Assert.Equal(NativeStatus.ParseError, ReaderApi.Next(reader, &table)); + NativeApi.LastParseError(out long line, out int column); + Assert.Equal(6, line); + Assert.Equal(1, column); + Assert.Equal(NativeStatus.Eof, ReaderApi.Next(reader, &table)); + ReaderApi.Close(reader); + } + } + + [Fact] + public void Reader_Closed_IsInvalidHandle() + { + fixed (byte* data = Content) + { + nint reader = Open(Layout(), ReaderOptions(2), data); + ReaderApi.Close(reader); + NativeTableRaw table = default; + + Assert.Equal(NativeStatus.InvalidHandle, ReaderApi.Next(reader, &table)); + } + } + + [Fact] + public void Reader_ZeroedOptions_UseDefaults() + { + var options = new NativeReaderOptionsRaw { StructSize = sizeof(NativeReaderOptionsRaw) }; + fixed (byte* data = "AB01\nAB02\n"u8) + { + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenMemory(data, 10, Layout(), &options, out nint reader)); + var batches = Drain(reader); + ReaderApi.Close(reader); + + Assert.Equal([2L], batches.Select(b => b.RowCount)); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeStreamReaderTests.cs b/tests/FixedWidthParser.Tests/Native/NativeStreamReaderTests.cs new file mode 100644 index 0000000..871b8c4 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeStreamReaderTests.cs @@ -0,0 +1,298 @@ +using System.Runtime.InteropServices; +using System.Text; +using Apache.Arrow; +using Apache.Arrow.C; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Arrow; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + /// + /// fw_reader_open_stream and fw_read_arrow_stream_stream over a real fw_stream whose callbacks + /// hand out a few bytes at a time: the batches equal a read of the same bytes from memory, header lines are skipped + /// without seeking, a failing callback's message reaches the caller, and release runs exactly once. + /// + public sealed unsafe class NativeStreamReaderTests + { + private static readonly byte[] Content = Encoding.UTF8.GetBytes( + "HEADER\n" + string.Concat(Enumerable.Range(1, 10).Select(i => i == 5 ? "ZZXX\r\n" : $"AB{i:D2}\n")) + "AB11"); + + /// What the callbacks read: at most bytes per call, failing at . + private sealed class Source(byte[] data, int chunk, int failAt = -1, long overReport = 0) + { + public int Position; + public int Released; + + public long Read(byte* buffer, long length) + { + if (failAt >= 0 && Position >= failAt) + { + byte[] message = "the disk went away"u8.ToArray(); + fixed (byte* m = message) + { + SourceErrors.Set(Encoding.UTF8.GetString(m, message.Length)); + } + return -1; + } + int count = (int)Math.Min(Math.Min(length, Chunk), data.Length - Position); + data.AsSpan(Position, count).CopyTo(new Span(buffer, count)); + Position += count; + return overReport > 0 ? length + overReport : count; + } + + public int Chunk => chunk; + } + + [UnmanagedCallersOnly] + private static long ReadCallback(void* userData, byte* buffer, long length) + { + return ((Source)GCHandle.FromIntPtr((IntPtr)userData).Target!).Read(buffer, length); + } + + [UnmanagedCallersOnly] + private static void ReleaseCallback(void* userData) + { + ((Source)GCHandle.FromIntPtr((IntPtr)userData).Target!).Released++; + } + + /// An fw_stream over ; the handle keeps it alive for the test. + private static CallbackReadStream Wrap(Source source, GCHandle handle) + { + var raw = new NativeStreamRaw + { + StructSize = sizeof(NativeStreamRaw), + UserData = (void*)GCHandle.ToIntPtr(handle), + Read = &ReadCallback, + Release = &ReleaseCallback, + }; + Assert.True(CallbackReadStream.TryCreate(&raw, "test", out var stream)); + return stream; + } + + private static NativeLayout Layout() + { + return Build(new Col("code", 0, 2, TString), new Col("n", 2, 2, TInt64)); + } + + private static NativeReaderOptionsRaw ReaderOptions(long batchRows, long headerLines = 1, bool skipInvalid = true) + { + return new NativeReaderOptionsRaw + { + StructSize = sizeof(NativeReaderOptionsRaw), + Read = Options(headerLines: (int)headerLines, skipInvalid: skipInvalid), + BatchRows = batchRows, + }; + } + + private static List Drain(nint reader, out int lastStatus) + { + var batches = new List(); + while (true) + { + NativeTableRaw table = default; + lastStatus = ReaderApi.Next(reader, &table); + if (lastStatus != NativeStatus.Ok) + { + return batches; + } + batches.Add(Collect(lastStatus, &table, default)); + } + } + + [Theory] + [InlineData(1, 3)] + [InlineData(7, 4)] + [InlineData(4096, 1000)] + public void Stream_EqualsMemory_AndReleasesOnce(int chunk, long batchRows) + { + var layout = Layout(); + var whole = FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)); + var source = new Source(Content, chunk); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(batchRows); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), layout, &options, out nint reader)); + var batches = Drain(reader, out int last); + Assert.Equal(0, source.Released); + ReaderApi.Close(reader); + + Assert.Equal(NativeStatus.Eof, last); + Assert.Equal(whole.Column(0), batches.SelectMany(b => b.Column(0))); + Assert.Equal(whole.Column(1), batches.SelectMany(b => b.Column(1))); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + + [Theory] + [InlineData("AB01\nAB02\n", 0)] + [InlineData("HEAD\nAB01\nAB02", 1)] + [InlineData("H1\r\n\nAB01\n", 2)] + [InlineData("ONLY A HEADER", 3)] + public void HeaderLinesAndByteOrderMark_MatchMemory(string text, long headerLines) + { + byte[] bytes = Encoding.UTF8.GetBytes(text); + var layout = Layout(); + var expected = FromMemory(bytes, layout, Options(headerLines: (int)headerLines)); + var source = new Source(bytes, 1); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(100, headerLines, skipInvalid: false); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), layout, &options, out nint reader)); + var batches = Drain(reader, out int last); + ReaderApi.Close(reader); + + Assert.True(last == NativeStatus.Eof, LastError()); + Assert.Equal(expected.Column(1), batches.SelectMany(b => b.Column(1))); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void ParseError_CountsTheSkippedHeaderLines() + { + var source = new Source(Content, 5); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(100, skipInvalid: false); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + Drain(reader, out int last); + NativeApi.LastParseError(out long line, out _); + ReaderApi.Close(reader); + + Assert.Equal(NativeStatus.ParseError, last); + Assert.Equal(6, line); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void FailingRead_ReportsTheCallbacksMessage() + { + var source = new Source(Content, 8, failAt: 30); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(2); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + Drain(reader, out int last); + + Assert.Equal(NativeStatus.Error, last); + Assert.Equal("the disk went away", LastError()); + ReaderApi.Close(reader); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void ReadReturningMoreThanAsked_IsAnError() + { + var source = new Source(Content, 4096, overReport: 1); + var handle = GCHandle.Alloc(source); + try + { + var options = ReaderOptions(100, headerLines: 0); + Assert.Equal(NativeStatus.Ok, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + Drain(reader, out int last); + ReaderApi.Close(reader); + + Assert.Equal(NativeStatus.Error, last); + Assert.Contains("fw_stream.read returned", LastError(), StringComparison.Ordinal); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void BadOptions_ReleaseTheStream() + { + var source = new Source(Content, 16); + var handle = GCHandle.Alloc(source); + try + { + var options = new NativeReaderOptionsRaw { StructSize = 3 }; + Assert.Equal(NativeStatus.InvalidArgument, ReaderApi.OpenStream(Wrap(source, handle), Layout(), &options, out nint reader)); + + Assert.Equal(0, reader); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + + [Fact] + public void RejectedStruct_IsNotReleased() + { + var raw = new NativeStreamRaw { StructSize = 0, Read = &ReadCallback, Release = &ReleaseCallback }; + Assert.False(CallbackReadStream.TryCreate(&raw, "fw_reader_open_stream", out _)); + Assert.Contains("struct_size is sizeof(fw_stream)", LastError(), StringComparison.Ordinal); + + raw = new NativeStreamRaw { StructSize = sizeof(NativeStreamRaw), Release = &ReleaseCallback }; + Assert.False(CallbackReadStream.TryCreate(&raw, "fw_reader_open_stream", out _)); + } + + [Fact] + public void ArrowStream_OverAStream_EqualsMemory_AndReleasesWithIt() + { + var layout = Layout(); + var whole = FromMemory(Content, layout, Options(headerLines: 1, skipInvalid: true)); + var source = new Source(Content, 3); + var handle = GCHandle.Alloc(source); + try + { + var stream = (CArrowArrayStream*)NativeMemory.AllocZeroed((nuint)sizeof(CArrowArrayStream)); + var options = ReaderOptions(4); + Assert.Equal(NativeStatus.Ok, ArrowApi.OpenStreamStream(Wrap(source, handle), layout, &options, (ArrowArrayStream*)stream)); + var imported = CArrowArrayStreamImporter.ImportArrayStream(stream); + var quantities = new List(); + try + { + // An unsafe context cannot await; the native stream completes every read synchronously. + while (imported.ReadNextRecordBatchAsync(TestContext.Current.CancellationToken).AsTask().GetAwaiter().GetResult() is { } batch) + { + using (batch) + { + var column = (Int64Array)batch.Column(1); + quantities.AddRange(Enumerable.Range(0, column.Length).Select(i => column.GetValue(i))); + } + } + } + finally + { + imported.Dispose(); + } + + Assert.Equal(whole.Column(1), quantities.Cast()); + Assert.Equal(1, source.Released); + } + finally + { + handle.Free(); + } + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs new file mode 100644 index 0000000..53b0761 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeTestLayouts.cs @@ -0,0 +1,121 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; + +namespace FixedWidthParser.Tests.Native +{ + /// One column as a C caller would describe it; unset fields stay zero, the C defaults. + internal sealed record Col(string Name, int Start, int Length, int Type) + { + public bool Nullable { get; init; } + public int Scale { get; init; } + public string? Format { get; init; } + public int TrimChar { get; init; } + public int TrimMode { get; init; } + public int Alignment { get; init; } + public int Padding { get; init; } + public int Overflow { get; init; } + public int Dictionary { get; init; } + public int StructSize { get; init; } = Marshal.SizeOf(); + } + + /// Builds s from s through the same entry point fw_layout_create uses. + internal static unsafe class NativeTestLayouts + { + internal const int TString = 0; + internal const int TInt64 = 1; + internal const int TFloat64 = 2; + internal const int TDecimal = 3; + internal const int TDate = 4; + internal const int TTime = 5; + internal const int TTimestamp = 6; + + internal static int Create(out NativeLayout? layout, params Col[] columns) + { + return Create(out layout, null, columns); + } + + internal static int Create(out NativeLayout? layout, NativeLayoutOptionsRaw? options, params Col[] columns) + { + var allocations = new List(); + try + { + var specs = columns.Select(c => ToRaw(c, allocations)).ToArray(); + if (options is { } value) + { + return NativeLayout.TryCreate(specs, &value, out layout); + } + return NativeLayout.TryCreate(specs, null, out layout); + } + finally + { + allocations.ForEach(Marshal.FreeHGlobal); + } + } + + internal static NativeLayout Build(params Col[] columns) + { + int status = Create(out var layout, columns); + Assert.True(status == NativeStatus.Ok, $"status {status}: {LastError()}"); + return layout!; + } + + internal static NativeLayout Build(NativeLayoutOptionsRaw options, params Col[] columns) + { + int status = Create(out var layout, options, columns); + Assert.True(status == NativeStatus.Ok, $"status {status}: {LastError()}"); + return layout!; + } + + internal static NativeLayoutOptionsRaw LayoutOptions(int lineLength = 0, int filler = 0) + { + return new NativeLayoutOptionsRaw { StructSize = sizeof(NativeLayoutOptionsRaw), LineLength = lineLength, Filler = filler }; + } + + internal static string LastError() + { + var buffer = new byte[4096]; + NativeApi.LastError(buffer, out int length); + return Encoding.UTF8.GetString(buffer, 0, length); + } + + private static NativeColumnSpecRaw ToRaw(Col column, List allocations) + { + var (name, nameLength) = Utf8(column.Name, allocations); + var (format, formatLength) = Utf8(column.Format, allocations); + return new NativeColumnSpecRaw + { + StructSize = column.StructSize, + Name = (byte*)name, + NameLen = nameLength, + Start = column.Start, + Length = column.Length, + Type = column.Type, + Nullable = column.Nullable ? 1 : 0, + Scale = column.Scale, + Format = (byte*)format, + FormatLen = formatLength, + TrimChar = column.TrimChar, + TrimMode = column.TrimMode, + Alignment = column.Alignment, + Padding = column.Padding, + Overflow = column.Overflow, + Dictionary = column.Dictionary, + }; + } + + private static (IntPtr Pointer, int Length) Utf8(string? text, List allocations) + { + if (text is null) + { + return (IntPtr.Zero, 0); + } + byte[] bytes = Encoding.UTF8.GetBytes(text); + IntPtr memory = Marshal.AllocHGlobal(Math.Max(1, bytes.Length)); + allocations.Add(memory); + Marshal.Copy(bytes, 0, memory, bytes.Length); + return (memory, bytes.Length); + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs b/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs new file mode 100644 index 0000000..75d2ab3 --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeTestReads.cs @@ -0,0 +1,135 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; + +namespace FixedWidthParser.Tests.Native +{ + /// A published fw_table copied into managed arrays, then freed. + internal sealed class ReadOutcome + { + public int Status { get; init; } + public long RowCount { get; init; } + public NativeColumnRaw[] Columns { get; init; } = []; + public List Values { get; init; } = []; + public long LinesRead { get; init; } + public long LinesSkipped { get; init; } + public string Error { get; init; } = ""; + + public object?[] Column(int index) + { + return Values[index]; + } + } + + internal static unsafe class NativeTestReads + { + internal static NativeReadOptionsRaw Options(long headerLines = 0, bool skipInvalid = false, int degreeOfParallelism = 1, int encoding = 0) + { + return new NativeReadOptionsRaw + { + StructSize = sizeof(NativeReadOptionsRaw), + HeaderLines = headerLines, + SkipInvalid = skipInvalid ? 1 : 0, + DegreeOfParallelism = degreeOfParallelism, + Encoding = encoding, + }; + } + + internal static ReadOutcome FromMemory(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null, int chunkSize = 0) + { + NativeReadOptionsRaw value = options ?? default; + NativeTableRaw table = default; + var result = new NativeReadResultRaw { StructSize = sizeof(NativeReadResultRaw) }; + fixed (byte* data = content) + { + int status = ReadApi.ReadMemory(data, content.Length, layout, options is null ? null : &value, &table, &result, chunkSize); + return Collect(status, &table, result); + } + } + + internal static ReadOutcome FromFile(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null) + { + string path = Path.GetTempFileName(); + try + { + File.WriteAllBytes(path, content); + NativeReadOptionsRaw value = options ?? default; + NativeTableRaw table = default; + var result = new NativeReadResultRaw { StructSize = sizeof(NativeReadResultRaw) }; + int status = ReadApi.ReadFile(Encoding.UTF8.GetBytes(path), layout, options is null ? null : &value, &table, &result); + return Collect(status, &table, result); + } + finally + { + File.Delete(path); + } + } + + internal static ReadOutcome[] Both(string content, NativeLayout layout, NativeReadOptionsRaw? options = null, Encoding? encoding = null) + { + byte[] bytes = (encoding ?? Encoding.UTF8).GetBytes(content); + return [FromMemory(bytes, layout, options), FromFile(bytes, layout, options)]; + } + + internal static ReadOutcome Collect(int status, NativeTableRaw* table, NativeReadResultRaw result) + { + try + { + return new ReadOutcome + { + Status = status, + RowCount = table->RowCount, + Columns = [.. Enumerable.Range(0, table->ColumnCount).Select(i => Column(table, i))], + Values = [.. Enumerable.Range(0, table->ColumnCount).Select(i => Copy(Column(table, i)))], + LinesRead = result.LinesRead, + LinesSkipped = result.LinesSkipped, + Error = status == NativeStatus.Ok ? "" : NativeTestLayouts.LastError(), + }; + } + finally + { + NativeTable.Free(table); + } + } + + private static NativeColumnRaw Column(NativeTableRaw* table, int index) + { + return table->Columns[index]; + } + + internal static object?[] Copy(NativeColumnRaw column) + { + var values = new object?[column.Length]; + for (int row = 0; row < column.Length; row++) + { + values[row] = IsNull(column, row) ? null : Value(column, row); + } + return values; + } + + private static bool IsNull(NativeColumnRaw column, int row) + { + return column.Validity is not null && (column.Validity[row >> 3] & (1 << (row & 7))) == 0; + } + + private static string Text(NativeColumnRaw column, int row) + { + int* offsets = (int*)column.Values; + int length = offsets[row + 1] - offsets[row]; + return length == 0 ? "" : Encoding.UTF8.GetString(column.Data + offsets[row], length); + } + + private static object Value(NativeColumnRaw column, int row) + { + return column.Type switch + { + NativeType.String => Text(column, row), + NativeType.Float64 => ((double*)column.Values)[row], + NativeType.Date => ((int*)column.Values)[row], + _ => ((long*)column.Values)[row], + }; + } + } +} diff --git a/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs b/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs new file mode 100644 index 0000000..75abb9e --- /dev/null +++ b/tests/FixedWidthParser.Tests/Native/NativeWriteTests.cs @@ -0,0 +1,381 @@ +using System.Runtime.InteropServices; +using System.Text; +using FixedWidthParser.Native; +using FixedWidthParser.Native.Layouts; +using FixedWidthParser.Native.Reading; +using FixedWidthParser.Native.Writing; +using static FixedWidthParser.Tests.Native.NativeTestLayouts; +using static FixedWidthParser.Tests.Native.NativeTestReads; + +namespace FixedWidthParser.Tests.Native +{ + public sealed unsafe class NativeWriteTests + { + private const string EveryType = + "Ann " + "00042" + " 12.500" + "00012345" + "20231024" + "133005" + "20231024133000" + "\n" + + " " + " " + " -0.5" + "-0000001" + "19700101" + "000000" + "19700101000000" + "\n"; + + private static NativeLayout EveryTypeLayout() + { + return Build( + new Col("name", 0, 6, TString) { Nullable = true }, + new Col("qty", 6, 5, TInt64) { Nullable = true }, + new Col("price", 11, 8, TFloat64), + new Col("amount", 19, 8, TDecimal) { Scale = 2 }, + new Col("day", 27, 8, TDate) { Format = "yyyyMMdd" }, + new Col("time", 35, 6, TTime) { Format = "HHmmss" }, + new Col("stamp", 41, 14, TTimestamp) { Format = "yyyyMMddHHmmss" }); + } + + private static NativeWriteOptionsRaw WriteOptions(int degree = 1, long batchRows = 0, int encoding = 0) + { + return new NativeWriteOptionsRaw { StructSize = sizeof(NativeWriteOptionsRaw), DegreeOfParallelism = degree, BatchRows = batchRows, Encoding = encoding }; + } + + /// Reads into a table the caller frees. + private static NativeTableRaw ReadTable(byte[] content, NativeLayout layout, NativeReadOptionsRaw? options = null) + { + NativeTableRaw table = default; + NativeReadOptionsRaw value = options ?? default; + fixed (byte* data = content) + { + int status = ReadApi.ReadMemory(data, content.Length, layout, options is null ? null : &value, &table, null); + Assert.True(status == NativeStatus.Ok, LastError()); + } + return table; + } + + private static (int Status, byte[] Bytes, string Error) WriteMemory(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw? options = null) + { + NativeWriteOptionsRaw value = options ?? default; + NativeBufferRaw buffer = default; + int status = WriteApi.WriteMemory(table, layout, options is null ? null : &value, &buffer); + byte[] bytes = buffer.Data is null ? [] : new ReadOnlySpan(buffer.Data, (int)buffer.Length).ToArray(); + WriteApi.FreeBuffer(&buffer); + return (status, bytes, status == NativeStatus.Ok ? "" : LastError()); + } + + private static byte[] RoundTrip(string content, NativeLayout readLayout, NativeLayout writeLayout, NativeWriteOptionsRaw? options = null) + { + var table = ReadTable(Encoding.UTF8.GetBytes(content), readLayout); + try + { + var (status, bytes, error) = WriteMemory(&table, writeLayout, options); + Assert.True(status == NativeStatus.Ok, error); + return bytes; + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void Write_EveryType_RoundTripsThroughRead() + { + var layout = EveryTypeLayout(); + var original = FromMemory(Encoding.UTF8.GetBytes(EveryType), layout); + + byte[] written = RoundTrip(EveryType, layout, layout); + var reread = FromMemory(written, layout); + + Assert.Equal(original.RowCount, reread.RowCount); + for (int i = 0; i < 7; i++) + { + Assert.Equal(original.Column(i), reread.Column(i)); + } + Assert.Equal(2 * 56, written.Length); + } + + [Fact] + public void Write_ScaledZeroPadded_RoundTrips() + { + var layout = Build(new Col("preabe", 0, 13, TFloat64) { Scale = 2, Padding = '0' }, new Col("voltot", 13, 18, TDecimal) { Scale = 2, Padding = '0' }); + const string Content = + "0000000123456" + "000000000000123456" + "\n" + + "-000000012345" + "-00000000000000001" + "\n" + + "0000000000000" + "000000000000000000" + "\n"; + + Assert.Equal(Content, Encoding.UTF8.GetString(RoundTrip(Content, layout, layout))); + } + + [Fact] + public void Write_Null_FillsWithTheTrimCharacter() + { + var layout = Build(new Col("n", 0, 4, TInt64) { Nullable = true, TrimChar = '0', Padding = '0' }, new Col("s", 4, 3, TString) { Nullable = true }); + + Assert.Equal("0012abc\n0000 \n", Encoding.UTF8.GetString(RoundTrip("0012abc\n0000 \n", layout, layout))); + } + + [Fact] + public void Write_StringTruncate_KeepsWholeCharacters() + { + var read = Build(new Col("s", 0, 7, TString)); + var write = Build(new Col("s", 0, 2, TString) { Overflow = 1 }); + + Assert.Equal("A \n", Encoding.UTF8.GetString(RoundTrip("Açores\n", read, write))); + } + + [Fact] + public void Write_StringOverflowThrow_IsError() + { + var read = Build(new Col("s", 0, 6, TString)); + var write = Build(new Col("s", 0, 3, TString)); + var table = ReadTable("ABCDEF\n"u8.ToArray(), read); + try + { + var (status, _, error) = WriteMemory(&table, write); + + Assert.Equal(NativeStatus.Error, status); + Assert.Contains("exceeds the width 3", error, StringComparison.Ordinal); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void Write_NumberOverflow_IsError() + { + var read = Build(new Col("n", 0, 5, TInt64)); + var write = Build(new Col("n", 0, 2, TInt64)); + var table = ReadTable("12345\n"u8.ToArray(), read); + try + { + Assert.Equal(NativeStatus.Error, WriteMemory(&table, write).Status); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void Write_TypeMismatch_IsInvalidArgument() + { + var read = Build(new Col("n", 0, 5, TInt64)); + var table = ReadTable("12345\n"u8.ToArray(), read); + try + { + var (status, _, error) = WriteMemory(&table, Build(new Col("n", 0, 5, TString))); + + Assert.Equal(NativeStatus.InvalidArgument, status); + Assert.Contains("'n'", error, StringComparison.Ordinal); + Assert.Equal(NativeStatus.InvalidArgument, WriteMemory(&table, Build(new Col("a", 0, 2, TInt64), new Col("b", 2, 2, TInt64))).Status); + } + finally + { + NativeTable.Free(&table); + } + } + + public static TheoryData Batches() + { + return new TheoryData { { 1, 1 }, { 1, 3 }, { 4, 1 }, { 4, 3 }, { 4, 8192 }, { 0, 0 } }; + } + + [Theory] + [MemberData(nameof(Batches))] + public void Write_Parallel_IsByteIdenticalToSerial(int degree, long batchRows) + { + var layout = EveryTypeLayout(); + string content = string.Concat(Enumerable.Repeat(EveryType, 40)); + + byte[] serial = RoundTrip(content, layout, layout, WriteOptions(degree: 1)); + byte[] parallel = RoundTrip(content, layout, layout, WriteOptions(degree, batchRows)); + + Assert.Equal(serial, parallel); + } + + [Fact] + public void Write_File_EqualsMemory() + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(EveryType), layout); + string path = Path.GetTempFileName(); + try + { + var options = WriteOptions(degree: 4, batchRows: 1); + Assert.Equal(NativeStatus.Ok, WriteApi.WriteFile(Encoding.UTF8.GetBytes(path), &table, layout, &options)); + + Assert.Equal(WriteMemory(&table, layout).Bytes, File.ReadAllBytes(path)); + } + finally + { + NativeTable.Free(&table); + File.Delete(path); + } + } + + [Theory] + [InlineData(1, 0)] + [InlineData(4, 1)] + [InlineData(4, 7)] + [InlineData(16, 3)] + public void WriteStream_EqualsMemory(int degree, long batchRows) + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat(EveryType, 25))), layout); + try + { + var options = WriteOptions(degree, batchRows); + using var sink = new MemoryStream(); + Assert.Equal(NativeStatus.Ok, WriteApi.WriteStream(&table, layout, &options, sink)); + + Assert.Equal(WriteMemory(&table, layout).Bytes, sink.ToArray()); + } + finally + { + NativeTable.Free(&table); + } + } + + private sealed class Collected + { + public readonly List Bytes = []; + public int Calls; + public int FailAfter = int.MaxValue; + } + + [UnmanagedCallersOnly] + private static int SinkWrite(void* userData, byte* buffer, long length) + { + var collected = (Collected)GCHandle.FromIntPtr((IntPtr)userData).Target!; + if (++collected.Calls > collected.FailAfter) + { + SourceErrors.Set("the disk is full"); + return -1; + } + collected.Bytes.AddRange(new ReadOnlySpan(buffer, (int)length).ToArray()); + return 0; + } + + private static int WriteThroughSink(NativeTableRaw* table, NativeLayout layout, NativeWriteOptionsRaw options, Collected collected) + { + var handle = GCHandle.Alloc(collected); + try + { + var raw = new NativeSinkRaw { StructSize = sizeof(NativeSinkRaw), UserData = (void*)GCHandle.ToIntPtr(handle), Write = &SinkWrite }; + Assert.True(CallbackWriteStream.TryCreate(&raw, out var sink)); + using (sink) + { + return WriteApi.WriteStream(table, layout, &options, sink); + } + } + finally + { + handle.Free(); + } + } + + [Fact] + public void WriteStream_CallbackSink_GetsEveryBatchInOrder() + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat(EveryType, 10))), layout); + try + { + var collected = new Collected(); + Assert.Equal(NativeStatus.Ok, WriteThroughSink(&table, layout, WriteOptions(degree: 4, batchRows: 3), collected)); + + Assert.Equal(WriteMemory(&table, layout).Bytes, collected.Bytes); + Assert.Equal(7, collected.Calls); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void WriteStream_FailingSink_ReportsItsMessage() + { + var layout = EveryTypeLayout(); + var table = ReadTable(Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat(EveryType, 10))), layout); + try + { + var collected = new Collected { FailAfter = 2 }; + Assert.Equal(NativeStatus.Error, WriteThroughSink(&table, layout, WriteOptions(degree: 4, batchRows: 3), collected)); + + Assert.Equal("the disk is full", LastError()); + Assert.Equal(3, collected.Calls); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void WriteStream_SinkWithoutWrite_IsRejected() + { + var raw = new NativeSinkRaw { StructSize = sizeof(NativeSinkRaw) }; + Assert.False(CallbackWriteStream.TryCreate(&raw, out _)); + Assert.Contains("sizeof(fw_sink)", LastError(), StringComparison.Ordinal); + } + + [Fact] + public void Write_Latin1_EncodesText() + { + var layout = Build(new Col("s", 0, 6, TString)); + + byte[] written = RoundTrip("AÇÃO \n", layout, layout, WriteOptions(encoding: 28591)); + + Assert.Equal(Encoding.Latin1.GetBytes("AÇÃO \n"), written); + } + + [Fact] + public void Write_NewLineAndFiller_AreHonored() + { + var read = Build(new Col("a", 0, 2, TString), new Col("b", 2, 2, TString)); + var write = Build(LayoutOptions(lineLength: 8, filler: '.'), new Col("a", 0, 2, TString), new Col("b", 4, 2, TString)); + byte[] newline = "\r\n"u8.ToArray(); + fixed (byte* nl = newline) + { + var options = WriteOptions() with { NewLine = nl, NewLineLen = 2 }; + + Assert.Equal("ab..cd..\r\n", Encoding.UTF8.GetString(RoundTrip("abcd\n", read, write, options))); + } + } + + [Fact] + public void Write_TemporalWithoutFormat_IsIso() + { + var read = Build(new Col("d", 0, 8, TDate) { Format = "yyyyMMdd" }); + var write = Build(new Col("d", 0, 10, TDate)); + + Assert.Equal("2023-10-24\n", Encoding.UTF8.GetString(RoundTrip("20231024\n", read, write))); + } + + [Fact] + public void Write_EmptyTable_IsEmpty() + { + var layout = Build(new Col("a", 0, 2, TString)); + + Assert.Empty(RoundTrip("", layout, layout)); + } + + [Fact] + public void Write_BadOptionsStructSize_IsInvalidArgument() + { + var layout = Build(new Col("a", 0, 2, TString)); + var table = ReadTable("ab\n"u8.ToArray(), layout); + try + { + Assert.Equal(NativeStatus.InvalidArgument, WriteMemory(&table, layout, WriteOptions() with { StructSize = 4 }).Status); + } + finally + { + NativeTable.Free(&table); + } + } + + [Fact] + public void BufferStruct_HasTheHeaderLayout() + { + Assert.Equal(16, Marshal.SizeOf()); + Assert.Equal(32, Marshal.SizeOf()); + } + } +} diff --git a/tests/FixedWidthParser.Tests/NullableColumnTests.cs b/tests/FixedWidthParser.Tests/NullableColumnTests.cs index 3359d15..24fda02 100644 --- a/tests/FixedWidthParser.Tests/NullableColumnTests.cs +++ b/tests/FixedWidthParser.Tests/NullableColumnTests.cs @@ -1,109 +1,57 @@ -using System.Text; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Covers nullable value-type columns (T?): a blank (all-spaces) column parses to - /// without invoking the underlying parser, and a present value round-trips - /// exactly as the non-nullable T would. Exercises reflection parse/write (char + UTF-8) and - /// the generated parity, plus the combination with a custom converter. + /// Nullable value-type columns (T?): a blank column parses to without + /// invoking the underlying parser, and a present value round-trips exactly as T would. /// public class NullableColumnTests { - // NullableModel is Age[0,5) + Amount[5,15): build lines by padding each field to its column - // width, rather than hand-counting spaces in a literal (an easy off-by-one to get wrong). - private static string Line(string age, string amount) + // NullableModel is Age[0,5) + Amount[5,15). + private static byte[] Line(string age, string amount) { - return age.PadRight(5) + amount.PadRight(10); + return Bytes(age.PadRight(5) + amount.PadRight(10)); } - private static readonly string Blank = Line("", ""); - [Fact] - public void Reflection_Char_BlankColumn_ParsesToNull() + public void BlankColumn_ParsesToNull() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(Blank, Inv, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse(Line("", ""), FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Null(model.Age); Assert.Null(model.Amount); } [Fact] - public void Reflection_Char_PresentValue_Parses() + public void PresentValue_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(Line("30", "123.45"), Inv, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse(Line("30", "123.45"), FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(30, model.Age); Assert.Equal(123.45m, model.Amount); } [Fact] - public void Reflection_Char_InvalidValue_ReturnsFalse() + public void InvalidValue_ReturnsFalse() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse(Line("XX", ""), Inv, null, out _); - - Assert.False(ok); + Assert.False(FixedWidth.TryParse(Line("XX", ""), FixedWidthLayout.FromAttributes(), Ctx, out _)); } [Fact] - public void Reflection_Utf8_BlankColumn_ParsesToNull() + public void Write_Null_WritesBlank() { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes(Blank); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NullableModel { Age = null, Amount = null }); - Assert.True(ok); - Assert.Null(model.Age); - Assert.Null(model.Amount); + Assert.Equal(new string(' ', 15), line); } [Fact] - public void Reflection_Utf8_PresentValue_Parses() + public void RoundTrip_PreservesNullAndValue() { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes(Line("30", "123.45")); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(30, model.Age); - Assert.Equal(123.45m, model.Amount); - } - - [Fact] - public void Reflection_Write_Null_WritesBlank() - { - var writer = new FixedWidthWriter(); - var model = new NullableModel { Age = null, Amount = null }; - - string line = WriteOne(writer, model); - - Assert.Equal(Blank, line); - } - - [Fact] - public void Reflection_RoundTrip_PreservesNullAndValue() - { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); var original = new NullableModel { Age = 42, Amount = null }; - string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + string line = WriteOne(FixedWidthLayout.FromAttributes(), original); - Assert.True(ok); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var parsed)); Assert.Equal(original.Age, parsed.Age); Assert.Null(parsed.Amount); } @@ -112,95 +60,42 @@ public void Reflection_RoundTrip_PreservesNullAndValue() [InlineData(" ")] [InlineData("30 123.45 ")] [InlineData("XX ")] - public void Generated_Char_MatchesReflection(string line) + public void Generated_MatchesReflection(string line) { - var reflection = new FixedWidthParser(); - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Age, g.Age); - Assert.Equal(r.Amount, g.Amount); - } - } - - [Theory] - [InlineData(" ")] - [InlineData("30 123.45 ")] - public void Generated_Utf8_MatchesReflection(string line) - { - byte[] bytes = Encoding.UTF8.GetBytes(line); - var reflection = new Utf8FixedWidthParser(); - - bool okR = reflection.TryParse(bytes, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(bytes, Inv, null, out var g); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); Assert.Equal(r.Age, g.Age); Assert.Equal(r.Amount, g.Amount); } - [Fact] - public void NullableConverter_Blank_ParsesToNull() + [Theory] + [InlineData(" ")] + [InlineData("12345 ")] + public void NullableConverter_Generated_MatchesReflection(string line) { - var parser = new FixedWidthParser(); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); - bool ok = parser.TryParse(" ", Inv, null, out var model); - - Assert.True(ok); - Assert.Null(model.Amount); + Assert.True(okR); + Assert.True(okG); + Assert.Equal(r.Amount, g.Amount); } [Fact] public void NullableConverter_PresentValue_Parses() { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("12345 ", Inv, null, out var model); - - Assert.True(ok); + Assert.True(FixedWidth.TryParse("12345 "u8, FixedWidthLayout.FromAttributes(), Ctx, out var model)); Assert.Equal(new CentsValue(12345), model.Amount); } [Fact] public void NullableConverter_Write_Null_WritesBlank() { - var writer = new FixedWidthWriter(); - var model = new NullableConverterModel { Amount = null }; - - string line = WriteOne(writer, model); + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NullableConverterModel { Amount = null }); Assert.Equal(new string(' ', 8), line); } - - [Fact] - public void NullableConverter_Generated_Char_MatchesReflection() - { - var reflection = new FixedWidthParser(); - const string line = "12345 "; - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Amount, g.Amount); - } - - [Fact] - public void NullableConverter_Generated_Char_Blank_MatchesReflection() - { - var reflection = new FixedWidthParser(); - const string line = " "; - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Null(r.Amount); - Assert.Null(g.Amount); - } } } diff --git a/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs new file mode 100644 index 0000000..5510bf3 --- /dev/null +++ b/tests/FixedWidthParser.Tests/ParallelAggregateTests.cs @@ -0,0 +1,322 @@ +using System.Collections.Concurrent; +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// Parallel aggregation must equal one sequential pass, whatever the source and partition size. + public sealed class ParallelAggregateTests : IDisposable + { + public enum Source + { + File, + Stream, + } + + private readonly string _path = Path.GetTempFileName(); + + public void Dispose() + { + File.Delete(_path); + } + + public sealed class Ages : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in NameAgeModel model) + { + Values.Add(model.Age); + } + + public void Merge(Ages following) + { + Values.AddRange(following.Values); + } + } + + public sealed class GenAges : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in GenNameAgeModel model) + { + Values.Add(model.Age); + } + + public void Merge(GenAges following) + { + Values.AddRange(following.Values); + } + } + + /// Stalls on the first line and counts the accumulators not yet merged into another. + public sealed class SlowFirstChunk : IRecordAccumulator + { + public static int Unmerged; + public static int MaxUnmerged; + + public SlowFirstChunk() + { + int unmerged = Interlocked.Increment(ref Unmerged); + int max; + while (unmerged > (max = Volatile.Read(ref MaxUnmerged)) && Interlocked.CompareExchange(ref MaxUnmerged, unmerged, max) != max) + { + } + } + + public List Values { get; } = []; + + public void Add(in NameAgeModel model) + { + if (model.Name == "Name1" && model.Age == 1) + { + Thread.Sleep(300); + } + Values.Add(model.Age); + } + + public void Merge(SlowFirstChunk following) + { + Values.AddRange(following.Values); + Interlocked.Decrement(ref Unmerged); + } + } + + public sealed class CancelsItself : IRecordAccumulator + { + public void Add(in NameAgeModel model) + { + if (model.Age == 7) + { + throw new OperationCanceledException(); + } + } + + public void Merge(CancelsItself following) + { + } + } + + public static TheoryData SourcesAndChunkSizes() + { + var data = new TheoryData(); + foreach (var source in Enum.GetValues()) + { + foreach (int chunkSize in new[] { 1, 7, 10, 64, 4096, 0 }) + { + data.Add(source, chunkSize); + } + } + return data; + } + + public static TheoryData Sources() + { + return new TheoryData(Enum.GetValues()); + } + + // A BOM, CRLF and LF breaks, empty lines and no final line break: every boundary a partition can cut. + private static string Content(int lines, int badLine = 0) + { + var text = new StringBuilder(""); + for (int i = 1; i <= lines; i++) + { + string age = i == badLine ? "XX " : (i % 1000).ToString("D3", Inv); + text.Append("Name").Append(i % 10).Append(' ').Append(age); + if (i < lines) + { + text.Append(i % 3 == 0 ? "\r\n" : "\n"); + } + if (i % 7 == 0) + { + text.Append('\n'); + } + } + return text.ToString(); + } + + private List Sequential(FixedWidthReaderOptions? options = null) + { + return [.. FixedWidth.ReadFile(_path, FixedWidthLayout.FromAttributes(), options).Select(m => m.Age)]; + } + + private Task Aggregate(Source source, FixedWidthReaderOptions options) + { + var layout = FixedWidthLayout.FromAttributes(); + var ct = TestContext.Current.CancellationToken; + return source switch + { + Source.File => FixedWidth.AggregateFileAsync(_path, layout, options, ct), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, options, ct), + }; + } + + private Task AggregateGenerated(Source source, FixedWidthReaderOptions options) + { + var ct = TestContext.Current.CancellationToken; + return source switch + { + Source.File => FixedWidth.AggregateFileAsync(_path, options, ct), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), options: options, cancellationToken: ct), + }; + } + + [Theory] + [MemberData(nameof(SourcesAndChunkSizes))] + public async Task EveryPartitionSize_EqualsTheSequentialRead(Source source, int chunkSize) + { + File.WriteAllText(_path, Content(2000)); + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; + + var runtime = await Aggregate(source, parallel); + var generated = await AggregateGenerated(source, parallel); + + var expected = Sequential(); + Assert.Equal(2000, expected.Count); + Assert.Equal(expected, runtime.Values); + Assert.Equal(expected, generated.Values); + } + + // Hundreds of chunks finish while the first one stalls; only a few slots' worth may wait for it. + [Fact] + public async Task SlowEarlyChunk_BoundsTheResultsWaitingToMerge() + { + File.WriteAllText(_path, Content(2000)); + var options = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = 64 }; + + var result = await FixedWidth.AggregateAsync( + File.OpenRead(_path), FixedWidthLayout.FromAttributes(), leaveOpen: false, options, TestContext.Current.CancellationToken); + + Assert.Equal(Sequential(), result.Values); + Assert.InRange(SlowFirstChunk.MaxUnmerged, 1, 4 * 4 + 1); + } + + // Not the caller's cancellation: it fails its chunk like any exception instead of stalling the producer. + [Fact] + public async Task AccumulatorThrowingOperationCanceled_FailsTheAggregation() + { + File.WriteAllText(_path, Content(2000)); + var options = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = 64 }; + + await Assert.ThrowsAsync(() => FixedWidth.AggregateAsync( + File.OpenRead(_path), FixedWidthLayout.FromAttributes(), leaveOpen: false, options, TestContext.Current.CancellationToken)); + } + + [Theory] + [MemberData(nameof(Sources))] + public async Task OneWorker_EqualsTheSequentialRead(Source source) + { + File.WriteAllText(_path, Content(500)); + + var result = await Aggregate(source, new FixedWidthReaderOptions { DegreeOfParallelism = 1 }); + + Assert.Equal(Sequential(), result.Values); + } + + [Theory] + [MemberData(nameof(Sources))] + public async Task EmptySource_ReturnsAnEmptyAccumulator(Source source) + { + File.WriteAllBytes(_path, []); + + var result = await Aggregate(source, new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = 8 }); + + Assert.Empty(result.Values); + } + + [Theory] + [MemberData(nameof(SourcesAndChunkSizes))] + public async Task ParseFailure_ReportsTheSequentialLineNumber(Source source, int chunkSize) + { + File.WriteAllText(_path, Content(2000, badLine: 1537)); + var expected = Assert.Throws(() => Sequential()); + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = 4, ChunkSize = chunkSize }; + + var actual = await Assert.ThrowsAsync(() => Aggregate(source, parallel)); + + Assert.Equal(expected.LineNumber, actual.LineNumber); + Assert.Equal(expected.Message, actual.Message); + } + + [Theory] + [MemberData(nameof(Sources))] + public async Task OnParseError_SkipsEachBadLineOnce_WithItsLineNumber(Source source) + { + File.WriteAllText(_path, Content(2000, badLine: 1537) + "\nbad line\n" + Content(3, badLine: 2)[1..]); + var sequentialErrors = new List(); + var expected = Sequential(new FixedWidthReaderOptions { OnParseError = ex => { sequentialErrors.Add(ex.LineNumber); return true; } }); + var parallelErrors = new ConcurrentBag(); + var options = new FixedWidthReaderOptions + { + OnParseError = ex => { parallelErrors.Add(ex.LineNumber); return true; }, + DegreeOfParallelism = 4, + ChunkSize = 9, + }; + + var result = await Aggregate(source, options); + + Assert.Equal(3, sequentialErrors.Count); + Assert.Equal(expected, result.Values); + Assert.Equal(sequentialErrors, parallelErrors.Order()); + } + + [Theory] + [MemberData(nameof(SourcesAndChunkSizes))] + public async Task LineTooLong_ReportsTheSequentialLineNumber(Source source, int chunkSize) + { + File.WriteAllText(_path, Content(300) + "\n" + new string('9', 50) + "\n" + Content(3)[1..]); + var options = new FixedWidthReaderOptions { MaxLineLength = 20, DegreeOfParallelism = 4, ChunkSize = chunkSize }; + var expected = Assert.Throws(() => Sequential(options)); + + var actual = await Assert.ThrowsAsync(() => Aggregate(source, options)); + + Assert.Equal(expected.Message, actual.Message); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task Stream_IsDisposedUnlessLeftOpen(bool leaveOpen) + { + File.WriteAllText(_path, Content(10)); + await using var stream = File.OpenRead(_path); + + await FixedWidth.AggregateAsync(stream, FixedWidthLayout.FromAttributes(), leaveOpen, new FixedWidthReaderOptions { DegreeOfParallelism = 2, ChunkSize = 16 }, TestContext.Current.CancellationToken); + + Assert.Equal(leaveOpen, stream.CanRead); + } + + [Theory] + [MemberData(nameof(Sources))] + public async Task CancelledToken_Throws(Source source) + { + File.WriteAllText(_path, Content(10)); + using var cts = new CancellationTokenSource(); + await cts.CancelAsync(); + var layout = FixedWidthLayout.FromAttributes(); + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = 2 }; + + await Assert.ThrowsAnyAsync(() => source switch + { + Source.File => FixedWidth.AggregateFileAsync(_path, layout, parallel, cts.Token), + _ => FixedWidth.AggregateAsync(File.OpenRead(_path), layout, leaveOpen: false, parallel, cts.Token), + }); + } + + // Arguments are validated before the stream is taken over, so the caller still owns it. + [Fact] + public async Task NegativeDegreeOfParallelism_Throws() + { + File.WriteAllText(_path, Content(10)); + var layout = FixedWidthLayout.FromAttributes(); + var parallel = new FixedWidthReaderOptions { DegreeOfParallelism = -1 }; + var ct = TestContext.Current.CancellationToken; + await using var stream = File.OpenRead(_path); + + await Assert.ThrowsAsync(() => FixedWidth.AggregateFileAsync(_path, layout, parallel, ct)); + await Assert.ThrowsAsync(() => FixedWidth.AggregateAsync(stream, layout, leaveOpen: false, parallel, ct)); + Assert.True(stream.CanRead); + } + } +} diff --git a/tests/FixedWidthParser.Tests/ParserConstructionTests.cs b/tests/FixedWidthParser.Tests/ParserConstructionTests.cs deleted file mode 100644 index 1bb460b..0000000 --- a/tests/FixedWidthParser.Tests/ParserConstructionTests.cs +++ /dev/null @@ -1,61 +0,0 @@ -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Tests -{ - /// - /// Build-time paths of the runtime parsers. Both - /// and build their column plan in a static constructor - /// and capture any error, rethrowing it from the public constructor via - /// . These cover the - /// struct-without-declared-constructor branch of BuildModelFactory (default - /// initialization) and the error capture/rethrow (the layout-validation branches are covered by - /// for the char parser). - /// - public class ParserConstructionTests - { - // ----------------------------- char parser ----------------------------- - - [Fact] - public void Parser_StructWithoutDeclaredConstructor_Parses() - { - var parser = new FixedWidthParser(); - - Assert.True(parser.TryParse("ABC", null, null, out var model)); - Assert.Equal("ABC", model.Code); - } - - [Fact] - public void Parser_BuildError_IsRethrownOnEveryConstruction() - { - // The error is captured once in the static constructor and rethrown on each construction. - Assert.Throws(() => new FixedWidthParser()); - Assert.Throws(() => new FixedWidthParser()); - } - - // ----------------------------- UTF-8 parser ----------------------------- - - [Fact] - public void Utf8Parser_StructWithoutDeclaredConstructor_Parses() - { - var parser = new Utf8FixedWidthParser(); - - Assert.True(parser.TryParse("ABC"u8, null, null, out var model)); - Assert.Equal("ABC", model.Code); - } - - [Fact] - public void Utf8Parser_BuildError_IsRethrownOnEveryConstruction() - { - Assert.Throws(() => new Utf8FixedWidthParser()); - Assert.Throws(() => new Utf8FixedWidthParser()); - } - - [Fact] - public void Utf8Parser_OverlappingColumns_ThrowsOnConstruction() - { - var ex = Assert.Throws(() => new Utf8FixedWidthParser()); - - Assert.Contains("Overlapping", ex.Message, StringComparison.Ordinal); - } - } -} diff --git a/tests/FixedWidthParser.Tests/ParserTests.cs b/tests/FixedWidthParser.Tests/ParserTests.cs index 6d947a3..0074eaa 100644 --- a/tests/FixedWidthParser.Tests/ParserTests.cs +++ b/tests/FixedWidthParser.Tests/ParserTests.cs @@ -1,5 +1,4 @@ using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -9,10 +8,10 @@ public class ParserTests [Fact] public void TryParse_WellFormedLine_ParsesAllColumns() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - // Name Age Salary - bool ok = parser.TryParse("John Doe 30 60000.00 ", Inv, null, out var model); + // Name Age Salary + bool ok = FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, Ctx, out var model); Assert.True(ok); Assert.Equal("John Doe", model.Name); @@ -23,9 +22,9 @@ public void TryParse_WellFormedLine_ParsesAllColumns() [Fact] public void TryParse_TrimsTrailingSpacesOnStringColumns() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - parser.TryParse("Al 25 100.00 ", Inv, null, out var model); + FixedWidth.TryParse("Al 25 100.00 "u8, layout, Ctx, out var model); Assert.Equal("Al", model.Name); } @@ -33,20 +32,19 @@ public void TryParse_TrimsTrailingSpacesOnStringColumns() [Fact] public void TryParse_InvalidNumber_ReturnsFalse() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("John Doe XX 60000.00 ", Inv, null, out _); + bool ok = FixedWidth.TryParse("John Doe XX 60000.00 "u8, layout, Ctx, out _); Assert.False(ok); } [Fact] - public void TryParse_ShortLine_NumericColumnMissing_ReturnsFalse() + public void TryParse_ShortLine_ReturnsFalse() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - // The line ends before the Age column (start 10): the numeric processor fails. - bool ok = parser.TryParse("John", Inv, null, out _); + bool ok = FixedWidth.TryParse("John"u8, layout, Ctx, out _); Assert.False(ok); } @@ -54,9 +52,9 @@ public void TryParse_ShortLine_NumericColumnMissing_ReturnsFalse() [Fact] public void TryParse_ShortLine_StringColumnMissing_ReturnsFalse() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("42", Inv, null, out var model); + bool ok = FixedWidth.TryParse("42"u8, layout, Ctx, out var model); Assert.False(ok); Assert.Equal(default, model); @@ -65,23 +63,58 @@ public void TryParse_ShortLine_StringColumnMissing_ReturnsFalse() [Fact] public void TryParse_WithStringPool_ReturnsInternedInstances() { - var parser = new FixedWidthParser(); - var pool = new StringPool(); + var layout = FixedWidthLayout.FromAttributes(); + var context = CtxWith(pool: new StringPool()); - parser.TryParse("Alice 25 100.00 ", Inv, pool, out var first); - parser.TryParse("Alice 40 200.00 ", Inv, pool, out var second); + FixedWidth.TryParse("Alice 25 100.00 "u8, layout, context, out var first); + FixedWidth.TryParse("Alice 40 200.00 "u8, layout, context, out var second); Assert.Equal("Alice", first.Name); Assert.Same(first.Name, second.Name); } + // More values than the per-thread cache in front of the pool holds, so sets fill, evict and collide: + // every value must still decode right. (The pool evicts too, so instances are not compared here.) + [Fact] + public void TryParse_WithStringPool_ManyValues_DecodeRight() + { + var layout = FixedWidthLayout.FromAttributes(); + var pool = new StringPool(); + var context = CtxWith(pool: pool); + var names = Enumerable.Range(0, 5000).Select(i => i % 3 == 0 ? "é" + i : "N" + i).ToArray(); + + for (int round = 0; round < 2; round++) + { + foreach (string name in names) + { + byte[] line = [.. Bytes(name), .. Enumerable.Repeat((byte)' ', 10 - Bytes(name).Length), .. "25 100.00 "u8]; + Assert.True(FixedWidth.TryParse(line, layout, context, out var model)); + Assert.Equal(name, model.Name); + } + } + } + + [Fact] + public void TryParse_WithStringPool_OtherEncodingsAndStrictUtf8_DecodeAsWithoutIt() + { + var layout = FixedWidthLayout.FromAttributes(); + var latin1 = CtxWith(pool: new StringPool(), encoding: System.Text.Encoding.Latin1); + var strict = CtxWith(pool: new StringPool(), encoding: new System.Text.UTF8Encoding(false, true)); + + Assert.True(FixedWidth.TryParse([0xC9, 0x41, 0x42], layout, latin1, out var model)); + Assert.Equal("ÉAB", model.Code); + Assert.True(FixedWidth.TryParse("éA"u8, layout, strict, out model)); + Assert.Equal("éA", model.Code); + Assert.Throws(() => FixedWidth.TryParse([0xC3, 0x41, 0x42], layout, strict, out _)); + } + [Fact] public void TryParse_WithoutPool_ReturnsDistinctInstances() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - parser.TryParse("Alice 25 100.00 ", Inv, null, out var first); - parser.TryParse("Alice 25 100.00 ", Inv, null, out var second); + FixedWidth.TryParse("Alice 25 100.00 "u8, layout, Ctx, out var first); + FixedWidth.TryParse("Alice 25 100.00 "u8, layout, Ctx, out var second); Assert.Equal(first.Name, second.Name); Assert.NotSame(first.Name, second.Name); @@ -90,9 +123,9 @@ public void TryParse_WithoutPool_ReturnsDistinctInstances() [Fact] public void TryParse_FieldBasedModel_ParsesFields() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("ABC 12 ", Inv, null, out var product); + bool ok = FixedWidth.TryParse("ABC 12 "u8, layout, Ctx, out var product); Assert.True(ok); Assert.Equal("ABC", product.Code); @@ -102,12 +135,41 @@ public void TryParse_FieldBasedModel_ParsesFields() [Fact] public void TryParse_FloatColumn_Parses() { - var parser = new FixedWidthParser(); + var layout = FixedWidthLayout.FromAttributes(); - bool ok = parser.TryParse("3.14 ", Inv, null, out var measurement); + bool ok = FixedWidth.TryParse("3.14 "u8, layout, Ctx, out var measurement); Assert.True(ok); Assert.Equal(3.14f, measurement.Value, 2); } + + [Fact] + public void TryParse_StructWithoutDeclaredConstructor_Parses() + { + var layout = FixedWidthLayout.FromAttributes(); + + Assert.True(FixedWidth.TryParse("ABC"u8, layout, Ctx, out var model)); + Assert.Equal("ABC", model.Code); + } + + [Fact] + public void Constructor_BuildError_IsRethrownOnEveryConstruction() + { + // The error is captured once in the static constructor and rethrown on each construction. + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); + } + + [Fact] + public void TryParse_RefStructModel_Parses() + { + var layout = FixedWidthLayout.FromAttributes(); + + bool ok = FixedWidth.TryParse("John Doe 30 60000.00 "u8, layout, Ctx, out var model); + + Assert.True(ok); + Assert.Equal("John Doe", model.Name); + Assert.Equal(30, model.Age); + } } } diff --git a/tests/FixedWidthParser.Tests/PipeReaderTests.cs b/tests/FixedWidthParser.Tests/PipeReaderTests.cs deleted file mode 100644 index d15389f..0000000 --- a/tests/FixedWidthParser.Tests/PipeReaderTests.cs +++ /dev/null @@ -1,271 +0,0 @@ -using System.IO.Pipelines; -using System.Text; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the overloads of - /// (reflection) and (source-generated): line-ending - /// handling, BOM skipping, empty-line skipping, multi-segment lines (the pooled scratch-copy path), - /// cancellation, reader-completion behavior, and parity with the stream reader. - /// - public class PipeReaderTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - -#pragma warning disable IDISP001 // ownership of the stream transfers to the PipeReader / reader - private static PipeReader Pipe(string text, int bufferSize = -1) - { - var stream = new MemoryStream(Encoding.UTF8.GetBytes(text)); - return bufferSize > 0 - ? PipeReader.Create(stream, new StreamPipeReaderOptions(bufferSize: bufferSize)) - : PipeReader.Create(stream); - } - - private static MemoryStream StreamOf(string text) - { - return new MemoryStream(Encoding.UTF8.GetBytes(text)); - } -#pragma warning restore IDISP001 - - private static async Task> CollectAsync(IAsyncEnumerable source) - { - var list = new List(); - await foreach (var item in source.ConfigureAwait(false)) - { - list.Add(item); - } - return list; - } - - [Fact] - public async Task ReadAsync_Pipe_ParsesAll() - { - var reader = new FixedWidthByteReader(Inv); - - var people = await CollectAsync(reader.ReadAsync(Pipe(TwoPeople))).ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public async Task ReadAsync_Pipe_Generated_ParsesAll() - { - var people = await CollectAsync( - FixedWidthParser.FixedWidthUtf8.ReadAsync(Pipe(TwoPeople), formatProvider: Inv)) - .ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line - public async Task ReadAsync_Pipe_HandlesLineEndingsAndTrailingNewline(string text) - { - var reader = new FixedWidthByteReader(); - - var codes = (await CollectAsync(reader.ReadAsync(Pipe(text))).ConfigureAwait(true)).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_Pipe_SkipsEmptyLines() - { - var reader = new FixedWidthByteReader(); - - var codes = (await CollectAsync(reader.ReadAsync(Pipe("ABC\n\n\nDEF\n"))).ConfigureAwait(true)).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_Pipe_SkipsLeadingBom() - { - var reader = new FixedWidthByteReader(); - - var codes = (await CollectAsync(reader.ReadAsync(Pipe("ABC\nDEF\n"))).ConfigureAwait(true)).Select(m => m.Code).ToList(); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public async Task ReadAsync_Pipe_MultiSegmentLine_ParsesViaScratch() - { - // A tiny pipe buffer (8 bytes) forces each 25-byte record to span several segments, so the - // line is non-contiguous and must be copied into the pooled scratch buffer before parsing. - var reader = new FixedWidthByteReader(Inv); - - var people = await CollectAsync(reader.ReadAsync(Pipe(TwoPeople, bufferSize: 8))).ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal(60000.00, people[0].Salary, 2); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(28, people[1].Age); - } - - [Fact] - public async Task ReadAsync_Pipe_MatchesStreamReader() - { - // Same content, two sources: the pipe path must produce exactly what the stream path does. - var byteReader = new FixedWidthByteReader(Inv); - -#pragma warning disable IDISP004 // the reader owns and disposes the stream when iteration completes - var fromStream = await CollectAsync(byteReader.ReadAsync(StreamOf(TwoPeople))).ConfigureAwait(true); -#pragma warning restore IDISP004 - var fromPipe = await CollectAsync(byteReader.ReadAsync(Pipe(TwoPeople))).ConfigureAwait(true); - - Assert.Equal(fromStream.Count, fromPipe.Count); - for (int i = 0; i < fromStream.Count; i++) - { - Assert.Equal(fromStream[i].Name, fromPipe[i].Name); - Assert.Equal(fromStream[i].Age, fromPipe[i].Age); - Assert.Equal(fromStream[i].Salary, fromPipe[i].Salary, 2); - } - } - - [Fact] - public async Task ReadAsync_Pipe_HonorsCancellation() - { - var reader = new FixedWidthByteReader(); - using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); - - await Assert.ThrowsAnyAsync(async () => - { - await foreach (var _ in reader.ReadAsync(Pipe("ABC\nDEF\n")).WithCancellation(cts.Token).ConfigureAwait(false)) - { - Assert.Fail("Enumeration should have been cancelled before yielding a record."); - } - }).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_CompletesReaderByDefault() - { - var reader = new FixedWidthByteReader(); - var pipe = Pipe("ABC\nDEF\n"); - - _ = await CollectAsync(reader.ReadAsync(pipe)).ConfigureAwait(true); - - // The reader was completed, so a further read is no longer allowed. - await Assert.ThrowsAsync(async () => await pipe.ReadAsync().ConfigureAwait(false)).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_LeaveOpen_DoesNotCompleteReader() - { - var reader = new FixedWidthByteReader(); - var pipe = Pipe("ABC\nDEF\n"); - - _ = await CollectAsync(reader.ReadAsync(pipe, leaveOpen: true)).ConfigureAwait(true); - - // Not completed: a further read succeeds and reports end-of-data. - var result = await pipe.ReadAsync().ConfigureAwait(true); - Assert.True(result.IsCompleted); - pipe.AdvanceTo(result.Buffer.End); - await pipe.CompleteAsync().ConfigureAwait(true); - } - - [Fact] - public void ReadAsync_Pipe_NullReader_Throws() - { - var reader = new FixedWidthByteReader(); - - Assert.Throws(() => reader.ReadAsync((PipeReader)null!)); - } - - [Fact] - [System.Diagnostics.CodeAnalysis.SuppressMessage("IDisposableAnalyzers.Correctness", "IDISP016:Don't use disposed instance", - Justification = "The test deliberately uses the enumerator after disposal to verify it throws.")] - public async Task ReadAsync_Pipe_MoveNextAfterDispose_Throws() - { - var reader = new FixedWidthByteReader(); - var enumerator = reader.ReadAsync(Pipe("ABC\nDEF\n")).GetAsyncEnumerator(); - await enumerator.DisposeAsync().ConfigureAwait(true); - - await Assert.ThrowsAsync(async () => await enumerator.MoveNextAsync().ConfigureAwait(false)).ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_AssemblesAcrossManySmallWrites() - { - // Feeding one byte at a time forces the BOM check and every line to span multiple reads, - // exercising the "need more data" paths (both before and within a line). - var pipe = new Pipe(); - var reader = new FixedWidthByteReader(); - - var produce = Task.Run(async () => - { - foreach (var b in Encoding.UTF8.GetBytes("ABC\nDEF\n")) - { - await pipe.Writer.WriteAsync(new[] { b }).ConfigureAwait(false); - await pipe.Writer.FlushAsync().ConfigureAwait(false); - } - await pipe.Writer.CompleteAsync().ConfigureAwait(false); - }); - - var codes = (await CollectAsync(reader.ReadAsync(pipe.Reader)).ConfigureAwait(true)).Select(m => m.Code).ToList(); - await produce.ConfigureAwait(true); - - Assert.Equal(["ABC", "DEF"], codes); - } - - [Fact] - public Task ReadAsync_Pipe_UnparseableLine_ThrowsFormat() - { - var reader = new FixedWidthByteReader(); - - // "A" is shorter than CodeModel's 3-wide column, so the trailing line fails to parse. - return Assert.ThrowsAsync(async () => - await CollectAsync(reader.ReadAsync(Pipe("A"))).ConfigureAwait(false)); - } - - [Fact] - public async Task ReadAsync_Pipe_CancelPendingRead_ThrowsOperationCanceledException() - { - var pipe = new Pipe(); - var reader = new FixedWidthByteReader(); - - var readTask = CollectAsync(reader.ReadAsync(pipe.Reader)); - - pipe.Reader.CancelPendingRead(); - - await Assert.ThrowsAnyAsync(async () => await readTask.ConfigureAwait(false)).ConfigureAwait(true); - await pipe.Writer.CompleteAsync().ConfigureAwait(true); - } - - [Fact] - public async Task ReadAsync_Pipe_UnparseableLine_WithLeaveOpen_LeavesReaderUsable() - { - var reader = new FixedWidthByteReader(); - var pipe = Pipe("A\nDEF\n"); - - var enumerator = reader.ReadAsync(pipe, leaveOpen: true).GetAsyncEnumerator(); - - await Assert.ThrowsAsync(async () => await enumerator.MoveNextAsync().ConfigureAwait(false)).ConfigureAwait(true); - - await enumerator.DisposeAsync().ConfigureAwait(true); - - var result = await pipe.ReadAsync().ConfigureAwait(true); - var text = Encoding.UTF8.GetString(result.Buffer); - Assert.Contains("DEF", text); - pipe.AdvanceTo(result.Buffer.End); - await pipe.CompleteAsync().ConfigureAwait(true); - } - } -} diff --git a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs index 2e69651..58f06d6 100644 --- a/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs +++ b/tests/FixedWidthParser.Tests/PropertyRoundTripTests.cs @@ -1,8 +1,4 @@ -using System.Text; using CsCheck; -using FixedWidthParser.Attributes; -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -31,13 +27,13 @@ public class PropertyRoundTripTests [Fact] public void RoundTrip_TextAndNumber_IsLossless() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); GenTextNumber.Sample(original => { string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Text, parsed.Text); @@ -48,14 +44,14 @@ public void RoundTrip_TextAndNumber_IsLossless() [Fact] public void RoundTrip_Decimal_IsExact() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); GenMoney.Sample(amount => { var original = new MoneyModel { Amount = amount }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Amount, parsed.Amount); @@ -65,43 +61,20 @@ public void RoundTrip_Decimal_IsExact() [Fact] public void RoundTrip_Double_F2_PreservesToTwoDecimals() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); // Values with at most two fractional digits, formatted "F2": survives to two decimals. Gen.Long[-9_999_999, 9_999_999].Select(cents => cents / 100.0).Sample(value => { var original = new DoubleF2Model { Value = value }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Value, parsed.Value, 2); }); } - - [Fact] - public void CharAndByteParsers_AgreeOnSameLine() - { - // The char writer emits ASCII (letters + digits), so byte offsets equal char offsets and the - // UTF-8 byte parser must produce exactly what the char parser does for the same line. - var writer = new FixedWidthWriter(); - var charParser = new FixedWidthParser(); - var byteParser = new Utf8FixedWidthParser(); - - GenTextNumber.Sample(original => - { - string line = WriteOne(writer, original); - - bool charOk = charParser.TryParse(line, Inv, null, out var fromChar); - bool byteOk = byteParser.TryParse(Encoding.UTF8.GetBytes(line), Inv, null, out var fromByte); - - Assert.True(charOk); - Assert.True(byteOk); - Assert.Equal(fromChar.Text, fromByte.Text); - Assert.Equal(fromChar.Number, fromByte.Number); - }); - } } // ----- Models whose layout makes the generated domain round-trip losslessly ----- diff --git a/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs new file mode 100644 index 0000000..065edef --- /dev/null +++ b/tests/FixedWidthParser.Tests/ReaderOptionsTests.cs @@ -0,0 +1,179 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// + /// : the column a names, + /// , + /// and validation. + /// + public class ReaderOptionsTests + { + private const string BadAge = + "John Doe 30 60000.00 \n" + + "Jane XX 55000.00 \n" + + "Max 41 72000.00 \n"; + + [Fact] + public void ParseError_NamesColumnAndRawValue_OnBothPaths() + { + var reflected = Assert.Throws(() => FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes()).ToList()); + var generated = Assert.Throws(() => FixedWidth.Read(StreamOf(BadAge)).ToList()); + + foreach (var ex in new[] { reflected, generated }) + { + Assert.Equal(2, ex.LineNumber); + Assert.Equal("Age", ex.ColumnName); + Assert.Equal(10, ex.ColumnStart); + Assert.Equal(5, ex.ColumnLength); + Assert.Equal("XX ", ex.RawValue); + Assert.Contains("column 'Age'", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public void ParseError_ShortLine_SaysHowLongTheLayoutIs() + { + var reflected = Assert.Throws(() => FixedWidth.Read(StreamOf("John\n"), FixedWidthLayout.FromAttributes()).ToList()); + var generated = Assert.Throws(() => FixedWidth.Read(StreamOf("John\n")).ToList()); + + foreach (var ex in new[] { reflected, generated }) + { + Assert.Null(ex.ColumnName); + Assert.Equal(-1, ex.ColumnStart); + Assert.Contains("is 4 bytes long and the layout needs 25", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public void ParseError_IsAFormatException() + { + Assert.ThrowsAny(() => FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes()).ToList()); + } + + [Fact] + public void OnParseError_ReturningTrue_SkipsTheLine() + { + var errors = new List(); + var options = new FixedWidthReaderOptions { OnParseError = ex => { errors.Add(ex); return true; } }; + + var reflected = FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes(), options: options).Select(p => p.Name).ToList(); + var generated = FixedWidth.Read(StreamOf(BadAge), options: options).Select(p => p.Name).ToList(); + + Assert.Equal(["John Doe", "Max"], reflected); + Assert.Equal(reflected, generated); + Assert.Equal(2, errors.Count); + Assert.All(errors, ex => Assert.Equal(2, ex.LineNumber)); + } + + [Fact] + public void OnParseError_ReturningFalse_Throws() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => false }; + + Assert.Throws(() => FixedWidth.Read(StreamOf(BadAge), FixedWidthLayout.FromAttributes(), options: options).ToList()); + } + + [Fact] + public void OnParseError_SkippingTheLastLine_EndsCleanly() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + + var codes = FixedWidth.Read(StreamOf("ABC\nA"), FixedWidthLayout.FromAttributes(), options: options).Select(m => m.Code).ToList(); + + Assert.Equal(["ABC"], codes); + } + + [Fact] + public async Task OnParseError_SkipsOnAsyncReads() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + var layout = FixedWidthLayout.FromAttributes(); + + var fromStream = await FixedWidth.ReadAsync(StreamOf(BadAge), layout, options: options).Select(p => p.Name).ToListAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["John Doe", "Max"], fromStream); + } + + [Fact] + public void MaxLineLength_UnterminatedLine_Throws() + { + // No line break at all: the buffer would otherwise keep doubling. + var options = new FixedWidthReaderOptions { BufferSize = 16, MaxLineLength = 100 }; + var source = new MemoryStream(new byte[10_000]); + + var ex = Assert.Throws(() => FixedWidth.Read(source, FixedWidthLayout.FromAttributes(), options: options).ToList()); + + Assert.Contains("MaxLineLength", ex.Message, StringComparison.Ordinal); + Assert.Contains("Line 1", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void MaxLineLength_TerminatedLongLine_Throws() + { + var options = new FixedWidthReaderOptions { MaxLineLength = 5 }; + + var ex = Assert.Throws(() => FixedWidth.Read(StreamOf("ABC\nABCDEFGH\n"), FixedWidthLayout.FromAttributes(), options: options).ToList()); + + Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void MaxLineLength_LineAtTheLimit_IsRead() + { + var options = new FixedWidthReaderOptions { BufferSize = 2, MaxLineLength = 3 }; + + var codes = FixedWidth.Read(StreamOf("ABC\r\nDEF"), FixedWidthLayout.FromAttributes(), options: options).Select(m => m.Code).ToList(); + + Assert.Equal(["ABC", "DEF"], codes); + } + + [Fact] + public async Task MaxLineLength_UnterminatedLine_ThrowsOnAsyncRead() + { + var options = new FixedWidthReaderOptions { BufferSize = 16, MaxLineLength = 100 }; + + await Assert.ThrowsAsync(async () => + { + await foreach (var _ in FixedWidth.ReadAsync(new MemoryStream(new byte[10_000]), FixedWidthLayout.FromAttributes(), options: options)) + { + } + }); + } + + [Theory] + [InlineData(0, 10)] + [InlineData(10, 0)] + public void Options_OutOfRange_Throw(int bufferSize, int maxLineLength) + { + var options = new FixedWidthReaderOptions { BufferSize = bufferSize, MaxLineLength = maxLineLength }; + + Assert.Throws(() => FixedWidth.Read(StreamOf("ABC"), FixedWidthLayout.FromAttributes(), options: options)); + Assert.Throws(() => FixedWidth.Read(StreamOf("ABC"), options: options)); + } + + [Fact] + public void Latin1Encoding_DoesNotSkipWhatLooksLikeAUtf8Bom() + { + // In Latin-1, EF BB BF are the characters "", not a byte order mark. + var options = new FixedWidthReaderOptions { Encoding = Encoding.Latin1 }; + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "\nABC\n"u8]; + + var codes = FixedWidth.Read(new MemoryStream(bytes), FixedWidthLayout.FromAttributes(), options: options).Select(m => m.Code).ToList(); + + Assert.Equal(["", "ABC"], codes); + } + + [Fact] + public void Utf8Text_WithMultiByteCharacters_KeepsBytePositions() + { + // "Zoë" is 4 bytes in UTF-8; the layout counts bytes, so the name column must be 6 wide here + // and the age still starts at byte 6. + var person = FixedWidth.Read(StreamOf("Zoë 42 \n"), FixedWidthLayout.FromAttributes()).Single(); + + Assert.Equal("Zoë", person.Name); + Assert.Equal(42, person.Age); + } + } +} diff --git a/tests/FixedWidthParser.Tests/ReaderStreamTests.cs b/tests/FixedWidthParser.Tests/ReaderStreamTests.cs deleted file mode 100644 index 0fdb206..0000000 --- a/tests/FixedWidthParser.Tests/ReaderStreamTests.cs +++ /dev/null @@ -1,136 +0,0 @@ -using System.Text; -using FixedWidthParser.Readers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ - /// - /// Covers the Stream overloads of (sync and async), - /// including custom encoding and the leaveOpen behavior. - /// - public class ReaderStreamTests - { - private const string TwoPeople = - "John Doe 30 60000.00 \n" + - "Jane 28 55000.00 "; - - private static MemoryStream Utf8(string text) - { - return new(Encoding.UTF8.GetBytes(text)); - } - - // ----------------------------- Synchronous ----------------------------- - - [Fact] - public void Read_Stream_ParsesAll() - { - var reader = new FixedWidthReader(Inv); - using var stream = Utf8(TwoPeople); - - var people = reader.Read(stream).ToList(); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - Assert.Equal(55000.00, people[1].Salary, 2); - } - - [Fact] - public void Read_Stream_CustomEncoding_IsHonored() - { - // 'Á' (0xC1) is a single byte in Latin1 but invalid as standalone UTF-8. - var reader = new FixedWidthReader(); - using var stream = new MemoryStream(Encoding.Latin1.GetBytes("Áb \n")); - - var codes = reader.Read(stream, Encoding.Latin1).Select(m => m.Code).ToList(); - - Assert.Equal(["Áb"], codes); - } - - [Fact] - public void Read_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthReader(); - using var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream, leaveOpen: true).ToList(); - - Assert.True(stream.CanRead); // not disposed - } - - [Fact] - public void Read_Stream_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = reader.Read(stream).ToList(); // default leaveOpen: false - - Assert.False(stream.CanRead); // disposed together with the StreamReader - } - - [Fact] - public void Read_NullStream_Throws() - { - var reader = new FixedWidthReader(); - - Assert.Throws(() => reader.Read((Stream)null!)); - } - - // ----------------------------- Asynchronous ----------------------------- - - [Fact] - public async Task ReadAsync_Stream_ParsesAll() - { - var reader = new FixedWidthReader(Inv); - await using var stream = Utf8(TwoPeople); - - var people = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); - } - - [Fact] - public async Task ReadAsync_Stream_CustomEncoding_IsHonored() - { - var reader = new FixedWidthReader(); - await using MemoryStream stream = new(Encoding.Latin1.GetBytes("Áb \n")); - - var codes = await reader.ReadAsync(stream, Encoding.Latin1).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - - Assert.Equal(["Áb"], codes); - } - - [Fact] - public async Task ReadAsync_Stream_LeaveOpenTrue_KeepsStreamOpen() - { - var reader = new FixedWidthReader(); - await using var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream, leaveOpen: true).ToListAsync().ConfigureAwait(true); - - Assert.True(stream.CanRead); - } - - [Fact] - public async Task ReadAsync_Stream_LeaveOpenFalse_DisposesStream() - { - var reader = new FixedWidthReader(); - var stream = Utf8("ABC\nDEF\n"); - - _ = await reader.ReadAsync(stream).ToListAsync().ConfigureAwait(true); - - Assert.False(stream.CanRead); - } - - [Fact] - public void ReadAsync_NullStream_Throws() - { - var reader = new FixedWidthReader(); - - Assert.Throws(() => reader.ReadAsync((Stream)null!)); - } - } -} diff --git a/tests/FixedWidthParser.Tests/ReaderTests.cs b/tests/FixedWidthParser.Tests/ReaderTests.cs index 2290c7d..94ecb3f 100644 --- a/tests/FixedWidthParser.Tests/ReaderTests.cs +++ b/tests/FixedWidthParser.Tests/ReaderTests.cs @@ -1,13 +1,16 @@ using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; +using FixedWidthParser.Infrastructure; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { + /// + /// Reading through an attribute layout over streams and files, sync and async: line endings, BOM, + /// buffer growth, error reporting, leaveOpen and string pooling. + /// public class ReaderTests { - // Lines in the PersonModel layout (Name[0,10) Age[10,5) Salary[15,10)), 25 chars each. + // Lines in the PersonModel layout (Name[0,10) Age[10,5) Salary[15,10)), 25 bytes each. private const string TwoPeople = "John Doe 30 60000.00 \n" + "Jane 28 55000.00 "; @@ -15,27 +18,23 @@ public class ReaderTests [Fact] public void Read_MultipleLines_ParsesAll() { - var reader = new FixedWidthReader(Inv); - - var people = reader.Read(new StringReader(TwoPeople)).ToList(); + var people = FixedWidth.Read(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes()).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); Assert.Equal(30, people[0].Age); Assert.Equal(60000.00, people[0].Salary, 2); Assert.Equal("Jane", people[1].Name); - Assert.Equal(28, people[1].Age); + Assert.Equal(55000.00, people[1].Salary, 2); } [Theory] - [InlineData("ABC\nDEF\n")] // LF + trailing newline - [InlineData("ABC\r\nDEF\r\n")] // CRLF - [InlineData("ABC\nDEF")] // no trailing newline on the last line + [InlineData("ABC\nDEF\n")] + [InlineData("ABC\r\nDEF\r\n")] + [InlineData("ABC\nDEF")] public void Read_HandlesLineEndingsAndTrailingNewline(string text) { - var reader = new FixedWidthReader(); - - var codes = reader.Read(new StringReader(text)).Select(m => m.Code).ToList(); + var codes = FixedWidth.Read(StreamOf(text), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -43,9 +42,17 @@ public void Read_HandlesLineEndingsAndTrailingNewline(string text) [Fact] public void Read_SkipsEmptyLines() { - var reader = new FixedWidthReader(); + var codes = FixedWidth.Read(StreamOf("ABC\n\n\nDEF\n"), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToList(); - var codes = reader.Read(new StringReader("ABC\n\n\nDEF\n")).Select(m => m.Code).ToList(); + Assert.Equal(["ABC", "DEF"], codes); + } + + [Fact] + public void Read_SkipsLeadingUtf8Bom() + { + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; + + var codes = FixedWidth.Read(new MemoryStream(bytes), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], codes); } @@ -53,36 +60,62 @@ public void Read_SkipsEmptyLines() [Fact] public void Read_LineLongerThanBuffer_StillParses() { - // A tiny bufferSize forces buffer compaction and growth (25-char lines). - var reader = new FixedWidthReader(Inv, bufferSize: 4); + // A tiny buffer forces compaction and growth (25-byte lines). + var readerOptions = new FixedWidthReaderOptions { BufferSize = 4 }; + var layout = FixedWidthLayout.FromAttributes(); - var people = reader.Read(new StringReader(TwoPeople)).ToList(); + var people = FixedWidth.Read(StreamOf(TwoPeople), layout, options: readerOptions).ToList(); Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); - Assert.Equal("Jane", people[1].Name); Assert.Equal(55000.00, people[1].Salary, 2); } [Fact] public void Read_InvalidLine_ThrowsWithLineNumber() { - var reader = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; // invalid Age on line 2 + var layout = FixedWidthLayout.FromAttributes(); - var ex = Assert.Throws( - () => reader.Read(new StringReader(text)).ToList()); + var ex = Assert.Throws( + () => FixedWidth.Read(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 "), layout).ToList()); + Assert.Equal(2, ex.LineNumber); Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); } + [Fact] + public void Read_LeaveOpenTrue_KeepsStreamOpen() + { + using var stream = StreamOf("ABC\nDEF\n"); + + _ = FixedWidth.Read(stream, FixedWidthLayout.FromAttributes(), leaveOpen: true).ToList(); + + Assert.True(stream.CanRead); + } + + [Fact] + public void Read_LeaveOpenFalse_DisposesStream() + { + var stream = StreamOf("ABC\nDEF\n"); + + _ = FixedWidth.Read(stream, FixedWidthLayout.FromAttributes()).ToList(); + + Assert.False(stream.CanRead); + } + + [Fact] + public void Read_NullStream_Throws() + { + Assert.Throws(() => FixedWidth.Read(null!, FixedWidthLayout.FromAttributes())); + } + [Fact] public void Read_WithStringPool_InternsRepeatedValues() { - var pool = new StringPool(); - var reader = new FixedWidthReader(stringPool: pool); + var readerOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + var layout = FixedWidthLayout.FromAttributes(); - var codes = reader.Read(new StringReader("ABC\nABC\n")).ToList(); + var codes = FixedWidth.Read(StreamOf("ABC\nABC\n"), layout, options: readerOptions).ToList(); Assert.Equal("ABC", codes[0].Code); Assert.Same(codes[0].Code, codes[1].Code); @@ -91,11 +124,10 @@ public void Read_WithStringPool_InternsRepeatedValues() [Fact] public void GetEnumerator_IsStruct_ForAllocationFreeForeach() { - Assert.True(typeof(RecordEnumeratorCore, TextReaderSource>).IsValueType); + Assert.True(typeof(RecordEnumerator>).IsValueType); - var reader = new FixedWidthReader(); int count = 0; - foreach (var _ in reader.Read(new StringReader("ABC\nDEF\n"))) + foreach (var _ in FixedWidth.Read(StreamOf("ABC\nDEF\n"), FixedWidthLayout.FromAttributes())) { count++; } @@ -103,6 +135,17 @@ public void GetEnumerator_IsStruct_ForAllocationFreeForeach() Assert.Equal(2, count); } + [Fact] + public void TryParse_UsesTheReaderOptions() + { + var readerOptions = new FixedWidthReaderOptions { FormatProvider = System.Globalization.CultureInfo.GetCultureInfo("de-DE") }; + var layout = FixedWidthLayout.FromAttributes(); + + Assert.True(FixedWidth.TryParse("1234,56 "u8, layout, readerOptions.Context, out var model)); + Assert.Equal(1234.56m, model.Amount); + Assert.False(FixedWidth.TryParse("short"u8, layout, readerOptions.Context, out _)); + } + [Fact] public void ReadFile_IsReEnumerable() { @@ -110,10 +153,10 @@ public void ReadFile_IsReEnumerable() try { File.WriteAllText(path, "ABC\nDEF\n"); - var reader = new FixedWidthReader(); + var layout = FixedWidthLayout.FromAttributes(); - var first = reader.ReadFile(path).Select(m => m.Code).ToList(); - var second = reader.ReadFile(path).Select(m => m.Code).ToList(); + var first = FixedWidth.ReadFile(path, layout).Select(m => m.Code).ToList(); + var second = FixedWidth.ReadFile(path, layout).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF"], first); Assert.Equal(first, second); @@ -124,16 +167,44 @@ public void ReadFile_IsReEnumerable() } } - // ----------------------- Asynchronous reading ----------------------- + [Fact] + public void Writer_To_File_To_Reader_RoundTrip() + { + string path = Path.GetTempFileName(); + try + { + PersonModel[] models = + [ + new() { Name = "John Doe", Age = 30, Salary = 60000 }, + new() { Name = "Jane", Age = 28, Salary = 55000 }, + ]; + using (var fs = File.Create(path)) + { + FixedWidth.WriteMany(fs, models.AsSpan(), FixedWidthLayout.FromAttributes()); + } + + var parsed = FixedWidth.ReadFile(path, FixedWidthLayout.FromAttributes()).ToList(); + + Assert.Equal(2, parsed.Count); + Assert.Equal("John Doe", parsed[0].Name); + Assert.Equal(60000, parsed[0].Salary, 2); + Assert.Equal("Jane", parsed[1].Name); + } + finally + { + File.Delete(path); + } + } + + // ----------------------------- Asynchronous ----------------------------- [Fact] - public async Task ReadAsync_MultipleLines_ParsesAll() + public async Task ReadAsync_Stream_ParsesAll() { - var reader = new FixedWidthReader(Inv); - var people = await reader.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); + var people = await FixedWidth.ReadAsync(StreamOf(TwoPeople), FixedWidthLayout.FromAttributes()).ToListAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, people.Count); Assert.Equal("John Doe", people[0].Name); - Assert.Equal(30, people[0].Age); Assert.Equal("Jane", people[1].Name); Assert.Equal(55000.00, people[1].Salary, 2); } @@ -141,38 +212,72 @@ public async Task ReadAsync_MultipleLines_ParsesAll() [Fact] public async Task ReadAsync_LineLongerThanBuffer_StillParses() { - var reader = new FixedWidthReader(Inv, bufferSize: 4); - var people = await reader.ReadAsync(new StringReader(TwoPeople)).ToListAsync().ConfigureAwait(true); + var readerOptions = new FixedWidthReaderOptions { BufferSize = 4 }; + var layout = FixedWidthLayout.FromAttributes(); + + var people = await FixedWidth.ReadAsync(StreamOf(TwoPeople), layout, options: readerOptions).ToListAsync(TestContext.Current.CancellationToken); + Assert.Equal(2, people.Count); - Assert.Equal("John Doe", people[0].Name); Assert.Equal(55000.00, people[1].Salary, 2); } + [Fact] + public async Task ReadAsync_SkipsLeadingUtf8Bom() + { + byte[] bytes = [0xEF, 0xBB, 0xBF, .. "ABC\nDEF\n"u8]; + + var codes = await FixedWidth.ReadAsync(new MemoryStream(bytes), FixedWidthLayout.FromAttributes()).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); + + Assert.Equal(["ABC", "DEF"], codes); + } + [Fact] public async Task ReadAsync_InvalidLine_ThrowsWithLineNumber() { - var reader = new FixedWidthReader(Inv); - var text = "John Doe 30 60000.00 \nJane XX 55000.00 "; + var layout = FixedWidthLayout.FromAttributes(); - var ex = await Assert.ThrowsAsync(async () => + var ex = await Assert.ThrowsAsync(async () => { - await foreach (var _ in reader.ReadAsync(new StringReader(text)).ConfigureAwait(false)) { } - }).ConfigureAwait(true); + await foreach (var _ in FixedWidth.ReadAsync(StreamOf("John Doe 30 60000.00 \nJane XX 55000.00 "), layout)) + { + } + }); - Assert.Contains("Line 2", ex.Message, StringComparison.Ordinal); + Assert.Equal(2, ex.LineNumber); } [Fact] public async Task ReadAsync_CancelledToken_Throws() { - var reader = new FixedWidthReader(); using var cts = new CancellationTokenSource(); - await cts.CancelAsync().ConfigureAwait(true); + await cts.CancelAsync(); await Assert.ThrowsAnyAsync(async () => { - await foreach (var _ in reader.ReadAsync(new StringReader("ABC\nDEF\n")).WithCancellation(cts.Token).ConfigureAwait(false)) { } - }).ConfigureAwait(true); + await foreach (var _ in FixedWidth.ReadAsync(StreamOf("ABC\nDEF\n"), FixedWidthLayout.FromAttributes()).WithCancellation(cts.Token)) + { + } + }); + } + + [Fact] + public async Task ReadAsync_LeaveOpen_ControlsDisposal() + { + var layout = FixedWidthLayout.FromAttributes(); + using var kept = StreamOf("ABC\n"); + var owned = StreamOf("ABC\n"); + + _ = await FixedWidth.ReadAsync(kept, layout, leaveOpen: true).ToListAsync(TestContext.Current.CancellationToken); + _ = await FixedWidth.ReadAsync(owned, layout).ToListAsync(TestContext.Current.CancellationToken); + + Assert.True(kept.CanRead); + Assert.False(owned.CanRead); + } + + [Fact] + public void ReadAsync_NullStream_Throws() + { + Assert.Throws(() => FixedWidth.ReadAsync((Stream)null!, FixedWidthLayout.FromAttributes())); } [Fact] @@ -181,10 +286,11 @@ public async Task ReadFileAsync_MatchesSyncReadFile() string path = Path.GetTempFileName(); try { - await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n").ConfigureAwait(true); - var reader = new FixedWidthReader(); - var asyncCodes = await reader.ReadFileAsync(path).Select(m => m.Code).ToListAsync().ConfigureAwait(true); - var syncCodes = reader.ReadFile(path).Select(m => m.Code).ToList(); + await File.WriteAllTextAsync(path, "ABC\nDEF\nGHI\n", TestContext.Current.CancellationToken); + var layout = FixedWidthLayout.FromAttributes(); + + var asyncCodes = await FixedWidth.ReadFileAsync(path, layout).Select(m => m.Code).ToListAsync(TestContext.Current.CancellationToken); + var syncCodes = FixedWidth.ReadFile(path, layout).Select(m => m.Code).ToList(); Assert.Equal(["ABC", "DEF", "GHI"], asyncCodes); Assert.Equal(syncCodes, asyncCodes); @@ -196,36 +302,14 @@ public async Task ReadFileAsync_MatchesSyncReadFile() } [Fact] - public void Writer_To_File_To_Reader_RoundTrip() + public async Task ReadAsync_WithStringPool_InternsRepeatedValues() { - string path = Path.GetTempFileName(); - try - { - var models = new[] - { - new PersonModel { Name = "John Doe", Age = 30, Salary = 60000 }, - new PersonModel { Name = "Jane", Age = 28, Salary = 55000 }, - }; - var writer = new FixedWidthWriter(); - using (var fs = File.Create(path)) - { - writer.WriteMany(fs, models.AsSpan(), Inv); - } + var readerOptions = new FixedWidthReaderOptions { StringPool = new StringPool() }; + var layout = FixedWidthLayout.FromAttributes(); - var reader = new FixedWidthReader(Inv); - var parsed = reader.ReadFile(path).ToList(); + var codes = await FixedWidth.ReadAsync(StreamOf("ABC\nABC\n"), layout, options: readerOptions).ToListAsync(TestContext.Current.CancellationToken); - Assert.Equal(2, parsed.Count); - Assert.Equal("John Doe", parsed[0].Name); - Assert.Equal(30, parsed[0].Age); - Assert.Equal(60000, parsed[0].Salary, 2); - Assert.Equal("Jane", parsed[1].Name); - Assert.Equal(55000, parsed[1].Salary, 2); - } - finally - { - File.Delete(path); - } + Assert.Same(codes[0].Code, codes[1].Code); } } } diff --git a/tests/FixedWidthParser.Tests/RefStructReadTests.cs b/tests/FixedWidthParser.Tests/RefStructReadTests.cs new file mode 100644 index 0000000..937a471 --- /dev/null +++ b/tests/FixedWidthParser.Tests/RefStructReadTests.cs @@ -0,0 +1,202 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// Ref struct models read through every enumerator and aggregate, generated and through a layout alike. + public sealed class RefStructReadTests : IDisposable + { + private const string Content = "Ann 30 1000.00 \nBob 41 2000.50 \r\nCid 52 3000.25 "; + private const string BadContent = "Ann 30 1000.00 \nBob XX 2000.50 \nCid 52 3000.25 "; + + private static readonly FixedWidthLayout Layout = FixedWidthLayout.FromAttributes(); + + private readonly string _path = Path.GetTempFileName(); + + public RefStructReadTests() + { + File.WriteAllText(_path, Content); + } + + public void Dispose() + { + File.Delete(_path); + } + + public sealed class GenAges : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in GenRefPersonModel model) + { + Values.Add(model.Age); + } + + public void Merge(GenAges following) + { + Values.AddRange(following.Values); + } + } + + public sealed class Ages : IRecordAccumulator + { + public List Values { get; } = []; + + public void Add(in RefPersonModel model) + { + Values.Add(model.Age); + } + + public void Merge(Ages following) + { + Values.AddRange(following.Values); + } + } + + private static MemoryStream Stream(string content) + { + return new MemoryStream(Encoding.UTF8.GetBytes(content)); + } + + [Fact] + public void Read_Generated_YieldsEveryRecord() + { + var names = new List(); + foreach (var person in FixedWidth.Read(Stream(Content))) + { + names.Add($"{person.Name}:{person.Age}:{person.Salary.ToString(Inv)}"); + } + + Assert.Equal(["Ann:30:1000", "Bob:41:2000.5", "Cid:52:3000.25"], names); + } + + [Fact] + public void ReadFile_Layout_YieldsEveryRecord() + { + var ages = new List(); + foreach (var person in FixedWidth.ReadFile(_path, Layout)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public void Read_ThroughIEnumerable_YieldsEveryRecord() + { + IEnumerable records = FixedWidth.Read(Stream(Content)); + var ages = new List(); + using var enumerator = records.GetEnumerator(); + while (enumerator.MoveNext()) + { + ages.Add(enumerator.Current.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public void Read_CurrentReadTwice_ReturnsTheSameRecord() + { + using var enumerator = FixedWidth.Read(Stream(Content)).GetEnumerator(); + Assert.True(enumerator.MoveNext()); + + Assert.Equal(enumerator.Current.Name, enumerator.Current.Name); + Assert.Equal(30, enumerator.Current.Age); + } + + [Fact] + public async Task ReadAsync_Generated_YieldsEveryRecord() + { + var ages = new List(); + await foreach (var person in FixedWidth.ReadAsync(Stream(Content))) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public async Task ReadFileAsync_Layout_YieldsEveryRecord() + { + var ages = new List(); + await foreach (var person in FixedWidth.ReadFileAsync(_path, Layout)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 41, 52], ages); + } + + [Fact] + public void Read_BadLineWithoutHandler_Throws() + { + var ex = Assert.Throws(() => + { + foreach (var person in FixedWidth.Read(Stream(BadContent))) + { + _ = person.Age; + } + }); + + Assert.Equal(2, ex.LineNumber); + Assert.Equal("Age", ex.ColumnName); + } + + [Fact] + public async Task ReadAsync_BadLineSkipped_YieldsTheRest() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + var ages = new List(); + await foreach (var person in FixedWidth.ReadAsync(Stream(BadContent), Layout, options: options)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 52], ages); + } + + [Fact] + public void Read_BadLineSkipped_YieldsTheRest() + { + var options = new FixedWidthReaderOptions { OnParseError = _ => true }; + var ages = new List(); + foreach (var person in FixedWidth.Read(Stream(BadContent), options: options)) + { + ages.Add(person.Age); + } + + Assert.Equal([30, 52], ages); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public async Task AggregateFileAsync_FoldsEveryRecord(int degreeOfParallelism) + { + var options = new FixedWidthReaderOptions { DegreeOfParallelism = degreeOfParallelism, ChunkSize = 7 }; + + var generated = await FixedWidth.AggregateFileAsync(_path, options); + var layout = await FixedWidth.AggregateFileAsync(_path, Layout, options); + + Assert.Equal([30, 41, 52], generated.Values); + Assert.Equal([30, 41, 52], layout.Values); + } + + [Theory] + [InlineData(1)] + [InlineData(4)] + public async Task AggregateAsync_FoldsEveryRecord(int degreeOfParallelism) + { + var options = new FixedWidthReaderOptions { DegreeOfParallelism = degreeOfParallelism, ChunkSize = 7 }; + + var generated = await FixedWidth.AggregateAsync(Stream(Content), options: options); + var layout = await FixedWidth.AggregateAsync(Stream(Content), Layout, options: options); + + Assert.Equal([30, 41, 52], generated.Values); + Assert.Equal([30, 41, 52], layout.Values); + } + } +} diff --git a/tests/FixedWidthParser.Tests/RefStructTests.cs b/tests/FixedWidthParser.Tests/RefStructTests.cs deleted file mode 100644 index d444052..0000000 --- a/tests/FixedWidthParser.Tests/RefStructTests.cs +++ /dev/null @@ -1,27 +0,0 @@ -using FixedWidthParser.Parsers; -using static FixedWidthParser.Tests.TestHelpers; - -namespace FixedWidthParser.Tests -{ -#if NET9_0_OR_GREATER - /// - /// Confirms the parser's allows ref struct constraint is real: a ref struct model is - /// built and parsed at runtime (Func/expression trees accept ref struct on .NET 10). - /// - public class RefStructTests - { - [Fact] - public void Parser_SupportsRefStructModel() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("John Doe 30 60000.00 ", Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("John Doe", model.Name); - Assert.Equal(30, model.Age); - Assert.Equal(60000.00, model.Salary, 2); - } - } -#endif -} diff --git a/tests/FixedWidthParser.Tests/RoundTripTests.cs b/tests/FixedWidthParser.Tests/RoundTripTests.cs index b20d9fa..0df97dc 100644 --- a/tests/FixedWidthParser.Tests/RoundTripTests.cs +++ b/tests/FixedWidthParser.Tests/RoundTripTests.cs @@ -1,5 +1,3 @@ -using FixedWidthParser.Parsers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -21,12 +19,12 @@ public static IEnumerable PersonCases() [MemberData(nameof(PersonCases))] public void RoundTrip_Person_PreservesValues(string name, int age, double salary) { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); var original = new PersonModel { Name = name, Age = age, Salary = salary }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Name, parsed.Name); @@ -37,12 +35,12 @@ public void RoundTrip_Person_PreservesValues(string name, int age, double salary [Fact] public void RoundTrip_FieldModel_PreservesValues() { - var writer = new FixedWidthWriter(); - var parser = new FixedWidthParser(); + var writer = FixedWidthLayout.FromAttributes(); + var parser = FixedWidthLayout.FromAttributes(); var original = new ProductModel { Code = "ABCDE", Quantity = 1234 }; string line = WriteOne(writer, original); - bool ok = parser.TryParse(line, Inv, null, out var parsed); + bool ok = FixedWidth.TryParse(Bytes(line), parser, Ctx, out var parsed); Assert.True(ok); Assert.Equal(original.Code, parsed.Code); diff --git a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs index 1718416..3b942bd 100644 --- a/tests/FixedWidthParser.Tests/SourceGenParityTests.cs +++ b/tests/FixedWidthParser.Tests/SourceGenParityTests.cs @@ -1,28 +1,26 @@ -using System.Globalization; +using System.Text; using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; +using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { /// - /// Verifies the source-generated TryParse (via ) - /// produces identical results to the runtime, reflection-based . + /// The source-generated TryParse (via ) produces the + /// same results as the type's layout, including what each reports + /// about a rejected line. /// public class SourceGenParityTests { - private static readonly CultureInfo Inv = CultureInfo.InvariantCulture; - [Theory] - [InlineData("John Doe 30 6000000.00")] // clean, full width - [InlineData("Jane 28 55000.00 ")] // trailing spaces in the numeric column - [InlineData("John")] // short line: numeric columns out of bounds - [InlineData("")] // empty line - [InlineData("Ann abc 10.5 ")] // non-numeric age: must be rejected by both + [InlineData("John Doe 30 6000000.00")] + [InlineData("Jane 28 55000.00 ")] + [InlineData("John")] + [InlineData("")] + [InlineData("Ann abc 10.5 ")] public void Person_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); if (okR) @@ -36,8 +34,7 @@ public void Person_GeneratedMatchesReflection(string line) [Fact] public void Person_ParsesExpectedValues() { - bool ok = FixedWidth.TryParse("John Doe 30 6000000.00", Inv, null, out var g); - Assert.True(ok); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, Ctx, out var g)); Assert.Equal("John Doe", g.Name); Assert.Equal(30, g.Age); Assert.Equal(6000000.00, g.Salary); @@ -46,19 +43,15 @@ public void Person_ParsesExpectedValues() [Theory] [InlineData("ABCDE0042")] [InlineData("X 7 ")] - [InlineData("AB")] // short: Quantity out of bounds -> rejected by both + [InlineData("AB")] public void Product_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Code, g.Code); - Assert.Equal(r.Quantity, g.Quantity); - } + Assert.Equal(r.Code, g.Code); + Assert.Equal(r.Quantity, g.Quantity); } [Theory] @@ -67,37 +60,21 @@ public void Product_GeneratedMatchesReflection(string line) [InlineData("notanum ")] public void Measurement_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Value, g.Value); - } + Assert.Equal(r.Value, g.Value); } - [Fact] - public void TrailingString_ShortLineGeneratedMatchesReflection() - { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse("42", Inv, null, out _); - bool okG = FixedWidth.TryParse("42", Inv, null, out _); - - Assert.False(okR); - Assert.False(okG); - } - - [Fact] - public void TrailingString_FullLineGeneratedMatchesReflection() + [Theory] + [InlineData("42")] + [InlineData("42 short ")] + public void TrailingString_GeneratedMatchesReflection(string line) { - const string line = "42 short "; - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); - Assert.True(okR); Assert.Equal(okR, okG); Assert.Equal(r.Id, g.Id); Assert.Equal(r.Note, g.Note); @@ -109,49 +86,90 @@ public void TrailingString_FullLineGeneratedMatchesReflection() [InlineData("bad ")] public void Decimal_GeneratedMatchesReflection(string line) { - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Amount, g.Amount); - } + Assert.Equal(r.Amount, g.Amount); } -#if NET9_0_OR_GREATER [Fact] - public void RefStruct_GeneratedMatchesReflection() + public void DecodedParsable_GeneratedMatchesReflection() { - const string line = "John Doe 30 6000000.00"; - var reflection = new FixedWidthParser(); - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + bool okR = FixedWidth.TryParse("hey "u8, FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse("hey "u8, Ctx, out var g); Assert.True(okR); - Assert.Equal(okR, okG); + Assert.True(okG); + Assert.Equal("HEY", r.Word.Text); + Assert.Equal(r.Word, g.Word); + } + + [Fact] + public void Latin1_GeneratedMatchesReflection() + { + // "Antônio" doesn't fit; "Joséé" is 5 characters and 5 bytes in Latin-1, 7 bytes in UTF-8. + var context = CtxWith(encoding: Encoding.Latin1); + byte[] line = Encoding.Latin1.GetBytes("Joséé 42 "); + + Assert.True(FixedWidth.TryParse(line, FixedWidthLayout.FromAttributes(), context, out var r)); + Assert.True(FixedWidth.TryParse(line, context, out var g)); + + Assert.Equal("Joséé", r.Name); + Assert.Equal(42, r.Age); + Assert.Equal(r.Name, g.Name); + Assert.Equal(r.Age, g.Age); + } + + [Fact] + public void RefStruct_GeneratedMatchesReflection() + { + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, FixedWidthLayout.FromAttributes(), Ctx, out var r)); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, Ctx, out var g)); + Assert.Equal(r.Name, g.Name); Assert.Equal(r.Age, g.Age); Assert.Equal(r.Salary, g.Salary); } -#endif [Fact] public void StringPool_InternsAndMatchesReflection() { - const string line = "John Doe 30 6000000.00"; var pool = new StringPool(); - var reflection = new FixedWidthParser(); + var context = CtxWith(pool: pool); - bool okR = reflection.TryParse(line, Inv, pool, out var r); - bool okG = FixedWidth.TryParse(line, Inv, pool, out var g); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, FixedWidthLayout.FromAttributes(), context, out var r)); + Assert.True(FixedWidth.TryParse("John Doe 30 6000000.00"u8, context, out var g)); - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Name, g.Name); - // Same pooled instance is returned for the same content. + Assert.Same(r.Name, g.Name); Assert.Same(pool.GetOrAdd("John Doe".AsSpan()), g.Name); } + + [Theory] + [InlineData("0042ABCtag01XYzzzJohn ")] + [InlineData(" 7 Z Q Ann ")] + [InlineData("xxxxABCtag01XYzzzJohn ")] + public void InheritedColumns_GeneratedMatchesReflection(string line) + { + bool okR = FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var r); + bool okG = FixedWidth.TryParse(Bytes(line), Ctx, out var g); + + Assert.Equal(okR, okG); + if (okR) + { + Assert.Equal((r.Id, r.Code, r.Tag, r.Label, r.Name), (g.Id, g.Code, g.Tag, g.Label, g.Name)); + } + } + + [Fact] + public void InheritedColumns_ParseAndFormatThroughGeneratedModel() + { + Assert.True(FixedWidth.TryParse("0042ABCtag01XYzzzJohn "u8, Ctx, out var g)); + Assert.Equal((42, "ABC", "tag01", "XY", "John"), (g.Id, g.Code, g.Tag, g.Label, g.Name)); + + var model = new GenDerivedModel { Id = 42, Code = "ABC", Tag = "tag01", Label = "XY", Name = "John" }; + Assert.Equal(WriteOne(FixedWidthLayout.FromAttributes(), model), Format(model)); + Assert.Equal("42 ABCtag01XY John ", Format(model)); + } } } diff --git a/tests/FixedWidthParser.Tests/TestHelpers.cs b/tests/FixedWidthParser.Tests/TestHelpers.cs index 1c82f6f..9723673 100644 --- a/tests/FixedWidthParser.Tests/TestHelpers.cs +++ b/tests/FixedWidthParser.Tests/TestHelpers.cs @@ -1,6 +1,6 @@ using System.Globalization; using System.Text; -using FixedWidthParser.Writers; +using CommunityToolkit.HighPerformance.Buffers; namespace FixedWidthParser.Tests { @@ -8,32 +8,58 @@ internal static class TestHelpers { public static readonly CultureInfo Inv = CultureInfo.InvariantCulture; - /// Writes a model and returns the line without the trailing newline. - public static string WriteOne(FixedWidthWriter writer, in T model, IFormatProvider? formatProvider = null) + /// Invariant culture, UTF-8, no pool. + public static readonly FixedWidthContext Ctx = default; + + public static FixedWidthContext CtxWith(IFormatProvider? provider = null, StringPool? pool = null, Encoding? encoding = null) + { + return new FixedWidthContext { FormatProvider = provider!, StringPool = pool, Encoding = encoding! }; + } + + public static byte[] Bytes(string text) + { + return Encoding.UTF8.GetBytes(text); + } + + public static MemoryStream StreamOf(string content) + { + return new MemoryStream(Encoding.UTF8.GetBytes(content)); + } + + /// Formats a generated model and returns the line as UTF-8 text. + public static string Format(in T model, in FixedWidthContext context = default) where T : IFixedWidthModel + { + Span destination = stackalloc byte[4096]; + Assert.True(FixedWidth.TryFormat(in model, destination, in context, out int written)); + return Encoding.UTF8.GetString(destination[..written]); + } + + /// Writes a model through an attribute or fluent layout and returns the line without the trailing newline. + public static string WriteOne(FixedWidthLayout layout, in T model, FixedWidthWriterOptions? options = null) { using var ms = new MemoryStream(); - writer.Write(ms, in model, formatProvider ?? Inv); + FixedWidth.Write(ms, in model, layout, options); return Decode(ms).TrimEnd('\r', '\n'); } - public static async Task WriteOneAsync(FixedWidthWriter writer, T model) + public static async Task WriteOneAsync(FixedWidthLayout layout, T model, FixedWidthWriterOptions? options = null) { await using var ms = new MemoryStream(); - await writer.WriteAsync(ms, model, Inv).ConfigureAwait(false); + await FixedWidth.WriteAsync(ms, model, layout, options, TestContext.Current.CancellationToken).ConfigureAwait(false); return Decode(ms).TrimEnd('\r', '\n'); } - public static string WriteMany(FixedWidthWriter writer, ReadOnlySpan models) + public static string WriteMany(FixedWidthLayout layout, ReadOnlySpan models, FixedWidthWriterOptions? options = null) { using var ms = new MemoryStream(); - writer.WriteMany(ms, models, Inv); + FixedWidth.WriteMany(ms, models, layout, options); return Decode(ms); } - public static string WriteMany(FixedWidthWriter writer, IEnumerable models) + public static string WriteMany(FixedWidthLayout layout, IEnumerable models, FixedWidthWriterOptions? options = null) { using var ms = new MemoryStream(); - writer.WriteMany(ms, models, Inv); + FixedWidth.WriteMany(ms, models, layout, options); return Decode(ms); } diff --git a/tests/FixedWidthParser.Tests/TestModels.cs b/tests/FixedWidthParser.Tests/TestModels.cs index 9737595..9f97fe3 100644 --- a/tests/FixedWidthParser.Tests/TestModels.cs +++ b/tests/FixedWidthParser.Tests/TestModels.cs @@ -1,39 +1,18 @@ using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; -using FixedWidthParser.Attributes; -using FixedWidthParser.Processors; +using FixedWidthParser.Infrastructure; namespace FixedWidthParser.Tests { /// Custom value type with no ISpanParsable — only parseable via a FixedColumnAttribute.Converter. public readonly record struct CentsValue(long Cents); - /// - /// Stateless / for - /// : stores cents as a plain integer. Implements both directions (parse/format) - /// and both element types (char/byte), so one attribute round-trips through every path. - /// - public sealed class CentsConverter : IFixedWidthConverter, IUtf8FixedWidthConverter + /// Stateless for : stores cents as a plain integer. + public sealed class CentsConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out CentsValue value) - { - if (!long.TryParse(field, formatProvider, out long cents)) - { - value = default; - return false; - } - value = new CentsValue(cents); - return true; - } - - public bool TryFormat(CentsValue value, Span destination, IFormatProvider? formatProvider, out int written) - { - return value.Cents.TryFormat(destination, out written, provider: formatProvider); - } - - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out CentsValue value) + public bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out CentsValue value) { - if (!long.TryParse(field, formatProvider, out long cents)) + if (!long.TryParse(column, context.FormatProvider, out long cents)) { value = default; return false; @@ -42,27 +21,25 @@ public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, return true; } - public bool TryFormat(CentsValue value, Span destination, IFormatProvider? formatProvider, out int written) + public bool TryFormat(CentsValue value, Span destination, in FixedWidthContext context, out int bytesWritten) { - return value.Cents.TryFormat(destination, out written, provider: formatProvider); + return value.Cents.TryFormat(destination, out bytesWritten, provider: context.FormatProvider); } } - /// Converter that only implements the char-side interface — used to prove a converter can - /// support one direction/element type without the other being required. + /// Converter for int, attached to a string column to prove the mismatch is rejected. public sealed class IntOnlyConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out int value) + public bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out int value) { - return int.TryParse(field, formatProvider, out value); + return int.TryParse(column, context.FormatProvider, out value); } - public bool TryFormat(int value, Span destination, IFormatProvider? formatProvider, out int written) + public bool TryFormat(int value, Span destination, in FixedWidthContext context, out int bytesWritten) { - return value.TryFormat(destination, out written, provider: formatProvider); + return value.TryFormat(destination, out bytesWritten, provider: context.FormatProvider); } } - /// Single custom-converted column, exercised by both the reflection parser/writer and the generator. public readonly record struct CentsConverterModel { @@ -332,28 +309,27 @@ public readonly record struct WideUtf8NumericModel [StructLayout(LayoutKind.Auto)] public readonly record struct BigValue(int Length); - /// Char-side-only converter for ; emits N 'x' chars on format. + /// Converter for ; emits N 'x' bytes on format. public sealed class BigConverter : IFixedWidthConverter { - public bool TryParse(ReadOnlySpan field, IFormatProvider? formatProvider, out BigValue value) + public bool TryParse(ReadOnlySpan column, in FixedWidthContext context, out BigValue value) { - value = new BigValue(field.Trim().Length); + value = new BigValue(column.Trim((byte)' ').Length); return true; } - public bool TryFormat(BigValue value, Span destination, IFormatProvider? formatProvider, out int written) + public bool TryFormat(BigValue value, Span destination, in FixedWidthContext context, out int bytesWritten) { if (destination.Length < value.Length) { - written = 0; + bytesWritten = 0; return false; } - destination[..value.Length].Fill('x'); - written = value.Length; + destination[..value.Length].Fill((byte)'x'); + bytesWritten = value.Length; return true; } } - [StructLayout(LayoutKind.Auto)] public readonly record struct ZeroPaddedLeadingModel { @@ -385,4 +361,104 @@ public readonly record struct TruncateInvalidModel { [FixedColumn(0, 5, Overflow = OverflowBehavior.Truncate)] public int Value { get; init; } } + + /// Padding set to a non-ASCII character: the runtime writer must refuse it. + public readonly record struct NonAsciiPaddingModel + { + [FixedColumn(0, 5, Alignment = Alignment.Right, Padding = 'é')] public int Value { get; init; } + } + + /// A column type with no parser of any kind and no converter. + public readonly record struct UnsupportedTypeModel + { + [FixedColumn(0, 10)] public Uri? Link { get; init; } + } + + /// Parses only from characters (no ), so the byte path decodes it first. + public readonly record struct Shout(string Text) : ISpanParsable, ISpanFormattable + { + public static Shout Parse(ReadOnlySpan s, IFormatProvider? provider) + { + return new Shout(s.ToString().ToUpperInvariant()); + } + + public static bool TryParse(ReadOnlySpan s, IFormatProvider? provider, out Shout result) + { + result = Parse(s, provider); + return true; + } + + [ExcludeFromCodeCoverage] + public static Shout Parse(string s, IFormatProvider? provider) + { + return Parse(s.AsSpan(), provider); + } + + [ExcludeFromCodeCoverage] + public static bool TryParse(string? s, IFormatProvider? provider, out Shout result) + { + return TryParse(s.AsSpan(), provider, out result); + } + + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + charsWritten = Text.Length; + return Text.AsSpan().TryCopyTo(destination); + } + + [ExcludeFromCodeCoverage] + public string ToString(string? format, IFormatProvider? formatProvider) + { + return Text; + } + } + + public readonly record struct ShoutModel + { + [FixedColumn(0, 6)] public Shout Word { get; init; } + } + + /// A Latin-1 name column followed by a number, to prove positions stay bytes in single-byte encodings. + public readonly record struct NameAgeModel + { + [FixedColumn(0, 6)] public string Name { get; init; } + [FixedColumn(6, 3)] public int Age { get; init; } + } + + /// A currency column, whose symbol may be non-ASCII depending on the culture. + public readonly record struct PriceModel + { + [FixedColumn(0, 8, Alignment = Alignment.Right, Format = "C")] public decimal Price { get; init; } + } + + /// Settable as a class, so a fluent layout and its attributes can bind the same type. + public sealed record class MutablePersonModel + { + [FixedColumn(0, 10)] public string Name { get; set; } = ""; + [FixedColumn(10, 5)] public int Age { get; set; } + [FixedColumn(15, 10)] public double Salary { get; set; } + } + + /// Settable as a struct: a fluent setter must reach the returned model, not a copy. + public record struct MutablePersonStruct + { + [FixedColumn(0, 10)] public string Name { get; set; } + [FixedColumn(10, 5)] public int Age { get; set; } + [FixedColumn(15, 10)] public double Salary { get; set; } + } + + /// One settable column per formatting and parsing option, for fluent-versus-attribute parity. + public sealed record class MutableOptionsModel + { + [FixedColumn(0, 5, Alignment = Alignment.Right, Padding = '0')] public int ZeroPadded { get; set; } + [FixedColumn(5, 8, Format = "F2")] public double Fixed2 { get; set; } + [FixedColumn(13, 4, Alignment = Alignment.Right, Overflow = OverflowBehavior.Truncate)] public string Truncated { get; set; } = ""; + [FixedColumn(17, 5, TrimChar = '*')] public int Starred { get; set; } + [FixedColumn(22, 5)] public int? NullableAge { get; set; } + [FixedColumn(27, 10)] public decimal? NullableAmount { get; set; } + [FixedColumn(37, 8, Converter = typeof(CentsConverter))] public CentsValue Cents { get; set; } + [FixedColumn(45, 8, Format = "yyyyMMdd")] public DateOnly Date { get; set; } + [FixedColumn(53, 8, Format = "yyyyMMdd")] public DateOnly? NullableDate { get; set; } + [FixedColumn(61, 8, Converter = typeof(CentsConverter))] public CentsValue? NullableCents { get; set; } + } } diff --git a/tests/FixedWidthParser.Tests/TrimCharTests.cs b/tests/FixedWidthParser.Tests/TrimCharTests.cs index 56835f5..4bb90b0 100644 --- a/tests/FixedWidthParser.Tests/TrimCharTests.cs +++ b/tests/FixedWidthParser.Tests/TrimCharTests.cs @@ -1,120 +1,36 @@ -using System.Text; -using FixedWidthParser.Parsers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { - /// - /// Covers FixedColumnAttribute.TrimChar: the character trimmed from the end of a column on - /// the READ side (parsing) is now configurable, instead of hardcoded ' '. Exercises reflection - /// parse (char + UTF-8), generated parity, and the non-ASCII-TrimChar guard on the UTF-8 byte path - /// (mirrors the existing decimal-separator ASCII guard in CultureHelpers). - /// + /// FixedColumnAttribute.TrimChar: the character trimmed from a column when parsing. public class TrimCharTests { - [Fact] - public void Reflection_Char_AsteriskTrim_ParsesUnderlyingValue() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("42***", Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(42, model.Value); - } - - [Fact] - public void Reflection_Char_HashTrim_StringColumn() - { - var parser = new FixedWidthParser(); - - bool ok = parser.TryParse("AB######", Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("AB", model.Code); - } - - [Fact] - public void Reflection_Utf8_AsteriskTrim_ParsesUnderlyingValue() - { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes("42***"); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal(42, model.Value); - } - - [Fact] - public void Reflection_Utf8_HashTrim_StringColumn() - { - var parser = new Utf8FixedWidthParser(); - byte[] bytes = Encoding.UTF8.GetBytes("AB######"); - - bool ok = parser.TryParse(bytes, Inv, null, out var model); - - Assert.True(ok); - Assert.Equal("AB", model.Code); - } - - [Fact] - public void Reflection_Utf8_NonAsciiTrimChar_Throws() - { - Assert.Throws(() => new Utf8FixedWidthParser()); - } - [Theory] - [InlineData("42***")] - [InlineData("1****")] - public void Generated_Char_MatchesReflection(string line) + [InlineData("42***", 42)] + [InlineData("1****", 1)] + public void AsteriskTrim_ParsesUnderlyingValue_OnBothPaths(string line, int expected) { - var reflection = new FixedWidthParser(); - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); + Assert.True(FixedWidth.TryParse(Bytes(line), FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse(Bytes(line), Ctx, out var generated)); - Assert.Equal(okR, okG); - Assert.Equal(r.Value, g.Value); + Assert.Equal(expected, reflected.Value); + Assert.Equal(expected, generated.Value); } [Fact] - public void Generated_Utf8_MatchesReflection() + public void HashTrim_StringColumn_OnBothPaths() { - byte[] bytes = Encoding.UTF8.GetBytes("42***"); - var reflection = new Utf8FixedWidthParser(); + Assert.True(FixedWidth.TryParse("AB######"u8, FixedWidthLayout.FromAttributes(), Ctx, out var reflected)); + Assert.True(FixedWidth.TryParse("AB######"u8, Ctx, out var generated)); - bool okR = reflection.TryParse(bytes, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(bytes, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Value, g.Value); + Assert.Equal("AB", reflected.Code); + Assert.Equal("AB", generated.Code); } [Fact] - public void Generated_Char_HashTrim_MatchesReflection() + public void NonAsciiTrimChar_Throws() { - const string line = "AB######"; - var reflection = new FixedWidthParser(); - - bool okR = reflection.TryParse(line, Inv, null, out var r); - bool okG = FixedWidth.TryParse(line, Inv, null, out var g); - - Assert.Equal(okR, okG); - Assert.Equal(r.Code, g.Code); - } - - [Fact] - public void Generated_Utf8_NonAsciiTrimChar_Throws() - { - // The guard runs in the generated model's static field initializer (__trim0), so it - // surfaces wrapped in a TypeInitializationException on first use of the type. - byte[] bytes = Encoding.UTF8.GetBytes(" 42"); - - var ex = Assert.Throws( - () => FixedWidthUtf8.TryParse(bytes, Inv, null, out _)); - - Assert.IsType(ex.InnerException); + Assert.Throws(() => FixedWidth.TryParse(ReadOnlySpan.Empty, FixedWidthLayout.FromAttributes(), Ctx, out _)); } } } diff --git a/tests/FixedWidthParser.Tests/Utf8SourceGenParityTests.cs b/tests/FixedWidthParser.Tests/Utf8SourceGenParityTests.cs deleted file mode 100644 index 0e9f3d3..0000000 --- a/tests/FixedWidthParser.Tests/Utf8SourceGenParityTests.cs +++ /dev/null @@ -1,164 +0,0 @@ -using System.Globalization; -using System.Text; -using CommunityToolkit.HighPerformance.Buffers; -using FixedWidthParser.Parsers; - -namespace FixedWidthParser.Tests -{ - /// - /// Verifies the source-generated UTF-8 TryParse (via ) - /// produces identical results to the runtime, reflection-based . - /// Mirrors over the byte path. All fixtures are ASCII, so byte - /// offsets equal character offsets. - /// - public class Utf8SourceGenParityTests - { - private static readonly CultureInfo Inv = CultureInfo.InvariantCulture; - - // u8 literals can't appear in [InlineData]; encode the ASCII fixture at the call site instead. - private static byte[] U8(string s) - { - return Encoding.UTF8.GetBytes(s); - } - - - [Theory] - [InlineData("John Doe 30 6000000.00")] // clean, full width - [InlineData("Jane 28 55000.00 ")] // trailing spaces in the numeric column - [InlineData("John")] // short line: numeric columns out of bounds - [InlineData("")] // empty line - [InlineData("Ann abc 10.5 ")] // non-numeric age: must be rejected by both - public void Person_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Name, g.Name); - Assert.Equal(r.Age, g.Age); - Assert.Equal(r.Salary, g.Salary); - } - } - - [Fact] - public void Person_ParsesExpectedValues() - { - bool ok = FixedWidthUtf8.TryParse("John Doe 30 6000000.00"u8, Inv, null, out var g); - Assert.True(ok); - Assert.Equal("John Doe", g.Name); - Assert.Equal(30, g.Age); - Assert.Equal(6000000.00, g.Salary); - } - - [Theory] - [InlineData("ABCDE0042")] - [InlineData("X 7 ")] - [InlineData("AB")] // short: Quantity out of bounds -> rejected by both - public void Product_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Code, g.Code); - Assert.Equal(r.Quantity, g.Quantity); - } - } - - [Theory] - [InlineData("3.14159 ")] - [InlineData(" -2.5 ")] - [InlineData("notanum ")] - public void Measurement_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Value, g.Value); - } - } - - [Fact] - public void TrailingString_ShortLineGeneratedMatchesReflection() - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse("42"u8, Inv, null, out _); - bool okG = FixedWidthUtf8.TryParse("42"u8, Inv, null, out _); - - Assert.False(okR); - Assert.False(okG); - } - - [Fact] - public void TrailingString_FullLineGeneratedMatchesReflection() - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse("42 short "u8, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse("42 short "u8, Inv, null, out var g); - - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Id, g.Id); - Assert.Equal(r.Note, g.Note); - } - - [Theory] - [InlineData("123.45 ")] - [InlineData("-0.01 ")] - [InlineData("bad ")] - public void Decimal_GeneratedMatchesReflection(string line) - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse(U8(line), Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse(U8(line), Inv, null, out var g); - - Assert.Equal(okR, okG); - if (okR) - { - Assert.Equal(r.Amount, g.Amount); - } - } - -#if NET9_0_OR_GREATER - [Fact] - public void RefStruct_GeneratedMatchesReflection() - { - var reflection = new Utf8FixedWidthParser(); - bool okR = reflection.TryParse("John Doe 30 6000000.00"u8, Inv, null, out var r); - bool okG = FixedWidthUtf8.TryParse("John Doe 30 6000000.00"u8, Inv, null, out var g); - - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Name, g.Name); - Assert.Equal(r.Age, g.Age); - Assert.Equal(r.Salary, g.Salary); - } -#endif - - [Fact] - public void StringPool_InternsAndMatchesReflection() - { - var pool = new StringPool(); - var reflection = new Utf8FixedWidthParser(); - - bool okR = reflection.TryParse("John Doe 30 6000000.00"u8, Inv, pool, out var r); - bool okG = FixedWidthUtf8.TryParse("John Doe 30 6000000.00"u8, Inv, pool, out var g); - - Assert.True(okR); - Assert.Equal(okR, okG); - Assert.Equal(r.Name, g.Name); - // Same pooled instance is returned for the same content (UTF-8 decoded once). - Assert.Same(pool.GetOrAdd("John Doe".AsSpan()), g.Name); - } - } -} diff --git a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs index fa0b24b..6009b21 100644 --- a/tests/FixedWidthParser.Tests/WriterAsyncTests.cs +++ b/tests/FixedWidthParser.Tests/WriterAsyncTests.cs @@ -1,14 +1,9 @@ using System.Text; -using FixedWidthParser.Readers; -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests { - /// - /// Covers the asynchronous batch write path (WriteManyAsync) for both the Stream and - /// StreamWriter overloads, checking output parity with the synchronous path. - /// + /// WriteManyAsync: output parity with the synchronous path. public class WriterAsyncTests { private static PersonModel[] SampleModels() @@ -21,52 +16,28 @@ private static PersonModel[] SampleModels() } [Fact] - public async Task WriteManyAsync_Stream_MatchesSyncOutput() + public async Task WriteManyAsync_MatchesSyncOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = SampleModels(); - string sync = WriteMany(writer, (IEnumerable)models); + string sync = WriteMany(layout, (IEnumerable)models); await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, models, Inv).ConfigureAwait(true); - string asyncResult = Encoding.UTF8.GetString(ms.ToArray()); + await FixedWidth.WriteManyAsync(ms, models, layout, null, TestContext.Current.CancellationToken); - Assert.Equal(sync, asyncResult); + Assert.Equal(sync, Encoding.UTF8.GetString(ms.ToArray())); } [Fact] - public async Task WriteManyAsync_StreamWriter_MatchesSyncOutput() + public async Task WriteManyAsync_LeavesStreamOpen_AndRoundTripsThroughReader() { - var writer = new FixedWidthWriter(); - var models = SampleModels(); - - string sync = WriteMany(writer, (IEnumerable)models); - await using var ms = new MemoryStream(); - await using (var sw = new StreamWriter(ms, leaveOpen: true)) - { - await writer.WriteManyAsync(sw, models, Inv).ConfigureAwait(true); - } - string asyncResult = Encoding.UTF8.GetString(ms.ToArray()); + await FixedWidth.WriteManyAsync(ms, SampleModels(), FixedWidthLayout.FromAttributes(), null, TestContext.Current.CancellationToken); - Assert.Equal(sync, asyncResult); - } - - [Fact] - public async Task WriteManyAsync_Stream_LeavesStreamOpen_AndRoundTripsThroughReader() - { - var writer = new FixedWidthWriter(); - var reader = new FixedWidthReader(Inv); - var models = SampleModels(); - - await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, models, Inv).ConfigureAwait(true); - - // The Stream overload uses leaveOpen: true, so the stream stays usable. Assert.True(ms.CanRead); ms.Position = 0; - var parsed = reader.Read(ms, leaveOpen: true).ToList(); + var parsed = FixedWidth.Read(ms, FixedWidthLayout.FromAttributes(), leaveOpen: true).ToList(); Assert.Equal(3, parsed.Count); Assert.Equal("John Doe", parsed[0].Name); @@ -75,26 +46,25 @@ public async Task WriteManyAsync_Stream_LeavesStreamOpen_AndRoundTripsThroughRea } [Fact] - public async Task WriteManyAsync_Stream_EmptyCollection_WritesNothing() + public async Task WriteManyAsync_ManyRecords_SpansSeveralBatches() { - var writer = new FixedWidthWriter(); + // 5,000 records of 27 bytes exceed the 64 KB batch, so the buffer is flushed mid-way. + var models = Enumerable.Range(0, 5000).Select(i => new PersonModel { Name = "P" + i, Age = i % 100, Salary = i }).ToList(); + var writerOptions = new FixedWidthWriterOptions { NewLine = "\r\n" }; + var layout = FixedWidthLayout.FromAttributes(); await using var ms = new MemoryStream(); - await writer.WriteManyAsync(ms, [], Inv).ConfigureAwait(true); + await FixedWidth.WriteManyAsync(ms, models, layout, writerOptions, TestContext.Current.CancellationToken); - Assert.Equal(0, ms.Length); + Assert.Equal(5000 * 27, ms.Length); + Assert.Equal(WriteMany(layout, models, writerOptions), Encoding.UTF8.GetString(ms.ToArray())); } [Fact] - public async Task WriteManyAsync_StreamWriter_EmptyCollection_WritesNothing() + public async Task WriteManyAsync_EmptyCollection_WritesNothing() { - var writer = new FixedWidthWriter(); - await using var ms = new MemoryStream(); - await using (var sw = new StreamWriter(ms, leaveOpen: true)) - { - await writer.WriteManyAsync(sw, [], Inv).ConfigureAwait(true); - } + await FixedWidth.WriteManyAsync(ms, [], FixedWidthLayout.FromAttributes(), null, TestContext.Current.CancellationToken); Assert.Equal(0, ms.Length); } diff --git a/tests/FixedWidthParser.Tests/WriterEncodingTests.cs b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs new file mode 100644 index 0000000..be03828 --- /dev/null +++ b/tests/FixedWidthParser.Tests/WriterEncodingTests.cs @@ -0,0 +1,120 @@ +using System.Text; +using static FixedWidthParser.Tests.TestHelpers; + +namespace FixedWidthParser.Tests +{ + /// The writer's encoding, byte-based truncation and line break. + public class WriterEncodingTests + { + [Fact] + public void Latin1_WritesOneBytePerCharacter() + { + var writerOptions = new FixedWidthWriterOptions { Encoding = Encoding.Latin1, NewLine = "\n" }; + var layout = FixedWidthLayout.FromAttributes(); + using var ms = new MemoryStream(); + + FixedWidth.Write(ms, new NameAgeModel { Name = "José", Age = 7 }, layout, writerOptions); + + Assert.Equal(Encoding.Latin1.GetBytes("José 7 \n"), ms.ToArray()); + } + + [Fact] + public void Utf8_MultiByteCharacters_CountAsBytes() + { + // "José" is 5 bytes in UTF-8, so one pad byte is left in the 6-byte column. + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NameAgeModel { Name = "José", Age = 7 }); + + Assert.Equal("José 7 ", line); + } + + [Fact] + public void Utf8_Truncation_NeverSplitsACharacter() + { + // "ABCDEé" needs 7 bytes; the 6-byte column keeps "ABCDE" and pads, instead of half of 'é'. + string line = WriteOne(FixedWidthLayout.FromAttributes(), new NameAgeModel { Name = "ABCDEé", Age = 1 }); + + Assert.Equal("ABCDE 1 ", line); + } + + [Fact] + public void Utf8_RightAlignedTruncation_KeepsWholeTrailingCharacters() + { + // Right-aligned width 4 keeps the rightmost whole characters: "éF" is 3 bytes, "DéF" would be 4. + string line = WriteOne(FixedWidthLayout.FromAttributes(), new RightTruncateModel { Code = "ABCDéF" }); + + Assert.Equal("DéF", line); + } + + [Fact] + public void Generated_MatchesReflection_ForLatin1() + { + var context = CtxWith(encoding: Encoding.Latin1); + var writerOptions = new FixedWidthWriterOptions { Encoding = Encoding.Latin1 }; + var layout = FixedWidthLayout.FromAttributes(); + Span generated = stackalloc byte[9]; + Span reflected = stackalloc byte[9]; + + Assert.True(FixedWidth.TryFormat(new GenNameAgeModel { Name = "Joséé", Age = 42 }, generated, context, out int g)); + Assert.True(FixedWidth.TryFormat(new NameAgeModel { Name = "Joséé", Age = 42 }, layout, reflected, writerOptions.Context, out int r)); + + Assert.Equal(9, g); + Assert.Equal(9, r); + Assert.True(generated.SequenceEqual(reflected)); + Assert.Equal("Joséé 42 ", Encoding.Latin1.GetString(generated)); + } + + // Numbers format straight to UTF-8; a non-ASCII result must still be re-encoded for other encodings. + [Theory] + [InlineData("utf-8", " é5.00")] + [InlineData("iso-8859-1", " é5.00")] + public void NonAsciiFormattedValue_UsesTheConfiguredEncoding(string encodingName, string expected) + { + var encoding = Encoding.GetEncoding(encodingName); + var culture = (System.Globalization.CultureInfo)Inv.Clone(); + culture.NumberFormat.CurrencySymbol = "é"; + var writerOptions = new FixedWidthWriterOptions { Encoding = encoding, FormatProvider = culture }; + var layout = FixedWidthLayout.FromAttributes(); + Span generated = stackalloc byte[8]; + Span reflected = stackalloc byte[8]; + + Assert.True(FixedWidth.TryFormat(new GenPriceModel { Price = 5m }, generated, CtxWith(culture, encoding: encoding), out _)); + Assert.True(FixedWidth.TryFormat(new PriceModel { Price = 5m }, layout, reflected, writerOptions.Context, out _)); + + Assert.Equal(encoding.GetBytes(expected), generated.ToArray()); + Assert.Equal(encoding.GetBytes(expected), reflected.ToArray()); + } + + [Fact] + public void NewLine_IsConfigurable() + { + var writerOptions = new FixedWidthWriterOptions { NewLine = "\r\n" }; + var layout = FixedWidthLayout.FromAttributes(); + + Assert.Equal("ABC\r\nDEF\r\n", WriteMany(layout, new[] { new CodeModel { Code = "ABC" }, new CodeModel { Code = "DEF" } }, writerOptions)); + } + + [Fact] + public void TryFormat_DestinationTooSmall_ReturnsFalse() + { + var layout = FixedWidthLayout.FromAttributes(); + + Assert.False(FixedWidth.TryFormat(new CodeModel { Code = "ABC" }, layout, new byte[2], Ctx, out int written)); + Assert.Equal(0, written); + } + + [Fact] + public void WriteMany_ManyRecords_SpansSeveralBatches() + { + var models = Enumerable.Range(0, 20_000).Select(i => new CodeModel { Code = (i % 1000).ToString("D3", Inv) }).ToArray(); + var writerOptions = new FixedWidthWriterOptions { NewLine = "\n" }; + var layout = FixedWidthLayout.FromAttributes(); + + string fromSpan = WriteMany(layout, models.AsSpan(), writerOptions); + string fromEnumerable = WriteMany(layout, (IEnumerable)models, writerOptions); + + Assert.Equal(20_000 * 4, fromSpan.Length); + Assert.Equal(fromSpan, fromEnumerable); + Assert.EndsWith("999\n", fromSpan, StringComparison.Ordinal); + } + } +} diff --git a/tests/FixedWidthParser.Tests/WriterFormattingTests.cs b/tests/FixedWidthParser.Tests/WriterFormattingTests.cs index f8ac549..1651cee 100644 --- a/tests/FixedWidthParser.Tests/WriterFormattingTests.cs +++ b/tests/FixedWidthParser.Tests/WriterFormattingTests.cs @@ -1,4 +1,3 @@ -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -11,9 +10,9 @@ public class WriterFormattingTests [Fact] public void Write_RightAligned_PadsOnLeftWithSpaces() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new RightAlignedModel { Value = 30 }); + string line = WriteOne(layout, new RightAlignedModel { Value = 30 }); Assert.Equal(" 30", line); } @@ -21,9 +20,9 @@ public void Write_RightAligned_PadsOnLeftWithSpaces() [Fact] public void Write_ZeroPaddedRightAligned_FillsWithZeros() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new ZeroPaddedModel { Value = 30 }); + string line = WriteOne(layout, new ZeroPaddedModel { Value = 30 }); Assert.Equal("00030", line); } @@ -31,9 +30,9 @@ public void Write_ZeroPaddedRightAligned_FillsWithZeros() [Fact] public void Write_FormatString_IsApplied() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new FormattedModel { Amount = 1234.5 }); + string line = WriteOne(layout, new FormattedModel { Amount = 1234.5 }); // "F2" → "1234.50" (7 chars), left-aligned in 8 → one padding char. Assert.Equal("1234.50 ", line); @@ -42,9 +41,9 @@ public void Write_FormatString_IsApplied() [Fact] public void Write_RightAlignedString_PadsOnLeft() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new RightStringModel { Code = "AB" }); + string line = WriteOne(layout, new RightStringModel { Code = "AB" }); Assert.Equal(" AB", line); } @@ -52,10 +51,10 @@ public void Write_RightAlignedString_PadsOnLeft() [Fact] public void Write_NumericOverflow_ThrowsByDefault() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var ex = Assert.Throws( - () => WriteOne(writer, new NarrowModel { Value = 12345 })); + () => WriteOne(layout, new NarrowModel { Value = 12345 })); Assert.Contains("Value", ex.Message, StringComparison.Ordinal); } @@ -63,17 +62,17 @@ public void Write_NumericOverflow_ThrowsByDefault() [Fact] public void Write_NumericOverflow_TruncatesWhenOptedIn() { - Assert.Throws(() => new FixedWidthWriter()); + Assert.Throws(() => FixedWidth.WriteMany(Stream.Null, Array.Empty(), FixedWidthLayout.FromAttributes())); } [Fact] public void Write_FormattedValueLargerThanStackBuffer_UsesArrayPoolFallback() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // 700 chars > 256 (stack buffer) and > 512 (first rented buffer): exercises the // ArrayPool fallback and its grow (size *= 2) path. - string line = WriteOne(writer, new WideValueModel { Value = new RepeatedChar(700) }); + string line = WriteOne(layout, new WideValueModel { Value = new RepeatedChar(700) }); Assert.Equal(new string('X', 700) + new string(' ', 100), line); } @@ -81,10 +80,10 @@ public void Write_FormattedValueLargerThanStackBuffer_UsesArrayPoolFallback() [Fact] public void Write_FormattedValueJustOverStackBuffer_FitsFirstRentedBuffer() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // 300 chars > 256 (stack) but <= 512: fallback succeeds on the first rented buffer. - string line = WriteOne(writer, new WideValueModel { Value = new RepeatedChar(300) }); + string line = WriteOne(layout, new WideValueModel { Value = new RepeatedChar(300) }); Assert.Equal(new string('X', 300) + new string(' ', 500), line); } @@ -92,13 +91,13 @@ public void Write_FormattedValueJustOverStackBuffer_FitsFirstRentedBuffer() [Fact] public void Write_ValueThatNeverFits_ThrowsClearError() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // 1<<21 chars exceeds the formatter's grow ceiling (1<<20), so TryFormat never succeeds. // The grow loop must fail fast with a clear, named error rather than overflowing its size // counter into a negative/huge ArrayPool rent. var ex = Assert.Throws( - () => WriteOne(writer, new WideValueModel { Value = new RepeatedChar(1 << 21) })); + () => WriteOne(layout, new WideValueModel { Value = new RepeatedChar(1 << 21) })); Assert.Contains(nameof(WideValueModel.Value), ex.Message, StringComparison.Ordinal); Assert.Contains("could not be formatted", ex.Message, StringComparison.Ordinal); @@ -107,10 +106,10 @@ public void Write_ValueThatNeverFits_ThrowsClearError() [Fact] public void Write_RightAlignedOverflow_TruncatesKeepingRightmostChars() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); // "ABCDEF" (6) in a right-aligned width-4 column → keeps the last 4 chars. - string line = WriteOne(writer, new RightTruncateModel { Code = "ABCDEF" }); + string line = WriteOne(layout, new RightTruncateModel { Code = "ABCDEF" }); Assert.Equal("CDEF", line); } @@ -119,9 +118,9 @@ public void Write_RightAlignedOverflow_TruncatesKeepingRightmostChars() public void Write_StringOverflow_TruncatesByDefault() { // Default string behavior preserved (PersonModel.Name has width 10). - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); + string line = WriteOne(layout, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); Assert.Equal("VeryLongNa", line[..10]); } diff --git a/tests/FixedWidthParser.Tests/WriterTests.cs b/tests/FixedWidthParser.Tests/WriterTests.cs index 5de4c73..feaf3f7 100644 --- a/tests/FixedWidthParser.Tests/WriterTests.cs +++ b/tests/FixedWidthParser.Tests/WriterTests.cs @@ -1,4 +1,3 @@ -using FixedWidthParser.Writers; using static FixedWidthParser.Tests.TestHelpers; namespace FixedWidthParser.Tests @@ -8,9 +7,9 @@ public class WriterTests [Fact] public void Write_ProducesFixedWidthLayout() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "John Doe", Age = 30, Salary = 60000 }); + string line = WriteOne(layout, new PersonModel { Name = "John Doe", Age = 30, Salary = 60000 }); Assert.Equal(25, line.Length); Assert.Equal("John Doe ", line[..10]); @@ -21,9 +20,9 @@ public void Write_ProducesFixedWidthLayout() [Fact] public void Write_StringLongerThanColumn_Truncates() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); + string line = WriteOne(layout, new PersonModel { Name = "VeryLongName", Age = 1, Salary = 0 }); Assert.Equal("VeryLongNa", line[..10]); } @@ -31,9 +30,9 @@ public void Write_StringLongerThanColumn_Truncates() [Fact] public void Write_EmptyString_PadsWithSpaces() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new PersonModel { Name = string.Empty, Age = 1, Salary = 0 }); + string line = WriteOne(layout, new PersonModel { Name = string.Empty, Age = 1, Salary = 0 }); Assert.Equal(" ", line[..10]); } @@ -41,9 +40,9 @@ public void Write_EmptyString_PadsWithSpaces() [Fact] public void Write_FieldBasedModel_Writes() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new ProductModel { Code = "AB", Quantity = 7 }); + string line = WriteOne(layout, new ProductModel { Code = "AB", Quantity = 7 }); Assert.Equal("AB ", line[..5]); Assert.Equal("7 ", line[5..9]); @@ -52,15 +51,15 @@ public void Write_FieldBasedModel_Writes() [Fact] public void WriteMany_SpanAndEnumerable_ProduceSameOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = new[] { new PersonModel { Name = "Alice", Age = 25, Salary = 100 }, new PersonModel { Name = "Bob", Age = 40, Salary = 200 }, }; - string spanOutput = WriteMany(writer, models.AsSpan()); - string enumerableOutput = WriteMany(writer, (IEnumerable)models); + string spanOutput = WriteMany(layout, models.AsSpan()); + string enumerableOutput = WriteMany(layout, (IEnumerable)models); Assert.Equal(enumerableOutput, spanOutput); } @@ -68,13 +67,13 @@ public void WriteMany_SpanAndEnumerable_ProduceSameOutput() [Fact] public async Task WriteAsync_MatchesSyncOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var model = new PersonModel { Name = "Alice", Age = 25, Salary = 100 }; #pragma warning disable S6966 // Awaitable method should be used - string sync = WriteOne(writer, model); + string sync = WriteOne(layout, model); #pragma warning restore S6966 // Awaitable method should be used - string asyncResult = await WriteOneAsync(writer, model).ConfigureAwait(true); + string asyncResult = await WriteOneAsync(layout, model).ConfigureAwait(true); Assert.Equal(sync, asyncResult); } @@ -84,9 +83,9 @@ public async Task WriteAsync_MatchesSyncOutput() [Fact] public void Write_LineLongerThan1024_UsesArrayPoolPath() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); - string line = WriteOne(writer, new WideLineModel { Name = "Wide", Number = 42 }); + string line = WriteOne(layout, new WideLineModel { Name = "Wide", Number = 42 }); Assert.Equal(WideLineModel.LineLength, line.Length); Assert.Equal("Wide", line[..1200].TrimEnd()); @@ -96,14 +95,14 @@ public void Write_LineLongerThan1024_UsesArrayPoolPath() [Fact] public void WriteMany_Span_LineLongerThan1024_UsesArrayPoolPath() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = new[] { new WideLineModel { Name = "Alice", Number = 1 }, new WideLineModel { Name = "Bob", Number = 2 }, }; - string output = WriteMany(writer, models.AsSpan()); + string output = WriteMany(layout, models.AsSpan()); var lines = output.Split('\n', StringSplitOptions.RemoveEmptyEntries) .Select(l => l.TrimEnd('\r')) @@ -119,15 +118,15 @@ public void WriteMany_Span_LineLongerThan1024_UsesArrayPoolPath() [Fact] public void WriteMany_SpanAndEnumerable_LineLongerThan1024_ProduceSameOutput() { - var writer = new FixedWidthWriter(); + var layout = FixedWidthLayout.FromAttributes(); var models = new[] { new WideLineModel { Name = "Alice", Number = 1 }, new WideLineModel { Name = "Bob", Number = 2 }, }; - string spanOutput = WriteMany(writer, models.AsSpan()); - string enumerableOutput = WriteMany(writer, (IEnumerable)models); + string spanOutput = WriteMany(layout, models.AsSpan()); + string enumerableOutput = WriteMany(layout, (IEnumerable)models); Assert.Equal(enumerableOutput, spanOutput); }