diff --git a/.github/workflows/nightly-performance.yml b/.github/workflows/nightly-performance.yml index 14548e1..292a6a7 100644 --- a/.github/workflows/nightly-performance.yml +++ b/.github/workflows/nightly-performance.yml @@ -1,11 +1,12 @@ name: Nightly Performance -# Runs the on-premise performance examples, writes their throughput figures into -# results_.json in the common-ci schema, and uploads them so the shared -# steps/compare-performance.ps1 can chart them on the documentation performance -# graphs. The Rust on-premise engines call the device-detection-cxx and -# ip-intelligence-cxx libraries through FFI, so the figures track native -# performance. +# Runs the on-premise performance examples against this checkout's crates. Each +# example writes its results JSON, the shared +# steps/publish-performance-results.ps1 validates it and puts it at +# results_.json, and the shared steps/compare-performance.ps1 charts it on +# the documentation performance graphs. The Rust on-premise engines call the +# device-detection-cxx and ip-intelligence-cxx libraries through FFI, so the +# figures track native performance. # # This mirrors the per-language performance steps in common-ci's nightly # workflows. It is kept self-contained here (rather than calling the shared @@ -56,14 +57,14 @@ jobs: with: path: rust submodules: recursive - - # 2. common-ci, checked out for the shared comparison step - # (steps/compare-performance.ps1). The Rust performance adapter itself - # now lives in this repo at ci/run-performance-tests.ps1 (see - # ci/missing-ci-scripts.md), because it serves only this repository and - # a folder named rust in common-ci collides with the clone of a - # repository named rust (51Degrees/rust#50); only the comparison step - # is shared. + # The adapter versions the crates from the release tags. + fetch-depth: 0 + + # 2. common-ci, checked out for the shared steps that publish and compare + # the results (steps/publish-performance-results.ps1 and + # steps/compare-performance.ps1). The Rust performance adapter lives in + # this repo at ci/run-performance-tests.ps1 (see + # ci/missing-ci-scripts.md), because it serves only this repository. - name: Checkout common-ci uses: actions/checkout@v4 with: @@ -132,8 +133,9 @@ jobs: with: workspaces: rust - # 5. Run the performance examples and write results_.json files under - # rust/test-results/performance-summary in the common-ci schema. + # 5. Run the performance examples against this checkout's crates and + # publish each results_.json under + # rust/test-results/performance-summary through the shared step. - name: Run performance and write results shell: pwsh run: ./rust/ci/run-performance-tests.ps1 -RepoName rust @@ -183,15 +185,6 @@ jobs: @{ Name = "DeviceDetection-OnPremise"; RunPerformance = $true }, @{ Name = "IpIntelligence-OnPremise"; RunPerformance = $true } ) - # The performance run resolves examples/Cargo.lock against source.toml - # (the local-path patch), which rewrites this tracked lockfile. The - # shared compare step publishes graphs by switching to an orphan images - # branch, and git aborts that switch while a tracked file has - # uncommitted changes. Restore the committed lockfile so the tree is - # clean before the switch. Only this one file is touched by the run; - # git's "changes would be overwritten" error enumerates every colliding - # tracked file, and it lists only examples/Cargo.lock. - git -C rust checkout -- examples/Cargo.lock ./common-ci/steps/compare-performance.ps1 ` -RepoName 'rust' ` -OrgName '51Degrees' ` diff --git a/AGENTS.md b/AGENTS.md index 1f407c2..63f57f3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -161,9 +161,12 @@ is ignored by git and listed in the `fodid` manifest's `include`, so the bundled asset and `UPSTREAM_PINNED_REF` to the same commit in one pull request. - `nightly-performance.yml` — nightly (and on-demand) on-premise throughput - run that feeds the documentation performance graphs. Its Rust adapter lives - in this repo at `ci/run-performance-tests.ps1`; only the shared comparison - step (`compare-performance.ps1`) comes from common-ci. See + run that feeds the documentation performance graphs. Its adapter, + `ci/run-performance-tests.ps1`, runs the performance examples against this + checkout's crates through `ci/test-examples-against-source.ps1`. Each + example writes its own results JSON, and common-ci provides the steps that + validate and compare it (`steps/publish-performance-results.ps1` and + `steps/compare-performance.ps1`). See [ci/missing-ci-scripts.md](ci/missing-ci-scripts.md) for how this relates to the org-standard reusable-workflow contract and the planned direction. diff --git a/Cargo.lock b/Cargo.lock index 5dd5a14..d8d991c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -783,6 +783,7 @@ dependencies = [ "fiftyone-native", "fiftyone-pipeline-core", "fiftyone-pipeline-engines", + "flate2", ] [[package]] @@ -842,6 +843,7 @@ name = "fiftyone-pipeline-core" version = "4.5.2" dependencies = [ "ahash", + "log", "thiserror 2.0.18", ] @@ -877,6 +879,7 @@ dependencies = [ "reqwest", "thiserror 2.0.18", "tiny_http", + "tokio", "uuid", ] diff --git a/README.md b/README.md index 291428c..69ce600 100644 --- a/README.md +++ b/README.md @@ -115,10 +115,12 @@ path, so cloud-only users and most of CI build without a C compiler. `.data_update_license_keys(..)`, and that refreshes the engine when the data file changes on disk. The Distributor serves nothing without a licence key, so a build with polling on and no key fails with an error that says so, as - it does in .NET, and a deployment of the free Lite file turns polling off - with `.auto_update(false)`. `.file_system_watcher(false)` turns the watcher - off, and `pipeline.resource::()` reaches the service for - a check on demand. + it does in .NET, and a deployment of the free Lite file turns automatic + updates off with `.auto_update(false)`, which stops the polling and the file + watcher together, as the specification's master switch does. + `.file_system_watcher(false)` turns only the watcher off, and + `pipeline.resource::()` reaches the service for a check + on demand. - JavaScript minification in `fiftyone-javascript-builder` is opt-in behind the `minify` feature and off by default. See ARCHITECTURE.md for why. diff --git a/ci/missing-ci-scripts.md b/ci/missing-ci-scripts.md index e750d22..3ef6e7e 100644 --- a/ci/missing-ci-scripts.md +++ b/ci/missing-ci-scripts.md @@ -60,9 +60,9 @@ Rust consumer, so there is nothing to share. Every other language keeps its - Repoints `nightly-performance.yml` to run `./rust/ci/run-performance-tests.ps1` (the workflow checks this repo out into a `rust/` subdirectory) instead of `./common-ci/rust/run-performance-tests.ps1`. -- Keeps the `Checkout common-ci` step, because the comparison step - `steps/compare-performance.ps1` is genuinely shared and still comes from - common-ci. +- Keeps the `Checkout common-ci` step, because the steps that publish and + compare the results, `steps/publish-performance-results.ps1` and + `steps/compare-performance.ps1`, are shared and come from common-ci. Moving the adapter's home is the structural part of the change. Alongside it, this repo's copy has since diverged from the common-ci original with @@ -71,11 +71,8 @@ repository-specific fixes (the ASN data fetch, the `dryrun` input, the fail the job), so the behaviour of the nightly run is not identical to the old `common-ci/rust` path. -The copy in `common-ci/rust/run-performance-tests.ps1` is intentionally **left -in place** for now and will be removed in a separate, manually raised common-ci -PR. This repo's copy is the one CI uses, and it is now the source of truth: the -two copies are no longer identical, so the common-ci copy should not be edited -in place — it is only awaiting deletion. +common-ci no longer carries a `rust` folder, so this repo's copy is the only +one. ## The remaining gap (Phase 2, deferred) diff --git a/ci/run-performance-tests.ps1 b/ci/run-performance-tests.ps1 index 51e9587..5d6f0e8 100644 --- a/ci/run-performance-tests.ps1 +++ b/ci/run-performance-tests.ps1 @@ -5,11 +5,21 @@ param( [string]$RepoName = ".", # Where the results files are written, relative to the repo directory. This is # the path the nightly workflow uploads and the compare-performance step reads. - [string]$OutputDir = "test-results/performance-summary" + [string]$OutputDir = "test-results/performance-summary", + # The common-ci checkout whose steps/publish-performance-results.ps1 + # validates each results file and copies it to $OutputDir. CI checks it out + # beside this repository. + [string]$CommonCi = (Join-Path $RepoName "../common-ci") ) $ErrorActionPreference = "Stop" $PSNativeCommandUseErrorActionPreference = $true +# Checked first, so a missing checkout fails before the long build. +$publishStep = Join-Path $CommonCi "steps/publish-performance-results.ps1" +if (!(Test-Path -Path $publishStep -PathType Leaf)) { + Write-Error "'$publishStep' was not found. Check out 51Degrees/common-ci beside this repository, or pass -CommonCi." +} + # The ip-intelligence-cxx checkout is a sibling of the repo directory (CI # checks this repo out into $RepoName next to it; a local run passes "." from # the repo root, whose parent is the workspace holding the sibling checkout). @@ -39,83 +49,41 @@ try { Pop-Location } -# Runs the 51Degrees Rust on-premise performance examples in release and writes -# their throughput figures into results_.json files, in the same -# `{ HigherIsBetter = @{ metric = value } }` shape the shared -# steps/compare-performance.ps1 consumes. The Rust on-premise engines call the +# Runs the 51Degrees Rust on-premise performance examples in release. Each +# example writes its own results JSON, in the shape the shared +# steps/compare-performance.ps1 reads, and the shared +# steps/publish-performance-results.ps1 validates it and copies it to +# results_.json under $OutputDir. The Rust on-premise engines call the # device-detection-cxx and ip-intelligence-cxx libraries through FFI, so the # figures track the native C/C++ performance. +# +# Name is the configuration the graph history is keyed on, and the metric +# names inside each file are its series, so neither may change. +$performanceExamples = @( + @{ Name = "DeviceDetection-OnPremise"; Package = "device-detection-examples"; Bin = "dd-onprem-performance" } + @{ Name = "IpIntelligence-OnPremise"; Package = "ip-intelligence-examples"; Bin = "ipi-onprem-performance" } +) -Push-Location $RepoName -try { - $summaryDir = Join-Path (Get-Location) $OutputDir - New-Item -ItemType Directory -Force -Path $summaryDir | Out-Null - - # The example crates live in their own workspace under examples/ and depend - # on the published crates from crates.io by default. The nightly must - # benchmark this checkout's engine code, not the released packages, so the - # cargo commands run from examples/ with `--config source.toml`, the patch - # file that points every fiftyone-* dependency at its local path. - $examplesDir = Join-Path (Get-Location) "examples" +# Absolute, because the examples run from the examples directory. +$resultsDir = Join-Path (Resolve-Path $RepoName).Path "test-results/performance" +foreach ($example in $performanceExamples) { + $example.Results = Join-Path $resultsDir "results_$($example.Name).json" + # A file left by an earlier run would otherwise pass for this run's figures. + Remove-Item -Path $example.Results -Force -ErrorAction SilentlyContinue +} - # Run one performance example and parse the throughput it prints. The - # examples take the highest throughput figure they report (the multi-threaded - # pass), which is the headline number for the product. - function Get-Throughput { - param( - [Parameter(Mandatory)][string]$Package, - [Parameter(Mandatory)][string]$Bin, - [Parameter(Mandatory)][string]$Pattern - ) - Write-Host "Running performance example '$Bin'..." - Push-Location $examplesDir - try { - # The native exit code check is turned off for this one call so the - # example output is printed before any failure is raised. With it on, - # a non-zero cargo exit threw at the assignment, Write-Host below - # never ran, and the job log showed only that cargo ended - # with a non-zero exit code, with nothing the example or the - # compiler had said. That is what hid why ipi-onprem-performance was - # failing every night. - $PSNativeCommandUseErrorActionPreference = $false - $output = cargo run --release --config source.toml -p $Package --bin $Bin 2>&1 | Out-String - $exitCode = $LASTEXITCODE - } finally { - Pop-Location - } - Write-Host $output - if ($exitCode -ne 0) { - Write-Error "'$Bin' exited with code $exitCode. Its output is above." - } - $found = [regex]::Matches($output, $Pattern) - if ($found.Count -eq 0) { - Write-Error "Could not parse a throughput figure from '$Bin' output" - } - # Take the last match so the multi-threaded figure wins where an example - # prints both a single- and a multi-threaded result. - return [double]$found[$found.Count - 1].Groups[1].Value +# The examples depend on the published crates. ci/test-examples-against-source.ps1 +# points them at this checkout for the run and puts the manifests and the lock +# back afterwards, so the figures are for the code being tested and not for the +# last release. The closure carries the list above into that script. +$runExamples = { + foreach ($example in $performanceExamples) { + Write-Host "Running performance example '$($example.Bin)'..." + cargo run --release --config source.toml -p $example.Package --bin $example.Bin -- --json-output $example.Results } +}.GetNewClosure() +& (Join-Path $PSScriptRoot "test-examples-against-source.ps1") -Command $runExamples - # Device Detection on-premise (Hash): detections per second. - $ddDetectionsPerSecond = Get-Throughput ` - -Package "device-detection-examples" ` - -Bin "dd-onprem-performance" ` - -Pattern "Detections per second\s*:\s*(\d+)" - @{ HigherIsBetter = @{ DetectionsPerSecond = $ddDetectionsPerSecond } } | - ConvertTo-Json -Depth 5 | - Out-File (Join-Path $summaryDir "results_DeviceDetection-OnPremise.json") -Encoding utf8 - - # IP Intelligence on-premise (IP graph): lookups per second. - $ipiLookupsPerSecond = Get-Throughput ` - -Package "ip-intelligence-examples" ` - -Bin "ipi-onprem-performance" ` - -Pattern "Throughput:\s*(\d+)\s*lookups/sec" - @{ HigherIsBetter = @{ LookupsPerSecond = $ipiLookupsPerSecond } } | - ConvertTo-Json -Depth 5 | - Out-File (Join-Path $summaryDir "results_IpIntelligence-OnPremise.json") -Encoding utf8 - - Write-Host "Wrote performance results to '$summaryDir':" - Get-ChildItem $summaryDir -Filter "results_*.json" | ForEach-Object { Write-Host " - $($_.Name)" } -} finally { - Pop-Location +foreach ($example in $performanceExamples) { + & $publishStep -SourceFile $example.Results -Name $example.Name -RepoName $RepoName -SummaryDir $OutputDir } diff --git a/ci/test-examples-against-source.ps1 b/ci/test-examples-against-source.ps1 index 317c63e..080ec10 100644 --- a/ci/test-examples-against-source.ps1 +++ b/ci/test-examples-against-source.ps1 @@ -8,7 +8,8 @@ The examples depend on the published crates, and examples/source.toml patches each one to its path in this tree. Cargo uses a patch only when its version is the one it would pick, so this first writes a version above every published release into the crates, then moves each patched crate in the examples' lock to -the copy in this tree, then checks the lock and runs the tests. +the copy in this tree, then checks the lock and runs the command, which by +default builds and tests every example. The manifests and the lock are put back as they were when the script ends, so a run leaves the working tree unchanged. @@ -20,10 +21,16 @@ such as 4.5.9 after 4.5.8. .PARAMETER Root The workspace root. Defaults to the repository this script sits in. + +.PARAMETER Command +What to run in examples/ while the crates point at this tree. Each cargo +command in it needs `--config source.toml`. ci/run-performance-tests.ps1 passes +the performance examples here, so the nightly figures are for this tree. #> param ( [string]$Version, - [string]$Root = (Join-Path $PSScriptRoot "..") + [string]$Root = (Join-Path $PSScriptRoot ".."), + [scriptblock]$Command = { cargo test --config source.toml } ) $ErrorActionPreference = "Stop" $PSNativeCommandUseErrorActionPreference = $true @@ -41,7 +48,7 @@ if (-not $Version) { } $Version = "$($released.Major).$($released.Minor).$($released.Build + 1)" } -Write-Output "Testing the examples against this tree's crates at $Version." +Write-Output "Running the examples against this tree's crates at $Version." $examples = Join-Path $Root "examples" $patched = Get-Content -Path (Join-Path $examples "source.toml") | @@ -94,7 +101,7 @@ try { throw "These crates still come from crates.io, not this tree:`n$($fromRegistry -join "`n")" } - cargo test --config source.toml + & $Command } finally { Pop-Location foreach ($path in $originals.Keys) { diff --git a/cloud-request-engine/src/constants.rs b/cloud-request-engine/src/constants.rs index 03e532a..0fe4c4d 100644 --- a/cloud-request-engine/src/constants.rs +++ b/cloud-request-engine/src/constants.rs @@ -98,3 +98,10 @@ pub const PROCESS_STARTED_KEY: &str = "process-started"; /// The element-data field holding the advisory warning messages, if any, that /// the cloud service returned in the response's `warnings` array. pub const WARNINGS_KEY: &str = "warnings"; + +/// The top-level member of a data response that carries 51Did values. +/// +/// The cloud creates each 51Did afresh for the request that asked for it, so a +/// response carrying this member answers that one request and is never shared +/// with another through the response cache. +pub const IDENTIFIER_MEMBER: &str = "fodid"; diff --git a/cloud-request-engine/src/engine.rs b/cloud-request-engine/src/engine.rs index 221cc8a..db519f4 100644 --- a/cloud-request-engine/src/engine.rs +++ b/cloud-request-engine/src/engine.rs @@ -22,6 +22,7 @@ //! The cloud request engine and its builder. +use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -35,6 +36,7 @@ use fiftyone_pipeline_core::{ use fiftyone_pipeline_engines::{ AspectEngine, AspectPropertyMetaData, EngineDeployment, MissingPropertyResult, }; +use serde::de::IgnoredAny; use crate::constants; use crate::data::CloudRequestData; @@ -151,7 +153,11 @@ struct Endpoints { /// /// Repeated request failures within a window trip a [`RecoveryGate`], which then /// short-circuits requests for a recovery period so a slow or failing cloud -/// cannot stall consumer requests. See the +/// cannot stall consumer requests. A failure counts when the request did not +/// complete, or the service answered with a status of 500 or more, a 429, or a +/// success status whose body could not be used. A status from 400 to 499 other +/// than 429 answers the request's own content, which any visitor can choose, +/// so it does not count, because the gate is shared by every visitor. See the /// [recovery-mode section](https://github.com/51Degrees/specifications/blob/main/pipeline-specification/pipeline-elements/cloud-request-engine.md#recovery-mode). /// /// # Example @@ -207,8 +213,10 @@ pub struct CloudRequestEngine { /// An optional response cache, keyed by the request's evidence. When present, /// an identical request is served from the cache instead of calling the cloud. - /// A short-lived host (for example a `wasm32-wasip1` edge instance) can supply - /// a cache backed by its own store, so responses survive across cold starts. + /// A response carrying a 51Did is never stored or served, see + /// [`CloudRequestEngineBuilder::cache`]. A short-lived host (for example a + /// `wasm32-wasip1` edge instance) can supply a cache backed by its own store, + /// so responses survive across cold starts. response_cache: Option>>, /// The core property metadata: `cloud-response`, `json-response` and @@ -456,6 +464,21 @@ fn response_carries(body: &serde_json::Value, asked: &str) -> bool { } } +/// Whether a response body carries 51Did values, being a top-level member +/// named [`constants::IDENTIFIER_MEMBER`] in any case. +/// +/// Such a body answers the one request it was made for, so it is never stored +/// in the response cache or served from it. A body that is not a JSON object +/// cannot be checked, so it counts as carrying them. +fn carries_identifier(json: &str) -> bool { + match serde_json::from_str::>(json) { + Ok(members) => members + .keys() + .any(|name| name.eq_ignore_ascii_case(constants::IDENTIFIER_MEMBER)), + Err(_) => true, + } +} + /// Validate that a resolved endpoint is a well-formed absolute http(s) URL. /// /// The base endpoint can come from the builder's `endpoint`/`*_endpoint` setters, @@ -486,21 +509,24 @@ fn validate_endpoint_url(url: &str, what: &str) -> Result<()> { /// /// The form holds the credential, the asked-for property list when there is one, /// and the stripped evidence fields. The pairs are sorted so the key is -/// independent of evidence insertion order, then joined, so two requests with the -/// same credential, property list and evidence map to the same key. Including the -/// credential and the property list means a cache shared between engines never -/// returns one engine's response for another's request, which matters because two -/// engines asking for different properties get different answers to the same -/// evidence. +/// independent of evidence insertion order. Each name and each value is then +/// written after its length in bytes, so no value can be read as the end of its +/// own pair and the start of another, whatever characters it holds. Two requests +/// therefore share a key only when their credential, property list and evidence +/// are the same. Including the credential and the property list means a cache +/// shared between engines never returns one engine's response for another's +/// request, which matters because two engines asking for different properties +/// get different answers to the same evidence. fn cache_key(form: &[(String, String)]) -> String { let mut pairs: Vec<&(String, String)> = form.iter().collect(); pairs.sort(); let mut key = String::new(); for (name, value) in pairs { - key.push_str(name); - key.push('='); - key.push_str(value); - key.push('\n'); + for part in [name, value] { + key.push_str(&part.len().to_string()); + key.push(':'); + key.push_str(part); + } } key } @@ -534,10 +560,12 @@ impl FlowElement for CloudRequestEngine { // serve a stored response without calling the cloud. The lookup runs // before the recovery gate, so a cache hit also avoids being blocked // during a recovery period. A short-lived host supplies its own cache so - // hits survive across cold starts. + // hits survive across cold starts. A stored body carrying 51Did values, + // which a store shared with other writers can hold, was made for another + // request, so it is not served and the cloud is asked instead. let cache_key = self.response_cache.as_ref().map(|_| cache_key(&form)); if let (Some(cache), Some(key)) = (&self.response_cache, &cache_key) { - if let Some(body) = cache.get(key) { + if let Some(body) = cache.get(key).filter(|body| !carries_identifier(body)) { // The body validated when it was stored, so re-validating it // reproduces the same JSON and warnings a live response would, // without a network call. @@ -578,9 +606,12 @@ impl FlowElement for CloudRequestEngine { )?; self.add_missing_values_warning(&mut parsed); - // Store the successful response so the next identical request is a hit. + // Store the successful response so the next identical request is a hit, + // unless it carries 51Did values, which were created for this request. if let (Some(cache), Some(key)) = (&self.response_cache, cache_key) { - cache.put(key, parsed.json.clone()); + if !carries_identifier(&parsed.json) { + cache.put(key, parsed.json.clone()); + } } if let Some(cloud) = data.get_mut_cloud() { @@ -861,6 +892,14 @@ impl CloudRequestEngineBuilder { /// credential, a different property list or different evidence never /// collides. The cached value is the raw JSON response body. /// + /// A response carrying 51Did values is never stored, and one found in the + /// cache is never served. The cloud creates a 51Did afresh for each request + /// that asks for one, and two visitors can present the same evidence, so a + /// stored copy would hand one visitor the 51Did created for another. A + /// request that carries no `id.usage` evidence asks for no 51Did, so in a + /// pipeline that asks for one on some requests the cache still serves the + /// others, while each request that asks for one reaches the cloud. + /// /// Any [`fiftyone_caching::PutCache`] works, so a consumer chooses the policy: /// the in-process [`fiftyone_caching::LruCache`] for a long-lived host, or a /// custom implementation backed by the host's own store on a short-lived host @@ -1252,12 +1291,25 @@ fn send_and_validate_inner( Ok(parsed) } Err(error) => { - recovery.record_failure(); + if signals_service_trouble(response.status) { + recovery.record_failure(); + } Err(error) } } } +/// Whether a failed answer with this status counts towards a recovery period. +/// +/// The gate is shared by every request the engine makes, so only an answer +/// that says the service is failing or asks callers to slow down counts, being +/// a status of 500 or more, a 429, or a success status whose body could not be +/// used. A status from 400 to 499 answers the request's own content, which one +/// visitor can choose, such as a user agent too long for the service. +fn signals_service_trouble(status: u16) -> bool { + status == 429 || !(400..500).contains(&status) +} + /// Fetch the accepted evidence keys from the cloud, mapping any failure to an /// [`Error::CloudRequest`]. The evidence-keys body is a flat JSON array, so /// error-message checking is disabled for it. @@ -1458,6 +1510,33 @@ mod tests { assert_ne!(a, other_key, "different resource key keys differently"); } + #[test] + fn cache_key_cannot_be_matched_by_a_value_shaped_like_another_field() { + // A query value can hold a line break and an equals sign, so one value + // must not be able to spell out the rest of another request's key. + let crafted = cache_key(&[ + ("resource".to_owned(), "rk".to_owned()), + ( + "user-agent".to_owned(), + "UA\nx-device-user-agent=Other".to_owned(), + ), + ]); + let both = cache_key(&[ + ("resource".to_owned(), "rk".to_owned()), + ("user-agent".to_owned(), "UA".to_owned()), + ("x-device-user-agent".to_owned(), "Other".to_owned()), + ]); + assert_ne!(crafted, both); + + // Nor can a value carry the start of the next field's length. + let split = cache_key(&[("a".to_owned(), "1:b".to_owned())]); + let joined = cache_key(&[ + ("a".to_owned(), "1".to_owned()), + ("b".to_owned(), "".to_owned()), + ]); + assert_ne!(split, joined); + } + #[test] fn strip_prefix_removes_known_prefix() { assert_eq!(strip_prefix("query.user-agent"), "user-agent"); @@ -1655,6 +1734,22 @@ mod tests { assert!(!response_carries(&body, "javascriptProperties.device")); } + #[test] + fn carries_identifier_finds_the_member_in_any_case() { + assert!(carries_identifier( + r#"{"device":{"ismobile":true},"fodid":{"idrandglobal":"x"}}"# + )); + assert!(carries_identifier(r#"{"FODid":{"idprobglobal":null}}"#)); + // A member of another product that merely mentions the name is not one. + assert!(!carries_identifier( + r#"{"device":{"ismobile":true,"fodid":"text"},"javascriptProperties":[]}"# + )); + assert!(!carries_identifier(r#"{"device":{"ismobile":true}}"#)); + // What cannot be read as an object cannot be checked, so it is not shared. + assert!(carries_identifier("[]")); + assert!(carries_identifier("not json")); + } + #[cfg(not(feature = "reqwest-client"))] #[test] fn build_without_client_errors_when_reqwest_disabled() { diff --git a/cloud-request-engine/src/lib.rs b/cloud-request-engine/src/lib.rs index 0625350..a190793 100644 --- a/cloud-request-engine/src/lib.rs +++ b/cloud-request-engine/src/lib.rs @@ -141,7 +141,11 @@ //! //! Repeated request failures within a window trip a recovery gate, which //! short-circuits requests for a recovery period so a slow or failing cloud -//! cannot stall consumer requests. +//! cannot stall consumer requests. Only a failure that says the service is in +//! trouble counts, being a request that did not complete, a status of 500 or +//! more, a 429, or a success status whose body could not be used. A status +//! from 400 to 499 answers the request's own content, which any visitor can +//! choose, so it does not count. //! //! ## Element data shape //! diff --git a/cloud-request-engine/tests/mock_client.rs b/cloud-request-engine/tests/mock_client.rs index 9e480a7..9573252 100644 --- a/cloud-request-engine/tests/mock_client.rs +++ b/cloud-request-engine/tests/mock_client.rs @@ -456,6 +456,95 @@ fn recovery_mode_blocks_after_repeated_data_failures() { ); } +/// A transport answering as the cloud service did on 26 October 2026, with a +/// 404 and a plain text body for a `user-agent` of 32,000 bytes or more, and +/// JSON for anything else. With `failing` set it answers every request 503. +struct SizeLimitedCloud { + failing: bool, +} + +impl CloudHttpClient for SizeLimitedCloud { + fn send(&self, request: &CloudHttpRequest) -> Result { + if self.failing { + return FakeClient::with_status(503, "Service Unavailable"); + } + let user_agent = request + .form + .iter() + .find(|(name, _)| name == "user-agent") + .map_or(0, |(_, value)| value.len()); + if user_agent >= 32_000 { + FakeClient::with_status(404, "Status code: 404, 'Error' method not found") + } else { + FakeClient::ok(r#"{"device":{"ismobile":false}}"#) + } + } +} + +/// A pipeline over one engine with the default recovery settings, ten +/// failures in 100 seconds opening 60 seconds of recovery. +fn recovery_pipeline(cloud: SizeLimitedCloud) -> Arc { + let state = CloudEngineState { + evidence_keys: vec![EvidenceKeyEntry { + key: "query.user-agent".to_owned(), + order: 0, + }], + accessible_properties: LicensedProducts::default(), + }; + let engine = CloudRequestEngine::builder() + .resource_key("rk") + .http_client(Arc::new(cloud)) + .set_state(state) + .build() + .unwrap(); + Pipeline::builder() + .add_element(Arc::new(engine)) + .suppress_process_exceptions(true) + .build() + .unwrap() +} + +/// The error messages a request with this user agent records. +fn errors_for(pipeline: &Arc, user_agent: &str) -> Vec { + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("query.user-agent", user_agent) + .build(), + ); + data.process().unwrap(); + data.errors() + .iter() + .map(|error| error.source.to_string()) + .collect() +} + +#[test] +fn answers_to_one_visitors_own_content_leave_the_next_visitor_served() { + // The gate is shared by every visitor, so a visitor sending evidence the + // service refuses must not be able to open a recovery period for the rest. + let pipeline = recovery_pipeline(SizeLimitedCloud { failing: false }); + let oversized = "A".repeat(32_000); + for _ in 0..10 { + assert_eq!(errors_for(&pipeline, &oversized).len(), 1); + } + let next = errors_for(&pipeline, "Mozilla/5.0"); + assert!(next.is_empty(), "the next visitor was refused: {next:?}"); +} + +#[test] +fn service_errors_still_open_a_recovery_period() { + let pipeline = recovery_pipeline(SizeLimitedCloud { failing: true }); + for _ in 0..10 { + assert_eq!(errors_for(&pipeline, "Mozilla/5.0").len(), 1); + } + let next = errors_for(&pipeline, "Mozilla/5.0"); + assert!( + next.iter() + .any(|error| error.contains("temporarily restricted")), + "ten 503 answers did not open a recovery period: {next:?}" + ); +} + /// A consumer-supplied response cache, recording its lookups and stores so the /// tests can assert how it was used. It stands in for any [`PutCache`], for /// example one a `wasm32-wasip1` host would back with its own key/value store. @@ -627,6 +716,108 @@ fn host_provided_cache_outlives_the_engine_for_short_lived_instances() { ); } +#[test] +fn a_response_carrying_a_51did_is_never_stored() { + // The cloud creates a 51Did for the one request that asked for it, so a body + // carrying one is not kept for the next request with the same evidence. + let client = Arc::new(FakeClient::default()); + client.set_evidence_keys(FakeClient::ok(r#"["header.user-agent","query.id.usage"]"#)); + client.set_data(FakeClient::ok( + r#"{"device":{"ismobile":true},"fodid":{"idrandglobal":"created-for-one-request"}}"#, + )); + + let cache = Arc::new(RecordingCache::default()); + let engine = CloudRequestEngine::builder() + .resource_key("test-resource-key") + .endpoint("https://cloud.example.test/api/v4/") + .http_client(client.clone()) + .cache(cache.clone()) + .build() + .unwrap(); + let after_build = client.request_count(); + let pipeline = Pipeline::builder() + .add_element(Arc::new(engine)) + .build() + .unwrap(); + let evidence = || { + Evidence::builder() + .add("header.user-agent", "UA") + .add("query.id.usage", "non-marketing") + .build() + }; + + for request in 1..=2 { + let mut data = pipeline.create_flow_data_with(evidence()); + data.process().unwrap(); + let cloud = data.get(CloudRequestEngine::DATA_KEY).unwrap(); + assert!(!cloud.cache_hit(), "request {request} is not a cache hit"); + } + assert_eq!(cache.puts(), 0, "no body carrying a 51Did was stored"); + assert_eq!( + client.request_count(), + after_build + 2, + "each request reached the cloud" + ); +} + +/// A cache holding one body under every key, as a store shared with other +/// writers can hold a body for the key a request makes. +struct StoredBodyCache(String); + +impl Cache for StoredBodyCache { + fn get(&self, _key: &String) -> Option { + Some(self.0.clone()) + } + fn len(&self) -> usize { + 1 + } +} + +impl PutCache for StoredBodyCache { + fn put(&self, _key: String, _value: String) {} +} + +#[test] +fn a_stored_response_carrying_a_51did_is_never_served() { + // The 51Did in a stored body was created for another request, so the engine + // asks the cloud rather than serving it, whatever case the member is in. + for member in ["fodid", "FODid"] { + let client = Arc::new(FakeClient::default()); + client.set_data(FakeClient::ok( + r#"{"device":{"ismobile":true},"fodid":{"idrandglobal":"created-for-this-request"}}"#, + )); + let stored = format!( + r#"{{"device":{{"ismobile":true}},"{member}":{{"idrandglobal":"created-for-another-request"}}}}"# + ); + let engine = CloudRequestEngine::builder() + .resource_key("rk") + .http_client(client.clone()) + .set_state(sample_state()) + .cache(Arc::new(StoredBodyCache(stored))) + .build() + .unwrap(); + let pipeline = Pipeline::builder() + .add_element(Arc::new(engine)) + .build() + .unwrap(); + + let mut data = pipeline + .create_flow_data_with(Evidence::builder().add("header.user-agent", "UA").build()); + data.process().unwrap(); + + let cloud = data.get(CloudRequestEngine::DATA_KEY).unwrap(); + assert!( + cloud + .json_response() + .is_some_and(|json| json.contains("created-for-this-request")), + "the {member} body in the store was not served, got {:?}", + cloud.json_response() + ); + assert!(!cloud.cache_hit(), "the {member} body is not a cache hit"); + assert_eq!(client.request_count(), 1, "the cloud was asked instead"); + } +} + /// Build a state snapshot with one accepted evidence key and one product /// property, the kind a consumer would persist and re-inject. fn sample_state() -> CloudEngineState { @@ -1068,3 +1259,88 @@ fn evidence_named_values_does_not_displace_the_engines_property_list() { assert_eq!(values.len(), 1, "one property list, got {values:?}"); assert_eq!(values[0], "device.ismobile", "the engine's list wins"); } + +/// A fake transport that answers each data request with the `user-agent` field +/// it was sent, so a test can tell which request a response was made for. +#[derive(Default)] +struct EchoClient { + data_requests: Mutex, +} + +impl CloudHttpClient for EchoClient { + fn send(&self, request: &CloudHttpRequest) -> Result { + *self.data_requests.lock().unwrap() += 1; + let user_agent = request + .form + .iter() + .find(|(name, _)| name == "user-agent") + .map(|(_, value)| value.clone()) + .unwrap_or_default(); + FakeClient::ok(&serde_json::json!({ "device": { "useragent": user_agent } }).to_string()) + } +} + +#[test] +fn evidence_that_differs_never_shares_a_cache_entry() { + // The web integration decodes the query string, so a query value can hold + // a line break followed by text shaped like another evidence field. That + // request must not be given the cache key of a visitor whose evidence + // really holds both values, or one is served the answer made for the other. + let client = Arc::new(EchoClient::default()); + let state = CloudEngineState { + evidence_keys: [ + "header.user-agent", + "query.user-agent", + "header.x-device-user-agent", + ] + .into_iter() + .map(|key| EvidenceKeyEntry { + key: key.to_owned(), + order: 0, + }) + .collect(), + accessible_properties: LicensedProducts::default(), + }; + let engine = CloudRequestEngine::builder() + .resource_key("rk") + .http_client(client.clone()) + .set_state(state) + .cache(Arc::new(RecordingCache::default())) + .build() + .unwrap(); + let pipeline = Pipeline::builder() + .add_element(Arc::new(engine)) + .build() + .unwrap(); + + let mut crafted = pipeline.create_flow_data_with( + Evidence::builder() + .add("query.user-agent", "Mozilla/5.0\nx-device-user-agent=Other") + .build(), + ); + crafted.process().unwrap(); + + let mut visitor = pipeline.create_flow_data_with( + Evidence::builder() + .add("header.user-agent", "Mozilla/5.0") + .add("header.x-device-user-agent", "Other") + .build(), + ); + visitor.process().unwrap(); + + let cloud = visitor.get(CloudRequestEngine::DATA_KEY).unwrap(); + assert_eq!( + cloud.json_response(), + Some(r#"{"device":{"useragent":"Mozilla/5.0"}}"#), + "the visitor was served the answer made for the other request" + ); + assert!( + !cloud.cache_hit(), + "the visitor's request is not a cache hit" + ); + assert_eq!( + *client.data_requests.lock().unwrap(), + 2, + "each request reached the cloud" + ); +} diff --git a/common-sys/src/lib.rs b/common-sys/src/lib.rs index 297e972..548b2e5 100644 --- a/common-sys/src/lib.rs +++ b/common-sys/src/lib.rs @@ -41,8 +41,8 @@ //! - **Evidence** ([`EvidenceKeyValuePairArray`], create, free, add and iterate, //! plus the [`EvidencePrefix`] enum and prefix mapping). //! - **Properties** ([`PropertiesRequired`]). -//! - **Memory** ([`fiftyoneDegreesMemoryStandardFree`]) so that strings -//! allocated by the library can be released through the matching allocator. +//! - **Memory** ([`fiftyoneDegreesFree`]) so that strings allocated by the +//! library can be released through the matching allocator. //! //! Internals such as collections, data sets and headers are exposed as opaque //! pointer types. Only the structures that actually cross the boundary in this @@ -407,7 +407,7 @@ extern "C" { // -- status.h -- /// Returns a newly allocated English message for the status code. The caller - /// must free the returned pointer with [`fiftyoneDegreesMemoryStandardFree`]. + /// must free the returned pointer with [`fiftyoneDegreesFree`]. /// /// # Safety /// `file_name` must be a valid null terminated string or null. @@ -419,8 +419,7 @@ extern "C" { // -- exceptions.h -- /// Returns a newly allocated English message describing the exception. The - /// caller must free the returned pointer with - /// [`fiftyoneDegreesMemoryStandardFree`]. + /// caller must free the returned pointer with [`fiftyoneDegreesFree`]. /// /// # Safety /// `exception` must point to a valid, set [`Exception`]. @@ -544,17 +543,25 @@ extern "C" { // -- memory.h -- - /// Frees memory allocated by the library through the standard allocator. + /// Frees memory through the C runtime's `free`, which is how the library + /// frees memory unless its allocator has been replaced. Release what the + /// library allocates with [`fiftyoneDegreesFree`] instead, which follows a + /// replaced allocator. /// - /// Used to release strings returned by + /// # Safety + /// `ptr` must be a pointer the C runtime's `malloc` allocated, or null. + pub fn fiftyoneDegreesMemoryStandardFree(ptr: *mut c_void); + + /// The function the library frees memory with, the counterpart of the + /// allocator it allocates with. Release the strings /// [`fiftyoneDegreesStatusGetMessage`] and - /// [`fiftyoneDegreesExceptionGetMessage`], which the default library - /// allocator produces. + /// [`fiftyoneDegreesExceptionGetMessage`] return with it. It is + /// [`fiftyoneDegreesMemoryStandardFree`] unless the allocator has been + /// replaced, as the library's memory tracking replaces it. /// /// # Safety - /// `ptr` must be a pointer the library allocated through its standard - /// allocator, or null. - pub fn fiftyoneDegreesMemoryStandardFree(ptr: *mut c_void); + /// Call it only with a pointer the library allocated, or null. + pub static mut fiftyoneDegreesFree: unsafe extern "C" fn(ptr: *mut c_void); // -- threading.h -- @@ -578,7 +585,7 @@ mod tests { // The library produces a human readable message for the code. assert!(!message.is_empty(), "status message should not be empty"); // Free through the matching allocator. - fiftyoneDegreesMemoryStandardFree(raw as *mut std::os::raw::c_void); + fiftyoneDegreesFree(raw as *mut std::os::raw::c_void); } } @@ -603,7 +610,7 @@ mod tests { assert!(!raw.is_null(), "exception message should be allocated"); let message = CStr::from_ptr(raw).to_string_lossy().into_owned(); assert!(!message.is_empty(), "exception message should not be empty"); - fiftyoneDegreesMemoryStandardFree(raw as *mut std::os::raw::c_void); + fiftyoneDegreesFree(raw as *mut std::os::raw::c_void); } } diff --git a/device-detection-onpremise/src/builder.rs b/device-detection-onpremise/src/builder.rs index 721957f..49fdea9 100644 --- a/device-detection-onpremise/src/builder.rs +++ b/device-detection-onpremise/src/builder.rs @@ -38,6 +38,7 @@ use fiftyone_pipeline_engines::{ }; use crate::engine::{open_manager, DeviceDetectionOnPremiseEngine}; +use crate::operational_copy::OperationalCopy; /// The download type the Distributor serves Hash data files as, sent in the /// update URL's `Type` parameter. @@ -72,6 +73,7 @@ pub struct DeviceDetectionOnPremiseEngineBuilder { data_source_tier: Option, auto_update: Option, concurrency: Option, + temp_dir: Option, } /// The optional automatic-update settings captured by the builder. @@ -95,6 +97,7 @@ impl DeviceDetectionOnPremiseEngineBuilder { data_source_tier: None, auto_update: None, concurrency: None, + temp_dir: None, } } @@ -119,6 +122,15 @@ impl DeviceDetectionOnPremiseEngineBuilder { self } + /// Set the folder for the engine's temporary files. With automatic updates + /// on, every profile but [`PerformanceProfile::InMemory`] reads a copy of + /// the data file made there, so an update can replace the data file itself. + /// Defaults to the system's temporary folder, [`std::env::temp_dir`]. + pub fn temp_dir_path(mut self, path: impl Into) -> Self { + self.temp_dir = Some(path.into()); + self + } + /// Restrict the engine to a single named property, adding to any already /// requested. With no restriction the engine loads every property the data /// file supports. @@ -201,7 +213,9 @@ impl DeviceDetectionOnPremiseEngineBuilder { /// no licence key, as .NET does. When automatic updates /// were configured with a service, the engine's data file is registered /// with that service before the engine is returned, so a startup poll (if - /// requested) and the polling schedule are in effect. + /// requested) and the polling schedule are in effect. A startup poll that + /// fails leaves the engine on the data file it loaded, as the + /// specification says, and the service logs the failure. pub fn build(self) -> Result> { // The data-file configuration the engine carries. When auto-update was // requested, use that configuration; otherwise build a static one with @@ -219,22 +233,35 @@ impl DeviceDetectionOnPremiseEngineBuilder { }; data_file_config.check_license_keys()?; - // Open the data set with the requested property restriction. - let manager = open_manager( + // Open the data set with the requested property restriction, from a + // copy of the data file when updates can replace the file while it is + // open. + let copy = OperationalCopy::when_needed( &self.data_file_path, + self.temp_dir.as_deref(), + self.profile, + &data_file_config, + ); + let manager = open_manager( + copy.as_ref() + .map_or(self.data_file_path.as_path(), OperationalCopy::path), self.profile, &self.requested_properties, self.concurrency, )?; - let engine = Arc::new(DeviceDetectionOnPremiseEngine::from_manager( - manager, - self.profile, - self.requested_properties, - data_file_config, - self.data_source_tier, - self.concurrency, - )); + let engine = Arc::new( + DeviceDetectionOnPremiseEngine::from_manager( + manager, + copy, + self.profile, + self.requested_properties, + data_file_config, + self.data_source_tier, + self.concurrency, + ) + .with_temp_dir(self.temp_dir), + ); // Register the data file with the update service if one was supplied. if let Some(service) = service { @@ -244,9 +271,9 @@ impl DeviceDetectionOnPremiseEngineBuilder { .cloned() .ok_or_else(|| Error::configuration("engine has no data file to register"))?; let as_engine: Arc = engine.clone(); - service - .register(as_engine, data_file) - .map_err(|e| Error::configuration(e.to_string()))?; + // A failed update on start-up leaves the engine on the data file it + // loaded, as the specification says, and the service logs it. + let _ = service.register(as_engine, data_file); } Ok(engine) diff --git a/device-detection-onpremise/src/engine.rs b/device-detection-onpremise/src/engine.rs index 56329da..c0cea15 100644 --- a/device-detection-onpremise/src/engine.rs +++ b/device-detection-onpremise/src/engine.rs @@ -30,7 +30,7 @@ use std::collections::HashMap; use std::ffi::CStr; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, PoisonError}; use arc_swap::ArcSwap; @@ -51,6 +51,9 @@ use fiftyone_pipeline_engines::{ EngineDeployment, OnPremiseAspectEngine, }; +use crate::operational_copy::OperationalCopy; +use crate::retained::Retained; + /// The separator used to join multiple values of a list-typed property into a /// single string when reading from the native results. The other ports use a /// vertical bar between values, which the shared device-data list accessor then @@ -143,10 +146,11 @@ const METRIC_PROPERTIES: &[(&str, PropertyValueType, &str)] = &[ /// never caches results. Only the cloud path, whose data is plain owned values, /// may use a results cache. pub struct DeviceDetectionOnPremiseEngine { - /// The loaded Hash data set. Swapped atomically on refresh. In-flight + /// The loaded Hash data set and the properties a detection reads from it, + /// swapped together atomically on refresh. In-flight /// [`fiftyone_native::dd::Results`] hold their own [`Arc`] to the manager /// they were created from, so a swap does not invalidate them. - manager: ArcSwap, + loaded: ArcSwap, /// The performance profile used to (re)load the data file, retained so a /// refresh reloads with the same profile. @@ -156,55 +160,101 @@ pub struct DeviceDetectionOnPremiseEngine { /// properties. Used when reloading the data set on refresh. requested_properties: Vec, - /// Core property metadata for [`FlowElement::properties`]. - properties: Vec, + /// The property metadata and tier of the loaded data set, for + /// [`FlowElement::properties`], [`AspectEngine::aspect_properties`] and + /// [`AspectEngine::data_source_tier`]. + metadata: Retained, - /// Aspect property metadata for [`AspectEngine::aspect_properties`]. - aspect_properties: Vec, + /// The tier the builder set, which every data set loaded reports, or + /// `None` to report the tier each data file names. + tier_override: Option, - /// The data file properties a detection reads on request, shared by every - /// detection's reader. - readable: Arc, + /// Held while a refresh loads a data set, so refreshes run one at a time. + refresh_lock: Mutex<()>, /// The evidence keys this engine reads, replaced on refresh. evidence_key_filter: HashEvidenceKeyFilter, - /// The data-source tier, for example `Lite`. - data_source_tier: String, - /// The expected concurrency the data set's handle pool was sized for, or /// `None` for the profile default. Carried so a reload re-opens the data set /// with the same pool size. concurrency: Option, + /// The folder a refresh copies the data file into, or `None` for the + /// system's temporary folder. + temp_dir: Option, + /// The single data file's run-time state, shared with the update service. data_files: Vec>, } +/// A loaded Hash data set, the properties a detection reads from it and the +/// copy of the data file it was read from, if any. +struct LoadedDataSet { + manager: Arc, + readable: Arc, + /// Held only to be dropped with the data set, and declared after the + /// manager, so it is removed after the data set that has it open is freed. + _copy: Option, +} + +/// The property metadata and tier of a loaded data set. +#[derive(PartialEq)] +struct EngineMetadata { + properties: Vec, + aspect_properties: Vec, + tier: String, +} + +/// The data set `manager` holds, read from `copy` when there is one, and its +/// metadata, reporting `tier_override` when it is set, otherwise the tier the +/// data file names. +fn describe( + manager: Arc, + copy: Option, + tier_override: &Option, +) -> (LoadedDataSet, EngineMetadata) { + // An explicit builder override wins, then the tier read from the native + // data file header (Lite, Enterprise, TAC and so on), then the safe Lite + // default when the native name is absent. + let tier = tier_override + .clone() + .or_else(|| manager.data_set_name()) + .unwrap_or_else(|| DEFAULT_DATA_SOURCE_TIER.to_owned()); + let (properties, aspect_properties) = build_property_metadata(&manager, &Some(tier.clone())); + let readable = Arc::new(ReadableProperties::new(&aspect_properties)); + ( + LoadedDataSet { + manager, + readable, + _copy: copy, + }, + EngineMetadata { + properties, + aspect_properties, + tier, + }, + ) +} + impl DeviceDetectionOnPremiseEngine { /// Build an engine from an already-loaded manager and its configuration. /// - /// Called by the builder once it has opened the data file. Derives the - /// property and evidence-key metadata from the data set, records the - /// data-file publish time from the file on disk, and assembles the engine. + /// Called by the builder once it has opened the data file, or `copy` of + /// it. Derives the property and evidence-key metadata from the data set, + /// records the data-file publish time from the file on disk, and assembles + /// the engine. pub(crate) fn from_manager( manager: Arc, + copy: Option, profile: PerformanceProfile, requested_properties: Vec, data_file_config: DataFileConfiguration, data_source_tier: Option, concurrency: Option, ) -> Self { - // Resolve the data-source tier: an explicit builder override wins, then - // the tier read from the native data file header (Lite, Enterprise, TAC - // and so on), then the safe Lite default when the native name is absent. - let data_source_tier = data_source_tier - .or_else(|| manager.data_set_name()) - .unwrap_or_else(|| DEFAULT_DATA_SOURCE_TIER.to_owned()); - let (properties, aspect_properties) = - build_property_metadata(&manager, &Some(data_source_tier.clone())); - let readable = Arc::new(ReadableProperties::new(&aspect_properties)); let evidence_key_filter = HashEvidenceKeyFilter::new(named_evidence_keys(&manager)); + let (loaded, metadata) = describe(manager, copy, &data_source_tier); let data_file = Arc::new(AspectEngineDataFile::new(data_file_config)); // Record the publish time from the data file on disk, the best signal @@ -215,19 +265,26 @@ impl DeviceDetectionOnPremiseEngine { } DeviceDetectionOnPremiseEngine { - manager: ArcSwap::from(manager), + loaded: ArcSwap::from_pointee(loaded), profile, requested_properties, - properties, - aspect_properties, - readable, + metadata: Retained::new(metadata), + tier_override: data_source_tier, + refresh_lock: Mutex::new(()), evidence_key_filter, - data_source_tier, concurrency, + temp_dir: None, data_files: vec![data_file], } } + /// Copy the data file into `temp_dir` on refresh, the folder the builder + /// copied it into, or the system's temporary folder when `None`. + pub(crate) fn with_temp_dir(mut self, temp_dir: Option) -> Self { + self.temp_dir = temp_dir; + self + } + /// The performance profile the engine loads its data file with. pub fn performance_profile(&self) -> PerformanceProfile { self.profile @@ -238,13 +295,15 @@ impl DeviceDetectionOnPremiseEngine { /// the high-entropy values keys. The engine also accepts any `query.51d_` /// or `cookie.51d_` value, which the client-side JavaScript sends back. pub fn evidence_keys(&self) -> Vec { - named_evidence_keys(&self.manager.load()) + named_evidence_keys(&self.loaded.load().manager) } /// The names of the available (required) properties in the loaded data set, /// including the match-metric pseudo-properties. pub fn available_properties(&self) -> Vec { - self.aspect_properties + self.metadata + .current() + .aspect_properties .iter() .map(|p| p.name().to_owned()) .collect() @@ -260,8 +319,8 @@ impl DeviceDetectionOnPremiseEngine { // Snapshot the current data set. The snapshot is an `Arc`, so an // in-flight detection keeps the data set alive even if `refresh` swaps a // new one in mid-flight. - let manager = self.manager.load_full(); - let mut results = manager.create_results()?; + let loaded = self.loaded.load_full(); + let mut results = loaded.manager.create_results()?; // Marshal the evidence the engine understands, preferring decoded UACH // values, into one merged evidence set, then process it. @@ -308,9 +367,8 @@ impl DeviceDetectionOnPremiseEngine { ); Ok(device.with_reader(DetectionReader { - manager, results: Mutex::new(results), - readable: Arc::clone(&self.readable), + loaded, })) } @@ -356,18 +414,35 @@ impl DeviceDetectionOnPremiseEngine { /// Reload the data set from the data file on disk and atomically swap it in. /// /// Shared by [`OnPremiseAspectEngine::refresh`] and the builder's reload - /// path. A successful reload replaces the live manager; in-flight detections - /// keep using the manager they snapshotted. + /// path. A successful reload replaces the live data set, the properties a + /// detection reads, the evidence keys and the metadata, because a new data + /// file can carry different properties or a different tier. In-flight + /// detections keep using the data set they snapshotted. A failed reload + /// changes nothing. fn reload_from_path(&self, path: &Path) -> Result<()> { + let _refreshing = self + .refresh_lock + .lock() + .unwrap_or_else(PoisonError::into_inner); + let copy = self.data_files.first().and_then(|file| { + OperationalCopy::when_needed( + path, + self.temp_dir.as_deref(), + self.profile, + file.configuration(), + ) + }); let manager = open_manager( - path, + copy.as_ref().map_or(path, OperationalCopy::path), self.profile, &self.requested_properties, self.concurrency, )?; self.evidence_key_filter .replace(named_evidence_keys(&manager)); - self.manager.store(manager); + let (loaded, metadata) = describe(manager, copy, &self.tier_override); + self.loaded.store(Arc::new(loaded)); + self.metadata.replace(metadata); // Refresh the recorded publish time from the new file. if let Some(file) = self.data_files.first() { @@ -398,13 +473,13 @@ impl FlowElement for DeviceDetectionOnPremiseEngine { } fn properties(&self) -> &[PropertyMetaData] { - &self.properties + &self.metadata.current().properties } } impl AspectEngine for DeviceDetectionOnPremiseEngine { fn data_source_tier(&self) -> &str { - &self.data_source_tier + &self.metadata.current().tier } fn deployment(&self) -> EngineDeployment { @@ -412,7 +487,7 @@ impl AspectEngine for DeviceDetectionOnPremiseEngine { } fn aspect_properties(&self) -> &[AspectPropertyMetaData] { - &self.aspect_properties + &self.metadata.current().aspect_properties } } @@ -513,27 +588,27 @@ impl ReadableProperties { /// Reads the properties of one detection from its native results when they are /// requested. struct DetectionReader { - /// The data set snapshot the results came from, which the property indexes - /// belong to. - manager: Arc, /// The native results. Reading a value writes into them, so each read - /// holds the lock. + /// holds the lock. Declared first, so they are freed before the data set + /// they came from. results: Mutex, - readable: Arc, + /// The data set snapshot the results came from, which the property indexes + /// belong to. + loaded: Arc, } impl DevicePropertyReader for DetectionReader { fn property_names(&self) -> &[String] { - &self.readable.lowercase_names + &self.loaded.readable.lowercase_names } fn position(&self, name: &str) -> Option { - self.readable.by_name.get(name).copied() + self.loaded.readable.by_name.get(name).copied() } fn read_property(&self, position: usize) -> std::result::Result, String> { - let (name, value_type) = &self.readable.properties[position]; - let Some(index) = self.manager.required_property_index(name) else { + let (name, value_type) = &self.loaded.readable.properties[position]; + let Some(index) = self.loaded.manager.required_property_index(name) else { return Ok(None); }; let results = self.results.lock().unwrap_or_else(PoisonError::into_inner); diff --git a/device-detection-onpremise/src/lib.rs b/device-detection-onpremise/src/lib.rs index 6af7e57..2a8dc9d 100644 --- a/device-detection-onpremise/src/lib.rs +++ b/device-detection-onpremise/src/lib.rs @@ -105,6 +105,8 @@ mod builder; mod engine; +mod operational_copy; +mod retained; pub use builder::{DeviceDetectionOnPremiseEngineBuilder, DATA_DOWNLOAD_TYPE}; pub use engine::DeviceDetectionOnPremiseEngine; diff --git a/device-detection-onpremise/src/operational_copy.rs b/device-detection-onpremise/src/operational_copy.rs new file mode 100644 index 0000000..fe3b7cd --- /dev/null +++ b/device-detection-onpremise/src/operational_copy.rs @@ -0,0 +1,96 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! The copy of a data file an engine reads while updates can replace the file. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_engines::DataFileConfiguration; + +/// A copy of a data file that a data set is read from, removed when dropped. +/// +/// Every profile but `InMemory` keeps the data file open while the data set +/// lives, and Windows refuses to replace a file that is open. With automatic +/// updates on, the engine therefore reads a copy in its temporary folder, +/// which the specification calls the operational data copy, and an update can +/// replace the data file itself. Drop the copy after the data set read from +/// it, which closes it. +pub(crate) struct OperationalCopy { + path: PathBuf, +} + +impl OperationalCopy { + /// A copy of `data_file` in `temp_dir`, or in the system's temporary + /// folder when that is `None`, when `profile` keeps the data file open and + /// automatic updates are on, otherwise `None`. + /// + /// When the copy cannot be made the engine reads the data file itself, so + /// a full temporary folder does not stop it from building, and on Windows + /// an update then cannot replace the file while it is open. + pub(crate) fn when_needed( + data_file: &Path, + temp_dir: Option<&Path>, + profile: PerformanceProfile, + config: &DataFileConfiguration, + ) -> Option { + if profile == PerformanceProfile::InMemory || !config.automatic_updates_enabled { + return None; + } + let temp_dir = temp_dir.map_or_else(std::env::temp_dir, Path::to_path_buf); + OperationalCopy::of(data_file, &temp_dir).ok() + } + + /// Copy `data_file` into `temp_dir`, under a name no other copy in this + /// process has. + fn of(data_file: &Path, temp_dir: &Path) -> std::io::Result { + static NEXT: AtomicU64 = AtomicU64::new(0); + let mut name = data_file + .file_name() + .map(OsString::from) + .unwrap_or_default(); + name.push(format!( + ".{}.{}.operational", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + // Made before the copy, so a copy that fails part way is removed. + let copy = OperationalCopy { + path: temp_dir.join(name), + }; + std::fs::copy(data_file, ©.path)?; + Ok(copy) + } + + /// Where the copy is. + pub(crate) fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for OperationalCopy { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} diff --git a/device-detection-onpremise/src/retained.rs b/device-detection-onpremise/src/retained.rs new file mode 100644 index 0000000..17e5064 --- /dev/null +++ b/device-detection-onpremise/src/retained.rs @@ -0,0 +1,87 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Values a refresh replaces that the engine lends by reference. + +use std::sync::OnceLock; + +/// Every distinct value kept so far, oldest first, of which the newest is the +/// current one. +/// +/// The pipeline traits lend an engine's property lists and tier by reference +/// for as long as the engine lives, so a value a refresh replaces cannot be +/// freed while the engine lives. Each is kept instead. A refresh adds a value +/// only when it differs from the newest, which happens when a data file with +/// a different property set or tier is loaded. +pub(crate) struct Retained { + value: T, + next: OnceLock>>, +} + +impl Retained { + pub(crate) fn new(value: T) -> Self { + Retained { + value, + next: OnceLock::new(), + } + } + + fn newest_link(&self) -> &Retained { + let mut link = self; + while let Some(next) = link.next.get() { + link = next; + } + link + } + + /// The newest value. + pub(crate) fn current(&self) -> &T { + &self.newest_link().value + } + + /// Make `value` the current one, unless it equals the current one. The + /// caller serialises calls, so two never add at once. + pub(crate) fn replace(&self, value: T) { + let newest = self.newest_link(); + if newest.value != value { + let _ = newest.next.set(Box::new(Retained::new(value))); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_newest_distinct_value_is_current_and_older_ones_stay_lent() { + let retained = Retained::new(String::from("Lite")); + let lent = retained.current(); + retained.replace(String::from("Lite")); + assert!(retained.next.get().is_none(), "an equal value is not kept"); + retained.replace(String::from("Enterprise")); + assert_eq!(retained.current(), "Enterprise"); + assert_eq!(lent, "Lite"); + retained.replace(String::from("Lite")); + assert_eq!(retained.current(), "Lite"); + } +} diff --git a/device-detection-onpremise/tests/operational_copy.rs b/device-detection-onpremise/tests/operational_copy.rs new file mode 100644 index 0000000..e5ff92c --- /dev/null +++ b/device-detection-onpremise/tests/operational_copy.rs @@ -0,0 +1,254 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A data update replaces the data file of a running engine under every +//! profile, and the copy the engine reads leaves with it. The tests need the +//! Lite file `51DEGREES_DD_PATH` names, and skip with a note without it. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use fiftyone_device_detection_onpremise::{ + DeviceDetectionOnPremiseEngine, DeviceDetectionOnPremiseEngineBuilder, +}; +use fiftyone_device_detection_shared::{DeviceData, DEVICE_DATA_KEY}; +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_core::{Evidence, FlowData, FlowElement, Pipeline}; +use fiftyone_pipeline_engines::{ + AutoUpdateStatus, DataFileConfiguration, DataUpdateService, OnPremiseAspectEngine, +}; + +/// A mobile Safari on iPhone User-Agent. +const MOBILE_USER_AGENT: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"; + +fn lite_data_file() -> Option { + std::env::var("51DEGREES_DD_PATH") + .ok() + .map(PathBuf::from) + .filter(|path| path.is_file()) +} + +/// A new, empty folder holding a copy of `lite` named `name`. +fn folder_with(lite: &Path, test: &str, name: &str) -> (PathBuf, PathBuf) { + let dir = std::env::temp_dir().join(format!("fiftyone-dd-copy-{test}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(name); + std::fs::copy(lite, &path).unwrap(); + (dir, path) +} + +fn files_in(dir: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + +/// Serve `body` to one request, and return the URL. +fn serve_once(body: Vec) -> (String, std::thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/datafile", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = stream.read(&mut request); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + }); + (url, server) +} + +/// The data file configuration of automatic updates from `url`, which the +/// test polls itself. +fn updating_config(path: &Path, url: String) -> DataFileConfiguration { + DataFileConfiguration::builder(path) + .data_update_url(url) + .automatic_updates_enabled(true) + .polling_interval_seconds(24 * 60 * 60) + .max_randomisation_seconds(0) + .file_system_watcher_enabled(false) + .decompress_content(false) + .verify_md5(false) + .license_key_required_for_updates(false) + .build() +} + +/// An engine with automatic updates on from `url`, which the test polls +/// itself. +fn updating_engine( + path: &Path, + profile: PerformanceProfile, + url: String, + service: &Arc, +) -> Arc { + DeviceDetectionOnPremiseEngineBuilder::new(path) + .performance_profile(profile) + .auto_update(updating_config(path, url), Some(Arc::clone(service))) + .build() + .expect("the data file loads") +} + +fn detect(engine: &Arc) -> FlowData { + let element: Arc = engine.clone(); + let pipeline = Pipeline::builder().add_element(element).build().unwrap(); + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("header.user-agent", MOBILE_USER_AGENT) + .build(), + ); + data.process().unwrap(); + data +} + +fn is_mobile(data: &FlowData) -> Option { + data.get(DEVICE_DATA_KEY) + .unwrap() + .is_mobile() + .value() + .ok() + .copied() +} + +/// With automatic updates on, an update replaces the data file of a running +/// engine under every profile, including the ones that read the data file +/// while the data set lives, and the engine moves to the new file. +#[test] +fn an_update_replaces_the_data_file_under_every_profile() { + let Some(lite) = lite_data_file() else { + eprintln!("skipping operational copies: set 51DEGREES_DD_PATH to the Lite file"); + return; + }; + for profile in [ + PerformanceProfile::Default, + PerformanceProfile::Balanced, + PerformanceProfile::LowMemory, + PerformanceProfile::HighPerformance, + PerformanceProfile::InMemory, + ] { + let (dir, path) = folder_with(&lite, profile.as_str(), "51Degrees.hash"); + let (url, server) = serve_once(std::fs::read(&lite).unwrap()); + let service = Arc::new(DataUpdateService::new()); + let engine = updating_engine(&path, profile, url, &service); + let as_engine: Arc = engine.clone(); + + let status = service.check_for_update(&as_engine, None); + server.join().unwrap(); + + assert_eq!( + status.map_err(|error| error.to_string()), + Ok(AutoUpdateStatus::Success), + "{profile:?}" + ); + assert_eq!(is_mobile(&detect(&engine)), Some(true), "{profile:?}"); + drop(as_engine); + drop(engine); + drop(service); + assert_eq!(files_in(&dir), ["51Degrees.hash"], "{profile:?}"); + let _ = std::fs::remove_dir_all(&dir); + } +} + +/// The engine reads one copy at a time, and a copy goes once nothing reads +/// the data set it holds, even when a detection outlives a refresh. None is +/// left after the engine is dropped. +#[test] +fn each_copy_goes_with_the_data_set_read_from_it() { + let Some(lite) = lite_data_file() else { + eprintln!("skipping operational copies: set 51DEGREES_DD_PATH to the Lite file"); + return; + }; + let name = format!("copied-{}.hash", std::process::id()); + let (dir, path) = folder_with(&lite, "copies", &name); + let copies = || { + files_in(&std::env::temp_dir()) + .into_iter() + .filter(|file| file.starts_with(&name) && file.ends_with(".operational")) + .count() + }; + let service = Arc::new(DataUpdateService::new()); + let engine = updating_engine( + &path, + PerformanceProfile::Default, + String::from("http://127.0.0.1:9/unused"), + &service, + ); + assert_eq!(copies(), 1, "one copy for the data set loaded"); + + let early = detect(&engine); + for _ in 0..3 { + engine.refresh(None).expect("the data file reloads"); + } + assert_eq!( + copies(), + 2, + "the copy the early detection reads, and the newest" + ); + assert_eq!(is_mobile(&early), Some(true)); + drop(early); + assert_eq!(copies(), 1, "the early detection's copy goes with it"); + + drop(engine); + drop(service); + assert_eq!(copies(), 0, "no copy outlives the engine"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// The copy is made in the folder the builder names, on build and on each +/// refresh, and none is left there once the engine is dropped. +#[test] +fn the_copy_goes_in_the_folder_the_builder_names() { + let Some(lite) = lite_data_file() else { + eprintln!("skipping operational copies: set 51DEGREES_DD_PATH to the Lite file"); + return; + }; + let (dir, path) = folder_with(&lite, "named", "51Degrees.hash"); + let temp = dir.join("temp"); + std::fs::create_dir_all(&temp).unwrap(); + let copies = || files_in(&temp).len(); + + let engine = DeviceDetectionOnPremiseEngineBuilder::new(&path) + .temp_dir_path(&temp) + .auto_update( + updating_config(&path, String::from("http://127.0.0.1:9/unused")), + None, + ) + .build() + .expect("the data file loads"); + assert_eq!(copies(), 1, "the copy is in the named folder"); + + engine.refresh(None).expect("the data file reloads"); + assert_eq!(copies(), 1, "the refresh copies into the named folder too"); + + drop(engine); + assert_eq!(copies(), 0, "no copy outlives the engine"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/device-detection-onpremise/tests/refresh.rs b/device-detection-onpremise/tests/refresh.rs new file mode 100644 index 0000000..c7fea76 --- /dev/null +++ b/device-detection-onpremise/tests/refresh.rs @@ -0,0 +1,127 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A refresh to a data file with other properties or another tier leaves the +//! engine as an engine built on that file would be. +//! +//! The test needs the Lite file `51DEGREES_DD_PATH` names and a data file +//! with more properties beside it, `Enterprise-HashV41.hash` or +//! `TAC-HashV41.hash`, and skips with a note when either is missing. + +use std::path::PathBuf; +use std::sync::Arc; + +use fiftyone_device_detection_onpremise::{ + DeviceDetectionOnPremiseEngine, DeviceDetectionOnPremiseEngineBuilder, +}; +use fiftyone_device_detection_shared::DEVICE_DATA_KEY; +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_core::{ElementData, Evidence, FlowElement, Pipeline}; +use fiftyone_pipeline_engines::{AspectEngine, OnPremiseAspectEngine}; + +/// A mobile Safari on iPhone User-Agent. +const MOBILE_USER_AGENT: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"; + +/// A property the Lite file does not carry and the richer files do. +const RICHER_PROPERTY: &str = "HardwareModel"; + +/// The Lite file and a file with more properties beside it. +fn data_files() -> Option<(PathBuf, PathBuf)> { + let lite = PathBuf::from(std::env::var("51DEGREES_DD_PATH").ok()?); + let folder = lite.parent()?; + let richer = ["Enterprise-HashV41.hash", "TAC-HashV41.hash"] + .into_iter() + .map(|name| folder.join(name)) + .find(|path| path.is_file())?; + lite.is_file().then_some((lite, richer)) +} + +/// An engine that holds the whole data set in memory, so its data file can +/// be replaced on every platform. +fn engine(path: &PathBuf) -> Arc { + DeviceDetectionOnPremiseEngineBuilder::new(path) + .performance_profile(PerformanceProfile::InMemory) + .build() + .expect("the data file loads") +} + +/// The value of [`RICHER_PROPERTY`] for a mobile User-Agent, or the reason +/// there is none. +fn richer_value(engine: &Arc) -> Result { + let element: Arc = engine.clone(); + let pipeline = Pipeline::builder().add_element(element).build().unwrap(); + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("header.user-agent", MOBILE_USER_AGENT) + .build(), + ); + data.process().unwrap(); + let device = data.get(DEVICE_DATA_KEY).unwrap(); + device + .get(RICHER_PROPERTY) + .map(|value| format!("{value:?}")) + .map_err(|no_value| no_value.message) +} + +#[test] +fn a_refresh_reads_and_reports_the_properties_of_the_new_data_file() { + let Some((lite, richer)) = data_files() else { + eprintln!( + "skipping the refresh test: set 51DEGREES_DD_PATH to the Lite file, with \ + Enterprise-HashV41.hash or TAC-HashV41.hash beside it" + ); + return; + }; + let dir = std::env::temp_dir().join(format!("fiftyone-dd-refresh-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("51Degrees.hash"); + std::fs::copy(&lite, &path).unwrap(); + + let refreshed = engine(&path); + assert!( + richer_value(&refreshed).is_err(), + "the Lite file carries {RICHER_PROPERTY}" + ); + std::fs::copy(&richer, &path).unwrap(); + refreshed.refresh(None).expect("the richer file loads"); + let built = engine(&path); + + assert_eq!(refreshed.data_source_tier(), built.data_source_tier()); + assert_eq!( + refreshed.available_properties(), + built.available_properties() + ); + assert_eq!(refreshed.properties(), built.properties()); + assert_eq!(refreshed.aspect_properties(), built.aspect_properties()); + let value = richer_value(&refreshed); + assert!( + value.is_ok(), + "{RICHER_PROPERTY} after the refresh: {value:?}" + ); + assert_eq!(value, richer_value(&built)); + + drop(refreshed); + drop(built); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/device-detection-onpremise/tests/update_on_startup.rs b/device-detection-onpremise/tests/update_on_startup.rs new file mode 100644 index 0000000..c6608fa --- /dev/null +++ b/device-detection-onpremise/tests/update_on_startup.rs @@ -0,0 +1,101 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Update on start-up when the update cannot be fetched. The test needs the +//! Lite file `51DEGREES_DD_PATH` names, and skips with a note without it. + +use std::net::TcpListener; +use std::path::PathBuf; +use std::sync::Arc; + +use fiftyone_device_detection_onpremise::DeviceDetectionOnPremiseEngineBuilder; +use fiftyone_device_detection_shared::{DeviceData, DEVICE_DATA_KEY}; +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_core::{Evidence, FlowElement, Pipeline}; +use fiftyone_pipeline_engines::{DataFileConfiguration, DataUpdateService}; + +/// A mobile Safari on iPhone User-Agent. +const MOBILE_USER_AGENT: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"; + +/// With update on start-up on, an engine builds from the data file it has when +/// the update cannot be fetched, as the specification says, and detects with +/// it. The data file stays registered for later updates. +#[test] +fn an_engine_builds_from_its_data_file_when_the_update_on_start_up_fails() { + let Some(lite) = std::env::var("51DEGREES_DD_PATH") + .ok() + .map(PathBuf::from) + .filter(|path| path.is_file()) + else { + eprintln!("skipping update on start-up: set 51DEGREES_DD_PATH to the Lite file"); + return; + }; + let dir = std::env::temp_dir().join(format!("fiftyone-dd-startup-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("51Degrees.hash"); + std::fs::copy(&lite, &path).unwrap(); + // Nothing listens on a port once its listener is dropped. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/download", listener.local_addr().unwrap()); + drop(listener); + + let service = Arc::new(DataUpdateService::new()); + let config = DataFileConfiguration::builder(&path) + .data_update_url(url) + .data_update_license_keys(["KEY"]) + .update_on_startup(true) + .automatic_updates_enabled(false) + .file_system_watcher_enabled(false) + .build(); + let engine = DeviceDetectionOnPremiseEngineBuilder::new(&path) + .performance_profile(PerformanceProfile::InMemory) + .auto_update(config, Some(Arc::clone(&service))) + .build() + .expect("the engine uses the data file it has"); + + assert_eq!(service.registered_count(), 1); + let element: Arc = engine.clone(); + let pipeline = Pipeline::builder().add_element(element).build().unwrap(); + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("header.user-agent", MOBILE_USER_AGENT) + .build(), + ); + data.process().unwrap(); + assert_eq!( + data.get(DEVICE_DATA_KEY) + .unwrap() + .is_mobile() + .value() + .ok() + .copied(), + Some(true) + ); + + drop(data); + drop(pipeline); + drop(engine); + drop(service); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/device-detection-shared/src/data.rs b/device-detection-shared/src/data.rs index a1d9ad6..99a9766 100644 --- a/device-detection-shared/src/data.rs +++ b/device-detection-shared/src/data.rs @@ -732,6 +732,8 @@ mod tests { ); } + // WebAssembly has no threads to spawn. + #[cfg(not(target_family = "wasm"))] #[test] fn reads_from_several_threads_agree_and_read_once() { let (reader, reads) = CountingReader::new(); diff --git a/device-detection-sys/src/lib.rs b/device-detection-sys/src/lib.rs index 8dba6c4..9c9e09c 100644 --- a/device-detection-sys/src/lib.rs +++ b/device-detection-sys/src/lib.rs @@ -90,7 +90,7 @@ use std::os::raw::{c_char, c_int, c_void}; // Re-export the shared FFI types so a caller building a detection has a single // import surface. These are the exact types the Hash entry points consume. pub use fiftyone_common_sys::{ - fiftyoneDegreesExceptionGetMessage, fiftyoneDegreesMemoryStandardFree, + fiftyoneDegreesExceptionGetMessage, fiftyoneDegreesFree, fiftyoneDegreesMemoryStandardFree, fiftyoneDegreesResourceManagerFree, fiftyoneDegreesStatusGetMessage, EvidenceKeyValuePair, EvidenceKeyValuePairArray, EvidencePrefix, Exception, PropertiesAvailable, PropertiesRequired, ResourceHandle, ResourceManager, StatusCode, @@ -669,6 +669,17 @@ extern "C" { buffer: *mut c_char, length: usize, ) -> usize; + + /// Returns the number of handles in the file pool of the data set + /// managed by `manager` when any of its collections reads from the data + /// file after it is loaded, or zero when the whole data set is held in + /// memory. A read that finds every handle in use fails rather than + /// waiting for one, so this is how many calls can read the data file at + /// once. See `src/shim.c`. + /// + /// # Safety + /// `manager` must be an initialized Hash manager. + pub fn fiftyoneDegreesShimHashGetFileHandleCount(manager: *mut ResourceManager) -> u16; } #[cfg(test)] diff --git a/device-detection-sys/src/shim.c b/device-detection-sys/src/shim.c index b23cb39..3c05eae 100644 --- a/device-detection-sys/src/shim.c +++ b/device-detection-sys/src/shim.c @@ -352,3 +352,33 @@ size_t fiftyoneDegreesShimHashGetMatchedUserAgent( length, results->items[resultIndex].b.matchedUserAgent); } + +/* + * Returns the number of handles in the file pool of the data set managed by + * `manager` when any of its collections reads from the data file after it is + * loaded, or zero when the whole data set is held in memory. A read that + * finds every handle in use fails rather than waiting for one, so this is + * how many calls can read the data file at once. + */ +uint16_t fiftyoneDegreesShimHashGetFileHandleCount( + fiftyoneDegreesResourceManager *manager) { + fiftyoneDegreesDataSetHash *dataSet = + fiftyoneDegreesDataSetHashGet(manager); + if (dataSet == NULL) { + return 0; + } + const fiftyoneDegreesConfigHash *config = &dataSet->config; + const bool readsFile = dataSet->b.b.isInMemory == false && ( + config->strings.loaded == false || + config->components.loaded == false || + config->maps.loaded == false || + config->properties.loaded == false || + config->values.loaded == false || + config->profiles.loaded == false || + config->rootNodes.loaded == false || + config->nodes.loaded == false || + config->profileOffsets.loaded == false); + const uint16_t count = readsFile ? dataSet->b.b.filePool.pool.count : 0; + fiftyoneDegreesDataSetHashRelease(dataSet); + return count; +} diff --git a/device-detection/src/lib.rs b/device-detection/src/lib.rs index 5a3fad4..7efd9f8 100644 --- a/device-detection/src/lib.rs +++ b/device-detection/src/lib.rs @@ -90,10 +90,11 @@ //! Distributor serves nothing without the licence keys set with //! `.data_update_license_keys(..)`, so a build with polling on and no key //! fails with an error that says so, as it does in .NET, and a deployment of -//! the free Lite file turns polling off with `.auto_update(false)`. -//! `.file_system_watcher(false)` turns the watcher off, and -//! `pipeline.resource::()` reaches the service for a check -//! on demand. +//! the free Lite file turns automatic updates off with `.auto_update(false)`, +//! which stops the polling and the file watcher together, as the +//! specification's master switch does. `.file_system_watcher(false)` turns +//! only the watcher off, and `pipeline.resource::()` +//! reaches the service for a check on demand. //! //! ## Examples //! @@ -314,14 +315,17 @@ impl DeviceDetectionOnPremisePipelineBuilder { /// with each new file. The 51Degrees Distributor serves nothing without a /// licence key, so [`Self::build`] refuses with this on and no key set, as /// .NET does, and a deployment of the Lite file turns this off or sets - /// [`Self::data_update_license_keys`]. + /// [`Self::data_update_license_keys`]. This is the master switch for + /// automatic updates, so with it off the file-system watcher does not run + /// either. pub fn auto_update(mut self, enabled: bool) -> Self { self.auto_update = enabled; self } /// Enable or disable the file-system watcher, which refreshes the engine - /// when the data file changes on disk. On by default. + /// when the data file changes on disk. On by default, and it runs only + /// while [`Self::auto_update`] is on. pub fn file_system_watcher(mut self, enabled: bool) -> Self { self.file_system_watcher = enabled; self diff --git a/examples/device-detection-examples/src/bin/dd-onprem-update-data-file.rs b/examples/device-detection-examples/src/bin/dd-onprem-update-data-file.rs index 21f7dc8..3e716aa 100644 --- a/examples/device-detection-examples/src/bin/dd-onprem-update-data-file.rs +++ b/examples/device-detection-examples/src/bin/dd-onprem-update-data-file.rs @@ -76,9 +76,11 @@ pub fn run(options: ExampleOptions) -> Result<()> { let config = DataFileConfiguration::builder(&options.data_file) // Off so `build` does not block; see the startup section in the docs. .update_on_startup(false) - // Watch the file on disk and reload when it is replaced. + // Watch the file on disk and reload when it is replaced, while + // automatic updates are on. .file_system_watcher_enabled(true) - // Poll the distributor for a newer file on an interval. + // Poll the distributor for a newer file on an interval. Automatic + // updates are the master switch for polling and the watcher alike. .automatic_updates_enabled(options.license_key.is_some()) // A short interval purely so the example does not appear idle; a real // deployment uses around 30 minutes. @@ -164,9 +166,11 @@ fn describe_update_mechanisms(license_key: Option<&str>) { an Enterprise license key and a writable file location." ); println!( - "2. File-system watcher. Set file_system_watcher_enabled(true). The \ - service watches the data file on disk and reloads the engine when the \ - file is replaced, which is how a manually-downloaded file is picked up." + "2. File-system watcher. Set file_system_watcher_enabled(true), with \ + automatic_updates_enabled(true), the master switch for automatic \ + updates. The service watches the data file on disk and reloads the \ + engine when the file is replaced, which is how a manually-downloaded \ + file is picked up." ); println!( "3. Remote-update polling. Set automatic_updates_enabled(true), a \ @@ -266,10 +270,10 @@ mod tests { * configuration, building the engine blocks while the service polls the * 51Degrees distributor once, so the engine starts with the most recent data. * This needs an Enterprise license key and a writable file location. - * - File-system watcher. With `file_system_watcher_enabled(true)`, the service - * watches the data file on disk and reloads the engine whenever the file is - * replaced. This is how a file you download and drop into place yourself is - * picked up automatically. + * - File-system watcher. With `file_system_watcher_enabled(true)` and + * automatic updates on, the service watches the data file on disk and + * reloads the engine whenever the file is replaced. This is how a file you + * download and drop into place yourself is picked up automatically. * - Remote-update polling. With `automatic_updates_enabled(true)`, a polling * interval (around 30 minutes in production) and a randomisation window (around * 10 minutes, to stagger many instances), the service periodically polls the diff --git a/fodid-cloud/src/engine.rs b/fodid-cloud/src/engine.rs index be4b375..6783454 100644 --- a/fodid-cloud/src/engine.rs +++ b/fodid-cloud/src/engine.rs @@ -73,6 +73,14 @@ use crate::dto::map_fodid_object; /// the `id.usage` usage-policy value that drive the lookup. Its /// [`FlowElement::evidence_key_filter`] is therefore an empty whitelist. /// +/// # Response cache +/// +/// The cloud creates the 51Did values afresh for each request that asks for +/// them, so the request engine never stores a response carrying them in its +/// response cache and never serves one from it. Each visitor receives the 51Did +/// created for their own request, and a request that carries no `id.usage` +/// evidence, which asks for no 51Did, is still served from the cache. +/// /// # Metadata /// /// The property metadata is derived lazily from the request engine's diff --git a/fodid-cloud/tests/cloud_fodid.rs b/fodid-cloud/tests/cloud_fodid.rs index 497963d..7ebf859 100644 --- a/fodid-cloud/tests/cloud_fodid.rs +++ b/fodid-cloud/tests/cloud_fodid.rs @@ -30,14 +30,15 @@ //! OWID envelope, created here with the `owid` library, so the parsed path is //! exercised against real bytes rather than a hand-rolled string. -use std::sync::Arc; +use std::sync::{Arc, Mutex}; -use fiftyone_pipeline_core::{ElementData, Evidence, Pipeline}; +use fiftyone_pipeline_core::{ElementData, Evidence, FlowData, Pipeline}; use fiftyone_cloud_request_engine::{ - CloudHttpClient, CloudHttpRequest, CloudHttpResponse, CloudRequestEngine, HttpMethod, + CloudHttpClient, CloudHttpRequest, CloudHttpResponse, CloudRequestEngine, HttpMethod, LruCache, }; use fiftyone_fodid_cloud::{FodIdCloudEngine, FodIdData, FODID_DATA_KEY}; +use fiftyone_pipeline_engines::AspectData; use fodid::{Creator, Crypto, IdType, Owid, Usage}; /// A fake transport that answers the cloud endpoints from in-memory fixtures. @@ -289,6 +290,196 @@ fn response_without_fodid_block_yields_empty_data() { assert!(!fodid.id_prob_lic().has_value()); } +/// The flags byte of a random 51Did created for non-marketing use. Bits 6 and +/// 7 select the random type and bit 0 grants non-marketing. +const RANDOM_FLAGS: u8 = 0b0100_0001; + +/// A fake transport that answers the data endpoint as the cloud does. +/// +/// The cloud creates a new 51Did for every request that states a usage, and +/// leaves the `fodid` member out of a response to one that does not, while the +/// device answer is the same for the same evidence. Each 51Did created here is +/// a random one whose GUID carries the number of the request it answered, and +/// each is recorded in the order it was created. +struct CreatingCloud { + private_pem: String, + created: Mutex>, + data_requests: Mutex, +} + +impl CreatingCloud { + fn new() -> Self { + let crypto = Crypto::new(); + CreatingCloud { + private_pem: crypto.private_key_pem().expect("export private key"), + created: Mutex::new(Vec::new()), + data_requests: Mutex::new(0), + } + } + + /// The 51Dids created so far, oldest first. + fn created(&self) -> Vec { + self.created.lock().unwrap().clone() + } + + /// The number of requests that reached the data endpoint. + fn data_requests(&self) -> usize { + *self.data_requests.lock().unwrap() + } + + /// Create a signed random 51Did whose GUID starts with `sequence`. + fn create(&self, sequence: u32) -> String { + let mut payload = vec![0u8; 21]; + payload[0] = RANDOM_FLAGS; + payload[1..5].copy_from_slice(&LICENSE_ID.to_le_bytes()); + payload[5..9].copy_from_slice(&sequence.to_le_bytes()); + let signer = Crypto::new_sign_only(&self.private_pem).expect("import private key"); + let creator = Creator::new(TEST_DOMAIN, signer).expect("create creator"); + let owid = creator + .create(payload) + .expect("create and sign the envelope"); + owid.as_base64().expect("encode owid") + } +} + +impl CloudHttpClient for CreatingCloud { + fn send(&self, request: &CloudHttpRequest) -> Result { + let body = if request.url.contains("accessibleproperties") { + ACCESSIBLE_PROPERTIES.to_owned() + } else if request.url.contains("evidencekeys") { + EVIDENCE_KEYS.to_owned() + } else { + let sequence = { + let mut count = self.data_requests.lock().unwrap(); + *count += 1; + *count as u32 + }; + let states_usage = request.form.iter().any(|(name, _)| name == "id.usage"); + if states_usage { + let created = self.create(sequence); + self.created.lock().unwrap().push(created.clone()); + format!( + r#"{{"device":{{"ismobile":false}},"fodid":{{"idrandglobal":"{created}"}},"javascriptProperties":[]}}"# + ) + } else { + r#"{"device":{"ismobile":false},"javascriptProperties":[]}"#.to_owned() + } + }; + Ok(CloudHttpResponse { + status: 200, + body, + retry_after: None, + }) + } +} + +/// A pipeline of a request engine with the in-process response cache and the +/// identifier engine, over the creating fake cloud. +fn caching_pipeline(cloud: Arc) -> Arc { + let request_engine = Arc::new( + CloudRequestEngine::builder() + .resource_key("test-resource-key") + .http_client(cloud) + .cache(Arc::new(LruCache::::with_defaults())) + .build() + .expect("request engine builds"), + ); + let fodid_engine = FodIdCloudEngine::builder() + .cloud_request_engine(request_engine.clone()) + .build() + .expect("identifier engine builds"); + Pipeline::builder() + .add_element(request_engine) + .add_element(Arc::new(fodid_engine)) + .build() + .expect("pipeline builds") +} + +/// The random 51Did a processed flow data carries, raw and as its match key. +fn random_51did(data: &FlowData) -> (String, Vec) { + let fodid = data.get(FODID_DATA_KEY).expect("identifier data present"); + let raw = fodid + .id_rand_global() + .value() + .expect("a 51Did was received") + .clone(); + let parsed = fodid.id_rand_global_fod_id(); + let parsed = parsed.value().expect("the 51Did parses"); + (raw, parsed.match_key().to_vec()) +} + +#[test] +fn two_visitors_with_the_same_evidence_never_share_a_51did() { + // Two people on one browser version behind one network address present + // the same evidence, so their requests have the same response cache key. + // The cloud creates a different 51Did for each request, and each visitor + // must receive the one created for their own. + let cloud = Arc::new(CreatingCloud::new()); + let pipeline = caching_pipeline(cloud.clone()); + + let mut first = pipeline.create_flow_data_with(sample_evidence()); + first.process().expect("the first visitor is processed"); + let mut second = pipeline.create_flow_data_with(sample_evidence()); + second.process().expect("the second visitor is processed"); + + let (first_raw, first_key) = random_51did(&first); + let (second_raw, second_key) = random_51did(&second); + assert_ne!( + second_key, first_key, + "the second visitor was handed the 51Did created for the first" + ); + + let created = cloud.created(); + assert_eq!( + created.len(), + 2, + "the cloud created a 51Did for each visitor" + ); + assert_eq!(first_raw, created[0]); + assert_eq!(second_raw, created[1]); + let cloud_data = second + .get(CloudRequestEngine::DATA_KEY) + .expect("cloud data present"); + assert!( + !cloud_data.cache_hit(), + "a response carrying a 51Did is not served from the cache" + ); +} + +#[test] +fn requests_that_ask_for_no_51did_are_still_served_from_the_cache() { + // Without a usage the cloud creates no 51Did and the answer depends on the + // evidence alone, so the cache still saves the second call. + let cloud = Arc::new(CreatingCloud::new()); + let pipeline = caching_pipeline(cloud.clone()); + let evidence = || { + Evidence::builder() + .add("header.user-agent", "Mozilla/5.0") + .add("query.client-ip", "185.28.167.78") + .build() + }; + + let mut first = pipeline.create_flow_data_with(evidence()); + first.process().expect("the first request is processed"); + let mut second = pipeline.create_flow_data_with(evidence()); + second.process().expect("the second request is processed"); + + assert_eq!( + cloud.data_requests(), + 1, + "the second answer came from the cache" + ); + let cloud_data = second + .get(CloudRequestEngine::DATA_KEY) + .expect("cloud data present"); + assert!(cloud_data.cache_hit(), "the second request is a cache hit"); + let fodid = second.get(FODID_DATA_KEY).expect("identifier data present"); + assert!( + !fodid.id_rand_global().has_value(), + "no 51Did was asked for" + ); +} + /// Resolve a cloud resource key from the environment for the live test, /// honouring the aligned name first and then the CI-exported bespoke and basic /// names, so the live test runs from an explicit key or the keys CI exports. diff --git a/ip-intelligence-cloud/src/engine.rs b/ip-intelligence-cloud/src/engine.rs index 02835b5..3fa0c50 100644 --- a/ip-intelligence-cloud/src/engine.rs +++ b/ip-intelligence-cloud/src/engine.rs @@ -223,9 +223,11 @@ impl IpIntelligenceCloudEngine { /// Read the upstream cloud JSON, slice out the `ip` aspect and build the /// element data from it. /// - /// Returns an [`Error::PipelineConfiguration`] when there is no cloud JSON, - /// which almost always means no [`CloudRequestEngine`] ran before this - /// engine. + /// Returns an [`Error::PipelineConfiguration`] when there is no cloud + /// request data, which means no [`CloudRequestEngine`] ran before this + /// engine. A request engine that ran and failed has already recorded its + /// own error and left no response, so that gives empty data rather than a + /// second error. fn build_data(&self, data: &FlowData) -> Result { let cloud = data.get(CloudRequestEngine::DATA_KEY).ok_or_else(|| { Error::configuration( @@ -236,14 +238,10 @@ impl IpIntelligenceCloudEngine { ) })?; - let json = cloud.json_response().ok_or_else(|| { - Error::configuration( - "The cloud request engine returned no JSON response, so the \ - IP-intelligence cloud engine has nothing to parse.", - ) - })?; - - self.parse_json(json) + match cloud.json_response() { + Some(json) if !json.is_empty() => self.parse_json(json), + _ => Ok(IpIntelligenceDataBase::new(IP_DATA_KEY_NAME)), + } } /// Parse a full cloud response body into the element data, slicing out the @@ -649,4 +647,47 @@ mod tests { offenders.join("\n") ); } + + #[test] + fn a_failed_cloud_request_is_not_reported_as_a_configuration_error() { + // The request engine records its own error when its call fails and + // leaves no response. Nothing in the configuration is wrong, so this + // engine adds no error of its own, as the device engine adds none. + use fiftyone_pipeline_core::{Evidence, Pipeline}; + let request_engine = request_engine(); + let ip_engine = IpIntelligenceCloudEngine::builder() + .cloud_request_engine(request_engine.clone()) + .build() + .unwrap(); + let pipeline = Pipeline::builder() + .add_element(request_engine) + .add_element(Arc::new(ip_engine)) + .suppress_process_exceptions(true) + .build() + .unwrap(); + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("query.client-ip-51d", "185.28.167.78") + .build(), + ); + data.process().unwrap(); + + let reported: Vec = data + .errors() + .iter() + .map(|error| format!("{}: {}", error.element_data_key, error.source)) + .collect(); + assert_eq!( + reported.len(), + 1, + "only the request engine's own failure is reported, got {reported:?}" + ); + assert!( + !data + .errors() + .iter() + .any(|error| matches!(error.source, Error::PipelineConfiguration { .. })), + "a failed cloud request was reported as a configuration error: {reported:?}" + ); + } } diff --git a/ip-intelligence-onpremise/Cargo.toml b/ip-intelligence-onpremise/Cargo.toml index 6413b20..df7d952 100644 --- a/ip-intelligence-onpremise/Cargo.toml +++ b/ip-intelligence-onpremise/Cargo.toml @@ -21,6 +21,11 @@ fiftyone-native = { version = "4.5.2", path = "../native", default-features = fa arc-swap.workspace = true chrono.workspace = true +[dev-dependencies] +# Compresses a data file the way the Distributor serves it, for the update +# tests. +flate2.workspace = true + [package.metadata.docs.rs] # Build the documentation on docs.rs with every feature enabled, so the # cloud, on-premise and optional paths are all documented. diff --git a/ip-intelligence-onpremise/src/builder.rs b/ip-intelligence-onpremise/src/builder.rs index aa8512e..43964fd 100644 --- a/ip-intelligence-onpremise/src/builder.rs +++ b/ip-intelligence-onpremise/src/builder.rs @@ -30,7 +30,6 @@ use std::path::PathBuf; use std::sync::Arc; -use chrono::{TimeZone, Utc}; use fiftyone_native::ipi::Manager; use fiftyone_native::PerformanceProfile; use fiftyone_pipeline_core::{Error, Result}; @@ -39,7 +38,8 @@ use fiftyone_pipeline_engines::{ OnPremiseAspectEngine, DISTRIBUTOR_URL, }; -use crate::engine::{IpIntelligenceOnPremiseEngine, DEFAULT_DATA_SOURCE_TIER}; +use crate::engine::{record_published, IpIntelligenceOnPremiseEngine}; +use crate::operational_copy::OperationalCopy; /// The download type the Distributor serves IP Intelligence data files as, /// sent in the update URL's `Type` parameter. @@ -85,6 +85,7 @@ pub struct IpIntelligenceOnPremiseEngineBuilder { polling_interval_seconds: Option, max_randomisation_seconds: Option, data_update_service: Option>, + temp_dir: Option, } impl IpIntelligenceOnPremiseEngineBuilder { @@ -111,6 +112,7 @@ impl IpIntelligenceOnPremiseEngineBuilder { polling_interval_seconds: None, max_randomisation_seconds: None, data_update_service: None, + temp_dir: None, } } @@ -120,6 +122,15 @@ impl IpIntelligenceOnPremiseEngineBuilder { self } + /// Set the folder for the engine's temporary files. With automatic updates + /// on, every profile but [`PerformanceProfile::InMemory`] reads a copy of + /// the data file made there, so an update can replace the data file itself. + /// Defaults to the system's temporary folder, [`std::env::temp_dir`]. + pub fn temp_dir_path(mut self, path: impl Into) -> Self { + self.temp_dir = Some(path.into()); + self + } + /// Restrict the engine to a single named property, adding to any already /// requested. With no properties requested the engine populates every /// property the shared model surfaces. @@ -144,7 +155,7 @@ impl IpIntelligenceOnPremiseEngineBuilder { /// /// By default the engine reads the tier from the data file header (for /// example `Lite`, `Enterprise` or `TAC`), falling back to - /// [`DEFAULT_DATA_SOURCE_TIER`] only when the header cannot be read. This + /// [`crate::DEFAULT_DATA_SOURCE_TIER`] only when the header cannot be read. This /// override forces a specific tier, which an application can use when it /// knows better than the file metadata. pub fn data_source_tier(mut self, tier: impl Into) -> Self { @@ -156,15 +167,17 @@ impl IpIntelligenceOnPremiseEngineBuilder { /// /// When enabled, the [`DataUpdateService`] the engine is registered with /// polls the update URL on the polling interval and applies a downloaded - /// update by calling the engine's `refresh`. + /// update by calling the engine's `refresh`. This is the master switch for + /// automatic updates, so with it off the file-system watcher does not run + /// either. pub fn auto_update(mut self, enabled: bool) -> Self { self.auto_update_enabled = enabled; self } /// Enable or disable the file-system watcher for the data file. On by - /// default. When enabled, the update service refreshes the engine when the - /// file changes on disk. + /// default. When enabled, and [`Self::auto_update`] is on, the update + /// service refreshes the engine when the file changes on disk. pub fn file_system_watcher(mut self, enabled: bool) -> Self { self.file_system_watcher_enabled = enabled; self @@ -243,7 +256,9 @@ impl IpIntelligenceOnPremiseEngineBuilder { /// /// Registration starts the file-system watcher and schedules the polls the /// options configure, and polls once first when update on startup is on. - /// Refuses, before opening the file, when automatic updates or update on + /// A startup poll that fails leaves the engine on the data file it loaded, + /// as the specification says, and the service logs the failure. Refuses, + /// before opening the file, when automatic updates or update on /// startup are on and no licence key is set. pub fn build_shared(mut self) -> Result> { let service = self.data_update_service.take(); @@ -255,9 +270,9 @@ impl IpIntelligenceOnPremiseEngineBuilder { .cloned() .ok_or_else(|| Error::configuration("engine has no data file to register"))?; let as_engine: Arc = engine.clone(); - service - .register(as_engine, data_file) - .map_err(|e| Error::configuration(e.to_string()))?; + // A failed update on start-up leaves the engine on the data file it + // loaded, as the specification says, and the service logs it. + let _ = service.register(as_engine, data_file); } Ok(engine) } @@ -277,38 +292,33 @@ impl IpIntelligenceOnPremiseEngineBuilder { ) }; - let manager = Manager::open_with_properties( + // Open from a copy of the data file when updates can replace the file + // while it is open. + let copy = OperationalCopy::when_needed( &self.data_file_path, + self.temp_dir.as_deref(), + self.profile, + &configuration, + ); + let manager = Manager::open_with_properties( + copy.as_ref() + .map_or(self.data_file_path.as_path(), OperationalCopy::path), self.profile, property_refs.as_deref(), )?; - // Report the tier from an explicit override first, then the data file - // header, then the fixed default when the header carries no name. - let data_source_tier = self - .data_source_tier - .clone() - .or_else(|| manager.data_set_name()) - .unwrap_or_else(|| DEFAULT_DATA_SOURCE_TIER.to_owned()); - let data_file = Arc::new(AspectEngineDataFile::new(configuration)); - - // Record the data file's embedded publish date, read from its header, so - // the metadata reports it and the update service can use it for the - // `If-Modified-Since` decision. - if let Some((year, month, day)) = manager.data_set_published() { - if let Some(published) = Utc.with_ymd_and_hms(year, month, day, 0, 0, 0).single() { - data_file.set_data_published(published); - } - } + record_published(&manager, &data_file); Ok(IpIntelligenceOnPremiseEngine::from_parts( manager, + copy, self.profile, self.requested_properties, - data_source_tier, + self.data_source_tier, data_file, - )) + ) + .with_temp_dir(self.temp_dir)) } /// Build the data-file configuration that captures the update options. The diff --git a/ip-intelligence-onpremise/src/engine.rs b/ip-intelligence-onpremise/src/engine.rs index db39f5a..e412984 100644 --- a/ip-intelligence-onpremise/src/engine.rs +++ b/ip-intelligence-onpremise/src/engine.rs @@ -30,9 +30,11 @@ use std::collections::HashMap; use std::os::raw::c_int; +use std::path::PathBuf; use std::sync::{Arc, Mutex, PoisonError}; use arc_swap::ArcSwap; +use chrono::{TimeZone, Utc}; use fiftyone_ip_intelligence_shared::{ declared_property_value_type, generated_property_names, IpIntelligenceDataBase, @@ -50,6 +52,8 @@ use fiftyone_pipeline_engines::{ }; use crate::builder::IpIntelligenceOnPremiseEngineBuilder; +use crate::operational_copy::OperationalCopy; +use crate::retained::Retained; /// The default data-source tier reported by the engine. /// @@ -104,15 +108,57 @@ pub struct IpIntelligenceOnPremiseEngine { evidence_key_filter: DataFileEvidenceKeys, /// Core property metadata for the populated properties. properties: Vec, - /// Aspect property metadata for the populated properties. - aspect_properties: Vec, - /// The data-source tier reported by the engine. - data_source_tier: String, + /// The tier of the loaded data set and the aspect property metadata, which + /// names that tier, for [`AspectEngine::data_source_tier`] and + /// [`AspectEngine::aspect_properties`]. + metadata: Retained, + /// The tier the builder set, which every data set loaded reports, or + /// `None` to report the tier each data file names. + tier_override: Option, + /// Held while a refresh loads a data set, so refreshes run one at a time. + refresh_lock: Mutex<()>, + /// The folder a refresh copies the data file into, or `None` for the + /// system's temporary folder. + temp_dir: Option, /// The single data file this engine reads, as shared run-time state for the /// data update service. data_files: Vec>, } +/// The tier of a loaded data set and the aspect property metadata that names +/// it. +#[derive(PartialEq)] +struct TierMetadata { + tier: String, + aspect_properties: Vec, +} + +impl TierMetadata { + /// The metadata of the data set `manager` holds, reporting `tier_override` + /// when it is set, otherwise the tier the data file names, then + /// [`DEFAULT_DATA_SOURCE_TIER`] when it names none. + fn new(manager: &Manager, tier_override: &Option, populated: &[String]) -> Self { + let tier = tier_override + .clone() + .or_else(|| manager.data_set_name()) + .unwrap_or_else(|| DEFAULT_DATA_SOURCE_TIER.to_owned()); + TierMetadata { + aspect_properties: build_aspect_metadata(populated, &tier), + tier, + } + } +} + +/// Record the publish date in `manager`'s data file header on `data_file`, +/// for the metadata and the update service's `If-Modified-Since` header. +pub(crate) fn record_published(manager: &Manager, data_file: &AspectEngineDataFile) { + if let Some((year, month, day)) = manager.data_set_published() { + if let Some(published) = Utc.with_ymd_and_hms(year, month, day, 0, 0, 0).single() { + data_file.set_data_published(published); + } + } +} + impl IpIntelligenceOnPremiseEngine { /// Start building an engine for the `.ipi` data file at `data_file_path`. /// @@ -127,29 +173,43 @@ impl IpIntelligenceOnPremiseEngine { /// it has opened the data file and derived the metadata. pub(crate) fn from_parts( manager: Arc, + copy: Option, profile: PerformanceProfile, requested_properties: Vec, - data_source_tier: String, + tier_override: Option, data_file: Arc, ) -> Self { let populated_property_names = populated_property_names(&requested_properties); let properties = build_core_metadata(&populated_property_names); - let aspect_properties = build_aspect_metadata(&populated_property_names, &data_source_tier); + let metadata = TierMetadata::new(&manager, &tier_override, &populated_property_names); IpIntelligenceOnPremiseEngine { base: AspectEngineBase::new(), evidence_key_filter: DataFileEvidenceKeys::new(&manager), - loaded: ArcSwap::from_pointee(LoadedDataSet::new(manager, &populated_property_names)), + loaded: ArcSwap::from_pointee(LoadedDataSet::new( + manager, + copy, + &populated_property_names, + )), profile, requested_properties, populated_property_names, properties, - aspect_properties, - data_source_tier, + metadata: Retained::new(metadata), + tier_override, + refresh_lock: Mutex::new(()), + temp_dir: None, data_files: vec![data_file], } } + /// Copy the data file into `temp_dir` on refresh, the folder the builder + /// copied it into, or the system's temporary folder when `None`. + pub(crate) fn with_temp_dir(mut self, temp_dir: Option) -> Self { + self.temp_dir = temp_dir; + self + } + /// The evidence keys the loaded data file defines, in the order the native /// lookup tries them, lowercased. /// @@ -206,7 +266,7 @@ impl IpIntelligenceOnPremiseEngine { Ok( IpIntelligenceDataBase::new(IP_DATA_KEY_NAME).with_reader(LookupReader { results: Mutex::new(results), - readable: Arc::clone(&loaded.readable), + loaded, }), ) } @@ -214,8 +274,14 @@ impl IpIntelligenceOnPremiseEngine { /// Reopen the data file from disk and atomically swap it in. /// /// Shared by [`OnPremiseAspectEngine::refresh`]. A concurrent `process` call - /// keeps using the old data set until this swap completes. + /// keeps using the old data set until this swap completes. The evidence + /// keys, the tier and the publish date follow the new data file, which can + /// differ from the old one. A failed reload changes nothing. fn reload_from_disk(&self) -> Result<()> { + let _refreshing = self + .refresh_lock + .lock() + .unwrap_or_else(PoisonError::into_inner); let path = self .data_files .first() @@ -236,13 +302,35 @@ impl IpIntelligenceOnPremiseEngine { .collect(), ) }; - let manager = Manager::open_with_properties(&path, self.profile, property_refs.as_deref())?; + let copy = self.data_files.first().and_then(|file| { + OperationalCopy::when_needed( + &path, + self.temp_dir.as_deref(), + self.profile, + file.configuration(), + ) + }); + let manager = Manager::open_with_properties( + copy.as_ref().map_or(path.as_path(), OperationalCopy::path), + self.profile, + property_refs.as_deref(), + )?; let evidence_keys = DataFileEvidenceKeys::whitelist(&manager); + let metadata = TierMetadata::new( + &manager, + &self.tier_override, + &self.populated_property_names, + ); + if let Some(data_file) = self.data_files.first() { + record_published(&manager, data_file); + } self.loaded.store(Arc::new(LoadedDataSet::new( manager, + copy, &self.populated_property_names, ))); self.evidence_key_filter.keys.store(Arc::new(evidence_keys)); + self.metadata.replace(metadata); Ok(()) } } @@ -352,16 +440,24 @@ fn build_aspect_metadata(names: &[String], tier: &str) -> Vec, readable: Arc, + /// Held only to be dropped with the data set, and declared after the + /// manager, so it is removed after the data set that has it open is freed. + _copy: Option, } impl LoadedDataSet { - fn new(manager: Arc, populated: &[String]) -> Self { + fn new(manager: Arc, copy: Option, populated: &[String]) -> Self { let readable = Arc::new(ReadableNames::new(populated, &manager)); - LoadedDataSet { manager, readable } + LoadedDataSet { + manager, + readable, + _copy: copy, + } } } @@ -407,19 +503,20 @@ impl ReadableNames { /// requested. struct LookupReader { /// The native results. Reading a value writes into them, so each read - /// holds the lock. + /// holds the lock. Declared first, so they are freed before the data set + /// they came from. results: Mutex, - /// The properties of the data set the results came from. - readable: Arc, + /// The data set the results came from, with its properties. + loaded: Arc, } impl IpIntelligencePropertyReader for LookupReader { fn property_names(&self) -> &[String] { - &self.readable.lowercase_names + &self.loaded.readable.lowercase_names } fn position(&self, name: &str) -> Option { - self.readable.by_name.get(name).copied() + self.loaded.readable.by_name.get(name).copied() } fn read_property( @@ -427,8 +524,8 @@ impl IpIntelligencePropertyReader for LookupReader { position: usize, data: &mut IpIntelligenceDataBase, ) -> std::result::Result<(), String> { - let name = &self.readable.names[position]; - let index = self.readable.indexes[position]; + let name = &self.loaded.readable.names[position]; + let index = self.loaded.readable.indexes[position]; let results = self.results.lock().unwrap_or_else(PoisonError::into_inner); populate_property(data, &results, name, index).map_err(|error| error.to_string()) } @@ -552,7 +649,7 @@ impl FlowElement for IpIntelligenceOnPremiseEngine { impl AspectEngine for IpIntelligenceOnPremiseEngine { fn data_source_tier(&self) -> &str { - &self.data_source_tier + &self.metadata.current().tier } fn deployment(&self) -> EngineDeployment { @@ -560,7 +657,7 @@ impl AspectEngine for IpIntelligenceOnPremiseEngine { } fn aspect_properties(&self) -> &[AspectPropertyMetaData] { - &self.aspect_properties + &self.metadata.current().aspect_properties } } diff --git a/ip-intelligence-onpremise/src/lib.rs b/ip-intelligence-onpremise/src/lib.rs index 1151005..821fc3f 100644 --- a/ip-intelligence-onpremise/src/lib.rs +++ b/ip-intelligence-onpremise/src/lib.rs @@ -105,6 +105,8 @@ mod builder; mod engine; +mod operational_copy; +mod retained; pub use builder::{IpIntelligenceOnPremiseEngineBuilder, DATA_DOWNLOAD_TYPE}; #[allow(deprecated)] diff --git a/ip-intelligence-onpremise/src/operational_copy.rs b/ip-intelligence-onpremise/src/operational_copy.rs new file mode 100644 index 0000000..fe3b7cd --- /dev/null +++ b/ip-intelligence-onpremise/src/operational_copy.rs @@ -0,0 +1,96 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! The copy of a data file an engine reads while updates can replace the file. + +use std::ffi::OsString; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_engines::DataFileConfiguration; + +/// A copy of a data file that a data set is read from, removed when dropped. +/// +/// Every profile but `InMemory` keeps the data file open while the data set +/// lives, and Windows refuses to replace a file that is open. With automatic +/// updates on, the engine therefore reads a copy in its temporary folder, +/// which the specification calls the operational data copy, and an update can +/// replace the data file itself. Drop the copy after the data set read from +/// it, which closes it. +pub(crate) struct OperationalCopy { + path: PathBuf, +} + +impl OperationalCopy { + /// A copy of `data_file` in `temp_dir`, or in the system's temporary + /// folder when that is `None`, when `profile` keeps the data file open and + /// automatic updates are on, otherwise `None`. + /// + /// When the copy cannot be made the engine reads the data file itself, so + /// a full temporary folder does not stop it from building, and on Windows + /// an update then cannot replace the file while it is open. + pub(crate) fn when_needed( + data_file: &Path, + temp_dir: Option<&Path>, + profile: PerformanceProfile, + config: &DataFileConfiguration, + ) -> Option { + if profile == PerformanceProfile::InMemory || !config.automatic_updates_enabled { + return None; + } + let temp_dir = temp_dir.map_or_else(std::env::temp_dir, Path::to_path_buf); + OperationalCopy::of(data_file, &temp_dir).ok() + } + + /// Copy `data_file` into `temp_dir`, under a name no other copy in this + /// process has. + fn of(data_file: &Path, temp_dir: &Path) -> std::io::Result { + static NEXT: AtomicU64 = AtomicU64::new(0); + let mut name = data_file + .file_name() + .map(OsString::from) + .unwrap_or_default(); + name.push(format!( + ".{}.{}.operational", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + // Made before the copy, so a copy that fails part way is removed. + let copy = OperationalCopy { + path: temp_dir.join(name), + }; + std::fs::copy(data_file, ©.path)?; + Ok(copy) + } + + /// Where the copy is. + pub(crate) fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for OperationalCopy { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + } +} diff --git a/ip-intelligence-onpremise/src/retained.rs b/ip-intelligence-onpremise/src/retained.rs new file mode 100644 index 0000000..17e5064 --- /dev/null +++ b/ip-intelligence-onpremise/src/retained.rs @@ -0,0 +1,87 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Values a refresh replaces that the engine lends by reference. + +use std::sync::OnceLock; + +/// Every distinct value kept so far, oldest first, of which the newest is the +/// current one. +/// +/// The pipeline traits lend an engine's property lists and tier by reference +/// for as long as the engine lives, so a value a refresh replaces cannot be +/// freed while the engine lives. Each is kept instead. A refresh adds a value +/// only when it differs from the newest, which happens when a data file with +/// a different property set or tier is loaded. +pub(crate) struct Retained { + value: T, + next: OnceLock>>, +} + +impl Retained { + pub(crate) fn new(value: T) -> Self { + Retained { + value, + next: OnceLock::new(), + } + } + + fn newest_link(&self) -> &Retained { + let mut link = self; + while let Some(next) = link.next.get() { + link = next; + } + link + } + + /// The newest value. + pub(crate) fn current(&self) -> &T { + &self.newest_link().value + } + + /// Make `value` the current one, unless it equals the current one. The + /// caller serialises calls, so two never add at once. + pub(crate) fn replace(&self, value: T) { + let newest = self.newest_link(); + if newest.value != value { + let _ = newest.next.set(Box::new(Retained::new(value))); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_newest_distinct_value_is_current_and_older_ones_stay_lent() { + let retained = Retained::new(String::from("Lite")); + let lent = retained.current(); + retained.replace(String::from("Lite")); + assert!(retained.next.get().is_none(), "an equal value is not kept"); + retained.replace(String::from("Enterprise")); + assert_eq!(retained.current(), "Enterprise"); + assert_eq!(lent, "Lite"); + retained.replace(String::from("Lite")); + assert_eq!(retained.current(), "Lite"); + } +} diff --git a/ip-intelligence-onpremise/tests/operational_copy.rs b/ip-intelligence-onpremise/tests/operational_copy.rs new file mode 100644 index 0000000..7e5766a --- /dev/null +++ b/ip-intelligence-onpremise/tests/operational_copy.rs @@ -0,0 +1,185 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A data update replaces the data file of a running engine under every +//! profile, and the copy the engine reads leaves with it. The test needs the +//! ASN file `51DEGREES_IPI_PATH` names, and skips with a note without it. + +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use fiftyone_ip_intelligence_onpremise::{IpIntelligenceOnPremiseEngine, IP_DATA_KEY}; +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_core::{Evidence, FlowElement, Pipeline}; +use fiftyone_pipeline_engines::{AutoUpdateStatus, DataUpdateService, OnPremiseAspectEngine}; + +fn files_in(dir: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + names.sort(); + names +} + +/// Serve `body` to one request, and return the URL. +fn serve_once(body: Vec) -> (String, std::thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/datafile", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = stream.read(&mut request); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(head.as_bytes()); + let _ = stream.write_all(&body); + }); + (url, server) +} + +/// With automatic updates on, an update replaces the data file of a running +/// engine under every profile, and the engine moves to the new file. No copy +/// is left once the engine is dropped. +#[test] +fn an_update_replaces_the_data_file_under_every_profile() { + let Some(asn) = std::env::var("51DEGREES_IPI_PATH") + .ok() + .map(PathBuf::from) + .filter(|path| path.is_file()) + else { + eprintln!("skipping operational copies: set 51DEGREES_IPI_PATH to the ASN file"); + return; + }; + for profile in [ + PerformanceProfile::Default, + PerformanceProfile::Balanced, + PerformanceProfile::LowMemory, + PerformanceProfile::HighPerformance, + PerformanceProfile::InMemory, + ] { + let dir = std::env::temp_dir().join(format!( + "fiftyone-ipi-copy-{}-{}", + profile.as_str(), + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let name = format!("copied-{}-{}.ipi", profile.as_str(), std::process::id()); + let path = dir.join(&name); + std::fs::copy(&asn, &path).unwrap(); + // Compressed, as the Distributor serves a data file. + let compressed = { + let mut encoder = + flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + encoder.write_all(&std::fs::read(&asn).unwrap()).unwrap(); + encoder.finish().unwrap() + }; + let (url, server) = serve_once(compressed); + let service = Arc::new(DataUpdateService::new()); + let engine = IpIntelligenceOnPremiseEngine::builder(&path) + .performance_profile(profile) + .properties(["Asn"]) + .auto_update(true) + .file_system_watcher(false) + .polling_interval_seconds(24 * 60 * 60) + .max_randomisation_seconds(0) + .data_update_url(url) + .data_update_license_keys(["KEY"]) + .data_update_service(Arc::clone(&service)) + .build_shared() + .expect("the data file loads"); + let as_engine: Arc = engine.clone(); + + let status = service.check_for_update(&as_engine, None); + server.join().unwrap(); + + assert_eq!( + status.map_err(|error| error.to_string()), + Ok(AutoUpdateStatus::Success), + "{profile:?}" + ); + let element: Arc = engine.clone(); + let pipeline = Pipeline::builder().add_element(element).build().unwrap(); + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("server.client-ip", "1.1.1.1") + .build(), + ); + data.process().unwrap(); + assert!(data.get(IP_DATA_KEY).unwrap().string("Asn").value().is_ok()); + + drop(data); + drop(pipeline); + drop(as_engine); + drop(engine); + drop(service); + let left = files_in(&std::env::temp_dir()) + .into_iter() + .filter(|file| file.starts_with(&name)) + .count(); + assert_eq!(left, 0, "{profile:?}: copies left in the temporary folder"); + let _ = std::fs::remove_dir_all(&dir); + } +} + +/// The copy is made in the folder the builder names, on build and on each +/// refresh, and none is left there once the engine is dropped. +#[test] +fn the_copy_goes_in_the_folder_the_builder_names() { + let Some(asn) = std::env::var("51DEGREES_IPI_PATH") + .ok() + .map(PathBuf::from) + .filter(|path| path.is_file()) + else { + eprintln!("skipping operational copies: set 51DEGREES_IPI_PATH to the ASN file"); + return; + }; + let dir = std::env::temp_dir().join(format!("fiftyone-ipi-named-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let temp = dir.join("temp"); + std::fs::create_dir_all(&temp).unwrap(); + let path = dir.join("51Degrees.ipi"); + std::fs::copy(&asn, &path).unwrap(); + let copies = || files_in(&temp).len(); + + let engine = IpIntelligenceOnPremiseEngine::builder(&path) + .properties(["Asn"]) + .temp_dir_path(&temp) + .auto_update(true) + .file_system_watcher(false) + .data_update_license_keys(["KEY"]) + .build() + .expect("the data file loads"); + assert_eq!(copies(), 1, "the copy is in the named folder"); + + engine.refresh(None).expect("the data file reloads"); + assert_eq!(copies(), 1, "the refresh copies into the named folder too"); + + drop(engine); + assert_eq!(copies(), 0, "no copy outlives the engine"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/ip-intelligence-onpremise/tests/refresh.rs b/ip-intelligence-onpremise/tests/refresh.rs new file mode 100644 index 0000000..342c910 --- /dev/null +++ b/ip-intelligence-onpremise/tests/refresh.rs @@ -0,0 +1,82 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A refresh to a data file of another tier leaves the engine reporting what +//! an engine built on that file reports. +//! +//! The test needs the ASN file `51DEGREES_IPI_PATH` names and the Lite file +//! `51Degrees-LiteV41.ipi` beside it, and skips with a note when either is +//! missing. + +use std::path::PathBuf; + +use fiftyone_ip_intelligence_onpremise::IpIntelligenceOnPremiseEngine; +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_engines::{AspectEngine, OnPremiseAspectEngine}; + +/// The ASN file and the Lite file beside it. +fn data_files() -> Option<(PathBuf, PathBuf)> { + let asn = PathBuf::from(std::env::var("51DEGREES_IPI_PATH").ok()?); + let lite = asn.parent()?.join("51Degrees-LiteV41.ipi"); + (asn.is_file() && lite.is_file()).then_some((asn, lite)) +} + +/// An engine that holds the whole data set in memory, so its data file can +/// be replaced on every platform. +fn engine(path: &PathBuf) -> IpIntelligenceOnPremiseEngine { + IpIntelligenceOnPremiseEngine::builder(path) + .performance_profile(PerformanceProfile::InMemory) + .auto_update(false) + .build() + .expect("the data file loads") +} + +#[test] +fn a_refresh_reports_the_tier_and_publish_date_of_the_new_data_file() { + let Some((asn, lite)) = data_files() else { + eprintln!( + "skipping the refresh test: set 51DEGREES_IPI_PATH to the ASN file, with \ + 51Degrees-LiteV41.ipi beside it" + ); + return; + }; + let dir = std::env::temp_dir().join(format!("fiftyone-ipi-refresh-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("51Degrees.ipi"); + std::fs::copy(&asn, &path).unwrap(); + + let refreshed = engine(&path); + let asn_tier = refreshed.data_source_tier().to_owned(); + std::fs::copy(&lite, &path).unwrap(); + refreshed.refresh(None).expect("the Lite file loads"); + let built = engine(&path); + + assert_ne!(built.data_source_tier(), asn_tier, "the files share a tier"); + assert_eq!(refreshed.data_source_tier(), built.data_source_tier()); + assert_eq!(refreshed.aspect_properties(), built.aspect_properties()); + assert_eq!(refreshed.data_file_published(), built.data_file_published()); + + drop(refreshed); + drop(built); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/ip-intelligence-onpremise/tests/update_on_startup.rs b/ip-intelligence-onpremise/tests/update_on_startup.rs new file mode 100644 index 0000000..e58c8fe --- /dev/null +++ b/ip-intelligence-onpremise/tests/update_on_startup.rs @@ -0,0 +1,88 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Update on start-up when the update cannot be fetched. The test needs the +//! ASN file `51DEGREES_IPI_PATH` names, and skips with a note without it. + +use std::net::TcpListener; +use std::path::PathBuf; +use std::sync::Arc; + +use fiftyone_ip_intelligence_onpremise::{IpIntelligenceOnPremiseEngine, IP_DATA_KEY}; +use fiftyone_native::PerformanceProfile; +use fiftyone_pipeline_core::{Evidence, FlowElement, Pipeline}; +use fiftyone_pipeline_engines::DataUpdateService; + +/// With update on start-up on, an engine builds from the data file it has when +/// the update cannot be fetched, as the specification says, and looks up with +/// it. The data file stays registered for later updates. +#[test] +fn an_engine_builds_from_its_data_file_when_the_update_on_start_up_fails() { + let Some(asn) = std::env::var("51DEGREES_IPI_PATH") + .ok() + .map(PathBuf::from) + .filter(|path| path.is_file()) + else { + eprintln!("skipping update on start-up: set 51DEGREES_IPI_PATH to the ASN file"); + return; + }; + let dir = std::env::temp_dir().join(format!("fiftyone-ipi-startup-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("51Degrees.ipi"); + std::fs::copy(&asn, &path).unwrap(); + // Nothing listens on a port once its listener is dropped. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/download", listener.local_addr().unwrap()); + drop(listener); + + let service = Arc::new(DataUpdateService::new()); + let engine = IpIntelligenceOnPremiseEngine::builder(&path) + .performance_profile(PerformanceProfile::InMemory) + .properties(["Asn"]) + .auto_update(false) + .file_system_watcher(false) + .update_on_startup(true) + .data_update_url(url) + .data_update_license_keys(["KEY"]) + .data_update_service(Arc::clone(&service)) + .build_shared() + .expect("the engine uses the data file it has"); + + assert_eq!(service.registered_count(), 1); + let element: Arc = engine.clone(); + let pipeline = Pipeline::builder().add_element(element).build().unwrap(); + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("server.client-ip", "1.1.1.1") + .build(), + ); + data.process().unwrap(); + let asn_value = data.get(IP_DATA_KEY).unwrap().string("Asn"); + assert!(asn_value.value().is_ok(), "{asn_value:?}"); + + drop(data); + drop(pipeline); + drop(engine); + drop(service); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/ip-intelligence-shared/src/data.rs b/ip-intelligence-shared/src/data.rs index ec75199..1a4c1b8 100644 --- a/ip-intelligence-shared/src/data.rs +++ b/ip-intelligence-shared/src/data.rs @@ -993,6 +993,8 @@ mod tests { ); } + // WebAssembly has no threads to spawn. + #[cfg(not(target_family = "wasm"))] #[test] fn reads_from_several_threads_agree_and_read_once() { let (reader, reads) = CountingReader::new(); diff --git a/ip-intelligence-sys/src/lib.rs b/ip-intelligence-sys/src/lib.rs index c20b5c2..4b2d603 100644 --- a/ip-intelligence-sys/src/lib.rs +++ b/ip-intelligence-sys/src/lib.rs @@ -355,7 +355,7 @@ pub mod common { extern "C" { /// Returns a newly allocated English message describing the exception. - /// Free the returned pointer with [`fiftyoneDegreesMemoryStandardFree`]. + /// Free the returned pointer with [`fiftyoneDegreesFree`]. /// Resolved against this crate's own `common-cxx` build. /// /// The native symbol is `ipi_`-prefixed because this crate's private @@ -368,18 +368,33 @@ pub mod common { #[link_name = "ipi_fiftyoneDegreesExceptionGetMessage"] pub fn fiftyoneDegreesExceptionGetMessage(exception: *mut Exception) -> *const c_char; - /// Frees memory the library allocated through its standard allocator, - /// such as the string returned by - /// [`fiftyoneDegreesExceptionGetMessage`]. + /// Frees memory through the C runtime's `free`, which is how the + /// library frees memory unless its allocator has been replaced. Release + /// what the library allocates with [`fiftyoneDegreesFree`] instead, + /// which follows a replaced allocator. /// /// The native symbol is `ipi_`-prefixed for the same reason as /// [`fiftyoneDegreesExceptionGetMessage`] above. /// /// # Safety - /// `ptr` must be a pointer the library allocated, or null. + /// `ptr` must be a pointer the C runtime's `malloc` allocated, or null. #[link_name = "ipi_fiftyoneDegreesMemoryStandardFree"] pub fn fiftyoneDegreesMemoryStandardFree(ptr: *mut std::os::raw::c_void); + /// The function the library frees memory with, the counterpart of the + /// allocator it allocates with. Release the string + /// [`fiftyoneDegreesExceptionGetMessage`] returns with it. It is + /// [`fiftyoneDegreesMemoryStandardFree`] unless the allocator has been + /// replaced, as the library's memory tracking replaces it. + /// + /// The native symbol is `ipi_`-prefixed for the same reason as + /// [`fiftyoneDegreesExceptionGetMessage`] above. + /// + /// # Safety + /// Call it only with a pointer the library allocated, or null. + #[link_name = "ipi_fiftyoneDegreesFree"] + pub static mut fiftyoneDegreesFree: unsafe extern "C" fn(ptr: *mut std::os::raw::c_void); + /// Creates an evidence array with room for `capacity` pairs, or returns /// null when the allocation fails. Free it with /// [`fiftyoneDegreesEvidenceFree`]. @@ -630,13 +645,22 @@ pub struct StringBuilder { _private: [u8; 0], } -/// Opaque collection item, mirroring `fiftyoneDegreesCollectionItem`. +/// A value read from one of the data set's collections, mirroring +/// `fiftyoneDegreesCollectionItem`. /// -/// Used by [`fiftyoneDegreesIpiGetIpAddressAsString`] to point at a string in -/// the strings collection. The layout is library internal, so it is opaque. +/// It is laid out as the C structure is, because [`ProfilePercentage`] holds +/// one ahead of its weighting, and it is passed by pointer to +/// [`fiftyoneDegreesIpiGetIpAddressAsString`]. The library sets and releases +/// the item, so a caller only reads it. #[repr(C)] +#[derive(Debug, Clone, Copy)] pub struct CollectionItem { - _private: [u8; 0], + /// The item's data, including any memory allocated to hold it. + pub data: Data, + /// A handle the library releases the item with, such as a cache node. + pub handle: *mut c_void, + /// The collection the item came from. Library internal. + pub collection: *const c_void, } /// Callback used to iterate matching profiles, mirroring @@ -750,11 +774,10 @@ pub struct WeightedValuesCollection { /// (an alias of `fiftyoneDegreesWeightedItem`). /// /// It pairs a collection item holding the value with a `raw_weighting`. The -/// `item` field is opaque because its internals are library managed and released -/// when the owning results are freed. +/// item is library managed and released when the owning results are freed. #[repr(C)] pub struct ProfilePercentage { - /// Opaque collection item holding the value. Library managed. + /// Collection item holding the value. Library managed. pub item: CollectionItem, /// Raw weighting out of the library maximum (`65535 * 65535`). pub raw_weighting: u32, @@ -1224,12 +1247,43 @@ extern "C" { buffer: *mut c_char, length: usize, ) -> usize; + + /// Initializes `manager` from the data file `file_name` with a copy of + /// `config`, setting the expected concurrency of every collection in the + /// copy to `concurrency` when that is zero or more, or keeping the + /// configuration's own when it is negative. The expected concurrency + /// sizes the pool of file handles the collections that are not held in + /// memory read through. The configuration `config` points to is not + /// changed. Defined by this crate's `src/shim.c`. + /// + /// # Safety + /// As for [`fiftyoneDegreesIpiInitManagerFromFile`], and `config` must + /// point to a configuration. + pub fn fiftyoneDegreesShimIpiInitManagerFromFile( + manager: *mut ResourceManager, + config: *const ConfigIpi, + concurrency: i32, + properties: *mut PropertiesRequired, + file_name: *const c_char, + exception: *mut Exception, + ) -> StatusCode; + + /// Returns the number of handles in the file pool of the data set + /// managed by `manager` when any of its collections reads from the data + /// file after it is loaded, or zero when the whole data set is held in + /// memory. A read that finds every handle in use fails rather than + /// waiting for one, so this is how many calls can read the data file at + /// once. Defined by this crate's `src/shim.c`. + /// + /// # Safety + /// `manager` must be initialized. + pub fn fiftyoneDegreesShimIpiGetFileHandleCount(manager: *mut ResourceManager) -> u16; } #[cfg(test)] mod tests { use super::*; - use crate::common::{fiftyoneDegreesExceptionGetMessage, fiftyoneDegreesMemoryStandardFree}; + use crate::common::{fiftyoneDegreesExceptionGetMessage, fiftyoneDegreesFree}; use std::ffi::{CStr, CString}; use std::path::PathBuf; @@ -1255,6 +1309,83 @@ mod tests { path.is_file().then_some(path) } + /// The data file `51DEGREES_IPI_PATH` names, else the ASN file. + fn named_or_asn_data_file() -> Option { + std::env::var("51DEGREES_IPI_PATH") + .ok() + .map(PathBuf::from) + .filter(|path| path.is_file()) + .or_else(asn_data_file) + } + + /// The weighting of the first value the legacy getter returns is the + /// weighting the weighted value collection gives the same value, which + /// needs [`ProfilePercentage`] laid out as the C structure is. + #[test] + fn the_legacy_getter_reads_each_values_weighting() { + let Some(data_file) = named_or_asn_data_file() else { + eprintln!("no IP Intelligence data file found; skipping the legacy weighting"); + return; + }; + let file = CString::new(data_file.to_string_lossy().as_ref()).expect("no nul"); + unsafe { + let mut manager = ResourceManager::zeroed(); + let mut exception = Exception::cleared(); + let status = fiftyoneDegreesIpiInitManagerFromFile( + &mut manager, + std::ptr::addr_of_mut!(fiftyoneDegreesIpiInMemoryConfig), + std::ptr::null_mut(), + file.as_ptr(), + &mut exception, + ); + assert_eq!( + status, + StatusCode::Success, + "{}", + exception_message(&mut exception) + ); + let results = fiftyoneDegreesResultsIpiCreate(&mut manager); + assert!(!results.is_null(), "results should be allocated"); + let ip = CString::new("1.1.1.1").expect("no nul"); + fiftyoneDegreesResultsIpiFromIpAddressString( + results, + ip.as_ptr(), + ip.as_bytes().len(), + &mut exception, + ); + assert!(exception.is_okay(), "{}", exception_message(&mut exception)); + + let mut checked = 0; + let count = fiftyoneDegreesShimIpiGetRequiredPropertyCount(&mut manager); + for index in 0..count as c_int { + let first = fiftyoneDegreesResultsIpiGetValues(results, index, &mut exception); + assert!(exception.is_okay(), "{}", exception_message(&mut exception)); + if first.is_null() { + continue; + } + let legacy = (*first).raw_weighting; + let indexes = [index]; + let mut collection = fiftyoneDegreesResultsIpiGetValuesCollection( + results, + indexes.as_ptr(), + 1, + std::ptr::null_mut(), + &mut exception, + ); + assert!(exception.is_okay(), "{}", exception_message(&mut exception)); + assert!(collection.items_count > 0, "property {index} has values"); + let weighted = (**collection.items).raw_weighting; + fiftyoneDegreesWeightedValuesCollectionRelease(&mut collection); + assert_eq!(legacy, weighted, "property {index}"); + checked += 1; + } + assert!(checked > 0, "some property should have values"); + + fiftyoneDegreesResultsIpiFree(results); + fiftyoneDegreesResourceManagerFree(&mut manager); + } + } + /// The no value reason message getter links and returns a non-empty static /// description. This proves the IP Intelligence static library linked on the /// host toolchain even when the data file is absent. @@ -1448,7 +1579,7 @@ mod tests { return String::from("(exception set, no message)"); } let message = CStr::from_ptr(raw).to_string_lossy().into_owned(); - fiftyoneDegreesMemoryStandardFree(raw as *mut c_void); + fiftyoneDegreesFree(raw as *mut c_void); message } } diff --git a/ip-intelligence-sys/src/shim.c b/ip-intelligence-sys/src/shim.c index a757a30..c15dcca 100644 --- a/ip-intelligence-sys/src/shim.c +++ b/ip-intelligence-sys/src/shim.c @@ -272,3 +272,72 @@ size_t fiftyoneDegreesShimIpiGetUniqueHeaderName( fiftyoneDegreesDataSetIpiRelease(dataSet); return written; } + +/* + * Initialises `manager` from the data file `fileName` with a copy of + * `config`, setting the expected concurrency of every collection in the copy + * to `concurrency` when that is zero or more. The expected concurrency sizes + * the pool of file handles the collections that are not held in memory read + * through. The configuration `config` points to is not changed. + */ +fiftyoneDegreesStatusCode fiftyoneDegreesShimIpiInitManagerFromFile( + fiftyoneDegreesResourceManager *manager, + const fiftyoneDegreesConfigIpi *config, + int32_t concurrency, + fiftyoneDegreesPropertiesRequired *properties, + const char *fileName, + fiftyoneDegreesException *exception) { + fiftyoneDegreesConfigIpi copy = *config; + if (concurrency >= 0) { + const uint16_t expected = (uint16_t)concurrency; + copy.strings.concurrency = expected; + copy.components.concurrency = expected; + copy.maps.concurrency = expected; + copy.properties.concurrency = expected; + copy.values.concurrency = expected; + copy.profiles.concurrency = expected; + copy.graphs.concurrency = expected; + copy.profileGroups.concurrency = expected; + copy.profileOffsets.concurrency = expected; + copy.propertyTypes.concurrency = expected; + copy.graph.concurrency = expected; + } + return fiftyoneDegreesIpiInitManagerFromFile( + manager, + ©, + properties, + fileName, + exception); +} + +/* + * Returns the number of handles in the file pool of the data set managed by + * `manager` when any of its collections reads from the data file after it is + * loaded, or zero when the whole data set is held in memory. A read that + * finds every handle in use fails rather than waiting for one, so this is + * how many calls can read the data file at once. + */ +uint16_t fiftyoneDegreesShimIpiGetFileHandleCount( + fiftyoneDegreesResourceManager *manager) { + fiftyoneDegreesDataSetIpi *dataSet = + fiftyoneDegreesDataSetIpiGet(manager); + if (dataSet == NULL) { + return 0; + } + const fiftyoneDegreesConfigIpi *config = &dataSet->config; + const bool readsFile = dataSet->b.b.isInMemory == false && ( + config->strings.loaded == false || + config->components.loaded == false || + config->maps.loaded == false || + config->properties.loaded == false || + config->values.loaded == false || + config->profiles.loaded == false || + config->graphs.loaded == false || + config->profileGroups.loaded == false || + config->profileOffsets.loaded == false || + config->propertyTypes.loaded == false || + config->graph.loaded == false); + const uint16_t count = readsFile ? dataSet->b.b.filePool.pool.count : 0; + fiftyoneDegreesDataSetIpiRelease(dataSet); + return count; +} diff --git a/ip-intelligence-sys/src/symbol_prefix.h b/ip-intelligence-sys/src/symbol_prefix.h index 1140254..122bfde 100644 --- a/ip-intelligence-sys/src/symbol_prefix.h +++ b/ip-intelligence-sys/src/symbol_prefix.h @@ -166,6 +166,10 @@ #define fiftyoneDegreesMemoryTrackingMalloc ipi_fiftyoneDegreesMemoryTrackingMalloc #define fiftyoneDegreesMemoryTrackingMallocAligned ipi_fiftyoneDegreesMemoryTrackingMallocAligned #define fiftyoneDegreesMemoryTrackingReset ipi_fiftyoneDegreesMemoryTrackingReset +#define fiftyoneDegreesMutexClose ipi_fiftyoneDegreesMutexClose +#define fiftyoneDegreesMutexCreate ipi_fiftyoneDegreesMutexCreate +#define fiftyoneDegreesMutexLock ipi_fiftyoneDegreesMutexLock +#define fiftyoneDegreesMutexUnlock ipi_fiftyoneDegreesMutexUnlock #define fiftyoneDegreesNativeToFloat ipi_fiftyoneDegreesNativeToFloat #define fiftyoneDegreesOverrideHasValueForRequiredPropertyIndex ipi_fiftyoneDegreesOverrideHasValueForRequiredPropertyIndex #define fiftyoneDegreesOverrideProfileIds ipi_fiftyoneDegreesOverrideProfileIds diff --git a/ip-intelligence/src/lib.rs b/ip-intelligence/src/lib.rs index 3b4f845..1ac4bd1 100644 --- a/ip-intelligence/src/lib.rs +++ b/ip-intelligence/src/lib.rs @@ -65,10 +65,11 @@ //! Distributor serves nothing without the licence keys set with //! `.data_update_license_keys(..)`, so a build with polling on and no key //! fails with an error that says so, as it does in .NET, and a deployment -//! without a key turns polling off with `.auto_update(false)`. -//! `.file_system_watcher(false)` turns the watcher off, and -//! `pipeline.resource::()` reaches the service for a check -//! on demand. +//! without a key turns automatic updates off with `.auto_update(false)`, which +//! stops the polling and the file watcher together, as the specification's +//! master switch does. `.file_system_watcher(false)` turns only the watcher +//! off, and `pipeline.resource::()` reaches the service for +//! a check on demand. //! //! # Features //! diff --git a/ip-intelligence/src/on_premise.rs b/ip-intelligence/src/on_premise.rs index 699ca27..4fd1019 100644 --- a/ip-intelligence/src/on_premise.rs +++ b/ip-intelligence/src/on_premise.rs @@ -128,14 +128,17 @@ impl OnPremiseIpIntelligencePipelineBuilder { /// with each new file. The 51Degrees Distributor serves nothing without a /// licence key, so [`Self::build`] refuses with this on and no key set, as /// .NET does, and a deployment without one turns this off or sets - /// [`Self::data_update_license_keys`]. + /// [`Self::data_update_license_keys`]. This is the master switch for + /// automatic updates, so with it off the file-system watcher does not run + /// either. pub fn auto_update(mut self, enabled: bool) -> Self { self.auto_update = enabled; self } /// Enable or disable the file-system watcher, which refreshes the engine - /// when the data file changes on disk. On by default. + /// when the data file changes on disk. On by default, and it runs only + /// while [`Self::auto_update`] is on. pub fn file_system_watcher(mut self, enabled: bool) -> Self { self.file_system_watcher = enabled; self diff --git a/javascript-builder/src/element.rs b/javascript-builder/src/element.rs index e866381..d9c1d41 100644 --- a/javascript-builder/src/element.rs +++ b/javascript-builder/src/element.rs @@ -23,7 +23,6 @@ //! The JavaScript builder flow element. use std::collections::BTreeMap; -use std::sync::atomic::{AtomicBool, Ordering}; use fiftyone_json_builder::JSON_BUILDER_DATA_KEY; use fiftyone_pipeline_core::constants::{ @@ -141,10 +140,10 @@ fn safe_sequence(sequence: i32) -> i32 { /// bit integer, else `1`, because the template writes it as bare code. /// - **has delayed properties**: true when the JSON payload contains the /// `delayexecution` marker. -/// - **supports promises**: true when the device-detection `Promise` property is -/// `Full`. The check is latched off once the property proves unavailable. -/// - **supports fetch**: true when the device-detection `Fetch` property is -/// true. The check is latched off once the property proves unavailable. +/// - **supports promises**: true when this request's device-detection +/// `Promise` property is `Full`. +/// - **supports fetch**: true when this request's device-detection `Fetch` +/// property is true. /// - **user prompt**: true when the builder was given a cloud request engine /// whose licensed products include 51Did, through /// [`JavaScriptBuilderElementBuilder::set_cloud_request_engine`]. The @@ -190,13 +189,6 @@ pub struct JavaScriptBuilderElement { /// Whether the script carries the user prompt section, decided by the /// builder from the cloud request engine's licensed products. user_prompt: bool, - - /// Latches that short-circuit the Promise/Fetch property lookups once they - /// have proved unavailable. Stored atomically because - /// `FlowElement::process` takes `&self` and the element is shared across - /// threads. - promise_property_available: AtomicBool, - fetch_property_available: AtomicBool, } impl JavaScriptBuilderElement { @@ -246,8 +238,6 @@ impl JavaScriptBuilderElement { enable_cookies, minify, user_prompt, - promise_property_available: AtomicBool::new(true), - fetch_property_available: AtomicBool::new(true), } } @@ -422,60 +412,34 @@ impl JavaScriptBuilderElement { serde_json::to_string(¶meters).unwrap_or_else(|_| "{}".to_owned()) } - /// Determine whether a latched device-detection property satisfies a + /// Whether this request's value of a device-detection property satisfies a /// predicate. /// - /// Returns `false` immediately when the latch has already been cleared. The - /// property is otherwise looked up and the predicate applied to its value. - /// Once the property proves unavailable the latch is cleared so later - /// requests skip the lookup. - fn supports_property( - &self, - latch: &AtomicBool, + /// The value describes the visitor's browser, so it is read from each + /// request's flow data. A request with no value, because no element + /// produces the property or because this request has no answer for it, + /// does not satisfy the predicate, and the next request is read afresh. + fn property_satisfies( data: &FlowData, property: &str, predicate: impl Fn(&PropertyValue) -> bool, ) -> bool { - if !latch.load(Ordering::Relaxed) { - return false; - } - match data.get_evidence_or_property(property) { - Ok(value) => predicate(&value), - Err(_) => { - // The property is not available in this pipeline; latch off so - // we do not keep looking. - latch.store(false, Ordering::Relaxed); - false - } - } + data.get_evidence_or_property(property) + .is_ok_and(|value| predicate(&value)) } - /// Determine whether the client supports promises. - /// - /// Returns `true` only when the device-detection `Promise` property resolves - /// to `Full`. Once the property proves unavailable the latch is cleared so - /// later requests skip the lookup. - fn supports_promises(&self, data: &FlowData) -> bool { - self.supports_property( - &self.promise_property_available, - data, - PROMISE_PROPERTY, - |value| value.as_str() == Some(PROMISE_FULL_VALUE), - ) - } - - /// Determine whether the client supports the fetch API. - /// - /// Returns `true` only when the device-detection `Fetch` property resolves to - /// true. Once the property proves unavailable the latch is cleared so later - /// requests skip the lookup. - fn supports_fetch(&self, data: &FlowData) -> bool { - self.supports_property( - &self.fetch_property_available, - data, - FETCH_PROPERTY, - |value| value.as_bool() == Some(true), - ) + /// Whether the client supports promises, being true only when this + /// request's device-detection `Promise` property is `Full`. + fn supports_promises(data: &FlowData) -> bool { + Self::property_satisfies(data, PROMISE_PROPERTY, |value| { + value.as_str() == Some(PROMISE_FULL_VALUE) + }) + } + + /// Whether the client supports the fetch API, being true only when this + /// request's device-detection `Fetch` property is true. + fn supports_fetch(data: &FlowData) -> bool { + Self::property_satisfies(data, FETCH_PROPERTY, |value| value.as_bool() == Some(true)) } /// Read the JSON payload from the JSON builder's element data, or an empty @@ -494,8 +458,8 @@ impl JavaScriptBuilderElement { let object_name = self.resolve_object_name(data); let enable_cookies = self.resolve_enable_cookies(data); - let supports_promises = self.supports_promises(data); - let supports_fetch = self.supports_fetch(data); + let supports_promises = Self::supports_promises(data); + let supports_fetch = Self::supports_fetch(data); let json_object = Self::json_object(data); let parameters = Self::build_parameters(data); @@ -775,17 +739,10 @@ mod tests { } #[test] - fn promise_and_fetch_latch_off_when_unavailable() { - let element = JavaScriptBuilderElement::new(); + fn promise_and_fetch_are_unsupported_without_a_device_answer() { + // No device element is present, so neither property resolves. let data = flow_data_with(&[]); - // No device element present, so neither property resolves. - assert!(!element.supports_promises(&data)); - assert!(!element.supports_fetch(&data)); - // The latches have been cleared so subsequent calls skip the lookup. - assert!(!element.promise_property_available.load(Ordering::Relaxed)); - assert!(!element.fetch_property_available.load(Ordering::Relaxed)); - // A second call still reports no support. - assert!(!element.supports_promises(&data)); - assert!(!element.supports_fetch(&data)); + assert!(!JavaScriptBuilderElement::supports_promises(&data)); + assert!(!JavaScriptBuilderElement::supports_fetch(&data)); } } diff --git a/javascript-builder/tests/javascript_builder.rs b/javascript-builder/tests/javascript_builder.rs index 1f0298f..4adb123 100644 --- a/javascript-builder/tests/javascript_builder.rs +++ b/javascript-builder/tests/javascript_builder.rs @@ -42,8 +42,8 @@ use fiftyone_pipeline_engines_fiftyone::SequenceElement; // --------------------------------------------------------------------------- // A minimal device-like element that exposes the Promise and Fetch properties -// the JavaScript builder consults, so the promise/fetch latching can be driven -// through a real pipeline. +// the JavaScript builder consults, so its promise and fetch decisions can be +// driven through a real pipeline. // --------------------------------------------------------------------------- struct DeviceData { @@ -534,3 +534,107 @@ fn the_json_builder_leaves_out_this_builders_data() { assert!(fiftyone_json_builder::DEFAULT_ELEMENT_EXCLUSION_LIST .contains(&fiftyone_javascript_builder::JAVASCRIPT_BUILDER_ELEMENT_DATA_KEY)); } + +/// The user agent that makes [`CloudLikeDevice`] fail, as a device engine +/// does when its cloud request failed. +const FAILING_REQUEST: &str = "the cloud request for this one failed"; + +/// A device element that behaves like the cloud device engine: it adds device +/// data carrying full promise and fetch support, except on a request whose +/// cloud call failed, where it records the failure and adds no device data. +struct CloudLikeDevice { + filter: EvidenceKeyFilterWhitelist, + properties: Vec, +} + +impl CloudLikeDevice { + fn new() -> Self { + CloudLikeDevice { + filter: EvidenceKeyFilterWhitelist::new(["header.user-agent"]), + properties: vec![ + PropertyMetaData::new("Promise", "device", PropertyValueType::String), + PropertyMetaData::new("Fetch", "device", PropertyValueType::Bool), + ], + } + } +} + +impl FlowElement for CloudLikeDevice { + fn process(&self, data: &mut FlowData) -> Result<()> { + if data.evidence().get("header.user-agent") == Some(FAILING_REQUEST) { + return Err(fiftyone_pipeline_core::Error::CloudRequest { + status_code: 0, + retry_after_seconds: None, + message: "the request timed out".to_owned(), + }); + } + data.get_or_add(DeviceElement::KEY, || DeviceData { + promise: Some("Full".to_owned()), + fetch: Some(true), + })?; + Ok(()) + } + + fn data_key(&self) -> &str { + "device" + } + + fn evidence_key_filter(&self) -> &dyn EvidenceKeyFilter { + &self.filter + } + + fn properties(&self) -> &[PropertyMetaData] { + &self.properties + } +} + +#[test] +fn a_request_without_device_data_leaves_later_visitors_unaffected() { + // One builder serves every visitor. Whether a browser supports promises + // and fetch belongs to the request, so a request that got no device answer + // must not change the script the next visitor is given. + let pipeline = Pipeline::builder() + .add_element(Arc::new(CloudLikeDevice::new())) + .add_element(Arc::new(JsonBuilderElement::new())) + .add_element(Arc::new( + JavaScriptBuilderElement::builder() + .set_minify(false) + .set_protocol("https") + .unwrap() + .build(), + )) + .suppress_process_exceptions(true) + .build() + .expect("pipeline builds"); + let script = |user_agent: &str| { + let mut data = pipeline.create_flow_data_with( + Evidence::builder() + .add("header.host", "example.com") + .add("header.user-agent", user_agent) + .build(), + ); + data.process().expect("errors are suppressed"); + data.get(JAVASCRIPT_BUILDER_DATA_KEY) + .expect("javascript builder data present") + .javascript() + .to_owned() + }; + + let failed = script(FAILING_REQUEST); + assert!( + !failed.contains("this.promise = new Promise"), + "with no device answer the script claims no promise support" + ); + + let next = script("Mozilla/5.0"); + assert!( + next.contains("this.promise = new Promise"), + "the next visitor's browser supports promises, but the script was \ + rendered without them" + ); + assert!( + next.contains("fetch('https://example.com/51dpipeline/json'"), + "the next visitor's browser supports fetch, but the script was \ + rendered without it" + ); +} diff --git a/json-builder/src/element.rs b/json-builder/src/element.rs index 8f66626..0bc91d0 100644 --- a/json-builder/src/element.rs +++ b/json-builder/src/element.rs @@ -22,7 +22,7 @@ //! The JSON builder flow element. -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{HashMap, HashSet}; use fiftyone_pipeline_core::{ ElementData, EvidenceKeyFilter, EvidenceKeyFilterWhitelist, FlowData, FlowElement, Pipeline, @@ -153,8 +153,10 @@ impl PipelineConfig { /// (`element.property`) of every JavaScript-typed property present. It is /// only emitted while the request sequence number is below /// [`crate::MAX_JAVASCRIPT_ITERATIONS`]. This key keeps its original casing. -/// - `errors` is an object mapping an element data key to the list of error -/// messages recorded against it, present only when the flow data has errors. +/// - `errors` is an array of the messages of the errors the flow data +/// recorded, in the order they were recorded, present only when there are +/// any. The specification gives it this shape and the client script reads it +/// as one. /// /// # Determinism /// @@ -329,10 +331,10 @@ impl JsonBuilderElement { } } - // Flow errors, grouped by the element they relate to. + // Flow errors, as the list of their messages. let errors = Self::collect_errors(data); if !errors.is_empty() { - root.insert(ERRORS_KEY.to_owned(), Value::Object(errors)); + root.insert(ERRORS_KEY.to_owned(), Value::Array(errors)); } // `serde_json::to_string_pretty` preserves the insertion order because @@ -423,22 +425,13 @@ impl JsonBuilderElement { paths } - /// Group the flow data's errors by the element data key they relate to, - /// mapping each to its list of error messages. - fn collect_errors(data: &FlowData) -> Map { - let mut grouped: BTreeMap> = BTreeMap::new(); - for error in data.errors() { - grouped - .entry(error.element_data_key.clone()) - .or_default() - .push(error.source.to_string()); - } - let mut map = Map::new(); - for (key, messages) in grouped { - let array = messages.into_iter().map(Value::String).collect(); - map.insert(key, Value::Array(array)); - } - map + /// The messages of the flow data's errors, in the order they were + /// recorded. + fn collect_errors(data: &FlowData) -> Vec { + data.errors() + .iter() + .map(|error| Value::String(error.source.to_string())) + .collect() } } diff --git a/json-builder/tests/json_builder.rs b/json-builder/tests/json_builder.rs index cdcb34e..f8558d3 100644 --- a/json-builder/tests/json_builder.rs +++ b/json-builder/tests/json_builder.rs @@ -643,7 +643,9 @@ fn suppresses_javascript_properties_at_cap() { } #[test] -fn appends_flow_errors() { +fn flow_errors_are_a_list_of_messages() { + // The specification puts the flow errors under `errors` as a list of + // message strings, which is what the client script reads it as. let device = TestElement::new("device").with_value("ismobile", true); let failing = FailingElement::new("badengine"); @@ -653,18 +655,18 @@ fn appends_flow_errors() { ); let value = parse(&json); - let errors = value.get("errors").expect("errors object present"); - let messages = errors - .get("badengine") + let errors = value + .get("errors") .and_then(Value::as_array) - .expect("error list for the failing element"); + .unwrap_or_else(|| panic!("errors is not a list of messages: {json}")); + let messages: Vec<&str> = errors + .iter() + .map(|error| error.as_str().expect("each error is a message")) + .collect(); assert_eq!(messages.len(), 1); assert!( - messages[0] - .as_str() - .unwrap() - .contains("deliberate test failure"), - "error message carried through" + messages[0].contains("deliberate test failure"), + "error message carried through, got {messages:?}" ); } diff --git a/native/src/dd.rs b/native/src/dd.rs index 3a90764..62fb02c 100644 --- a/native/src/dd.rs +++ b/native/src/dd.rs @@ -34,7 +34,11 @@ //! The native resource manager is internally reference counted and lock //! protected (see `common-cxx/resource.h`), so a [`Manager`] is both [`Send`] //! and [`Sync`] and a single one may be shared across threads behind an -//! [`Arc`]. A native results structure is per-thread scratch, so [`Results`] is +//! [`Arc`]. A data set that reads from its file after it is loaded holds one +//! file handle for each concurrent operation it was opened to expect, and a +//! read that finds none free fails. So no more calls that can read the file +//! run at once than there are handles, and a call beyond them waits for one. +//! A native results structure is per-thread scratch, so [`Results`] is //! [`Send`] (it may be moved to another thread) but not [`Sync`] (it must not be //! used from two threads at once). This matches the pipeline rule that one flow //! data, and therefore one results structure, belongs to a single thread at a @@ -52,6 +56,7 @@ use fiftyone_pipeline_core::{constants, Error, Result}; use crate::buffer::with_value_string; use crate::evidence::with_native_evidence; +use crate::file_reads::{FileReadPermit, FileReads}; use crate::profile::PerformanceProfile; use crate::status::{status_name, status_to_result, NativeException, NativeStatus}; @@ -87,7 +92,7 @@ impl NativeException for sys::Exception { } unsafe fn free_message(ptr: *mut c_void) { - sys::fiftyoneDegreesMemoryStandardFree(ptr); + sys::fiftyoneDegreesFree(ptr); } } @@ -169,13 +174,16 @@ pub struct Manager { /// lookup. Every later lookup reads it with no lock, so concurrent readers /// never contend. property_index: OnceLock>, + /// Permits for the calls that can read the data file, one for each of its + /// file handles, or [`None`] when the data set is held in memory. + file_reads: Option, } // Safety: the native resource manager is internally reference counted and // lock protected per `common-cxx/resource.h`, so concurrent access from several // threads through a shared `&Manager` is sound. The cached property index is -// set once through a `OnceLock` and only read after that. Therefore the -// manager is both `Send` and `Sync`. +// set once through a `OnceLock` and only read after that, and the file read +// permits are lock protected. Therefore the manager is both `Send` and `Sync`. unsafe impl Send for Manager {} unsafe impl Sync for Manager {} @@ -205,10 +213,11 @@ impl Manager { /// concurrency. /// /// `concurrency` sizes the file-handle pool the file-backed collections use - /// under the `LowMemory` and `Balanced` profiles, so it should be at least - /// the number of threads that will process through the data set at once. - /// [`None`] keeps the profile's default. It has no effect on the in-memory - /// profiles, where every collection is already resident. + /// under the `LowMemory` and `Balanced` profiles, which is how many calls + /// can read the data file at once. A call beyond that waits for a handle, + /// so it should be the number of threads that will process through the + /// data set at once. [`None`] keeps the profile's default. It has no effect + /// on the in-memory profiles, where every collection is already resident. pub fn open_with_options( path: impl AsRef, profile: PerformanceProfile, @@ -216,14 +225,7 @@ impl Manager { concurrency: Option, ) -> Result> { let path = path.as_ref(); - let path_string = path.to_str().ok_or_else(|| Error::Native { - status: String::from("InvalidInput"), - message: format!("data file path is not valid UTF-8: {}", path.display()), - })?; - let path_c = CString::new(path_string).map_err(|_| Error::Native { - status: String::from("InvalidInput"), - message: String::from("data file path contains an interior nul byte"), - })?; + let path_c = crate::path::native_path(path)?; // A separated property list, kept alive for the duration of the init // call because `PropertiesRequired` only borrows the string pointer. @@ -286,9 +288,12 @@ impl Manager { // Safety: a successful init leaves a non-null, initialized manager. let manager = NonNull::new(manager_ptr).expect("init produced a non-null manager"); + // Safety: the manager was initialized above. + let handles = unsafe { sys::fiftyoneDegreesShimHashGetFileHandleCount(manager.as_ptr()) }; Ok(Arc::new(Manager { manager, property_index: OnceLock::new(), + file_reads: FileReads::for_handles(handles), })) } @@ -297,6 +302,12 @@ impl Manager { self.manager.as_ptr() } + /// A permit for a call that can read the data file, waiting while every + /// file handle is in use. [`None`] when the data set is held in memory. + fn file_read(&self) -> Option> { + self.file_reads.as_ref().map(FileReads::permit) + } + /// The number of required (available) properties in the loaded data set. pub fn property_count(&self) -> u32 { // Safety: the manager is initialized for the lifetime of `self`. @@ -370,6 +381,7 @@ impl Manager { /// caller can fall back to its own default rather than assert a tier. pub fn data_set_name(&self) -> Option { let mut buffer = vec![0u8; 128]; + let _file_read = self.file_read(); // Safety: the buffer is writable for its full length and the manager is // initialized for the lifetime of `self`. let written = unsafe { @@ -495,6 +507,7 @@ impl Results { pub fn process_evidence(&mut self, evidence: &fiftyone_pipeline_core::Evidence) -> Result<()> { let mut exception = sys::Exception::cleared(); let results = self.as_ptr(); + let _file_read = self.manager.file_read(); // Safety: the array the closure receives is valid only for the call, // which is exactly the window the native processor reads it in. unsafe { @@ -508,6 +521,7 @@ impl Results { /// Run a detection from a single User-Agent string. pub fn process_user_agent(&mut self, user_agent: &str) -> Result<()> { let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: the user agent bytes are valid for the duration of the call. unsafe { sys::fiftyoneDegreesResultsHashFromUserAgent( @@ -523,12 +537,21 @@ impl Results { /// Run a detection from a 51Degrees device id. Returns the number of valid /// profiles parsed from the id. pub fn process_device_id(&mut self, device_id: &str) -> Result { + // The native parser reads each profile id up to the first byte that + // is not a digit, whatever the length it is given, so the id is + // passed as a null terminated copy. A Rust string slice is not null + // terminated, and digits just past it would join the last profile id. + let device_id_c = CString::new(device_id).map_err(|_| Error::Native { + status: String::from("InvalidInput"), + message: String::from("device id contains an interior nul byte"), + })?; let mut exception = sys::Exception::cleared(); - // Safety: the device id bytes are valid for the duration of the call. + let _file_read = self.manager.file_read(); + // Safety: `device_id_c` is null terminated and valid for the call. let profiles = unsafe { sys::fiftyoneDegreesResultsHashFromDeviceId( self.as_ptr(), - device_id.as_ptr() as *const c_char, + device_id_c.as_ptr(), device_id.len(), &mut exception, ) @@ -551,6 +574,7 @@ impl Results { // avoids a growable allocation on this metric path. let mut buffer = [0u8; 256]; let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: the results are valid, the buffer is writable for its full // length, and the exception is valid. The getter null terminates within // the buffer and returns the buffer pointer, or null when it did not fit. @@ -583,6 +607,7 @@ impl Results { return Ok(false); }; let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: the results are valid and the index is a real required index. let has = unsafe { sys::fiftyoneDegreesResultsHashGetHasValues(self.as_ptr(), index, &mut exception) @@ -619,6 +644,7 @@ impl Results { let results = self.as_ptr(); let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: the writer only touches the buffer it is given and returns the // native available-character count. The property and separator strings // outlive every invocation. @@ -666,6 +692,7 @@ impl Results { } let results = self.as_ptr(); let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: the writer only touches the buffer it is given and returns the // native available-character count. The separator outlives every // invocation and the index is in range for this data set. @@ -1018,6 +1045,65 @@ mod tests { } } + /// A device id read from a slice ends where the slice ends, however the + /// bytes after it in memory read. + #[test] + fn a_device_id_ends_where_its_slice_ends() { + let Some(path) = data_file() else { + eprintln!("no Hash data file found; skipping device id slices"); + return; + }; + let manager = Manager::open(&path, PerformanceProfile::Default).expect("data file loads"); + let detected = detect(&manager, "header.user-agent", CHROME_ON_WINDOWS); + let device_id = detected + .device_id() + .expect("the device id reads") + .expect("a detection has a device id"); + let profiles = device_id.split('-').filter(|id| *id != "0").count() as i32; + + // Digits straight after the slice, as a larger buffer holding the id + // can have. + let held = format!("{device_id}5-1"); + let mut results = manager.create_results().expect("results allocate"); + let found = results + .process_device_id(&held[..device_id.len()]) + .expect("the device id is processed"); + assert_eq!(found, profiles, "{device_id}"); + assert_eq!( + results.device_id().expect("the device id reads"), + Some(device_id) + ); + } + + /// A data set that reads from its file has a file handle for each + /// concurrent operation it was opened to expect, and one held in memory + /// has none, so its calls take no permits. + #[test] + fn file_handles_follow_the_expected_concurrency() { + let Some(path) = data_file() else { + eprintln!("no Hash data file found; skipping file handles"); + return; + }; + for (profile, concurrency, handles) in [ + (PerformanceProfile::LowMemory, Some(3), 3), + (PerformanceProfile::Balanced, None, 10), + (PerformanceProfile::HighPerformance, Some(3), 0), + (PerformanceProfile::InMemory, None, 0), + ] { + let manager = Manager::open_with_options(&path, profile, None, concurrency) + .expect("data file loads"); + // Safety: the manager is initialized. + let count = unsafe { sys::fiftyoneDegreesShimHashGetFileHandleCount(manager.as_ptr()) }; + assert_eq!(count, handles, "{}", profile.as_str()); + assert_eq!( + manager.file_reads.is_some(), + handles > 0, + "{}", + profile.as_str() + ); + } + } + #[test] fn an_index_outside_the_data_set_is_an_error() { let Some(path) = data_file() else { diff --git a/native/src/file_reads.rs b/native/src/file_reads.rs new file mode 100644 index 0000000..734aebd --- /dev/null +++ b/native/src/file_reads.rs @@ -0,0 +1,123 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Keeps the calls that can read a data file within the handles its pool +//! holds. +//! +//! A data set that reads from its file after it is loaded holds a fixed pool +//! of file handles, one for each concurrent operation it was opened to +//! expect. A read that finds every handle in use does not wait for one. It +//! fails with `InsufficientHandles`, and in the IP Intelligence library a +//! failed read of a property's values ends the process. A permit taken before +//! each such call makes a caller beyond the size of the pool wait instead. + +use std::sync::{Condvar, Mutex, PoisonError}; + +/// The permits for the calls that can read a data file, one for each handle +/// in its pool. +pub(crate) struct FileReads { + available: Mutex, + returned: Condvar, +} + +impl FileReads { + /// Permits for a pool of `handles` file handles, or [`None`] when the data + /// set reads nothing from its file once it is loaded. + pub(crate) fn for_handles(handles: u16) -> Option { + (handles > 0).then(|| FileReads { + available: Mutex::new(usize::from(handles)), + returned: Condvar::new(), + }) + } + + /// Wait for a permit. It is returned when the permit is dropped. + pub(crate) fn permit(&self) -> FileReadPermit<'_> { + let mut available = self + .available + .lock() + .unwrap_or_else(PoisonError::into_inner); + while *available == 0 { + available = self + .returned + .wait(available) + .unwrap_or_else(PoisonError::into_inner); + } + *available -= 1; + FileReadPermit(self) + } +} + +/// A permit to make one call that can read a data file. +pub(crate) struct FileReadPermit<'a>(&'a FileReads); + +impl Drop for FileReadPermit<'_> { + fn drop(&mut self) { + *self + .0 + .available + .lock() + .unwrap_or_else(PoisonError::into_inner) += 1; + self.0.returned.notify_one(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::Arc; + + #[test] + fn a_data_set_held_in_memory_needs_no_permits() { + assert!(FileReads::for_handles(0).is_none()); + } + + /// No more callers hold a permit at once than there are handles. + #[test] + fn no_more_callers_read_at_once_than_there_are_handles() { + const HANDLES: u16 = 3; + let reads = Arc::new(FileReads::for_handles(HANDLES).expect("permits")); + let reading = Arc::new(AtomicUsize::new(0)); + let most = Arc::new(AtomicUsize::new(0)); + let threads: Vec<_> = (0..12) + .map(|_| { + let (reads, reading, most) = + (Arc::clone(&reads), Arc::clone(&reading), Arc::clone(&most)); + std::thread::spawn(move || { + for _ in 0..200 { + let _permit = reads.permit(); + let now = reading.fetch_add(1, Ordering::SeqCst) + 1; + most.fetch_max(now, Ordering::SeqCst); + std::thread::yield_now(); + reading.fetch_sub(1, Ordering::SeqCst); + } + }) + }) + .collect(); + for thread in threads { + thread.join().expect("a thread does not panic"); + } + let most = most.load(Ordering::SeqCst); + assert!(most <= usize::from(HANDLES), "{most} callers read at once"); + assert_eq!(*reads.available.lock().unwrap(), usize::from(HANDLES)); + } +} diff --git a/native/src/ipi.rs b/native/src/ipi.rs index 1b384f5..8480b72 100644 --- a/native/src/ipi.rs +++ b/native/src/ipi.rs @@ -49,7 +49,12 @@ //! //! As for Device Detection, [`Manager`] is [`Send`] and [`Sync`] (the native //! manager is reference counted and lock protected) and [`Results`] is [`Send`] -//! but not [`Sync`] (per-thread scratch). +//! but not [`Sync`] (per-thread scratch). A data set that reads from its file +//! after it is loaded holds one file handle for each concurrent operation it +//! was opened to expect, so no more calls that can read the file run at once +//! than there are handles, and a call beyond them waits for one. The native +//! library ends the process when a read of a property's values finds no free +//! handle. use std::collections::HashMap; use std::ffi::{CStr, CString}; @@ -63,6 +68,7 @@ use fiftyone_pipeline_core::{constants, Error, Evidence, Result}; use crate::buffer::with_value_string; use crate::evidence::parse_client_ip; +use crate::file_reads::{FileReadPermit, FileReads}; use crate::profile::PerformanceProfile; use crate::status::{status_name, status_to_result, NativeException, NativeStatus}; @@ -94,7 +100,7 @@ impl NativeException for sys::Exception { } unsafe fn free_message(ptr: *mut c_void) { - sys::common::fiftyoneDegreesMemoryStandardFree(ptr); + sys::common::fiftyoneDegreesFree(ptr); } } @@ -146,6 +152,9 @@ pub struct Manager { /// The required property name to index map, built on first use so a read /// by name costs one hash lookup rather than a native name search. property_index: OnceLock>, + /// Permits for the calls that can read the data file, one for each of its + /// file handles, or [`None`] when the data set is held in memory. + file_reads: Option, } /// One evidence key the data file defines, split into the parts the native @@ -161,7 +170,8 @@ struct EvidenceKey { // Safety: the native resource manager is internally reference counted and lock // protected per `common-cxx/resource.h`, so a shared `&Manager` is sound to use -// from several threads. Therefore the manager is both `Send` and `Sync`. +// from several threads, and the file read permits are lock protected. +// Therefore the manager is both `Send` and `Sync`. unsafe impl Send for Manager {} unsafe impl Sync for Manager {} @@ -178,16 +188,27 @@ impl Manager { path: impl AsRef, profile: PerformanceProfile, properties: Option<&[&str]>, + ) -> Result> { + Self::open_with_options(path, profile, properties, None) + } + + /// Open an IP Intelligence data file with the named properties and an + /// optional expected concurrency. + /// + /// `concurrency` sizes the file-handle pool the file-backed collections use + /// under the `LowMemory` and `Balanced` profiles, which is how many calls + /// can read the data file at once. A call beyond that waits for a handle, + /// so it should be the number of threads that will look up through the + /// data set at once. [`None`] keeps the profile's default. It has no effect + /// on the in-memory profiles, where every collection is already resident. + pub fn open_with_options( + path: impl AsRef, + profile: PerformanceProfile, + properties: Option<&[&str]>, + concurrency: Option, ) -> Result> { let path = path.as_ref(); - let path_string = path.to_str().ok_or_else(|| Error::Native { - status: String::from("InvalidInput"), - message: format!("data file path is not valid UTF-8: {}", path.display()), - })?; - let path_c = CString::new(path_string).map_err(|_| Error::Native { - status: String::from("InvalidInput"), - message: String::from("data file path contains an interior nul byte"), - })?; + let path_c = crate::path::native_path(path)?; let property_list = properties.map(|names| names.join(",")); let property_list_c = property_list @@ -209,11 +230,13 @@ impl Manager { let mut exception = sys::Exception::cleared(); // Safety: all pointers are valid for the call and the path is null - // terminated. + // terminated. The shim copies the configuration before setting the + // concurrency, so the shared global is not changed. let status = unsafe { - sys::fiftyoneDegreesIpiInitManagerFromFile( + sys::fiftyoneDegreesShimIpiInitManagerFromFile( manager_ptr, config_for(profile), + concurrency.map_or(-1, i32::from), &mut required, path_c.as_ptr(), &mut exception, @@ -237,10 +260,13 @@ impl Manager { let manager = NonNull::new(manager_ptr).expect("init produced a non-null manager"); // Safety: the manager was initialized above. let evidence_keys = unsafe { read_evidence_keys(manager.as_ptr()) }; + // Safety: as above. + let handles = unsafe { sys::fiftyoneDegreesShimIpiGetFileHandleCount(manager.as_ptr()) }; Ok(Arc::new(Manager { manager, evidence_keys, property_index: OnceLock::new(), + file_reads: FileReads::for_handles(handles), })) } @@ -248,6 +274,12 @@ impl Manager { self.manager.as_ptr() } + /// A permit for a call that can read the data file, waiting while every + /// file handle is in use. [`None`] when the data set is held in memory. + fn file_read(&self) -> Option> { + self.file_reads.as_ref().map(FileReads::permit) + } + /// The evidence keys the data file defines, in the order the native lookup /// tries them, lowercased. /// @@ -267,6 +299,7 @@ impl Manager { /// rather than a fixed default. pub fn data_set_name(&self) -> Option { let mut buffer = vec![0u8; 128]; + let _file_read = self.file_read(); // Safety: the buffer is writable for its full length and the manager is // initialized for the lifetime of `self`. let written = unsafe { @@ -499,6 +532,7 @@ impl Results { message: String::from("IP address contains an interior nul byte"), })?; let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: `ip_c` is a null terminated buffer valid for the duration of the // call, and the length passed excludes the terminator the parser reads. unsafe { @@ -540,6 +574,7 @@ impl Results { }); } + let _file_read = self.manager.file_read(); // Safety: the array is created, filled and freed here. The keys belong // to the manager, and the values to `pairs`, so both outlive the call. unsafe { @@ -582,6 +617,7 @@ impl Results { let results = self.as_ptr(); let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: the writer only touches its buffer and returns the native // available-character count. The strings outlive every invocation. let value = unsafe { @@ -621,6 +657,7 @@ impl Results { self.manager.check_property_index(index)?; let results = self.as_ptr(); let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: the writer only touches its buffer and returns the native // available-character count. The separator outlives every invocation // and the index is in range for this data set. @@ -706,6 +743,7 @@ impl Results { ) -> Result> { let indexes = [index]; let mut exception = sys::Exception::cleared(); + let _file_read = self.manager.file_read(); // Safety: `results` is valid and the index array is readable for its // single element. With no scratch buffer passed, the library converts // values to text in one of its own and frees it before returning. The @@ -1135,6 +1173,35 @@ mod tests { assert!(manager.required_property_index("NoSuchProperty").is_none()); } + /// A data set that reads from its file has a file handle for each + /// concurrent operation it was opened to expect, and one held in memory + /// has none, so its calls take no permits. + #[test] + fn file_handles_follow_the_expected_concurrency() { + let Some(data_file) = ipi_data_file() else { + eprintln!("no IP Intelligence data file found; skipping file handles"); + return; + }; + for (profile, concurrency, handles) in [ + (PerformanceProfile::LowMemory, Some(3), 3), + (PerformanceProfile::Balanced, None, 10), + (PerformanceProfile::HighPerformance, Some(3), 0), + (PerformanceProfile::InMemory, None, 0), + ] { + let manager = Manager::open_with_options(&data_file, profile, None, concurrency) + .expect("data file loads"); + // Safety: the manager is initialized. + let count = unsafe { sys::fiftyoneDegreesShimIpiGetFileHandleCount(manager.as_ptr()) }; + assert_eq!(count, handles, "{}", profile.as_str()); + assert_eq!( + manager.file_reads.is_some(), + handles > 0, + "{}", + profile.as_str() + ); + } + } + /// An index outside the data set's required properties is refused. #[test] fn an_index_outside_the_data_set_is_an_error() { diff --git a/native/src/lib.rs b/native/src/lib.rs index f13bf24..4394358 100644 --- a/native/src/lib.rs +++ b/native/src/lib.rs @@ -40,7 +40,9 @@ //! loaded data set. It is reference counted (returned inside an [`Arc`](std::sync::Arc)) and //! is [`Send`] and [`Sync`], because the native resource manager is internally //! reference counted and lock protected, so one loaded data set is shared -//! across threads cheaply. +//! across threads cheaply. When the data set reads from its file after it is +//! loaded, no more calls that can read the file run at once than it has file +//! handles, and a call beyond them waits for one. //! - A `Results` per engine ([`dd::Results`], [`ipi::Results`]) that owns a //! native results structure and holds an [`Arc`](std::sync::Arc) to its manager so the data //! set cannot be freed while a result still references it. It is [`Send`] but @@ -106,6 +108,8 @@ // it is only needed when at least one product feature is on. #[cfg(any(feature = "dd", feature = "ipi"))] mod buffer; +#[cfg(any(feature = "dd", feature = "ipi"))] +mod file_reads; mod status; // Device Detection is gated behind the `dd` feature and IP Intelligence behind @@ -121,6 +125,8 @@ pub mod dd; pub mod evidence; #[cfg(feature = "ipi")] pub mod ipi; +#[cfg(any(feature = "dd", feature = "ipi"))] +mod path; pub mod profile; pub use profile::PerformanceProfile; diff --git a/native/src/path.rs b/native/src/path.rs new file mode 100644 index 0000000..ceab57a --- /dev/null +++ b/native/src/path.rs @@ -0,0 +1,189 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A data file path in the form the native libraries open it. +//! +//! The native libraries open a data file with the C runtime's narrow file +//! functions. On Windows those read the name in the system's ANSI code page +//! rather than UTF-8, so a path holding a character outside ASCII is passed +//! in that code page, or by its short name when the code page cannot hold it. +//! Elsewhere the path's bytes are passed as they are. + +use std::ffi::CString; +use std::path::Path; + +use fiftyone_pipeline_core::Error; + +/// The path as the native libraries' file functions read it. +pub(crate) fn native_path(path: &Path) -> Result { + let bytes = encode(path).ok_or_else(|| Error::Native { + status: String::from("InvalidInput"), + message: format!( + "the data file path cannot be given to the native library: {}", + path.display() + ), + })?; + CString::new(bytes).map_err(|_| Error::Native { + status: String::from("InvalidInput"), + message: String::from("data file path contains an interior nul byte"), + }) +} + +#[cfg(unix)] +fn encode(path: &Path) -> Option> { + use std::os::unix::ffi::OsStrExt; + Some(path.as_os_str().as_bytes().to_vec()) +} + +#[cfg(not(any(unix, windows)))] +fn encode(path: &Path) -> Option> { + path.to_str().map(|text| text.as_bytes().to_vec()) +} + +#[cfg(windows)] +fn encode(path: &Path) -> Option> { + use std::os::windows::ffi::OsStrExt; + let wide: Vec = path.as_os_str().encode_wide().collect(); + windows::to_ansi(&wide).or_else(|| windows::to_ansi(&windows::short_name(&wide)?)) +} + +#[cfg(windows)] +mod windows { + use std::os::raw::{c_char, c_int}; + use std::ptr; + + const CP_ACP: u32 = 0; + const CP_UTF8: u32 = 65001; + const WC_NO_BEST_FIT_CHARS: u32 = 0x400; + + #[link(name = "kernel32")] + extern "system" { + fn GetACP() -> u32; + fn WideCharToMultiByte( + code_page: u32, + flags: u32, + wide: *const u16, + wide_length: c_int, + multi_byte: *mut c_char, + multi_byte_length: c_int, + default_char: *const c_char, + used_default_char: *mut c_int, + ) -> c_int; + fn GetShortPathNameW(long_path: *const u16, short_path: *mut u16, length: u32) -> u32; + } + + /// `wide` in the ANSI code page, or [`None`] when a character has no + /// exact equivalent in it. + pub(super) fn to_ansi(wide: &[u16]) -> Option> { + if wide.is_empty() { + return Some(Vec::new()); + } + let length = c_int::try_from(wide.len()).ok()?; + // Safety: reads only the constant code page. + let utf8 = unsafe { GetACP() } == CP_UTF8; + // A UTF-8 code page takes neither the flag nor the default character + // test, and holds every character. + let flags = if utf8 { 0 } else { WC_NO_BEST_FIT_CHARS }; + let mut used_default: c_int = 0; + let used_default_ptr = if utf8 { + ptr::null_mut() + } else { + &mut used_default as *mut c_int + }; + // Safety: `wide` holds `length` units, and the first call writes + // nothing because it is given no output buffer. + let needed = unsafe { + WideCharToMultiByte( + CP_ACP, + flags, + wide.as_ptr(), + length, + ptr::null_mut(), + 0, + ptr::null(), + used_default_ptr, + ) + }; + if needed <= 0 || used_default != 0 { + return None; + } + let mut bytes = vec![0u8; needed as usize]; + // Safety: `bytes` has room for the `needed` bytes the first call + // reported. + let written = unsafe { + WideCharToMultiByte( + CP_ACP, + flags, + wide.as_ptr(), + length, + bytes.as_mut_ptr() as *mut c_char, + needed, + ptr::null(), + used_default_ptr, + ) + }; + (written == needed && used_default == 0).then_some(bytes) + } + + /// The short name of the existing path `wide`, or [`None`] when it has + /// none. + pub(super) fn short_name(wide: &[u16]) -> Option> { + let mut long: Vec = wide.to_vec(); + long.push(0); + // Safety: `long` is null terminated, and a zero length output takes + // no write. + let needed = unsafe { GetShortPathNameW(long.as_ptr(), ptr::null_mut(), 0) }; + if needed == 0 { + return None; + } + let mut short = vec![0u16; needed as usize]; + // Safety: `short` has room for the `needed` units reported, the + // terminator included. + let written = unsafe { GetShortPathNameW(long.as_ptr(), short.as_mut_ptr(), needed) }; + if written == 0 || written >= needed { + return None; + } + short.truncate(written as usize); + Some(short) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_ascii_path_is_passed_as_it_is() { + let path = Path::new("data/51Degrees-LiteV4.1.hash"); + assert_eq!( + native_path(path) + .expect("an ASCII path converts") + .as_bytes(), + b"data/51Degrees-LiteV4.1.hash" + ); + } + + #[test] + fn a_nul_byte_is_refused() { + assert!(native_path(Path::new("data\0.hash")).is_err()); + } +} diff --git a/native/tests/data_file_path.rs b/native/tests/data_file_path.rs new file mode 100644 index 0000000..011391c --- /dev/null +++ b/native/tests/data_file_path.rs @@ -0,0 +1,74 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A data file opens from a folder whose name holds letters outside ASCII. + +use std::path::{Path, PathBuf}; + +use fiftyone_native::PerformanceProfile; + +fn data_file(variable: &str) -> Option { + let path = PathBuf::from(std::env::var(variable).ok()?); + path.is_file().then_some(path) +} + +/// Copy `source` into a new folder named with accented letters, open the copy +/// with `open`, and remove the folder again. +fn open_from_accented_folder( + source: &Path, + name: &str, + open: impl FnOnce(&Path) -> Result, +) { + let folder = std::env::temp_dir().join(format!("Données {name} {}", std::process::id())); + std::fs::create_dir_all(&folder).expect("a temporary folder"); + let copy = folder.join(source.file_name().expect("a file name")); + std::fs::copy(source, ©).expect("the data file copies"); + let opened = open(©).map(|_| ()); + std::fs::remove_dir_all(&folder).ok(); + if let Err(error) = opened { + panic!("{} does not open: {error}", copy.display()); + } +} + +#[cfg(feature = "dd")] +#[test] +fn a_hash_data_file_opens_from_a_folder_with_accented_letters() { + let Some(source) = data_file("51DEGREES_DD_PATH") else { + eprintln!("51DEGREES_DD_PATH names no Hash data file; skipping"); + return; + }; + open_from_accented_folder(&source, "dd", |path| { + fiftyone_native::dd::Manager::open(path, PerformanceProfile::LowMemory) + }); +} + +#[cfg(feature = "ipi")] +#[test] +fn an_ip_intelligence_data_file_opens_from_a_folder_with_accented_letters() { + let Some(source) = data_file("51DEGREES_IPI_PATH") else { + eprintln!("51DEGREES_IPI_PATH names no IP Intelligence data file; skipping"); + return; + }; + open_from_accented_folder(&source, "ipi", |path| { + fiftyone_native::ipi::Manager::open(path, PerformanceProfile::LowMemory) + }); +} diff --git a/native/tests/dd_memory.rs b/native/tests/dd_memory.rs new file mode 100644 index 0000000..d8af641 --- /dev/null +++ b/native/tests/dd_memory.rs @@ -0,0 +1,275 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Every Device Detection entry point leaves no native memory allocated. +//! +//! The test counts the blocks the native library allocates by giving it a +//! counting allocator, which every test in the same process would share. It +//! is therefore the only test in this file. + +#![cfg(feature = "dd")] + +mod support; + +use std::os::raw::{c_int, c_void}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use fiftyone_native::{dd, PerformanceProfile}; +use fiftyone_pipeline_core::Evidence; +use support::Leaks; + +extern "C" { + #[link_name = "fiftyoneDegreesMalloc"] + static mut MALLOC: unsafe extern "C" fn(usize) -> *mut c_void; + #[link_name = "fiftyoneDegreesMallocAligned"] + static mut MALLOC_ALIGNED: unsafe extern "C" fn(c_int, usize) -> *mut c_void; + #[link_name = "fiftyoneDegreesFree"] + static mut FREE: unsafe extern "C" fn(*mut c_void); + #[link_name = "fiftyoneDegreesFreeAligned"] + static mut FREE_ALIGNED: unsafe extern "C" fn(*mut c_void); + #[link_name = "fiftyoneDegreesMemoryStandardMalloc"] + fn standard_malloc(size: usize) -> *mut c_void; + #[link_name = "fiftyoneDegreesMemoryStandardMallocAligned"] + fn standard_malloc_aligned(alignment: c_int, size: usize) -> *mut c_void; + #[link_name = "fiftyoneDegreesMemoryStandardFree"] + fn standard_free(pointer: *mut c_void); + #[link_name = "fiftyoneDegreesMemoryStandardFreeAligned"] + fn standard_free_aligned(pointer: *mut c_void); +} + +unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + let pointer = standard_malloc(size); + support::allocated(pointer, size); + pointer +} + +unsafe extern "C" fn malloc_aligned(alignment: c_int, size: usize) -> *mut c_void { + let pointer = standard_malloc_aligned(alignment, size); + support::allocated(pointer, size); + pointer +} + +unsafe extern "C" fn free(pointer: *mut c_void) { + support::freed(pointer); + standard_free(pointer); +} + +unsafe extern "C" fn free_aligned(pointer: *mut c_void) { + support::freed(pointer); + standard_free_aligned(pointer); +} + +/// Route the native library's allocations through the counting allocator. +fn count_native_allocations() { + // Safety: no native call is running, and each replacement allocates and + // frees with the standard functions the library uses by default. + unsafe { + MALLOC = malloc; + MALLOC_ALIGNED = malloc_aligned; + FREE = free; + FREE_ALIGNED = free_aligned; + } +} + +const DESKTOP: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \ + AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"; +const MOBILE: &str = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) \ + AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1"; + +/// Base64 of a Chrome on Windows high entropy values result. +const GHEV: &str = "eyJicmFuZHMiOlt7ImJyYW5kIjoiR29vZ2xlIENocm9tZSIsInZlcnNpb24iOiIxMjQifV0s\ + ImZ1bGxWZXJzaW9uTGlzdCI6W3siYnJhbmQiOiJHb29nbGUgQ2hyb21lIiwidmVyc2lvbiI6IjEy\ + NC4wLjYzNjcuOTEifV0sIm1vYmlsZSI6ZmFsc2UsIm1vZGVsIjoiIiwicGxhdGZvcm0iOiJXaW5k\ + b3dzIiwicGxhdGZvcm1WZXJzaW9uIjoiMTUuMC4wIiwiYXJjaGl0ZWN0dXJlIjoieDg2IiwiYml0\ + bmVzcyI6IjY0In0="; + +/// The Hash data file `51DEGREES_DD_PATH` names. +fn data_file() -> Option { + let path = PathBuf::from(std::env::var("51DEGREES_DD_PATH").ok()?); + path.is_file().then_some(path) +} + +/// Evidence that takes every processing path: headers, client hints, a high +/// entropy values result, values measured in the browser and a query value. +fn rich_evidence() -> Evidence { + Evidence::builder() + .add("header.user-agent", DESKTOP) + .add("header.sec-ch-ua", "\"Google Chrome\";v=\"124\"") + .add("header.sec-ch-ua-mobile", "?0") + .add("header.sec-ch-ua-platform", "\"Windows\"") + .add("query.51d_gethighentropyvalues", GHEV) + .add("query.51d_screenpixelswidth", "1234") + .add("cookie.51d_screenpixelsheight", "567") + .add("query.user-agent", MOBILE) + .build() +} + +fn measure_profile(leaks: &mut Leaks, path: &Path, profile: PerformanceProfile) { + let label = |stage: &str| format!("{} {stage}", profile.as_str()); + let open = || dd::Manager::open(path, profile).expect("the data file loads"); + + leaks.measure(&label("open and free"), 2, || drop(open())); + + let manager = open(); + let names = manager.property_names(); + let indexes: Vec<_> = names + .iter() + .map(|name| manager.required_property_index(name).expect("an index")) + .collect(); + leaks.measure(&label("property names"), 5, || { + manager.property_names(); + }); + leaks.measure(&label("evidence keys"), 5, || { + manager.evidence_keys(); + }); + leaks.measure(&label("data set name"), 5, || { + manager.data_set_name(); + }); + leaks.measure(&label("create and free results"), 5, || { + drop(manager.create_results().expect("results allocate")); + }); + + let mut results = manager.create_results().expect("results allocate"); + let mut user_agent = 0; + leaks.measure(&label("process a User-Agent"), 20, || { + user_agent ^= 1; + let value = [DESKTOP, MOBILE][user_agent]; + results + .process_user_agent(value) + .expect("a User-Agent is processed"); + }); + let evidence = rich_evidence(); + leaks.measure(&label("process evidence"), 20, || { + results + .process_evidence(&evidence) + .expect("evidence is processed"); + }); + let read = |results: &dd::Results| { + for name in &names { + results.value_as_string(name, ",").expect("read by name"); + results.has_values(name).expect("has values"); + } + for index in &indexes { + results + .value_as_string_by_index(*index, c",") + .expect("read by index"); + } + results.match_metrics(); + results.device_id().expect("the device id"); + results.matched_user_agents(); + }; + leaks.measure(&label("read evidence results"), 3, || read(&results)); + results.process_user_agent(MOBILE).expect("a User-Agent"); + leaks.measure(&label("read User-Agent results"), 3, || read(&results)); + let mut unmatched = manager.create_results().expect("results allocate"); + unmatched + .process_user_agent("matches nothing") + .expect("a User-Agent"); + leaks.measure(&label("read results that match nothing"), 3, || { + read(&unmatched) + }); + drop(unmatched); + let device_id = results + .device_id() + .expect("the device id") + .expect("a device id"); + leaks.measure(&label("process a device id"), 20, || { + results.process_device_id(&device_id).expect("a device id"); + }); + leaks.measure(&label("read device id results"), 3, || read(&results)); + + // A new data set loaded while results from the old one are alive, as an + // engine refresh does. + leaks.measure(&label("reload with results alive"), 2, || { + let old = open(); + let mut alive = old.create_results().expect("results allocate"); + alive.process_user_agent(MOBILE).expect("a User-Agent"); + let new = open(); + drop(old); + assert_eq!( + alive.value_as_string("IsMobile", ",").expect("a value"), + Some(String::from("True")) + ); + let mut fresh = new.create_results().expect("results allocate"); + fresh.process_user_agent(DESKTOP).expect("a User-Agent"); + drop(alive); + drop(fresh); + drop(new); + }); + + drop(results); + drop(manager); +} + +#[test] +fn every_entry_point_leaves_no_native_memory_allocated() { + let Some(path) = data_file() else { + eprintln!("51DEGREES_DD_PATH names no Hash data file; skipping"); + return; + }; + count_native_allocations(); + + // The thread's pooled evidence array is created on first use and kept, + // so it is made before the baseline is read. + let warm: Arc = + dd::Manager::open(&path, PerformanceProfile::InMemory).expect("the data file loads"); + let mut results = warm.create_results().expect("results allocate"); + results + .process_evidence(&rich_evidence()) + .expect("evidence is processed"); + drop(results); + drop(warm); + let baseline = support::held(); + + let mut leaks = Leaks::default(); + for profile in [ + PerformanceProfile::InMemory, + PerformanceProfile::HighPerformance, + PerformanceProfile::LowMemory, + PerformanceProfile::Balanced, + PerformanceProfile::Default, + ] { + measure_profile(&mut leaks, &path, profile); + leaks.compare(&format!("{} everything freed", profile.as_str()), baseline); + } + leaks.measure("open a missing file", 3, || { + let opened = dd::Manager::open(path.with_extension("missing"), PerformanceProfile::Default); + assert!(opened.is_err(), "a missing file does not load"); + }); + let not_data = std::env::temp_dir().join("fiftyone-native-dd-not-a-data-file.hash"); + std::fs::write(¬_data, vec![7u8; 64 * 1024]).expect("a temporary file"); + for profile in [PerformanceProfile::InMemory, PerformanceProfile::LowMemory] { + leaks.measure( + &format!("{} open a file that is not a data file", profile.as_str()), + 3, + || { + let opened = dd::Manager::open(¬_data, profile); + assert!(opened.is_err(), "the file does not load"); + }, + ); + } + std::fs::remove_file(¬_data).ok(); + + leaks.compare("everything freed", baseline); + leaks.assert_none(); +} diff --git a/native/tests/ipi_memory.rs b/native/tests/ipi_memory.rs new file mode 100644 index 0000000..53e47ae --- /dev/null +++ b/native/tests/ipi_memory.rs @@ -0,0 +1,272 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Every IP Intelligence entry point leaves no native memory allocated. +//! +//! The test counts the blocks the native library allocates by giving it a +//! counting allocator, which every test in the same process would share. It +//! is therefore the only test in this file. + +#![cfg(feature = "ipi")] + +mod support; + +use std::ffi::CString; +use std::os::raw::{c_int, c_void}; +use std::path::{Path, PathBuf}; + +use fiftyone_ip_intelligence_sys as sys; +use fiftyone_native::{ipi, PerformanceProfile}; +use fiftyone_pipeline_core::Evidence; +use support::Leaks; + +extern "C" { + #[link_name = "ipi_fiftyoneDegreesMalloc"] + static mut MALLOC: unsafe extern "C" fn(usize) -> *mut c_void; + #[link_name = "ipi_fiftyoneDegreesMallocAligned"] + static mut MALLOC_ALIGNED: unsafe extern "C" fn(c_int, usize) -> *mut c_void; + #[link_name = "ipi_fiftyoneDegreesFree"] + static mut FREE: unsafe extern "C" fn(*mut c_void); + #[link_name = "ipi_fiftyoneDegreesFreeAligned"] + static mut FREE_ALIGNED: unsafe extern "C" fn(*mut c_void); + #[link_name = "ipi_fiftyoneDegreesMemoryStandardMalloc"] + fn standard_malloc(size: usize) -> *mut c_void; + #[link_name = "ipi_fiftyoneDegreesMemoryStandardMallocAligned"] + fn standard_malloc_aligned(alignment: c_int, size: usize) -> *mut c_void; + #[link_name = "ipi_fiftyoneDegreesMemoryStandardFree"] + fn standard_free(pointer: *mut c_void); + #[link_name = "ipi_fiftyoneDegreesMemoryStandardFreeAligned"] + fn standard_free_aligned(pointer: *mut c_void); +} + +unsafe extern "C" fn malloc(size: usize) -> *mut c_void { + let pointer = standard_malloc(size); + support::allocated(pointer, size); + pointer +} + +unsafe extern "C" fn malloc_aligned(alignment: c_int, size: usize) -> *mut c_void { + let pointer = standard_malloc_aligned(alignment, size); + support::allocated(pointer, size); + pointer +} + +unsafe extern "C" fn free(pointer: *mut c_void) { + support::freed(pointer); + standard_free(pointer); +} + +unsafe extern "C" fn free_aligned(pointer: *mut c_void) { + support::freed(pointer); + standard_free_aligned(pointer); +} + +/// Route the native library's allocations through the counting allocator. +fn count_native_allocations() { + // Safety: no native call is running, and each replacement allocates and + // frees with the standard functions the library uses by default. + unsafe { + MALLOC = malloc; + MALLOC_ALIGNED = malloc_aligned; + FREE = free; + FREE_ALIGNED = free_aligned; + } +} + +/// The IP Intelligence data file `51DEGREES_IPI_PATH` names. +fn data_file() -> Option { + let path = PathBuf::from(std::env::var("51DEGREES_IPI_PATH").ok()?); + path.is_file().then_some(path) +} + +fn measure_profile(leaks: &mut Leaks, path: &Path, profile: PerformanceProfile) { + let label = |stage: &str| format!("{} {stage}", profile.as_str()); + let open = || ipi::Manager::open(path, profile).expect("the data file loads"); + + leaks.measure(&label("open and free"), 2, || drop(open())); + + let manager = open(); + let names = manager.available_property_names(); + let indexes: Vec<_> = names + .iter() + .map(|name| manager.required_property_index(name).expect("an index")) + .collect(); + leaks.measure(&label("property names"), 5, || { + manager.available_property_names(); + }); + leaks.measure(&label("data set name and date"), 5, || { + manager.data_set_name(); + manager.data_set_published(); + }); + leaks.measure(&label("create and free results"), 5, || { + drop(manager.create_results().expect("results allocate")); + }); + + let mut results = manager.create_results().expect("results allocate"); + let mut address = 0; + leaks.measure(&label("process an address"), 20, || { + address ^= 1; + let value = ["1.1.1.1", "2606:4700:4700::1111"][address]; + results.process_ip(value).expect("an address is processed"); + }); + let evidence = Evidence::builder() + .add("server.client-ip", "9.9.9.9") + .add("query.client-ip-51d", "1.1.1.1:443") + .build(); + leaks.measure(&label("process evidence"), 20, || { + results + .process_evidence(&evidence) + .expect("evidence is processed"); + }); + leaks.measure(&label("read every value"), 3, || { + for name in &names { + results.value_as_string(name, ",").expect("read by name"); + results.values_weighted(name).expect("weighted by name"); + results.read_property(name).expect("typed by name"); + } + for index in &indexes { + results + .value_as_string_by_index(*index, c",") + .expect("read by index"); + results.values_weighted_by_index(*index).expect("weighted"); + results.read_property_by_index(*index).expect("typed"); + } + }); + // Addresses the data file may hold few or no values for. + leaks.measure(&label("read values of reserved addresses"), 3, || { + for address in [ + "0.0.0.0", + "10.0.0.1", + "127.0.0.1", + "240.0.0.1", + "::1", + "fe80::1", + ] { + results + .process_ip(address) + .expect("an address is processed"); + for index in &indexes { + results.values_weighted_by_index(*index).expect("weighted"); + results.read_property_by_index(*index).expect("typed"); + results + .value_as_string_by_index(*index, c",") + .expect("read by index"); + } + } + }); + leaks.measure(&label("process an address that is not one"), 5, || { + results + .process_ip("not an address") + .expect_err("text is not an address"); + }); + + // A new data set loaded while results from the old one are alive, as an + // engine refresh does. + leaks.measure(&label("reload with results alive"), 2, || { + let old = open(); + let mut alive = old.create_results().expect("results allocate"); + alive.process_ip("1.1.1.1").expect("an address"); + let new = open(); + drop(old); + let asn = alive.value_as_string("Asn", ",").expect("a value"); + assert!(asn.unwrap_or_default().contains("AS13335")); + let mut fresh = new.create_results().expect("results allocate"); + fresh.process_ip("9.9.9.9").expect("an address"); + drop(alive); + drop(fresh); + drop(new); + }); + + drop(results); + drop(manager); +} + +/// Load the data file with the native API, reload it in place and free it. +fn native_reload(path: &Path) { + let file = CString::new(path.to_str().expect("a UTF-8 path")).expect("no nul"); + let mut manager = sys::ResourceManager::zeroed(); + let mut exception = sys::Exception::cleared(); + // Safety: the manager is zeroed and every pointer is valid for the calls. + unsafe { + let status = sys::fiftyoneDegreesIpiInitManagerFromFile( + &mut manager, + std::ptr::addr_of_mut!(sys::fiftyoneDegreesIpiInMemoryConfig), + std::ptr::null_mut(), + file.as_ptr(), + &mut exception, + ); + assert_eq!(status, sys::StatusCode::Success); + assert!(exception.is_okay()); + let status = sys::fiftyoneDegreesIpiReloadManagerFromFile( + &mut manager, + file.as_ptr(), + &mut exception, + ); + assert_eq!(status, sys::StatusCode::Success); + assert!(exception.is_okay()); + sys::fiftyoneDegreesResourceManagerFree(&mut manager); + } +} + +#[test] +fn every_entry_point_leaves_no_native_memory_allocated() { + let Some(path) = data_file() else { + eprintln!("51DEGREES_IPI_PATH names no IP Intelligence data file; skipping"); + return; + }; + count_native_allocations(); + let baseline = support::held(); + + let mut leaks = Leaks::default(); + for profile in [ + PerformanceProfile::InMemory, + PerformanceProfile::HighPerformance, + PerformanceProfile::LowMemory, + PerformanceProfile::Balanced, + PerformanceProfile::Default, + ] { + measure_profile(&mut leaks, &path, profile); + leaks.compare(&format!("{} everything freed", profile.as_str()), baseline); + } + leaks.measure("native reload", 2, || native_reload(&path)); + leaks.measure("open a missing file", 3, || { + let opened = + ipi::Manager::open(path.with_extension("missing"), PerformanceProfile::Default); + assert!(opened.is_err(), "a missing file does not load"); + }); + let not_data = std::env::temp_dir().join("fiftyone-native-ipi-not-a-data-file.ipi"); + std::fs::write(¬_data, vec![7u8; 64 * 1024]).expect("a temporary file"); + for profile in [PerformanceProfile::InMemory, PerformanceProfile::LowMemory] { + leaks.measure( + &format!("{} open a file that is not a data file", profile.as_str()), + 3, + || { + let opened = ipi::Manager::open(¬_data, profile); + assert!(opened.is_err(), "the file does not load"); + }, + ); + } + std::fs::remove_file(¬_data).ok(); + + leaks.compare("everything freed", baseline); + leaks.assert_none(); +} diff --git a/native/tests/shared_manager.rs b/native/tests/shared_manager.rs new file mode 100644 index 0000000..f3e2a43 --- /dev/null +++ b/native/tests/shared_manager.rs @@ -0,0 +1,149 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A manager shared by more threads than its data set has file handles +//! serves every one of them. +//! +//! A data set that reads from its file after it is loaded holds a fixed pool +//! of file handles. Each test shares one opened with the `LowMemory` profile, +//! which reads everything from the file, across more threads than the pool +//! holds. The IP Intelligence library ends the process when a read of a +//! property's values finds no free handle, so a failure of that test shows +//! as this test binary exiting rather than as a failed assertion. + +use std::path::PathBuf; +use std::sync::{Arc, Barrier}; + +use fiftyone_native::PerformanceProfile; + +/// Threads sharing one manager, more than the handles in either pool below. +const THREADS: usize = 32; + +/// Detections or lookups each thread runs. +const ROUNDS: usize = 10; + +fn data_file(variable: &str) -> Option { + let path = PathBuf::from(std::env::var(variable).ok()?); + path.is_file().then_some(path) +} + +/// Run `work` on [`THREADS`] threads at once and return the errors they +/// report. +fn on_every_thread(work: impl Fn(usize) -> Vec + Send + Sync + 'static) -> Vec { + let work = Arc::new(work); + let start = Arc::new(Barrier::new(THREADS)); + let threads: Vec<_> = (0..THREADS) + .map(|thread| { + let (work, start) = (Arc::clone(&work), Arc::clone(&start)); + std::thread::spawn(move || { + start.wait(); + work(thread) + }) + }) + .collect(); + threads + .into_iter() + .flat_map(|thread| thread.join().expect("a thread does not panic")) + .collect() +} + +#[cfg(feature = "dd")] +#[test] +fn device_detection_serves_more_threads_than_file_handles() { + use fiftyone_native::dd; + let Some(path) = data_file("51DEGREES_DD_PATH") else { + eprintln!("51DEGREES_DD_PATH names no Hash data file; skipping"); + return; + }; + // Two handles, so the threads must share them. + let manager = + dd::Manager::open_with_options(&path, PerformanceProfile::LowMemory, None, Some(2)) + .expect("the data file loads"); + let names = manager.property_names(); + let user_agents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ + (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/605.1.15 \ + (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1", + ]; + let errors = on_every_thread(move |thread| { + let mut errors = Vec::new(); + for round in 0..ROUNDS { + let mut results = manager.create_results().expect("results allocate"); + let user_agent = user_agents[(thread + round) % user_agents.len()]; + if let Err(error) = results.process_user_agent(user_agent) { + errors.push(error.to_string()); + continue; + } + for name in &names { + if let Err(error) = results.value_as_string(name, ",") { + errors.push(error.to_string()); + } + } + } + errors + }); + assert!( + errors.is_empty(), + "{} calls failed, the first: {}", + errors.len(), + errors[0] + ); +} + +#[cfg(feature = "ipi")] +#[test] +fn ip_intelligence_serves_more_threads_than_file_handles() { + use fiftyone_native::ipi; + let Some(path) = data_file("51DEGREES_IPI_PATH") else { + eprintln!("51DEGREES_IPI_PATH names no IP Intelligence data file; skipping"); + return; + }; + // The profile's own pool of ten handles. + let manager = + ipi::Manager::open(&path, PerformanceProfile::LowMemory).expect("the data file loads"); + let names = manager.available_property_names(); + let addresses = ["1.1.1.1", "9.9.9.9", "2606:4700:4700::1111", "8.8.8.8"]; + let errors = on_every_thread(move |thread| { + let mut errors = Vec::new(); + for round in 0..ROUNDS { + let mut results = manager.create_results().expect("results allocate"); + let address = addresses[(thread + round) % addresses.len()]; + if let Err(error) = results.process_ip(address) { + errors.push(error.to_string()); + continue; + } + for name in &names { + if let Err(error) = results.values_weighted(name) { + errors.push(error.to_string()); + } + } + } + errors + }); + assert!( + errors.is_empty(), + "{} calls failed, the first: {}", + errors.len(), + errors[0] + ); +} diff --git a/native/tests/support/mod.rs b/native/tests/support/mod.rs new file mode 100644 index 0000000..9cfe39b --- /dev/null +++ b/native/tests/support/mod.rs @@ -0,0 +1,108 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Counts the blocks a native library holds through its pluggable +//! allocator, and measures stages of work against that count. +//! +//! The count is of blocks as well as bytes, so a block of no bytes that is +//! never freed still shows. A test file that uses this installs the +//! allocator before any native call, and holds only one test, because every +//! test in a process shares the native allocator. + +use std::collections::HashMap; +use std::os::raw::c_void; +use std::sync::{Mutex, PoisonError}; + +/// The size of every block allocated since counting started and not yet +/// freed, by address. +static LIVE: Mutex>> = Mutex::new(None); + +fn with_live(action: impl FnOnce(&mut HashMap)) { + let mut live = LIVE.lock().unwrap_or_else(PoisonError::into_inner); + action(live.get_or_insert_with(HashMap::new)); +} + +/// Record a block the native library allocated. +pub fn allocated(pointer: *mut c_void, size: usize) { + if !pointer.is_null() { + with_live(|live| { + live.insert(pointer as usize, size); + }); + } +} + +/// Record a block the native library freed. A block allocated before +/// counting started is ignored. +pub fn freed(pointer: *mut c_void) { + with_live(|live| { + live.remove(&(pointer as usize)); + }); +} + +/// The blocks and bytes held. +pub fn held() -> (usize, usize) { + let mut held = (0, 0); + with_live(|live| held = (live.len(), live.values().sum())); + held +} + +/// The stages that left blocks allocated. +#[derive(Default)] +pub struct Leaks(pub Vec); + +impl Leaks { + /// Run `stage` `repeats` times so caches and buffers that grow to fit + /// settle, then `repeats` more times, recording any blocks the second + /// run left allocated. + pub fn measure(&mut self, name: &str, repeats: usize, mut stage: impl FnMut()) { + for _ in 0..repeats { + stage(); + } + let before = held(); + for _ in 0..repeats { + stage(); + } + self.compare(&format!("{name} {repeats} times"), before); + } + + /// Record any change from `before` in the blocks held. + pub fn compare(&mut self, name: &str, before: (usize, usize)) { + let after = held(); + println!("{name}: {before:?} -> {after:?}"); + if after != before { + self.0.push(format!( + "{name}: {} blocks, {} bytes", + after.0 as i64 - before.0 as i64, + after.1 as i64 - before.1 as i64 + )); + } + } + + /// Fail with every stage that left blocks allocated. + pub fn assert_none(&self) { + assert!( + self.0.is_empty(), + "native memory left allocated:\n{}", + self.0.join("\n") + ); + } +} diff --git a/pipeline-core/Cargo.toml b/pipeline-core/Cargo.toml index 4221662..39e423f 100644 --- a/pipeline-core/Cargo.toml +++ b/pipeline-core/Cargo.toml @@ -17,6 +17,9 @@ thiserror.workspace = true # pipeline is used in high-traffic web servers, so a faster hasher than the # default SipHash is worthwhile. ahash.workspace = true +# Processing logs the errors it suppresses through the log facade, which does +# nothing until the application installs a logger. +log = "0.4" [package.metadata.docs.rs] # Build the documentation on docs.rs with every feature enabled, so the diff --git a/pipeline-core/src/error.rs b/pipeline-core/src/error.rs index f932c1e..a47391c 100644 --- a/pipeline-core/src/error.rs +++ b/pipeline-core/src/error.rs @@ -224,8 +224,13 @@ pub enum Error { /// /// Thrown at the end of processing when `suppress_process_exceptions` is /// `false`. Only errors whose [`FlowError::should_throw`] is `true` are - /// included. - #[error("{} error(s) occurred during pipeline processing", .0.len())] + /// included. The message names each one, in brackets after the count, so + /// the cause shows wherever the error is printed. + #[error( + "{} error(s) occurred during pipeline processing{}", + .0.len(), + describe_each(.0) + )] Aggregate(Vec), /// A pipeline operation was attempted that requires processing to have @@ -340,6 +345,12 @@ impl Error { } } +/// Each error of an aggregate in brackets, for the aggregate's message. Every +/// error removes credentials from its own text. +fn describe_each(errors: &[FlowError]) -> String { + errors.iter().map(|error| format!(" ({error})")).collect() +} + #[cfg(test)] mod tests { use super::*; @@ -426,11 +437,35 @@ mod tests { let aggregate = Error::Aggregate(vec![FlowError::new("cloud", cloud_error_carrying_a_key())]); assert!(!format!("{aggregate:?}").contains(NOT_A_KEY)); + assert!(!aggregate.to_string().contains(NOT_A_KEY)); let flow = FlowError::new("cloud", cloud_error_carrying_a_key()); assert!(!flow.to_string().contains(NOT_A_KEY)); assert!(!format!("{flow:?}").contains(NOT_A_KEY)); } + #[test] + fn an_aggregate_says_what_failed() { + let aggregate = Error::Aggregate(vec![ + FlowError::new("cloud", cloud_error_carrying_a_key()), + FlowError::new("device", Error::configuration("the engine failed")), + ]); + let shown = aggregate.to_string(); + assert!( + shown.starts_with("2 error(s) occurred during pipeline processing"), + "{shown}" + ); + assert!( + shown.contains("(error in element 'cloud': cloud request failed with status 400:"), + "the first error is named: {shown}" + ); + assert!( + shown.contains( + "(error in element 'device': pipeline configuration error: the engine failed)" + ), + "the second error is named: {shown}" + ); + } + #[test] fn a_no_value_error_is_redacted() { let error = NoValueError::new(format!("no value because '{NOT_A_KEY}' is refused")); diff --git a/pipeline-core/src/flow_data.rs b/pipeline-core/src/flow_data.rs index 12e060f..b4f1bef 100644 --- a/pipeline-core/src/flow_data.rs +++ b/pipeline-core/src/flow_data.rs @@ -93,7 +93,7 @@ impl FlowData { &self.errors } - /// True once [`FlowData::process`] has run. + /// True once [`FlowData::process`] has been called. pub fn is_processed(&self) -> bool { self.processed } @@ -103,8 +103,21 @@ impl FlowData { /// Each element is run in order. Whether errors propagate or are recorded /// is governed by the pipeline's `suppress_process_exceptions` flag (see /// [`crate::Pipeline::is_suppress_process_exceptions`] for the exact - /// behavior). Processing more than once is allowed but unusual. + /// behavior). + /// + /// A flow data is processed once, as the + /// [automated testing specification](https://github.com/51Degrees/specifications/blob/main/pipeline-specification/automated-testing.md#test-cases) + /// requires. A second call runs no element and returns + /// [`Error::PipelineConfiguration`], so create a new flow data for each + /// set of evidence. pub fn process(&mut self) -> Result<()> { + if self.processed { + return Err(Error::configuration( + "This flow data has already been processed. A flow data is \ + processed once, so create a new one for each set of evidence.", + )); + } + self.processed = true; // Clone the Arc so we do not hold an immutable borrow of `self` (via // `self.pipeline`) while `process` needs `&mut self`. let pipeline = Arc::clone(&self.pipeline); @@ -117,11 +130,6 @@ impl FlowData { self.errors.push(error); } - /// Mark this flow data as processed. Called by the pipeline. - pub(crate) fn set_processed(&mut self) { - self.processed = true; - } - /// Get the element data stored under the given string data key, if any. /// /// This is mechanism 1 of the diff --git a/pipeline-core/src/pipeline.rs b/pipeline-core/src/pipeline.rs index 9684186..aa1a938 100644 --- a/pipeline-core/src/pipeline.rs +++ b/pipeline-core/src/pipeline.rs @@ -76,9 +76,10 @@ impl Pipeline { /// Whether the pipeline suppresses processing exceptions. /// /// When `true`, processing (via [`crate::FlowData::process`]) records - /// per-element errors on the flow data and returns `Ok`. When `false`, it - /// runs every element and then returns `Err(Error::Aggregate(..))` if any - /// failed. This is the `SuppressProcessExceptions` flag from the + /// per-element errors on the flow data, logs each at debug level and + /// returns `Ok`. When `false`, it runs every element and then returns + /// `Err(Error::Aggregate(..))` if any failed. This is the + /// `SuppressProcessExceptions` flag from the /// [exception-handling specification](https://github.com/51Degrees/specifications/blob/main/pipeline-specification/features/exception-handling.md#process-function). pub fn is_suppress_process_exceptions(&self) -> bool { self.suppress_process_exceptions @@ -116,8 +117,8 @@ impl Pipeline { /// behavior depends on `suppress_process_exceptions`: /// /// - **Suppressed** (`true`): the error is recorded on the flow data via - /// [`FlowData::errors`] and the remaining elements still run. The method - /// returns `Ok(())`. + /// [`FlowData::errors`], logged at debug level, and the remaining + /// elements still run. The method returns `Ok(())`. /// - **Not suppressed** (`false`, the default): every element is still run /// so that all errors are gathered, then the method returns /// `Err(Error::Aggregate(..))` containing the errors whose @@ -129,14 +130,22 @@ impl Pipeline { pub(crate) fn process(&self, data: &mut FlowData) -> Result<()> { for element in &self.elements { if let Err(error) = element.process(data) { + // A suppressed error is logged as well as recorded, so it can + // be seen without reading the flow data. Debug level, because + // suppressing says the caller carries on past failures. + if self.suppress_process_exceptions { + log::debug!( + "Suppressed an error in element '{}': {error}", + element.data_key() + ); + } data.add_error(FlowError::new(element.data_key(), error)); } } - data.set_processed(); if !self.suppress_process_exceptions { - // Collect the errors that should propagate. We drain them off the - // flow data so they are reported once, through the returned error. + // Collect the errors that should propagate. They stay on the flow + // data as well, so a caller can read them from either place. let to_throw: Vec = data .errors() .iter() diff --git a/pipeline-core/tests/core_behaviour.rs b/pipeline-core/tests/core_behaviour.rs index 718b2ae..d247e2c 100644 --- a/pipeline-core/tests/core_behaviour.rs +++ b/pipeline-core/tests/core_behaviour.rs @@ -466,6 +466,46 @@ fn pipeline_runs_elements_in_order() { assert!(data.get_str("third").is_some()); } +#[test] +fn a_flow_data_is_processed_once() { + let (pipeline, log) = single_element_pipeline(); + let mut data = pipeline.create_flow_data(); + data.process().unwrap(); + + let second = data.process(); + assert!( + matches!(second, Err(Error::PipelineConfiguration { .. })), + "a second call fails, got {second:?}" + ); + assert_eq!( + *log.lock().unwrap(), + vec!["recording"], + "no element runs a second time" + ); + assert!(data.is_processed()); +} + +#[test] +fn a_failed_flow_data_is_not_processed_again() { + // A flow data whose processing failed is refused as well, so its errors + // are those of its one run. + let log = Arc::new(std::sync::Mutex::new(Vec::new())); + let pipeline = Pipeline::builder() + .add_element(Arc::new(FailingElement::new())) + .add_element(Arc::new(OrderElement::new("after", Arc::clone(&log), &[]))) + .build() + .unwrap(); + let mut data = pipeline.create_flow_data(); + assert!(matches!(data.process(), Err(Error::Aggregate(_)))); + + assert!(matches!( + data.process(), + Err(Error::PipelineConfiguration { .. }) + )); + assert_eq!(data.errors().len(), 1, "the errors are not recorded twice"); + assert_eq!(*log.lock().unwrap(), vec!["after"]); +} + #[test] fn pipeline_evidence_filter_unions_element_filters() { let log = Arc::new(std::sync::Mutex::new(Vec::new())); @@ -573,6 +613,24 @@ fn errors_propagate_when_not_suppressed() { assert_eq!(data.errors()[0].element_data_key, "failing"); } +#[test] +fn the_returned_error_names_the_element_and_the_cause() { + let pipeline = Pipeline::builder() + .add_element(Arc::new(FailingElement::new())) + .build() + .unwrap(); + let shown = pipeline + .create_flow_data() + .process() + .unwrap_err() + .to_string(); + assert_eq!( + shown, + "1 error(s) occurred during pipeline processing (error in element 'failing': \ + pipeline configuration error: deliberate failure)" + ); +} + #[test] fn errors_are_collected_and_ok_returned_when_suppressed() { let log = Arc::new(std::sync::Mutex::new(Vec::new())); diff --git a/pipeline-core/tests/logging.rs b/pipeline-core/tests/logging.rs new file mode 100644 index 0000000..a288a69 --- /dev/null +++ b/pipeline-core/tests/logging.rs @@ -0,0 +1,168 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! What processing logs. Kept in its own test binary because a logger is +//! installed once for the whole process. + +use std::sync::{Arc, Mutex}; + +use fiftyone_pipeline_core::{ + Error, EvidenceKeyFilter, EvidenceKeyFilterWhitelist, FlowData, FlowElement, Pipeline, + PropertyMetaData, Result, +}; + +/// Log records from every test in this binary, as "LEVEL message". +static LOGGED: Mutex> = Mutex::new(Vec::new()); + +/// Keeps every log record in [`LOGGED`]. +struct TestLogger; + +impl log::Log for TestLogger { + fn enabled(&self, _metadata: &log::Metadata) -> bool { + true + } + + fn log(&self, record: &log::Record) { + LOGGED + .lock() + .unwrap() + .push(format!("{} {}", record.level(), record.args())); + } + + fn flush(&self) {} +} + +static LOGGER: TestLogger = TestLogger; + +/// Send log records to [`LOGGED`]. Safe to call from every test. +fn capture_logs() { + let _ = log::set_logger(&LOGGER); + log::set_max_level(log::LevelFilter::Trace); +} + +/// The records that mention `text`, so tests running at once do not see each +/// other's. +fn logged_about(text: &str) -> Vec { + LOGGED + .lock() + .unwrap() + .iter() + .filter(|record| record.contains(text)) + .cloned() + .collect() +} + +/// An element that always fails with the error `make` builds. +struct FailingElement { + key: &'static str, + make: fn() -> Error, + filter: EvidenceKeyFilterWhitelist, +} + +impl FailingElement { + fn new(key: &'static str, make: fn() -> Error) -> Self { + FailingElement { + key, + make, + filter: EvidenceKeyFilterWhitelist::default(), + } + } +} + +impl FlowElement for FailingElement { + fn process(&self, _data: &mut FlowData) -> Result<()> { + Err((self.make)()) + } + fn data_key(&self) -> &str { + self.key + } + fn evidence_key_filter(&self) -> &dyn EvidenceKeyFilter { + &self.filter + } + fn properties(&self) -> &[PropertyMetaData] { + &[] + } +} + +fn process(element: FailingElement, suppress: bool) -> Result<()> { + Pipeline::builder() + .add_element(Arc::new(element)) + .suppress_process_exceptions(suppress) + .build() + .unwrap() + .create_flow_data() + .process() +} + +#[test] +fn a_suppressed_error_is_logged() { + capture_logs(); + process( + FailingElement::new("suppressed-failure", || { + Error::configuration("deliberate failure") + }), + true, + ) + .unwrap(); + + assert_eq!( + logged_about("'suppressed-failure'"), + vec![ + "DEBUG Suppressed an error in element 'suppressed-failure': pipeline \ + configuration error: deliberate failure" + ] + ); +} + +#[test] +fn a_suppressed_error_is_logged_without_credentials() { + const NOT_A_KEY: &str = "AQ-NOT-A-REAL-KEY-000000"; + capture_logs(); + process( + FailingElement::new("suppressed-cloud", || Error::CloudRequest { + status_code: 400, + retry_after_seconds: None, + message: format!("'{NOT_A_KEY}' could not be read as a valid resource key."), + }), + true, + ) + .unwrap(); + + let records = logged_about("'suppressed-cloud'"); + assert_eq!(records.len(), 1, "{records:?}"); + assert!(!records[0].contains(NOT_A_KEY), "{records:?}"); + assert!(records[0].contains("status 400"), "{records:?}"); +} + +#[test] +fn a_returned_error_is_left_to_the_caller() { + capture_logs(); + let result = process( + FailingElement::new("returned-failure", || { + Error::configuration("deliberate failure") + }), + false, + ); + + assert!(matches!(result, Err(Error::Aggregate(_)))); + assert!(logged_about("'returned-failure'").is_empty()); +} diff --git a/pipeline-engines-fiftyone/Cargo.toml b/pipeline-engines-fiftyone/Cargo.toml index fc04738..bfa8871 100644 --- a/pipeline-engines-fiftyone/Cargo.toml +++ b/pipeline-engines-fiftyone/Cargo.toml @@ -63,6 +63,9 @@ data-update = ["fiftyone-pipeline-engines/data-update"] # A local HTTP server is spun up in tests to capture the usage-sharing POST # without reaching the real 51Degrees endpoint. tiny_http = "0.12" +# An async runtime, to prove the usage-sharing element can be created and used +# inside one, as a web server does. +tokio = { workspace = true, features = ["rt"] } # Depend on this crate itself with share-usage-transport enabled, so the # usage-sharing tests exercise the real built-in reqwest sender (the feature is # off by default, which makes the sender a no-op). The examples workspace used diff --git a/pipeline-engines-fiftyone/src/constants.rs b/pipeline-engines-fiftyone/src/constants.rs index c827548..a043a6e 100644 --- a/pipeline-engines-fiftyone/src/constants.rs +++ b/pipeline-engines-fiftyone/src/constants.rs @@ -111,7 +111,8 @@ pub const SHARE_USAGE_DEFAULT_MAX_QUEUE_SIZE: usize = 1000; /// The default timeout in milliseconds used when adding to the queue. pub const SHARE_USAGE_DEFAULT_ADD_TIMEOUT_MS: u64 = 5; -/// The default timeout in milliseconds used when taking from the queue. +/// The default take timeout in milliseconds, how long the usage-sharing +/// element waits for another entry once a message holds the minimum entries. pub const SHARE_USAGE_DEFAULT_TAKE_TIMEOUT_MS: u64 = 100; /// The default repeat-evidence interval in minutes (the sliding-window size). diff --git a/pipeline-engines-fiftyone/src/sequence_element.rs b/pipeline-engines-fiftyone/src/sequence_element.rs index e46f273..6e57ce8 100644 --- a/pipeline-engines-fiftyone/src/sequence_element.rs +++ b/pipeline-engines-fiftyone/src/sequence_element.rs @@ -112,8 +112,9 @@ impl ElementData for SequenceData { /// /// - If `query.session-id` is absent from the evidence, a new GUID is generated /// using the `uuid` crate. -/// - If `query.sequence` is present, it is parsed and incremented. Otherwise the -/// sequence is set to `1`. +/// - If `query.sequence` is present, it is parsed and incremented, wrapping +/// after `i64::MAX` as the .NET and Java pipelines do. Otherwise, or when it +/// is not an integer, the sequence is set to `1`. /// /// Both values are written to a [`SequenceData`] under this element's data key. /// The element takes no configuration. @@ -170,14 +171,16 @@ impl FlowElement for SequenceElement { .map(str::to_owned) .unwrap_or_else(Self::new_session_id); - // If a sequence is supplied, increment it. Anything that does not parse - // as an integer is treated as a fresh sequence of 1, which keeps the - // element robust to malformed client input rather than failing the + // If a sequence is supplied, increment it, wrapping after the largest + // integer as the .NET and Java pipelines do. Anything that does not + // parse as an integer is treated as a fresh sequence of 1, which keeps + // the element robust to malformed client input rather than failing the // whole flow. - let sequence = match data.evidence().get(EVIDENCE_SEQUENCE) { - Some(raw) => raw.trim().parse::().map(|n| n + 1).unwrap_or(1), - None => 1, - }; + let sequence = data + .evidence() + .get(EVIDENCE_SEQUENCE) + .and_then(|raw| raw.trim().parse::().ok()) + .map_or(1, |sequence| sequence.wrapping_add(1)); data.get_or_add(Self::KEY, || SequenceData::new(session_id, sequence))?; Ok(()) diff --git a/pipeline-engines-fiftyone/src/share_usage_element.rs b/pipeline-engines-fiftyone/src/share_usage_element.rs index 1b029e0..706f75b 100644 --- a/pipeline-engines-fiftyone/src/share_usage_element.rs +++ b/pipeline-engines-fiftyone/src/share_usage_element.rs @@ -33,9 +33,10 @@ //! function does the minimum work on the request thread: apply the ignore //! filter, roll the share dice, consult the repeat-evidence tracker, extract //! the wanted evidence into a self-contained [`ShareUsageData`], then hand it -//! to a bounded channel. A single background thread consumes the channel, -//! accumulates a batch of the configured minimum entries, builds the XML and -//! sends it. +//! to a bounded channel. A single background thread consumes the channel. +//! Once it holds the configured minimum entries it takes more until none +//! arrives within the take timeout or it holds twice the minimum, as the .NET +//! and Java pipelines do, then builds the XML and sends it. //! //! Usage sharing is expendable. A failure (a full queue, a send error, an //! offline endpoint) is logged through the [`log`] facade and never disrupts @@ -76,7 +77,7 @@ use std::fmt::Write as _; use std::io::Write; use std::net::UdpSocket; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::mpsc::{sync_channel, Receiver, RecvTimeoutError, SyncSender, TrySendError}; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender, TrySendError}; use std::sync::{Arc, Mutex, OnceLock}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; @@ -104,6 +105,9 @@ use crate::share_usage_tracker::ShareUsageTracker; /// needs at least one record. pub const MIN_ENTRIES_PER_MESSAGE_FLOOR: usize = 1; +/// A message holds at most this many times the minimum entries per message. +const MAXIMUM_ENTRIES_MULTIPLE: usize = 2; + /// How long the built-in transport waits for the endpoint to answer. #[cfg(feature = "share-usage-transport")] const HTTP_TIMEOUT_SECONDS: u64 = 30; @@ -178,7 +182,8 @@ impl ShareUsageConfig { self.share_percentage } - /// The number of entries accumulated before a message is sent. + /// The number of entries accumulated before a message is sent. A message + /// holds up to twice this number. pub fn minimum_entries_per_message(&self) -> usize { self.minimum_entries_per_message } @@ -248,8 +253,9 @@ impl ShareUsageConfigBuilder { } /// Set the minimum entries per message. Values below - /// [`MIN_ENTRIES_PER_MESSAGE_FLOOR`] are raised to it. Returns `self` for - /// chaining. + /// [`MIN_ENTRIES_PER_MESSAGE_FLOOR`] are raised to it. A message is sent + /// once it holds this many entries and no more arrive within the take + /// timeout, or once it holds twice this many. Returns `self` for chaining. pub fn minimum_entries_per_message(mut self, minimum_entries_per_message: usize) -> Self { self.config.minimum_entries_per_message = minimum_entries_per_message.max(MIN_ENTRIES_PER_MESSAGE_FLOOR); @@ -263,15 +269,20 @@ impl ShareUsageConfigBuilder { } /// Set how long a request waits for room when the queue is full. The - /// request's data is discarded if no room appears in that time. Returns - /// `self` for chaining. + /// request's data is discarded if no room appears in that time. A timeout + /// too long to add to the clock, such as `Duration::MAX`, waits until there + /// is room. Returns `self` for chaining. pub fn add_timeout(mut self, add_timeout: Duration) -> Self { self.config.add_timeout = add_timeout; self } - /// Set the timeout used when taking from the queue. Returns `self` for - /// chaining. + /// Set how long the background thread waits for another entry once a + /// message holds the minimum entries per message. The message is sent when + /// none arrives in that time, or when it holds twice the minimum, as the + /// .NET and Java pipelines do. Below the minimum the thread waits with no + /// time limit, so an idle element uses no processor time. Returns `self` + /// for chaining. pub fn take_timeout(mut self, take_timeout: Duration) -> Self { self.config.take_timeout = take_timeout; self @@ -406,17 +417,45 @@ trait UsageSender: Send + Sync { /// The production transport: a blocking `reqwest` client posting gzip XML. /// Compiled only with the `share-usage-transport` feature, because reqwest does /// not build for wasm32-wasip1. +/// +/// The client is built by the first send, which runs on the background thread, +/// because building a blocking client inside an async runtime panics and an +/// element is often created inside one. #[cfg(feature = "share-usage-transport")] struct HttpUsageSender { - client: reqwest::blocking::Client, + client: OnceLock>, url: String, } +#[cfg(feature = "share-usage-transport")] +impl HttpUsageSender { + fn new(url: String) -> Self { + HttpUsageSender { + client: OnceLock::new(), + url, + } + } + + /// The client, built on first use. A client that cannot be built makes + /// every send fail with the reason, which is logged like any failed send. + fn client(&self) -> std::result::Result<&reqwest::blocking::Client, String> { + self.client + .get_or_init(|| { + reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(HTTP_TIMEOUT_SECONDS)) + .build() + .map_err(|e| format!("The HTTP client could not be built: {e}")) + }) + .as_ref() + .map_err(Clone::clone) + } +} + #[cfg(feature = "share-usage-transport")] impl UsageSender for HttpUsageSender { fn send(&self, body: Vec) -> std::result::Result<(), String> { let response = self - .client + .client()? .post(&self.url) .header("content-encoding", "gzip") .header("content-type", "text/xml") @@ -528,21 +567,16 @@ impl ShareUsageElement { /// off, the element is constructed with a no-op transport: it collects and /// batches evidence but sends nothing to 51Degrees. See the module-level /// documentation for the full rationale. + /// + /// The element can be created inside an async runtime, because the HTTP + /// client is built on the element's own background thread. pub fn new(config: ShareUsageConfig) -> Self { // With the built-in transport, post over a blocking reqwest client. // Without it (for example on wasm32-wasip1) fall back to a no-op sender // so the element still constructs and runs. #[cfg(feature = "share-usage-transport")] - let sender: Box = { - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(HTTP_TIMEOUT_SECONDS)) - .build() - .unwrap_or_default(); - Box::new(HttpUsageSender { - client, - url: config.share_usage_url.clone(), - }) - }; + let sender: Box = + Box::new(HttpUsageSender::new(config.share_usage_url.clone())); #[cfg(not(feature = "share-usage-transport"))] let sender: Box = Box::new(NoopUsageSender); @@ -836,20 +870,24 @@ fn add_to_queue( mut usage: ShareUsageData, timeout: Duration, ) -> std::result::Result<(), Discarded> { - let deadline = Instant::now() + timeout; + // A timeout too long to add to the clock, such as Duration::MAX, has no + // deadline, as with the standard library's own timeouts. + let deadline = Instant::now().checked_add(timeout); loop { match sender.try_send(usage) { Ok(()) => return Ok(()), Err(TrySendError::Disconnected(_)) => return Err(Discarded::Stopped), Err(TrySendError::Full(returned)) => { - let now = Instant::now(); - if now >= deadline { - return Err(Discarded::QueueFull); + let mut wait = Duration::from_millis(ADD_RETRY_INTERVAL_MILLISECONDS); + if let Some(deadline) = deadline { + let now = Instant::now(); + if now >= deadline { + return Err(Discarded::QueueFull); + } + wait = wait.min(deadline - now); } usage = returned; - std::thread::sleep( - Duration::from_millis(ADD_RETRY_INTERVAL_MILLISECONDS).min(deadline - now), - ); + std::thread::sleep(wait); } } } @@ -867,39 +905,34 @@ struct Worker { } impl Worker { - /// The consumer loop. Accumulates a batch then sends it. On channel - /// disconnection it flushes whatever remains and exits. + /// The consumer loop. Below the minimum entries it waits for each record + /// with no time limit, so an idle element uses no processor time. Once it + /// holds the minimum it takes more until none arrives within the take + /// timeout or it holds twice the minimum, then sends the message. Once the + /// element is dropped and every queued record has been received, it sends + /// what remains, even below the minimum, and exits. fn run(self) { let min_entries = self.config.minimum_entries_per_message; + let max_entries = min_entries.saturating_mul(MAXIMUM_ENTRIES_MULTIPLE); let take_timeout = self.config.take_timeout; - let mut batch: Vec = Vec::with_capacity(min_entries); - - loop { - match self.receiver.recv_timeout(take_timeout) { - Ok(item) => { - batch.push(item); - // Once a full batch is gathered, send it. - if batch.len() >= min_entries { - self.send_batch(std::mem::take(&mut batch)); - } - } - Err(RecvTimeoutError::Timeout) => { - // The producer paused. Keep accumulating; do not send a - // partial batch until shutdown, to keep batches efficient. - } - Err(RecvTimeoutError::Disconnected) => { - // The element is being dropped. Drain anything still - // buffered in the channel, then send the final (possibly - // partial) batch. - while let Ok(item) = self.receiver.try_recv() { - batch.push(item); - } - if !batch.is_empty() { - self.send_batch(std::mem::take(&mut batch)); - } - break; + let mut batch: Vec = Vec::with_capacity(max_entries); + + // `recv` fails only when the channel is closed and empty. + while let Ok(item) = self.receiver.recv() { + batch.push(item); + if batch.len() < min_entries { + continue; + } + while batch.len() < max_entries { + match self.receiver.recv_timeout(take_timeout) { + Ok(item) => batch.push(item), + Err(_) => break, } } + self.send_batch(std::mem::take(&mut batch)); + } + if !batch.is_empty() { + self.send_batch(batch); } } @@ -1575,6 +1608,32 @@ mod tests { assert_eq!(capture.devices(), 2); } + #[test] + fn a_message_takes_up_to_twice_the_minimum_within_the_take_timeout() { + // A take timeout far longer than processing five requests takes, so + // every record after the minimum arrives within it. + const TAKE_TIMEOUT_SECONDS: u64 = 5; + let capture = Capture::default(); + share( + ShareUsageConfig::builder() + .minimum_entries_per_message(2) + .take_timeout(Duration::from_secs(TAKE_TIMEOUT_SECONDS)) + .build(), + capture.clone(), + Vec::new(), + ["one", "two", "three", "four", "five"] + .into_iter() + .map(user_agent) + .collect(), + ); + let sizes: Vec = capture + .messages() + .iter() + .map(|xml| xml.matches("").count()) + .collect(); + assert_eq!(sizes, [4, 1], "twice the minimum, then the rest"); + } + #[test] fn a_full_queue_waits_for_the_add_timeout_then_discards() { const ADD_TIMEOUT_MILLISECONDS: u64 = 50; @@ -1642,6 +1701,36 @@ mod tests { )); } + #[test] + fn an_add_timeout_too_long_for_the_clock_waits_for_room() { + let (tx, rx) = sync_channel::(1); + assert!(add_to_queue(&tx, ShareUsageData::default(), Duration::MAX).is_ok()); + + // The queue of one is full, so the next add waits until it has room. + let taker = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(50)); + rx.recv().unwrap(); + rx + }); + assert!(add_to_queue(&tx, ShareUsageData::default(), Duration::MAX).is_ok()); + let rx = taker.join().unwrap(); + assert!(rx.try_recv().is_ok(), "the second record was queued"); + } + + #[test] + fn a_request_is_shared_with_the_longest_add_timeout() { + let capture = Capture::default(); + share( + ShareUsageConfig::builder() + .add_timeout(Duration::MAX) + .build(), + capture.clone(), + Vec::new(), + vec![user_agent("one")], + ); + assert_eq!(capture.devices(), 1); + } + #[test] fn a_repeated_failure_is_logged_once_until_the_interval_passes() { let throttle = LogThrottle::new(); diff --git a/pipeline-engines-fiftyone/tests/elements.rs b/pipeline-engines-fiftyone/tests/elements.rs index e45cf10..d00a6ad 100644 --- a/pipeline-engines-fiftyone/tests/elements.rs +++ b/pipeline-engines-fiftyone/tests/elements.rs @@ -97,6 +97,28 @@ fn sequence_treats_unparseable_sequence_as_one() { assert_eq!(sequence.sequence(), Some(1)); } +#[test] +fn sequence_wraps_after_the_largest_integer() { + // The sequence comes from the request, so the largest integer must not + // overflow when it is incremented. It wraps, as the .NET and Java + // pipelines do. + let pipeline = Pipeline::builder() + .add_element(Arc::new(SequenceElement::new())) + .build() + .unwrap(); + + for (raw, expected) in [(i64::MAX, i64::MIN), (i64::MAX - 1, i64::MAX)] { + let evidence = Evidence::builder() + .add("query.sequence", raw.to_string()) + .build(); + let mut data = pipeline.create_flow_data_with(evidence); + data.process().unwrap(); + + let sequence = data.get(SequenceElement::KEY).unwrap(); + assert_eq!(sequence.sequence(), Some(expected), "for {raw}"); + } +} + // -------------------------------------------------------------------------- // Set headers element // -------------------------------------------------------------------------- @@ -450,6 +472,44 @@ fn share_usage_deduplicates_identical_requests() { ); } +#[test] +fn share_usage_is_created_and_used_inside_an_async_runtime() { + // A web server usually builds its pipeline inside its async runtime, where + // building a blocking HTTP client panics, so the element must not build + // one as it is created. + let (url, bodies, request_count) = start_capture_server(); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + runtime.block_on(async { + let share = Arc::new(ShareUsageElement::new( + ShareUsageConfig::builder() + .share_usage_url(url) + .minimum_entries_per_message(1) + .build(), + )); + let pipeline = Pipeline::builder() + .add_element(Arc::clone(&share) as Arc) + .build() + .unwrap(); + pipeline + .create_flow_data_with( + Evidence::builder() + .add("header.user-agent", "async-agent") + .build(), + ) + .process() + .unwrap(); + wait_until(Duration::from_secs(5), || share.success_count() == 1); + assert_eq!(share.success_count(), 1, "the message was sent"); + drop(pipeline); + drop(share); + }); + + assert_eq!(request_count.load(Ordering::SeqCst), 1); + assert!(bodies.lock().unwrap()[0].contains("async-agent")); +} + /// Log records from every test in this binary, as "LEVEL message". static LOGGED: Mutex> = Mutex::new(Vec::new()); diff --git a/pipeline-engines-fiftyone/tests/idle_cpu.rs b/pipeline-engines-fiftyone/tests/idle_cpu.rs new file mode 100644 index 0000000..6f5b866 --- /dev/null +++ b/pipeline-engines-fiftyone/tests/idle_cpu.rs @@ -0,0 +1,119 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! An idle usage-sharing element uses no processor time, whatever its take +//! timeout. In a test binary of its own, so no other test's work is counted. + +#![cfg(any(windows, target_os = "linux"))] + +use std::time::Duration; + +use fiftyone_pipeline_engines_fiftyone::{ShareUsageConfig, ShareUsageElement}; + +/// How long the element is left with nothing to do. +const IDLE_MILLISECONDS: u64 = 1_000; + +/// The most processor time the process may use while the element is idle. A +/// thread that spins uses all of the idle time. +const MAX_BUSY_MILLISECONDS: u64 = 250; + +#[test] +fn an_idle_element_uses_no_processor_time() { + for take_timeout in [Duration::ZERO, Duration::from_millis(100)] { + let element = ShareUsageElement::new( + ShareUsageConfig::builder() + .take_timeout(take_timeout) + // Nothing is processed, so nothing is sent here. + .share_usage_url("http://127.0.0.1:9/") + .build(), + ); + let before = process_milliseconds(); + std::thread::sleep(Duration::from_millis(IDLE_MILLISECONDS)); + let busy = process_milliseconds() - before; + drop(element); + assert!( + busy < MAX_BUSY_MILLISECONDS, + "{busy} ms of processor time in {IDLE_MILLISECONDS} ms idle, with a take \ + timeout of {take_timeout:?}" + ); + } +} + +/// The processor time this process has used, user and kernel, in +/// milliseconds. +#[cfg(windows)] +fn process_milliseconds() -> u64 { + /// The `FILETIME` structure, in hundreds of nanoseconds. + #[repr(C)] + #[derive(Default)] + struct FileTime { + low: u32, + high: u32, + } + + #[link(name = "kernel32")] + extern "system" { + fn GetCurrentProcess() -> isize; + fn GetProcessTimes( + process: isize, + creation: *mut FileTime, + exit: *mut FileTime, + kernel: *mut FileTime, + user: *mut FileTime, + ) -> i32; + } + + let mut creation = FileTime::default(); + let mut exit = FileTime::default(); + let mut kernel = FileTime::default(); + let mut user = FileTime::default(); + // SAFETY: the pseudo handle of the current process is always valid, and + // the four pointers are to writable structures of the expected layout. + let succeeded = unsafe { + GetProcessTimes( + GetCurrentProcess(), + &mut creation, + &mut exit, + &mut kernel, + &mut user, + ) + }; + assert_ne!(succeeded, 0, "GetProcessTimes failed"); + let hundreds_of_nanoseconds = + |time: &FileTime| (u64::from(time.high) << 32) | u64::from(time.low); + (hundreds_of_nanoseconds(&kernel) + hundreds_of_nanoseconds(&user)) / 10_000 +} + +/// The processor time this process has used, user and kernel, in +/// milliseconds. +#[cfg(target_os = "linux")] +fn process_milliseconds() -> u64 { + // Fields 14 and 15 of /proc/self/stat are the user and kernel time in + // clock ticks, a hundredth of a second each. The process name in field 2 + // can hold spaces, so the fields are counted from its closing bracket, + // after which field 3 comes first. + let stat = std::fs::read_to_string("/proc/self/stat").expect("/proc/self/stat reads"); + let after_name = &stat[stat.rfind(')').expect("the name is bracketed") + 1..]; + let fields: Vec<&str> = after_name.split_whitespace().collect(); + let ticks = |field: usize| -> u64 { fields[field - 3].parse().expect("a tick count") }; + (ticks(14) + ticks(15)) * 10 +} diff --git a/pipeline-engines/src/data_file.rs b/pipeline-engines/src/data_file.rs index 087ec11..e325b21 100644 --- a/pipeline-engines/src/data_file.rs +++ b/pipeline-engines/src/data_file.rs @@ -104,9 +104,12 @@ pub struct DataFileConfiguration { /// polling. pub data_update_url: Option, /// Whether the service should periodically poll [`Self::data_update_url`]. + /// This is the master switch for automatic updates, so with it off there + /// is no file-system watcher either. pub automatic_updates_enabled: bool, /// Whether a file-system watcher should refresh the engine when the file at - /// [`Self::data_file_path`] changes on disk. + /// [`Self::data_file_path`] changes on disk. The watcher runs only when + /// [`Self::automatic_updates_enabled`] is on as well. pub file_system_watcher_enabled: bool, /// Whether the service should poll for an update once, synchronously, when /// the data file is first registered. @@ -231,13 +234,15 @@ impl DataFileConfigurationBuilder { self } - /// Enable or disable automatic remote polling. + /// Enable or disable automatic updates, the master switch for remote + /// polling and the file-system watcher. pub fn automatic_updates_enabled(mut self, enabled: bool) -> Self { self.config.automatic_updates_enabled = enabled; self } - /// Enable or disable the file-system watcher. + /// Enable or disable the file-system watcher, which runs only while + /// automatic updates are on. pub fn file_system_watcher_enabled(mut self, enabled: bool) -> Self { self.config.file_system_watcher_enabled = enabled; self diff --git a/pipeline-engines/src/data_update_service.rs b/pipeline-engines/src/data_update_service.rs index 6c6e83d..3739611 100644 --- a/pipeline-engines/src/data_update_service.rs +++ b/pipeline-engines/src/data_update_service.rs @@ -34,7 +34,9 @@ //! stagger many instances), sending an `If-Modified-Since` header so the //! server can answer `304 Not Modified` cheaply. //! - **Automatic update from file.** A [`notify`] file-system watcher refreshes -//! the engine when the data file changes on disk. +//! the engine when the data file changes on disk. Like polling, it runs only +//! while automatic updates are on, which the specification makes the master +//! switch for both. //! - **Programmatic update.** [`DataUpdateService::check_for_update`] triggers an //! immediate check on demand. //! @@ -46,6 +48,15 @@ //! `refresh` is required to be thread-safe with respect to concurrent `process` //! calls, so request handling continues with minimal impact during a refresh. //! +//! # Downloads +//! +//! A download is written beside the data file as it arrives, with its MD5 +//! taken as it is read, then decompressed into a second file there that +//! replaces the data file with a rename. The memory a download needs does not +//! grow with the size of the file, the HTTP client's timeout limits each wait +//! for more of the response rather than the whole download, and both files are +//! removed when an update fails part way. +//! //! # Scope of this baseline //! //! The service downloads, verifies, decompresses and applies updates, and runs @@ -55,15 +66,17 @@ //! and a `429` response surfaces the server's `Retry-After` in the error but the //! next poll still uses the configured interval rather than that delay. -use std::collections::hash_map::DefaultHasher; +use std::collections::hash_map::RandomState; use std::hash::{Hash, Hasher}; -use std::io::Read; -use std::path::Path; +use std::io::{BufReader, BufWriter, ErrorKind, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Condvar, Mutex}; use std::thread::JoinHandle; use std::time::{Duration, Instant}; use chrono::{DateTime, Utc}; +use fiftyone_pipeline_core::redact::redact_with; use md5::{Digest, Md5}; use crate::data_file::AspectEngineDataFile; @@ -141,13 +154,18 @@ impl DataUpdateService { /// - If the configuration sets `update_on_startup`, a synchronous remote /// poll runs now and its result is returned. A failed startup update is /// returned as an error so the caller can decide whether to proceed. - /// - If `file_system_watcher_enabled` and the file has a path, a watcher is - /// installed. + /// - If `automatic_updates_enabled` and `file_system_watcher_enabled` are + /// both on and the file has a path, a watcher is installed. Automatic + /// updates are the master switch for the watcher, as the specification + /// says. /// - The file is added to the scheduler's list so it is polled on its /// interval (when `automatic_updates_enabled`). /// /// Returns the status of the startup poll, or [`AutoUpdateStatus::NotNeeded`] - /// when no startup poll was requested. + /// when no startup poll was requested. A failed startup poll leaves the + /// engine on the data it loaded, as the specification says. It is logged + /// and returned as an error, and the file is registered all the same, so + /// the updates that follow still run. pub fn register( &self, engine: Arc, @@ -164,15 +182,28 @@ impl DataUpdateService { watcher: Mutex::new(None), }); - let mut startup_status = AutoUpdateStatus::NotNeeded; + let mut startup = Ok(AutoUpdateStatus::NotNeeded); - // Update on startup: poll once, synchronously. + // Update on startup: poll once, synchronously. A failure leaves the + // engine on the data it loaded, so the file is still registered for + // the updates that follow. if config.update_on_startup && !already_registered { - startup_status = self.check_for_update_from_url(®istration)?; + startup = self.check_for_update_from_url(®istration); + if let Err(error) = &startup { + log::warn!( + "An error occurred while updating the data file for {} on start-up. \ + The data it has is used. Error detail: {error}", + registration.engine.data_key() + ); + } } - // Install the file-system watcher if requested. - if config.file_system_watcher_enabled && data_file.data_file_path().is_some() { + // Install the file-system watcher if requested and automatic updates, + // its master switch, are on. + if config.automatic_updates_enabled + && config.file_system_watcher_enabled + && data_file.data_file_path().is_some() + { self.install_watcher(®istration); } @@ -196,7 +227,7 @@ impl DataUpdateService { self.ensure_scheduler(); self.inner.wake.notify_all(); - Ok(startup_status) + startup } /// Trigger an immediate update check for the engine's data file matching @@ -279,30 +310,32 @@ impl DataUpdateService { .unwrap_or_else(|_| Duration::from_secs(0)); return Instant::now() + wait - + self.randomisation(data_file.configuration().max_randomisation_seconds); + + Self::randomisation(data_file.configuration().max_randomisation_seconds); } } - Instant::now() + self.poll_interval(data_file) + Instant::now() + Self::poll_interval(data_file) } - /// The base polling interval for a file plus randomisation. - fn poll_interval(&self, data_file: &AspectEngineDataFile) -> Duration { + /// The polling interval for a file plus a random delay of up to its + /// maximum randomisation, as each poll waits. + fn poll_interval(data_file: &AspectEngineDataFile) -> Duration { let config = data_file.configuration(); Duration::from_secs(config.polling_interval_seconds) - + self.randomisation(config.max_randomisation_seconds) + + Self::randomisation(config.max_randomisation_seconds) } - /// A pseudo-random extra delay between zero and `max_seconds`. + /// A random delay between zero and `max_seconds`, so instances that start + /// together poll at different times. /// - /// A hash of the current time seeds the choice, which is enough to stagger - /// many instances without pulling in a random-number-generator dependency. - fn randomisation(&self, max_seconds: u64) -> Duration { + /// Each `RandomState` has its own keys, which the standard library draws + /// from the operating system, so the delay differs between calls and + /// between processes without a random number generator dependency. + fn randomisation(max_seconds: u64) -> Duration { if max_seconds == 0 { return Duration::from_secs(0); } - let mut hasher = DefaultHasher::new(); - Instant::now().elapsed().as_nanos().hash(&mut hasher); - std::thread::current().id().hash(&mut hasher); + let mut hasher = std::hash::BuildHasher::build_hasher(&RandomState::new()); + std::time::SystemTime::now().hash(&mut hasher); Duration::from_secs(hasher.finish() % max_seconds) } @@ -331,6 +364,10 @@ impl DataUpdateService { /// Install a file-system watcher that refreshes the engine when the data /// file changes on disk. + /// + /// The registration owns the watcher, so the watcher's callback holds the + /// registration weakly. A strong reference would keep the registration, + /// its engine and the watcher's thread alive after the service is dropped. fn install_watcher(&self, registration: &Arc) { use notify::{RecursiveMode, Watcher}; @@ -338,24 +375,29 @@ impl DataUpdateService { Some(p) => p.to_path_buf(), None => return, }; + let file_name = path.file_name().map(ToOwned::to_owned); - let reg = Arc::clone(registration); - // The service is held alive by the caller for as long as updates are - // wanted, so the closure captures only the registration (which owns the - // engine and data file) plus a clone of the HTTP client is not needed - // here, the watcher only refreshes from the file on disk. - let client = self.client.clone(); - let service = ServiceHandle { - client, - inner: Arc::clone(&self.inner), - }; + // The file as it is now is the one the engine loaded, so only a later + // change refreshes it. + if registration.data_file.last_applied_modified().is_none() { + if let Some(modified) = file_modified_time(&path) { + registration.data_file.set_last_applied_modified(modified); + } + } + let reg = Arc::downgrade(registration); let watcher = notify::recommended_watcher(move |res: notify::Result| { - if let Ok(event) = res { - if event.kind.is_modify() || event.kind.is_create() { - // Debounce duplicate events by comparing the file's - // last-modified time with the last we applied. - service.on_file_changed(®); + let Ok(event) = res else { + return; + }; + // The watch is on the folder, so skip events for other files in it. + let names_the_file = event + .paths + .iter() + .any(|changed| changed.file_name() == file_name.as_deref()); + if names_the_file && (event.kind.is_modify() || event.kind.is_create()) { + if let Some(reg) = reg.upgrade() { + ServiceHandle::on_file_changed(®); } } }); @@ -513,12 +555,7 @@ impl ServiceHandle { // Poll now. A failure is logged, and the next interval retries. let result = self.check_for_update_from_url(®istration); log_poll(®istration, result); - let interval = Duration::from_secs( - registration - .data_file - .configuration() - .polling_interval_seconds, - ); + let interval = DataUpdateService::poll_interval(®istration.data_file); let next = now + interval; *registration .next_poll @@ -535,7 +572,7 @@ impl ServiceHandle { /// Handle a file-system-watcher event by refreshing the engine from disk if /// the file is genuinely newer than the one already applied. - fn on_file_changed(&self, registration: &Arc) { + fn on_file_changed(registration: &Registration) { let path = match registration.data_file.data_file_path() { Some(p) => p.to_path_buf(), None => return, @@ -567,8 +604,32 @@ impl ServiceHandle { } } + /// True once the service has been dropped. + fn shutting_down(&self) -> bool { + *self + .inner + .shutting_down + .lock() + .unwrap_or_else(|e| e.into_inner()) + } + /// Poll the remote update URL, verify, decompress, write and refresh. + /// + /// Every failing exit comes through here, so no error leaves the service + /// carrying the data file's licence keys, however the poll failed. fn check_for_update_from_url(&self, registration: &Arc) -> DataUpdateResult { + let keys = ®istration + .data_file + .configuration() + .data_update_license_keys; + self.poll_update_url(registration).map_err(|mut error| { + error.message = redact_with(&error.message, keys).into_owned(); + error + }) + } + + /// Poll the remote update URL, verify, decompress, write and refresh. + fn poll_update_url(&self, registration: &Arc) -> DataUpdateResult { let data_file = ®istration.data_file; let config = data_file.configuration(); let url = match data_file.data_update_url() { @@ -597,7 +658,10 @@ impl ServiceHandle { let response = request.send().map_err(|e| { DataUpdateError::new( AutoUpdateStatus::HttpsError, - format!("error accessing data update service at '{url}': {e}"), + format!( + "error accessing data update service at '{url}': {}", + request_error_message(e) + ), ) })?; @@ -642,73 +706,70 @@ impl ServiceHandle { .and_then(|v| v.to_str().ok()) .map(str::to_owned); - let compressed = response.bytes().map_err(|e| { - DataUpdateError::new( - AutoUpdateStatus::StreamError, - format!("error reading data update response from '{url}': {e}"), - ) - })?; + // The body is read as it arrives, so the client's timeout limits each + // wait for more of it rather than the whole download, and its MD5 is + // taken as it is read. Reading stops once the service is dropped, so + // dropping it does not wait for a long download to finish. + let stop = || self.shutting_down(); + let mut body = Md5Reader::new(Stoppable { + inner: response, + stop: &stop, + }); - // Verify integrity if requested and a hash was supplied. - if config.verify_md5 { - if let Some(expected) = expected_md5.as_deref() { - let actual = md5_hex(&compressed); - if !actual.eq_ignore_ascii_case(expected) { - return Err(DataUpdateError::new( - AutoUpdateStatus::Md5ValidationFailed, - format!( - "integrity check failed: server MD5 '{expected}' does \ - not match downloaded '{actual}'" - ), - )); - } - } - } + let Some(path) = data_file.data_file_path() else { + // Memory data source: the engine takes the data as bytes. + let mut compressed = Vec::new(); + body.read_to_end(&mut compressed) + .map_err(|e| read_error(&url, e))?; + verify_md5(config.verify_md5, expected_md5.as_deref(), &body.hex())?; + let data = if config.decompress_content { + gzip_decompress(&compressed).map_err(|e| decompress_error(&url, e))? + } else { + compressed + }; + registration + .engine + .refresh_from_memory(Some(data_file.identifier()), &data) + .map_err(|e| { + DataUpdateError::new(AutoUpdateStatus::RefreshFailed, e.to_string()) + })?; + data_file.set_data_published(Utc::now()); + return Ok(AutoUpdateStatus::Success); + }; - // Decompress if requested. - let uncompressed = if config.decompress_content { - gzip_decompress(&compressed).map_err(|e| { - DataUpdateError::new( - AutoUpdateStatus::StreamError, - format!("error decompressing data update from '{url}': {e}"), - ) - })? + // File data source: the download, and the file it decompresses to, are + // written beside the data file so the new file replaces it with a + // rename. Each is removed when it goes out of scope, so an update that + // fails part way leaves nothing behind. + let download = TempFile::beside(path, "download"); + save_body(&mut body, download.path(), &url)?; + verify_md5(config.verify_md5, expected_md5.as_deref(), &body.hex())?; + let new_file = if config.decompress_content { + let decompressed = TempFile::beside(path, "new"); + gunzip_file(download.path(), decompressed.path(), &stop) + .map_err(|e| decompress_error(&url, e))?; + drop(download); + decompressed } else { - compressed.to_vec() + download }; - self.apply_update(registration, &uncompressed) - } - - /// Apply downloaded, decompressed data to the engine and (when on disk) the - /// file. - fn apply_update(&self, registration: &Arc, data: &[u8]) -> DataUpdateResult { - let data_file = ®istration.data_file; - let engine = ®istration.engine; - let identifier = Some(data_file.identifier()); - - if let Some(path) = data_file.data_file_path() { - // File data source: write the new bytes, then refresh from the - // file. Pause the watcher's effect by recording the modified time - // we are about to create as already-applied. - write_atomically(path, data).map_err(|e| { - DataUpdateError::new( - AutoUpdateStatus::NewFileCannotRename, - format!("error writing data file '{}': {e}", path.display()), - ) - })?; - if let Some(modified) = file_modified_time(path) { - data_file.set_last_applied_modified(modified); - } - engine.refresh(identifier).map_err(|e| { - DataUpdateError::new(AutoUpdateStatus::RefreshFailed, e.to_string()) - })?; - } else { - // Memory data source: hand the bytes to the engine directly. - engine.refresh_from_memory(identifier, data).map_err(|e| { - DataUpdateError::new(AutoUpdateStatus::RefreshFailed, e.to_string()) - })?; + // Record the new file's modified time before it replaces the data + // file, which the rename keeps, so the watcher does not refresh the + // engine a second time. + if let Some(modified) = file_modified_time(new_file.path()) { + data_file.set_last_applied_modified(modified); } + std::fs::rename(new_file.path(), path).map_err(|e| { + DataUpdateError::new( + AutoUpdateStatus::NewFileCannotRename, + format!("error replacing data file '{}': {e}", path.display()), + ) + })?; + registration + .engine + .refresh(Some(data_file.identifier())) + .map_err(|e| DataUpdateError::new(AutoUpdateStatus::RefreshFailed, e.to_string()))?; // Record that fresh data is now in use. We mark "now" as a conservative // lower bound so a subsequent If-Modified-Since is not stale. The true @@ -739,6 +800,21 @@ fn log_poll(registration: &Arc, result: DataUpdateResult) { } } +/// The message of a failed request and each of its causes, without the URL +/// the request error names. That URL is the formatted one, whose query +/// carries the licence keys. +fn request_error_message(error: reqwest::Error) -> String { + let error = error.without_url(); + let mut message = error.to_string(); + let mut cause = std::error::Error::source(&error); + while let Some(source) = cause { + message.push_str(": "); + message.push_str(&source.to_string()); + cause = source.source(); + } + message +} + /// Format a timestamp as an HTTP-date for the `If-Modified-Since` header. fn httpdate_format(when: DateTime) -> String { // RFC 7231 IMF-fixdate, for example "Sun, 06 Nov 1994 08:49:37 GMT". @@ -752,18 +828,6 @@ fn file_modified_time(path: &Path) -> Option> { Some(DateTime::::from(modified)) } -/// Compute the lowercase hex MD5 of a byte slice. -fn md5_hex(data: &[u8]) -> String { - let mut hasher = Md5::new(); - hasher.update(data); - let digest = hasher.finalize(); - let mut out = String::with_capacity(digest.len() * 2); - for byte in digest { - out.push_str(&format!("{byte:02x}")); - } - out -} - /// Decompress a gzip byte slice into the original bytes. fn gzip_decompress(data: &[u8]) -> std::io::Result> { let mut decoder = flate2::read::GzDecoder::new(data); @@ -772,13 +836,184 @@ fn gzip_decompress(data: &[u8]) -> std::io::Result> { Ok(out) } -/// Write bytes to a path via a sibling temporary file and a rename, so a reader -/// never sees a half-written file. -fn write_atomically(path: &Path, data: &[u8]) -> std::io::Result<()> { - let tmp = path.with_extension("tmp-download"); - std::fs::write(&tmp, data)?; - std::fs::rename(&tmp, path)?; - Ok(()) +/// The size of each read when a download is written to disk. +const COPY_BUFFER_BYTES: usize = 64 * 1024; + +/// Reads through to another reader, taking the MD5 of every byte read. +struct Md5Reader { + inner: R, + hasher: Md5, +} + +impl Md5Reader { + fn new(inner: R) -> Self { + Md5Reader { + inner, + hasher: Md5::new(), + } + } + + /// The lowercase hex MD5 of the bytes read so far. + fn hex(&self) -> String { + self.hasher + .clone() + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() + } +} + +impl Read for Md5Reader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let read = self.inner.read(buffer)?; + self.hasher.update(&buffer[..read]); + Ok(read) + } +} + +/// Reads through to another reader, and fails once `stop` answers true, so a +/// download or decompression in progress ends when the service is dropped. +struct Stoppable<'a, R> { + inner: R, + stop: &'a dyn Fn() -> bool, +} + +impl Read for Stoppable<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if (self.stop)() { + return Err(std::io::Error::other("the data update service was dropped")); + } + self.inner.read(buffer) + } +} + +/// A file beside the data file that is removed when it is dropped, unless it +/// has been renamed away first. +struct TempFile { + path: PathBuf, +} + +impl TempFile { + /// A new name beside `path`, which no other update in this process uses. + fn beside(path: &Path, suffix: &str) -> TempFile { + static NEXT: AtomicU64 = AtomicU64::new(0); + let mut name = path + .file_name() + .map(std::ffi::OsString::from) + .unwrap_or_default(); + name.push(format!( + ".{}.{}.{suffix}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + TempFile { + path: path.with_file_name(name), + } + } + + fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for TempFile { + fn drop(&mut self) { + // A file renamed into place is no longer here, so this finds nothing. + let _ = std::fs::remove_file(&self.path); + } +} + +/// Write everything `body` reads into a new file at `path`, flushed to disk. +/// A failed read names the configured URL, and a failed write names the file. +fn save_body(body: &mut impl Read, path: &Path, url: &str) -> Result<(), DataUpdateError> { + let write_error = |error: std::io::Error| { + DataUpdateError::new( + AutoUpdateStatus::StreamError, + format!("error writing data update to '{}': {error}", path.display()), + ) + }; + let mut file = std::fs::File::create(path).map_err(write_error)?; + let mut buffer = vec![0u8; COPY_BUFFER_BYTES]; + loop { + let read = match body.read(&mut buffer) { + Ok(0) => break, + Ok(read) => read, + Err(error) if error.kind() == ErrorKind::Interrupted => continue, + Err(error) => return Err(read_error(url, error)), + }; + file.write_all(&buffer[..read]).map_err(write_error)?; + } + file.sync_all().map_err(write_error) +} + +/// Decompress the gzip file at `from` into a new file at `to`, flushed to disk. +fn gunzip_file(from: &Path, to: &Path, stop: &dyn Fn() -> bool) -> std::io::Result<()> { + let mut decoder = flate2::read::GzDecoder::new(Stoppable { + inner: BufReader::new(std::fs::File::open(from)?), + stop, + }); + let mut output = BufWriter::with_capacity(COPY_BUFFER_BYTES, std::fs::File::create(to)?); + std::io::copy(&mut decoder, &mut output)?; + output + .into_inner() + .map_err(|error| error.into_error())? + .sync_all() +} + +/// Check the MD5 of the download against the one the server sent, when the +/// check is on and the server sent one. +fn verify_md5(check: bool, expected: Option<&str>, actual: &str) -> Result<(), DataUpdateError> { + match expected { + Some(expected) if check && !actual.eq_ignore_ascii_case(expected) => { + Err(DataUpdateError::new( + AutoUpdateStatus::Md5ValidationFailed, + format!( + "integrity check failed: server MD5 '{expected}' does not match \ + downloaded '{actual}'" + ), + )) + } + _ => Ok(()), + } +} + +/// The error for a response body that could not be read. +fn read_error(url: &str, error: std::io::Error) -> DataUpdateError { + DataUpdateError::new( + AutoUpdateStatus::StreamError, + format!( + "error reading data update response from '{url}': {}", + read_error_message(error) + ), + ) +} + +/// The error for a download that could not be decompressed. +fn decompress_error(url: &str, error: std::io::Error) -> DataUpdateError { + DataUpdateError::new( + AutoUpdateStatus::StreamError, + format!("error decompressing data update from '{url}': {error}"), + ) +} + +/// The message of an error reading a response. A request error inside it +/// names the formatted URL, so its message comes from +/// [`request_error_message`]. +fn read_error_message(error: std::io::Error) -> String { + if !error + .get_ref() + .is_some_and(|inner| inner.is::()) + { + return error.to_string(); + } + match error + .into_inner() + .map(|inner| inner.downcast::()) + { + Some(Ok(request)) => request_error_message(*request), + _ => String::from("the response could not be read"), + } } #[cfg(test)] @@ -788,7 +1023,9 @@ mod tests { #[test] fn md5_matches_known_vector() { // MD5("abc") = 900150983cd24fb0d6963f7d28e17f72 - assert_eq!(md5_hex(b"abc"), "900150983cd24fb0d6963f7d28e17f72"); + let mut reader = Md5Reader::new(&b"abc"[..]); + std::io::copy(&mut reader, &mut std::io::sink()).unwrap(); + assert_eq!(reader.hex(), "900150983cd24fb0d6963f7d28e17f72"); } #[test] @@ -819,4 +1056,140 @@ mod tests { let service = DataUpdateService::new(); assert_eq!(service.registered_count(), 0); } + + /// An engine with one data file and no update URL, so a poll of it makes + /// no request. + struct IdleEngine { + filter: fiftyone_pipeline_core::EvidenceKeyFilterWhitelist, + data_files: Vec>, + } + + impl fiftyone_pipeline_core::FlowElement for IdleEngine { + fn process( + &self, + _data: &mut fiftyone_pipeline_core::FlowData, + ) -> fiftyone_pipeline_core::Result<()> { + Ok(()) + } + fn data_key(&self) -> &str { + "idle" + } + fn evidence_key_filter(&self) -> &dyn fiftyone_pipeline_core::EvidenceKeyFilter { + &self.filter + } + fn properties(&self) -> &[fiftyone_pipeline_core::PropertyMetaData] { + &[] + } + } + + impl crate::AspectEngine for IdleEngine { + fn data_source_tier(&self) -> &str { + "Lite" + } + fn aspect_properties(&self) -> &[crate::AspectPropertyMetaData] { + &[] + } + } + + impl OnPremiseAspectEngine for IdleEngine { + fn data_files(&self) -> &[Arc] { + &self.data_files + } + fn refresh(&self, _identifier: Option<&str>) -> fiftyone_pipeline_core::Result<()> { + Ok(()) + } + } + + /// A data file with no update URL that polls every `interval_seconds` + /// plus up to `max_randomisation_seconds`. + fn polled_file(interval_seconds: u64, max_randomisation_seconds: u64) -> AspectEngineDataFile { + AspectEngineDataFile::new( + crate::DataFileConfiguration::memory_builder() + .polling_interval_seconds(interval_seconds) + .max_randomisation_seconds(max_randomisation_seconds) + .build(), + ) + } + + /// The delay before the first poll is drawn at random, so instances that + /// start together poll at different times. 200 draws from 600 seconds + /// give about 170 different delays. + #[test] + fn the_first_poll_is_delayed_at_random() { + let service = DataUpdateService::new(); + let data_file = polled_file(0, 600); + let delays: std::collections::BTreeSet = (0..200) + .map(|_| { + let start = Instant::now(); + service + .initial_poll_due(&data_file) + .duration_since(start) + .as_secs() + }) + .collect(); + eprintln!("{} different delays in 200", delays.len()); + assert!( + delays.len() > 100, + "{} different delays in 200: {delays:?}", + delays.len() + ); + } + + /// Each poll after the first also waits the polling interval plus a + /// random delay, as the specification says. + #[test] + fn each_later_poll_adds_a_random_delay() { + const INTERVAL_SECONDS: u64 = 1000; + let handle = ServiceHandle { + client: reqwest::blocking::Client::new(), + inner: Arc::new(Inner { + registrations: Mutex::new(Vec::new()), + wake: Condvar::new(), + shutting_down: Mutex::new(false), + }), + }; + let due = Instant::now(); + for _ in 0..5 { + let data_file = Arc::new(polled_file(INTERVAL_SECONDS, INTERVAL_SECONDS)); + let engine = Arc::new(IdleEngine { + filter: fiftyone_pipeline_core::EvidenceKeyFilterWhitelist::new( + Vec::::new(), + ), + data_files: vec![Arc::clone(&data_file)], + }); + handle + .inner + .registrations + .lock() + .unwrap() + .push(Arc::new(Registration { + engine, + data_file, + next_poll: Mutex::new(Some(due)), + watcher: Mutex::new(None), + })); + } + + handle.run_due_polls(); + + // Without a delay every next poll is due at most one interval after + // the polls ran. Five delays of under two seconds out of a thousand + // happen once in about 3 * 10^13 runs. + let undelayed = Instant::now() + Duration::from_secs(INTERVAL_SECONDS + 1); + let delayed = handle + .inner + .registrations + .lock() + .unwrap() + .iter() + .filter(|registration| { + registration + .next_poll + .lock() + .unwrap() + .is_some_and(|next| next > undelayed) + }) + .count(); + assert!(delayed > 0, "no poll after the first added a delay"); + } } diff --git a/pipeline-engines/src/on_premise_aspect_engine.rs b/pipeline-engines/src/on_premise_aspect_engine.rs index 73bc874..51d97cc 100644 --- a/pipeline-engines/src/on_premise_aspect_engine.rs +++ b/pipeline-engines/src/on_premise_aspect_engine.rs @@ -99,8 +99,9 @@ pub trait OnPremiseAspectEngine: AspectEngine { } /// The directory this engine uses for temporary copies of its data files, - /// if any. Used by the update service when downloading to a scratch - /// location before swapping the live file. + /// if any. The data update service does not read it, as it writes each + /// download beside the data file, so that the new file replaces the old + /// one with a rename. fn temp_data_dir(&self) -> Option<&std::path::Path> { None } diff --git a/pipeline-engines/tests/download_memory.rs b/pipeline-engines/tests/download_memory.rs new file mode 100644 index 0000000..0561ddd --- /dev/null +++ b/pipeline-engines/tests/download_memory.rs @@ -0,0 +1,207 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! A data file update is written to disk as it arrives, so the memory a +//! download needs does not grow with the size of the file. +//! +//! The test counts every allocation through its own global allocator, which +//! the other tests in the same binary would share, so it is the only test in +//! this file. +#![cfg(feature = "data-update")] + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use fiftyone_pipeline_core::{ + EvidenceKeyFilter, EvidenceKeyFilterWhitelist, FlowData, FlowElement, PropertyMetaData, Result, +}; +use fiftyone_pipeline_engines::{ + AspectEngine, AspectEngineDataFile, AspectPropertyMetaData, AutoUpdateStatus, + DataFileConfiguration, DataUpdateService, OnPremiseAspectEngine, +}; + +/// The system allocator, counting the bytes allocated now and the most +/// allocated at once. +struct Counting; + +static ALLOCATED: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let pointer = System.alloc(layout); + if !pointer.is_null() { + let now = ALLOCATED.fetch_add(layout.size(), Ordering::SeqCst) + layout.size(); + PEAK.fetch_max(now, Ordering::SeqCst); + } + pointer + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + System.dealloc(pointer, layout); + ALLOCATED.fetch_sub(layout.size(), Ordering::SeqCst); + } +} + +#[global_allocator] +static GLOBAL: Counting = Counting; + +/// The size of the data file the server sends. +const FILE_BYTES: usize = 32 * 1024 * 1024; + +/// The most a download may add to the memory in use. Well under the size of +/// the file, so holding the file in memory fails. +const MOST_ADDED_BYTES: usize = 8 * 1024 * 1024; + +/// An engine that reads nothing when it refreshes. +struct QuietEngine { + filter: EvidenceKeyFilterWhitelist, + data_files: Vec>, +} + +impl FlowElement for QuietEngine { + fn process(&self, _data: &mut FlowData) -> Result<()> { + Ok(()) + } + fn data_key(&self) -> &str { + "quiet" + } + fn evidence_key_filter(&self) -> &dyn EvidenceKeyFilter { + &self.filter + } + fn properties(&self) -> &[PropertyMetaData] { + &[] + } +} + +impl AspectEngine for QuietEngine { + fn data_source_tier(&self) -> &str { + "Lite" + } + fn aspect_properties(&self) -> &[AspectPropertyMetaData] { + &[] + } +} + +impl OnPremiseAspectEngine for QuietEngine { + fn data_files(&self) -> &[Arc] { + &self.data_files + } + fn refresh(&self, _identifier: Option<&str>) -> Result<()> { + Ok(()) + } +} + +/// Serve `body` once, as gzip when `gzip` is set, and return the URL. +fn serve_once(body: Vec, gzip: bool) -> (String, std::thread::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/datafile", listener.local_addr().unwrap()); + let handle = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = stream.read(&mut request); + let encoding = if gzip { + "Content-Type: application/gzip\r\n" + } else { + "" + }; + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\n{encoding}Connection: close\r\n\r\n", + body.len() + ); + stream.write_all(head.as_bytes()).unwrap(); + stream.write_all(&body).unwrap(); + }); + (url, handle) +} + +/// The most memory a check for an update added while downloading `body`. +fn memory_added_by_update(body: Vec, gzip: bool, test: &str) -> usize { + let dir = std::env::temp_dir().join(format!( + "fiftyone-download-memory-{test}-{}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("device.dat"); + std::fs::write(&path, b"version 1").unwrap(); + + let (url, server) = serve_once(body, gzip); + let config = DataFileConfiguration::builder(&path) + .data_update_url(url) + .automatic_updates_enabled(false) + .file_system_watcher_enabled(false) + .decompress_content(gzip) + .build(); + let engine: Arc = Arc::new(QuietEngine { + filter: EvidenceKeyFilterWhitelist::new(Vec::::new()), + data_files: vec![Arc::new(AspectEngineDataFile::new(config))], + }); + let service = DataUpdateService::new(); + + let before = ALLOCATED.load(Ordering::SeqCst); + PEAK.store(before, Ordering::SeqCst); + let status = service.check_for_update(&engine, None); + let added = PEAK.load(Ordering::SeqCst) - before; + eprintln!("{test}: a {FILE_BYTES} byte update added {added} bytes at most"); + + server.join().unwrap(); + assert_eq!( + status.map_err(|error| error.to_string()), + Ok(AutoUpdateStatus::Success) + ); + assert_eq!(std::fs::metadata(&path).unwrap().len(), FILE_BYTES as u64); + let _ = std::fs::remove_dir_all(&dir); + added +} + +#[test] +fn downloading_an_update_holds_little_of_the_file_in_memory() { + // Bytes that do not repeat, so the plain file is as large to send as it + // is to keep. + let plain: Vec = (0..FILE_BYTES) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8) + .collect(); + let added = memory_added_by_update(plain, false, "plain"); + assert!( + added < MOST_ADDED_BYTES, + "a {FILE_BYTES} byte download added {added} bytes" + ); + + // A file that compresses well, so most of its size appears only once it + // is decompressed. + let compressed = { + use flate2::write::GzEncoder; + use flate2::Compression; + let mut encoder = GzEncoder::new(Vec::new(), Compression::fast()); + encoder.write_all(&vec![51u8; FILE_BYTES]).unwrap(); + encoder.finish().unwrap() + }; + let added = memory_added_by_update(compressed, true, "gzip"); + assert!( + added < MOST_ADDED_BYTES, + "a {FILE_BYTES} byte file decompressed while adding {added} bytes" + ); +} diff --git a/pipeline-engines/tests/file_watcher.rs b/pipeline-engines/tests/file_watcher.rs new file mode 100644 index 0000000..17d9d9a --- /dev/null +++ b/pipeline-engines/tests/file_watcher.rs @@ -0,0 +1,198 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Tests of the data update service's file system watcher, which refreshes an +//! engine when its data file changes on disk. +#![cfg(feature = "data-update")] + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use fiftyone_pipeline_core::{ + EvidenceKeyFilter, EvidenceKeyFilterWhitelist, FlowData, FlowElement, PropertyMetaData, Result, +}; +use fiftyone_pipeline_engines::{ + AspectEngine, AspectEngineDataFile, AspectPropertyMetaData, DataFileConfiguration, + DataUpdateService, OnPremiseAspectEngine, +}; + +/// How long a test waits for a refresh it expects. +const REFRESH_WAIT: Duration = Duration::from_secs(10); + +/// How long a test waits before deciding a refresh it does not expect has +/// not happened. +const QUIET_WAIT: Duration = Duration::from_secs(2); + +/// How long the watcher is given to start before the file changes. +const WATCHER_START: Duration = Duration::from_millis(300); + +/// An engine that counts its refreshes. +struct CountingEngine { + filter: EvidenceKeyFilterWhitelist, + data_files: Vec>, + refreshes: Arc, +} + +impl FlowElement for CountingEngine { + fn process(&self, _data: &mut FlowData) -> Result<()> { + Ok(()) + } + fn data_key(&self) -> &str { + "counting" + } + fn evidence_key_filter(&self) -> &dyn EvidenceKeyFilter { + &self.filter + } + fn properties(&self) -> &[PropertyMetaData] { + &[] + } +} + +impl AspectEngine for CountingEngine { + fn data_source_tier(&self) -> &str { + "Lite" + } + fn aspect_properties(&self) -> &[AspectPropertyMetaData] { + &[] + } +} + +impl OnPremiseAspectEngine for CountingEngine { + fn data_files(&self) -> &[Arc] { + &self.data_files + } + fn refresh(&self, _identifier: Option<&str>) -> Result<()> { + self.refreshes.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +/// A new, empty folder for one test, holding a data file. +fn folder(test: &str) -> (PathBuf, PathBuf) { + let dir = std::env::temp_dir().join(format!("fiftyone-watcher-{test}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("device.dat"); + std::fs::write(&path, b"version 1").unwrap(); + (dir, path) +} + +/// An engine whose data file at `path` is watched, with automatic updates on +/// or off. There is no update URL, so a poll makes no request. +fn watched_engine(path: &Path, automatic_updates: bool) -> Arc { + let config = DataFileConfiguration::builder(path) + .automatic_updates_enabled(automatic_updates) + .file_system_watcher_enabled(true) + .build(); + Arc::new(CountingEngine { + filter: EvidenceKeyFilterWhitelist::new(Vec::::new()), + data_files: vec![Arc::new(AspectEngineDataFile::new(config))], + refreshes: Arc::new(AtomicUsize::new(0)), + }) +} + +fn register(service: &DataUpdateService, engine: &Arc) { + let data_file = Arc::clone(&engine.data_files[0]); + let engine: Arc = engine.clone(); + service.register(engine, data_file).unwrap(); + std::thread::sleep(WATCHER_START); +} + +/// The refresh count once it reaches `at_least`, or after `limit`. +fn refreshes_after(count: &AtomicUsize, at_least: usize, limit: Duration) -> usize { + let start = Instant::now(); + while count.load(Ordering::SeqCst) < at_least && start.elapsed() < limit { + std::thread::sleep(Duration::from_millis(20)); + } + count.load(Ordering::SeqCst) +} + +/// Dropping the service releases the engines registered with it and stops +/// their watchers, so the data set is freed and a later change to the file +/// refreshes nothing. +#[test] +fn dropping_the_service_releases_the_engine_and_stops_the_watcher() { + let (dir, path) = folder("drop"); + let engine = watched_engine(&path, true); + let refreshes = Arc::clone(&engine.refreshes); + let service = DataUpdateService::new(); + register(&service, &engine); + let released = Arc::downgrade(&engine); + + drop(engine); + drop(service); + + assert!( + released.upgrade().is_none(), + "the engine is still alive after the service and its caller dropped it" + ); + std::fs::write(&path, b"version 2").unwrap(); + assert_eq!(refreshes_after(&refreshes, 1, QUIET_WAIT), 0); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Automatic updates are the master switch for the watcher, as the +/// specification says. With them off a change to the file refreshes nothing, +/// and with them on it refreshes the engine. +#[test] +fn the_watcher_runs_only_with_automatic_updates_on() { + let (dir, path) = folder("switch"); + let service = DataUpdateService::new(); + + let off = watched_engine(&path, false); + register(&service, &off); + std::fs::write(&path, b"version 2").unwrap(); + assert_eq!( + refreshes_after(&off.refreshes, 1, QUIET_WAIT), + 0, + "the watcher refreshed an engine with automatic updates off" + ); + + let on = watched_engine(&path, true); + register(&service, &on); + std::fs::write(&path, b"version 3").unwrap(); + assert!(refreshes_after(&on.refreshes, 1, REFRESH_WAIT) >= 1); + let _ = std::fs::remove_dir_all(&dir); +} + +/// The watcher refreshes the engine when its data file changes, and not when +/// another file in the same folder does. +#[test] +fn the_watcher_ignores_other_files_in_the_folder() { + let (dir, path) = folder("others"); + let service = DataUpdateService::new(); + let engine = watched_engine(&path, true); + register(&service, &engine); + + std::fs::write(dir.join("application.log"), b"a line").unwrap(); + assert_eq!( + refreshes_after(&engine.refreshes, 1, QUIET_WAIT), + 0, + "another file in the folder refreshed the engine" + ); + + std::fs::write(&path, b"version 2").unwrap(); + assert!(refreshes_after(&engine.refreshes, 1, REFRESH_WAIT) >= 1); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/pipeline-engines/tests/remote_refresh.rs b/pipeline-engines/tests/remote_refresh.rs index df5478b..96ad6c1 100644 --- a/pipeline-engines/tests/remote_refresh.rs +++ b/pipeline-engines/tests/remote_refresh.rs @@ -32,7 +32,8 @@ //! built, so neither is this test. #![cfg(feature = "data-update")] -use std::io::Write; +use std::io::{Read, Write}; +use std::net::TcpListener; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -42,7 +43,7 @@ use fiftyone_pipeline_core::{ }; use fiftyone_pipeline_engines::{ AspectDataBase, AspectEngine, AspectEngineDataFile, AspectPropertyMetaData, AutoUpdateStatus, - DataFileConfiguration, DataUpdateService, OnPremiseAspectEngine, + DataFileConfiguration, DataUpdateService, FiftyOneUrlFormatter, OnPremiseAspectEngine, }; /// A minimal on-premise engine that records every refresh and the bytes it was @@ -259,3 +260,237 @@ fn update_from_memory_goes_to_engine() { assert_eq!(status, AutoUpdateStatus::Success); assert_eq!(*disk.lock().unwrap(), bytes); } + +/// The licence key the Distributor formatter writes into the update URL. +const LICENCE_KEY: &str = "LICENCE-KEY-NOT-FOR-LOGS"; + +/// An engine whose data file polls `url` through the Distributor formatter, +/// so each request carries [`LICENCE_KEY`] in its query. +fn engine_polling(url: &str, dir: &std::path::Path) -> Arc { + let path = dir.join("device.dat"); + std::fs::write(&path, b"current data").unwrap(); + let config = DataFileConfiguration::builder(&path) + .data_update_url(url) + .data_update_license_keys([LICENCE_KEY]) + .url_formatter(Some(Arc::new(FiftyOneUrlFormatter))) + .automatic_updates_enabled(false) + .file_system_watcher_enabled(false) + .build(); + Arc::new(TestEngine::new(Arc::new(AspectEngineDataFile::new(config)))) +} + +/// A poll that cannot connect fails with a message that names the configured +/// URL and the cause, and not the licence key the request carried. +#[test] +fn a_failed_connection_keeps_the_licence_key_out_of_the_error() { + // Nothing listens on a port once its listener is dropped. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/api/v2/download", listener.local_addr().unwrap()); + drop(listener); + let dir = std::env::temp_dir().join(format!("fiftyone-dut-key-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let engine = engine_polling(&url, &dir); + + let error = DataUpdateService::new() + .check_for_update(&engine, None) + .expect_err("nothing is listening"); + let message = error.to_string(); + assert_eq!(error.status, AutoUpdateStatus::HttpsError, "{message}"); + assert!(!message.contains(LICENCE_KEY), "{message}"); + assert!(message.contains(&url), "{message}"); + assert!(message.contains("error sending request: "), "{message}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A download whose body ends early fails with a message that names the +/// configured URL, and not the licence key the request carried. +#[test] +fn a_failed_download_keeps_the_licence_key_out_of_the_error() { + // A server that promises more bytes than it sends, then closes. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/api/v2/download", listener.local_addr().unwrap()); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = stream.read(&mut request); + let _ = stream.write_all( + b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\nConnection: close\r\n\r\nshort", + ); + }); + let dir = std::env::temp_dir().join(format!("fiftyone-dut-body-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let engine = engine_polling(&url, &dir); + + let error = DataUpdateService::new() + .check_for_update(&engine, None) + .expect_err("the body ends early"); + server.join().unwrap(); + let message = error.to_string(); + assert_eq!(error.status, AutoUpdateStatus::StreamError, "{message}"); + assert!(!message.contains(LICENCE_KEY), "{message}"); + assert!(message.contains(&url), "{message}"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// A download that keeps arriving completes even when it takes longer than +/// the HTTP client's timeout, which limits each wait for more of the response +/// and not the whole download. A data file of several gigabytes takes minutes +/// to arrive. +#[test] +fn a_steady_download_longer_than_the_client_timeout_completes() { + const CLIENT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(2); + const PARTS: usize = 10; + const PART_GAP: std::time::Duration = std::time::Duration::from_millis(500); + let body: Vec = (0..50 * 1024).map(|i| (i % 251) as u8).collect(); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/datafile", listener.local_addr().unwrap()); + let sent = body.clone(); + let server = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = stream.read(&mut request); + let head = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + sent.len() + ); + let _ = stream.write_all(head.as_bytes()); + for part in sent.chunks(sent.len().div_ceil(PARTS)) { + std::thread::sleep(PART_GAP); + if stream.write_all(part).and_then(|_| stream.flush()).is_err() { + return; + } + } + }); + + let dir = std::env::temp_dir().join(format!("fiftyone-dut-steady-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("device.dat"); + std::fs::write(&path, b"version 1").unwrap(); + let config = DataFileConfiguration::builder(&path) + .data_update_url(url) + .automatic_updates_enabled(false) + .file_system_watcher_enabled(false) + .decompress_content(false) + .build(); + let engine: Arc = + Arc::new(TestEngine::new(Arc::new(AspectEngineDataFile::new(config)))); + let client = reqwest::blocking::Client::builder() + .timeout(CLIENT_TIMEOUT) + .build() + .unwrap(); + + let status = DataUpdateService::with_client(client).check_for_update(&engine, None); + server.join().unwrap(); + assert_eq!( + status.map_err(|error| error.to_string()), + Ok(AutoUpdateStatus::Success) + ); + assert_eq!(std::fs::read(&path).unwrap(), body); + let _ = std::fs::remove_dir_all(&dir); +} + +/// An update whose data file cannot be replaced fails, and leaves no +/// temporary file beside the data file. +#[test] +fn a_failed_update_leaves_no_temporary_file() { + // A folder where the data file should be, so it cannot be replaced. + let dir = std::env::temp_dir().join(format!("fiftyone-dut-left-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let path = dir.join("device.dat"); + std::fs::create_dir_all(&path).unwrap(); + let server = tiny_http::Server::http("127.0.0.1:0").unwrap(); + let url = format!("http://{}/datafile", server.server_addr()); + let handle = std::thread::spawn(move || { + if let Ok(request) = server.recv() { + let _ = request.respond(tiny_http::Response::from_data(b"new data".to_vec())); + } + }); + let config = DataFileConfiguration::builder(&path) + .data_update_url(url) + .automatic_updates_enabled(false) + .file_system_watcher_enabled(false) + .decompress_content(false) + .build(); + let engine: Arc = + Arc::new(TestEngine::new(Arc::new(AspectEngineDataFile::new(config)))); + + let error = DataUpdateService::new() + .check_for_update(&engine, None) + .expect_err("a folder cannot be replaced by a file"); + handle.join().unwrap(); + let left: Vec = std::fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert_eq!( + error.status, + AutoUpdateStatus::NewFileCannotRename, + "{error}" + ); + assert_eq!(left, ["device.dat"]); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Dropping the service ends a scheduled download in progress, rather than +/// waiting for the rest of the data file, and leaves no temporary file. +#[test] +fn dropping_the_service_stops_a_download_in_progress() { + // A server that sends one byte every tenth of a second, so the whole of + // its body takes 30 seconds. + const BODY_BYTES: usize = 300; + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/datafile", listener.local_addr().unwrap()); + // Not joined, so the test cannot wait on a server the scheduler never + // reached. It ends when its next write finds the connection closed. + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0u8; 4096]; + let _ = stream.read(&mut request); + let head = + format!("HTTP/1.1 200 OK\r\nContent-Length: {BODY_BYTES}\r\nConnection: close\r\n\r\n"); + let _ = stream.write_all(head.as_bytes()); + for _ in 0..BODY_BYTES { + std::thread::sleep(std::time::Duration::from_millis(100)); + if stream.write_all(b"5").and_then(|_| stream.flush()).is_err() { + return; + } + } + }); + + let dir = std::env::temp_dir().join(format!("fiftyone-dut-drop-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("device.dat"); + std::fs::write(&path, b"version 1").unwrap(); + // Polled at once, by the service's scheduler. + let config = DataFileConfiguration::builder(&path) + .data_update_url(url) + .automatic_updates_enabled(true) + .polling_interval_seconds(0) + .max_randomisation_seconds(0) + .file_system_watcher_enabled(false) + .decompress_content(false) + .build(); + let data_file = Arc::new(AspectEngineDataFile::new(config)); + let engine: Arc = Arc::new(TestEngine::new(Arc::clone(&data_file))); + let service = DataUpdateService::new(); + service.register(engine, data_file).unwrap(); + std::thread::sleep(std::time::Duration::from_secs(1)); + + let dropping = std::time::Instant::now(); + drop(service); + let waited = dropping.elapsed(); + + let left: Vec = std::fs::read_dir(&dir) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + assert!( + waited < std::time::Duration::from_secs(5), + "dropping the service waited {waited:?} for the download" + ); + assert_eq!(left, ["device.dat"]); + assert_eq!(std::fs::read(&path).unwrap(), b"version 1"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/pipeline-engines/tests/update_on_startup.rs b/pipeline-engines/tests/update_on_startup.rs new file mode 100644 index 0000000..fdd10d0 --- /dev/null +++ b/pipeline-engines/tests/update_on_startup.rs @@ -0,0 +1,109 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! Tests of update on start-up when the update cannot be fetched. +#![cfg(feature = "data-update")] + +use std::net::TcpListener; +use std::sync::Arc; + +use fiftyone_pipeline_core::{ + EvidenceKeyFilter, EvidenceKeyFilterWhitelist, FlowData, FlowElement, PropertyMetaData, Result, +}; +use fiftyone_pipeline_engines::{ + AspectEngine, AspectEngineDataFile, AspectPropertyMetaData, AutoUpdateStatus, + DataFileConfiguration, DataUpdateService, OnPremiseAspectEngine, +}; + +/// An engine with data loaded, which a refresh leaves as it is. +struct LoadedEngine { + filter: EvidenceKeyFilterWhitelist, + data_files: Vec>, +} + +impl FlowElement for LoadedEngine { + fn process(&self, _data: &mut FlowData) -> Result<()> { + Ok(()) + } + fn data_key(&self) -> &str { + "loaded" + } + fn evidence_key_filter(&self) -> &dyn EvidenceKeyFilter { + &self.filter + } + fn properties(&self) -> &[PropertyMetaData] { + &[] + } +} + +impl AspectEngine for LoadedEngine { + fn data_source_tier(&self) -> &str { + "Lite" + } + fn aspect_properties(&self) -> &[AspectPropertyMetaData] { + &[] + } +} + +impl OnPremiseAspectEngine for LoadedEngine { + fn data_files(&self) -> &[Arc] { + &self.data_files + } + fn refresh(&self, _identifier: Option<&str>) -> Result<()> { + Ok(()) + } +} + +/// A failed update on start-up leaves the engine on the data it loaded, as the +/// specification says, so the data file is still registered for the updates +/// that follow, and the failure is reported. +#[test] +fn a_failed_update_on_start_up_still_registers_the_data_file() { + // Nothing listens on a port once its listener is dropped. + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("http://{}/datafile", listener.local_addr().unwrap()); + drop(listener); + let dir = std::env::temp_dir().join(format!("fiftyone-startup-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("device.dat"); + std::fs::write(&path, b"the data loaded").unwrap(); + let config = DataFileConfiguration::builder(&path) + .data_update_url(url) + .update_on_startup(true) + .automatic_updates_enabled(true) + .file_system_watcher_enabled(false) + .build(); + let data_file = Arc::new(AspectEngineDataFile::new(config)); + let engine: Arc = Arc::new(LoadedEngine { + filter: EvidenceKeyFilterWhitelist::new(Vec::::new()), + data_files: vec![Arc::clone(&data_file)], + }); + let service = DataUpdateService::new(); + + let error = service + .register(engine, data_file) + .expect_err("nothing is listening"); + + assert_eq!(error.status, AutoUpdateStatus::HttpsError); + assert_eq!(service.registered_count(), 1); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/pipeline-web/Cargo.toml b/pipeline-web/Cargo.toml index f77dd30..42f4ad5 100644 --- a/pipeline-web/Cargo.toml +++ b/pipeline-web/Cargo.toml @@ -24,9 +24,9 @@ fiftyone-pipeline-engines = { version = "4.5.2", path = "../pipeline-engines", d fiftyone-pipeline-engines-fiftyone = { version = "4.5.2", path = "../pipeline-engines-fiftyone" } fiftyone-json-builder = { version = "4.5.2", path = "../json-builder" } fiftyone-javascript-builder = { version = "4.5.2", path = "../javascript-builder" } -# ahash provides a fast hasher that can be seeded with fixed constants, giving a -# stable per-process digest for the web ETag. It is already a workspace -# dependency so it resolves to a single shared version. +# ahash provides a fast hasher that can be seeded with fixed constants, giving +# the web ETag the same digest in every process running the same build. It is +# already a workspace dependency so it resolves to a single shared version. ahash.workspace = true [package.metadata.docs.rs] diff --git a/pipeline-web/src/clientside.rs b/pipeline-web/src/clientside.rs index 7b5c99c..642f743 100644 --- a/pipeline-web/src/clientside.rs +++ b/pipeline-web/src/clientside.rs @@ -44,14 +44,16 @@ //! (JSON). //! - `Content-Length`: the UTF-8 byte length of the body. //! - `Cache-Control`: `private, max-age=1800` when the flow data recorded no -//! processing errors, otherwise `no-cache`. +//! processing errors, otherwise `no-cache`. A response to a request that +//! recorded errors carries no `ETag`, so a browser that kept it fetches it +//! again in full rather than having it confirmed by a `304`. //! - `Vary`: the comma-joined list of HTTP-header evidence keys in the //! evidence-key whitelist supplied through [`EndpointOptions`], with the //! `header.` prefix stripped and the names de-duplicated case-insensitively. //! Omitted when the list is empty or no whitelist was supplied. -//! - `ETag`: a stable per-process digest of the cache key the flow data's -//! evidence produces under the pipeline filter. The digest is opaque and -//! non-portable (see [`compute_etag`]). +//! - `ETag`: a digest of the cache key the flow data's evidence produces +//! under the pipeline filter, the same in every process running the same +//! build. The digest is opaque and non-portable (see [`compute_etag`]). //! - `Access-Control-Allow-Origin`: the request `Origin` header echoed back, //! but only when an origin is present and is not the literal string `null`. //! @@ -66,9 +68,9 @@ //! the endpoint through [`EndpointOptions`]. An adapter with its own view of //! the headers that change the result can pass a different list. -use std::hash::{Hash, Hasher}; +use std::hash::{BuildHasher, Hash, Hasher}; -use ahash::AHasher; +use ahash::RandomState; use fiftyone_javascript_builder::JAVASCRIPT_BUILDER_DATA_KEY; use fiftyone_json_builder::JSON_BUILDER_DATA_KEY; use fiftyone_pipeline_core::constants::EVIDENCE_HTTP_HEADER_PREFIX; @@ -89,8 +91,11 @@ pub const CONTENT_TYPE_JSON: &str = "application/json"; /// `max-age=1800` (30 minutes) so the client may reuse it for half an hour. pub const CACHE_CONTROL_CACHEABLE: &str = "private, max-age=1800"; -/// The `Cache-Control` value used when processing recorded one or more errors, -/// so a faulty result is never cached. +/// The `Cache-Control` value used when processing recorded one or more errors. +/// +/// A browser may keep such a response but must ask the server before using it +/// again, and the response carries no `ETag`, so asking again always gets a +/// full response and never a `304` confirming the faulty one. pub const CACHE_CONTROL_NO_CACHE: &str = "no-cache"; /// The first fixed seed for the ETag hasher (see [`compute_etag`]). @@ -220,7 +225,12 @@ fn build_response( } } - headers.push(("ETag".to_owned(), etag)); + // The validator stands for the evidence, not for whether processing + // worked, so a response to a request that recorded errors goes without + // one rather than being confirmed by a 304 once processing recovers. + if flow_data.errors().is_empty() { + headers.push(("ETag".to_owned(), etag)); + } if let Some(origin) = access_control_allow_origin(request) { headers.push(("Access-Control-Allow-Origin".to_owned(), origin)); @@ -269,15 +279,17 @@ pub fn vary_header(whitelist: &EvidenceKeyFilterWhitelist) -> String { names.join(",") } -/// Compute the stable per-process `ETag` for a flow data. +/// Compute the `ETag` for a flow data. /// /// The tag is a hex digest of the deterministic [`fiftyone_pipeline_core::DataKey`] /// the flow data's evidence produces under the pipeline filter, rendered as a /// quoted string per the HTTP `ETag` grammar. /// -/// The digest is produced with [`ahash::AHasher`] seeded with the four fixed -/// constants in this module, so it is **stable within a process and across -/// processes built from this code**, but it is deliberately **opaque and +/// The digest is produced by an [`ahash::RandomState`] hasher built from the +/// four fixed seeds in this module, so the same evidence gives the same tag in +/// every process running the same build. A browser revalidates against +/// whichever process answers, which after a restart or behind a load balancer +/// is not the one that made the tag. The tag is deliberately **opaque and /// non-portable**: callers must treat it as a meaningless validator and never /// parse it, compare it across language ports, or persist it expecting another /// implementation to reproduce it. Only equality against a value this same code @@ -285,15 +297,10 @@ pub fn vary_header(whitelist: &EvidenceKeyFilterWhitelist) -> String { pub fn compute_etag(flow_data: &FlowData, filter: &dyn EvidenceKeyFilter) -> String { let key = flow_data.generate_key(filter); - let mut hasher = AHasher::default(); - // Hash a fixed domain seed first so the digest is stable and seeded by this - // code rather than by the default hasher state. AHasher::default uses - // compile-time-fixed seeds (it is not randomised), so two runs of the same - // binary agree, and mixing our own constants documents and pins that intent. - ETAG_SEED_0.hash(&mut hasher); - ETAG_SEED_1.hash(&mut hasher); - ETAG_SEED_2.hash(&mut hasher); - ETAG_SEED_3.hash(&mut hasher); + // The seeds are given explicitly because ahash draws the seeds of its + // default hasher at run time on native targets, once per process. + let mut hasher = + RandomState::with_seeds(ETAG_SEED_0, ETAG_SEED_1, ETAG_SEED_2, ETAG_SEED_3).build_hasher(); // Hash the ordered entries. The key's entry order is already deterministic. for (name, value) in key.entries() { diff --git a/pipeline-web/src/lib.rs b/pipeline-web/src/lib.rs index 3600984..ad1ccae 100644 --- a/pipeline-web/src/lib.rs +++ b/pipeline-web/src/lib.rs @@ -58,12 +58,14 @@ //! `application/json` for JSON. //! - **`Content-Length`**: the UTF-8 byte length of the body. //! - **`Cache-Control`**: `private, max-age=1800` when processing recorded no -//! errors, else `no-cache`. +//! errors, else `no-cache` and no `ETag`, so a browser that kept the response +//! to a failed request fetches it again in full rather than having it +//! confirmed by a `304`. //! - **`Vary`**: the comma-joined HTTP-header evidence keys (prefix stripped, //! de-duplicated case-insensitively, sorted) taken from the supplied //! whitelist; omitted when empty. -//! - **`ETag`**: a stable, opaque, non-portable per-process digest of the -//! evidence cache key (see [`compute_etag`]). +//! - **`ETag`**: an opaque, non-portable digest of the evidence cache key, the +//! same in every process running the same build (see [`compute_etag`]). //! - **`Access-Control-Allow-Origin`**: the request `Origin` echoed back, unless //! it is absent or the literal string `null`. //! - A matching `If-None-Match` yields a bare `304 Not Modified` (empty body, no diff --git a/pipeline-web/tests/web_integration.rs b/pipeline-web/tests/web_integration.rs index c034aac..2ac1f65 100644 --- a/pipeline-web/tests/web_integration.rs +++ b/pipeline-web/tests/web_integration.rs @@ -499,6 +499,51 @@ fn non_matching_if_none_match_returns_200() { assert_eq!(response.status, WebResponse::STATUS_OK); } +/// Set in the environment of the second process the ETag test starts, which +/// then prints its ETag rather than comparing. +const ETAG_CHILD_VARIABLE: &str = "FIFTYONE_WEB_ETAG_CHILD"; + +#[test] +fn the_same_evidence_gives_the_same_etag_in_another_process() { + // A browser revalidates against whichever process answers, which is a + // different one after a restart or behind a load balancer, so the ETag for + // the same evidence must not depend on the process that computed it. + let web = web_pipeline(false); + let options = EndpointOptions::new(web.vary_whitelist().clone()); + let data = processed( + web.pipeline(), + Evidence::builder().add("header.user-agent", "abc").build(), + ); + let response = serve_json(&data, &TestRequest::default(), &options); + let etag = response.header("ETag").expect("an ETag").to_owned(); + + if std::env::var_os(ETAG_CHILD_VARIABLE).is_some() { + println!("etag={etag}"); + return; + } + let output = std::process::Command::new(std::env::current_exe().expect("the test binary")) + .args([ + "the_same_evidence_gives_the_same_etag_in_another_process", + "--exact", + "--nocapture", + "--test-threads=1", + ]) + .env(ETAG_CHILD_VARIABLE, "1") + .output() + .expect("the test binary runs a second time"); + // The test harness prints the test's name on the same line as its output, + // so the ETag is found anywhere in the second process's output. + let stdout = String::from_utf8_lossy(&output.stdout); + let other = stdout + .split_once("etag=") + .and_then(|(_, rest)| rest.split_whitespace().next()) + .unwrap_or_else(|| panic!("the second process printed no ETag:\n{stdout}")); + assert_eq!( + other, etag, + "the same evidence gave a different ETag in another process" + ); +} + // ---------------------------------------------------------------------------- // Access-Control-Allow-Origin echo (present / absent / null). // ---------------------------------------------------------------------------- @@ -678,3 +723,94 @@ fn vary_lists_a_header_only_the_element_knows() { assert!(vary.split(',').any(|h| h == "x-forwarded-for"), "{vary}"); assert!(vary.split(',').any(|h| h == "user-agent"), "{vary}"); } + +// ---------------------------------------------------------------------------- +// The response to a failed request. +// ---------------------------------------------------------------------------- + +/// A device element that fails while `failing` is set, standing in for an +/// engine whose cloud call failed on one request and worked on the next. +struct FlakyElement { + inner: TestElement, + failing: Arc, +} + +impl FlowElement for FlakyElement { + fn process(&self, data: &mut FlowData) -> Result<()> { + if self.failing.load(std::sync::atomic::Ordering::SeqCst) { + return Err(fiftyone_pipeline_core::Error::CloudRequest { + status_code: 0, + retry_after_seconds: None, + message: "the request timed out".to_owned(), + }); + } + self.inner.process(data) + } + fn data_key(&self) -> &str { + self.inner.data_key() + } + fn evidence_key_filter(&self) -> &dyn EvidenceKeyFilter { + self.inner.evidence_key_filter() + } + fn properties(&self) -> &[PropertyMetaData] { + self.inner.properties() + } +} + +#[test] +fn the_response_to_a_failed_request_is_not_kept_once_processing_recovers() { + // A browser keeps a response it may store and revalidates it with the + // response's ETag on the next page view. The response to a failed request + // must not be confirmed by a 304 once processing works again, or the + // browser goes on using the failed result for as long as the evidence + // stays the same. + let failing = Arc::new(std::sync::atomic::AtomicBool::new(true)); + let web = WebPipeline::build( + vec![Arc::new(FlakyElement { + inner: TestElement::new(false), + failing: Arc::clone(&failing), + })], + WebIntegrationOptions::default(), + ) + .expect("web pipeline builds"); + let options = EndpointOptions::new(web.vary_whitelist().clone()); + let evidence = || Evidence::builder().add("header.user-agent", "abc").build(); + + let failed = serve_javascript( + &processed(web.pipeline(), evidence()), + &TestRequest::default(), + &options, + ); + assert_eq!(failed.status, WebResponse::STATUS_OK); + assert_eq!(failed.header("Cache-Control"), Some("no-cache")); + assert!( + failed.header("ETag").is_none(), + "the response to a failed request carries no validator" + ); + + failing.store(false, std::sync::atomic::Ordering::SeqCst); + let revalidation = TestRequest { + headers: failed + .header("ETag") + .map(|etag| vec![("If-None-Match".to_owned(), etag.to_owned())]) + .unwrap_or_default(), + ..Default::default() + }; + let recovered = serve_javascript( + &processed(web.pipeline(), evidence()), + &revalidation, + &options, + ); + assert_eq!( + recovered.status, + WebResponse::STATUS_OK, + "the browser was told to keep the response to the failed request" + ); + assert!( + recovered + .body_str() + .unwrap_or_default() + .contains("ismobile"), + "the response once processing recovers carries the device data" + ); +}