fix(hubs): Relax reservation price backfill gate to recover MCA rows

docs-mslearn/toolkit/changelog.md

@@ -134,6 +134,7 @@ _Released June 2026_
     - Replaced `isnotempty(strcat(x_SkuMeterId, x_SkuOfferId))` with separate `isnotempty()` checks in FinOps hub ingestion scripts to avoid temporary string allocation.
   - Hardened the reservation price backfill to select the highest on-demand price (`max()` instead of `min()`) when duplicate price rows collapse under a single reservation price lookup key, preventing understated commitment discount savings ([#2189](https://github.com/microsoft/finops-toolkit/pull/2189)).
   - Fixed inflated savings in the Data Explorer dashboard Summary and Rate optimization pages by excluding commitment discount purchases from list cost, contracted cost, and savings ([#2315](https://github.com/microsoft/finops-toolkit/issues/2315)).
+  - Fixed the Reservations tab in the Rate optimization report showing no data for Microsoft Customer Agreement (MCA) accounts ([#1769](https://github.com/microsoft/finops-toolkit/issues/1769), [#2176](https://github.com/microsoft/finops-toolkit/issues/2176)).
   - Fixed hub redeploys and upgrades resetting data retention to the defaults and purging historical data ([#2206](https://github.com/microsoft/finops-toolkit/issues/2206), [#2348](https://github.com/microsoft/finops-toolkit/issues/2348)).
 
 ### [Power BI reports](power-bi/reports.md) v15

src/powershell/Tests/Unit/HubsReservationPriceBackfill.Tests.ps1

@@ -0,0 +1,82 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+<#
+    Regression coverage for the MCA reservation list-price backfill fix (#1769 / #2176):
+    the price backfill gate must not require x_SkuOfferId. x_SkuOfferId is EA-only -- for MCA it is blank on
+    both the cost and price side (confirmed against reporter data in #2176), so gating on it excluded every
+    MCA row from the price backfill before the join was even attempted. x_SkuOfferId stays in the lookup key
+    unchanged: for MCA it is blank on both sides (a no-op suffix), and for EA it keeps differentiating rows
+    (e.g. Production vs Dev/Test offers) exactly as before.
+
+    NOTE: a separate, unconfirmed hypothesis -- that x_BillingProfileId also needs normalizing to match between
+    the Costs and Prices transforms, because MCA cost rows may carry a full ARM billing-profile path where the
+    Prices transform normalizes to a bare ID -- is intentionally NOT included here. It wasn't raised or needed
+    in the reporter-confirmed #2176 investigation, and normalizing away the ARM path may not be the desired
+    fix (Prices' bare-ID form could itself be a Price Sheet export limitation rather than the canonical shape).
+    Needs review before changing.
+
+    Per-row behavioral coverage lives in the executable harness
+    Tests/assets/ReservationPriceBackfillKey.kql (PASS = 0 returned rows on any Kusto database).
+#>
+
+Describe 'HubsReservationPriceBackfill' {
+
+    BeforeDiscovery {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+        $scriptsPath = Join-Path $repoRoot 'src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts'
+        $ingestionFiles = @('IngestionSetup_v1_0.kql', 'IngestionSetup_v1_2.kql') | ForEach-Object {
+            @{ Name = $_; FullName = (Join-Path $scriptsPath $_) }
+        }
+    }
+
+    BeforeAll {
+        $repoRoot = (Resolve-Path "$PSScriptRoot/../../../..").Path
+        $harnessPath = Join-Path $repoRoot 'src/powershell/Tests/assets/ReservationPriceBackfillKey.kql'
+    }
+
+    Context 'Missing-price gate' {
+
+        It 'Should not require x_SkuOfferId to attempt the backfill: <Name>' -ForEach $ingestionFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content.Contains('isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)') | Should -BeFalse -Because 'x_SkuOfferId is EA-only and blank for every MCA row (cost and price side alike, confirmed in #2176); gating on it excludes every MCA row from the price backfill before the join is even attempted (#1769, #2176)'
+        }
+
+        It 'Should still require x_SkuMeterId to attempt the backfill: <Name>' -ForEach $ingestionFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content | Should -Match 'isnotempty\(x_SkuMeterId\)(?!\s+and\s+isnotempty\(x_SkuOfferId\))' -Because 'the meter ID is still required to identify the on-demand price to recover, without also requiring the EA-only offer ID (the negative lookahead rules out matching the pre-fix combined condition)'
+        }
+
+        It 'Should scope the offer-ID-less backfill to reservation usage rows: <Name>' -ForEach $ingestionFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content.Contains("and (isnotempty(x_SkuOfferId) or CommitmentDiscountCategory == 'Usage')") | Should -BeTrue -Because 'rows without an offer ID must only enter the backfill when they are reservation usage; otherwise every non-EA row would be imputed contracted and list values from effective when the price lookup misses'
+        }
+
+        It 'Should leave x_SkuOfferId in the reservation price lookup key: <Name>' -ForEach $ingestionFiles {
+            $content = Get-Content -Path $FullName -Raw
+            $content.Contains('x_SkuMeterId, x_SkuOfferId))') | Should -BeTrue -Because 'the key is unchanged by this fix: for MCA x_SkuOfferId is blank on both sides (a no-op suffix), and for EA it keeps differentiating rows (e.g. Production vs Dev/Test offers) as before'
+        }
+    }
+
+    Context 'Equivalence harness' {
+
+        It 'Should have the reservation price backfill key harness asset' {
+            Test-Path $harnessPath | Should -BeTrue
+        }
+
+        It 'Should assert expected divergence per row' {
+            $harness = Get-Content -Path $harnessPath -Raw
+            $harness | Should -Match '\| where \(old_matches != new_matches\) != expectedDivergent' -Because 'the harness must return only rows that violate the recorded old-vs-new expectation'
+        }
+
+        It 'Should cover an MCA blank offer ID case' {
+            $harness = Get-Content -Path $harnessPath -Raw
+            $harness | Should -Match 'mca-profile-1' -Because 'the fixture set must exercise the blank-offer-ID case that silently broke MCA reservation pricing (anchored on a fixture literal, not the header comment)'
+        }
+
+        It 'Should cover an EA offer ID differentiation case for regression safety' {
+            $harness = Get-Content -Path $harnessPath -Raw
+            $harness | Should -Match 'OFFER-EA-DEVTEST' -Because 'EA already matched before the fix; the fixture set must prove the fix does not regress it, including the case where offer ID legitimately differentiates two rows (anchored on a fixture literal, not the header comment)'
+        }
+    }
+}

src/powershell/Tests/assets/ReservationPriceBackfillKey.kql

@@ -0,0 +1,80 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//======================================================================================================================
+// Reservation price backfill gate regression harness (#1769 / #2176)
+//
+// Verifies, per row, whether a cost row backfills its list/contracted price from the Prices table, before and after
+// the fix. A row backfills only if (a) the missing-price gate lets it attempt the join, and (b) its
+// tmp_ReservationPriceLookupKey matches a price row's key:
+//
+//   old gate: isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)
+//   new gate: isnotempty(x_SkuMeterId) and (isnotempty(x_SkuOfferId) or CommitmentDiscountCategory == 'Usage')
+//             -- offer ID no longer required for reservation usage rows; all other rows keep the old requirement
+//
+//   key (unchanged by this fix): tolower(strcat(x_BillingProfileId, month, x_SkuMeterId, x_SkuOfferId))
+//
+// Confirmed against reporter data in #2176: for MCA, x_SkuOfferId is blank on BOTH the cost row (FOCUS
+// pass-through) and the price row (Price Sheet pass-through) -- it was never a key-matching problem, only a
+// gate problem. Relaxing the gate lets the (still offer-ID-suffixed) key match, because the blank offer ID is
+// an identical no-op suffix on both sides. For EA, x_SkuOfferId stays populated and continues to differentiate
+// rows sharing a profile/meter (e.g. Production vs Dev/Test offers) exactly as before.
+//
+// Scope: this harness only varies the GATE. The lookup key is identical before and after the fix, so it cannot detect
+// a key regression (that is covered by a content assertion in HubsReservationPriceBackfill.Tests.ps1). It also does not
+// model the month segment of the key, the x_SkuPriceType filter, or several price rows sharing one key.
+//
+// NOTE: this harness intentionally does not model x_BillingProfileId format (ARM path vs bare ID). Whether the
+// Costs and Prices transforms need to agree on that format is a separate, unconfirmed question -- see the test
+// file docstring.
+//
+// How to run: paste into any Kusto database (ADX or Fabric eventhouse; no table access required).
+//   - PASS = the query returns 0 rows.
+//
+// expectedDivergent = true rows are the intended behavior change: MCA rows that now backfill after the fix.
+// expectedDivergent = false rows assert everything else is unchanged: EA rows backfilled before and still do
+// (including where offer ID legitimately differentiates two rows), and rows with no corresponding price
+// (different meter/profile) stay unmatched either way.
+//======================================================================================================================
+
+let cases = datatable(
+    label:string,
+    billingProfileId:string,
+    priceBillingProfileId:string,
+    meterId:string,
+    priceMeterId:string,
+    commitmentCategory:string, // cost-side CommitmentDiscountCategory
+    offerId:string,        // cost-side x_SkuOfferId; blank for MCA (mapped from OfferId, which MCA exports never populate)
+    priceOfferId:string,   // price-side x_SkuOfferId; blank for MCA (pass-through from the Price Sheet export)
+    expectedDivergent:bool
+)
+[
+    // EA: offer ID populated on both sides -- gate passes and key matches before; must still backfill after.
+    'EA offer ID present, matching meter',                       'ea-profile-1', 'ea-profile-1', 'meter-1', 'meter-1', 'Usage', 'OFFER-EA-1', 'OFFER-EA-1', false,
+    // EA: same profile and meter, but genuinely different offer IDs (e.g. Production vs Dev/Test) -- the key must
+    // keep separating these both before and after the fix; confirms leaving x_SkuOfferId in the key still works.
+    'EA same profile/meter, different offer (Prod vs Dev/Test)', 'ea-profile-1', 'ea-profile-1', 'meter-1', 'meter-1', 'Usage', 'OFFER-EA-PROD', 'OFFER-EA-DEVTEST', false,
+    // MCA: offer ID blank on both cost and price rows (confirmed in #2176). Old gate excludes the row outright;
+    // new gate passes and, since the blank offer ID is an identical no-op suffix on both sides, the key matches.
+    'MCA blank offer ID on both sides',                          'mca-profile-1', 'mca-profile-1', 'meter-2', 'meter-2', 'Usage', '', '', true,
+    // MCA non-reservation row (blank offer ID, no commitment): stays outside the gate and is left untouched.
+    'MCA non-reservation row, blank offer ID',                   'mca-profile-1', 'mca-profile-1', 'meter-2', 'meter-2', '', '', '', false,
+    // Genuinely different meter within the same MCA profile: must stay unmatched under both old and new (gate
+    // failure under old, key mismatch under new) -- proves the fix does not backfill unrelated rows.
+    'MCA different meter, same profile',                         'mca-profile-1', 'mca-profile-1', 'meter-2', 'meter-3', 'Usage', '', '', false,
+    // Genuinely different billing profile: must not match under either, even post-fix.
+    'EA different profile entirely',                             'ea-profile-1', 'ea-profile-2', 'meter-1', 'meter-1', 'Usage', 'OFFER-EA-1', 'OFFER-EA-1', false,
+]
+// old: gate requires a non-empty offer ID
+| extend old_gate = isnotempty(meterId) and isnotempty(offerId)
+| extend old_costKey = tolower(strcat(billingProfileId, meterId, offerId))
+| extend old_priceKey = tolower(strcat(priceBillingProfileId, priceMeterId, priceOfferId))
+| extend old_matches = old_gate and old_costKey == old_priceKey
+// new: gate drops the offer ID requirement; key itself is unchanged
+| extend new_gate = isnotempty(meterId) and (isnotempty(offerId) or commitmentCategory == 'Usage')
+| extend new_costKey = tolower(strcat(billingProfileId, meterId, offerId))
+| extend new_priceKey = tolower(strcat(priceBillingProfileId, priceMeterId, priceOfferId))
+| extend new_matches = new_gate and new_costKey == new_priceKey
+| where (old_matches != new_matches) != expectedDivergent
+| project label, old_matches, new_matches, expectedDivergent
+| order by label asc

src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_0.kql

@@ -409,12 +409,13 @@ Costs_transform_v1_0()
         ConsumedQuantity
     )
     //
-    // Populate missing prices -- mapping to on-demand prices requires meter ID and offer ID
+    // Populate missing prices -- mapping to on-demand prices requires meter ID; offer ID is EA-only, so rows without one are recovered only when they are reservation usage (CommitmentDiscountCategory == 'Usage')
     | extend tmp_MissingPrices = ProviderName == 'Microsoft'
         and (ListUnitPrice == 0 or ContractedUnitPrice == 0)
         and x_EffectiveUnitPrice != 0
         and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused')
-        and isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)
+        and isnotempty(x_SkuMeterId)
+        and (isnotempty(x_SkuOfferId) or CommitmentDiscountCategory == 'Usage')
     | as allCosts
     | where tmp_MissingPrices
     | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId))

src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/scripts/IngestionSetup_v1_2.kql

@@ -413,12 +413,13 @@ Costs_transform_v1_2()
         ConsumedQuantity
     )
     //
-    // Populate missing prices -- mapping to on-demand prices requires meter ID and offer ID
+    // Populate missing prices -- mapping to on-demand prices requires meter ID; offer ID is EA-only, so rows without one are recovered only when they are reservation usage (CommitmentDiscountCategory == 'Usage')
     | extend tmp_MissingPrices = ProviderName == 'Microsoft'
         and (isempty(ListUnitPrice) or isempty(ContractedUnitPrice) or ListUnitPrice == 0 or ContractedUnitPrice == 0)
         and x_EffectiveUnitPrice != 0
         and not(CommitmentDiscountCategory == 'Spend' and CommitmentDiscountStatus == 'Unused')
-        and isnotempty(x_SkuMeterId) and isnotempty(x_SkuOfferId)
+        and isnotempty(x_SkuMeterId)
+        and (isnotempty(x_SkuOfferId) or CommitmentDiscountCategory == 'Usage')
     | as allCosts
     | where tmp_MissingPrices
     | extend tmp_ReservationPriceLookupKey = tolower(strcat(x_BillingProfileId, substring(ChargePeriodStart, 0, 7), x_SkuMeterId, x_SkuOfferId))