diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index d1f201360..3bb19e1ba 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -25,6 +25,11 @@ This article summarizes the features and enhancements in each release of the Fin The following section lists features and enhancements that are currently in development. +### [FinOps hubs](hubs/finops-hubs-overview.md) + +- **Added** + - Added a draft FOCUS 1.5 SKU Price preview (`SkuPrices_v1_5` and `SkuPricesWide_v1_5`) with validation and scenario scripts under `Analytics/preview/SkuPrices_v1_5`. The preview is not deployed with FinOps hubs; run it manually against a hub ([#2339](https://github.com/microsoft/finops-toolkit/pull/2339)). + -->
diff --git a/docs-mslearn/toolkit/hubs/data-model.md b/docs-mslearn/toolkit/hubs/data-model.md index 004f5e11b..0705a6e94 100644 --- a/docs-mslearn/toolkit/hubs/data-model.md +++ b/docs-mslearn/toolkit/hubs/data-model.md @@ -683,6 +683,15 @@ Columns in the **Prices** managed dataset include: | x_TotalUnitPriceDiscount | Real | Source: Hubs add-on. | | x_TotalUnitPriceDiscountPercent | Real | Source: Hubs add-on. | +### FOCUS 1.5 SKU prices preview + +A draft of the FOCUS 1.5 SKU Price dataset is available in the `src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5` folder of the [FinOps toolkit repository](https://github.com/microsoft/finops-toolkit). It isn't deployed with FinOps hubs and may change before release. The preview builds 2 tables from **Prices_final_v1_2** in the **Ingestion** database: + +- **SkuPrices_v1_5** has one row per SKU price and price type (`x_UnitPriceType` of List, Base, Contracted, or Effective) with effective start and end dates. +- **SkuPricesWide_v1_5** has the same prices with each price type in its own column. + +To try it, run `Build-SkuPrices.ps1` against your hub, then `Test-SkuPrices.ps1` to validate the results. See the README in that folder for mapping decisions and known gaps. +
## PricingUnits table diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 new file mode 100644 index 000000000..cd7c72e1c --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 @@ -0,0 +1,92 @@ +<# +.SYNOPSIS +Rebuilds the FOCUS 1.5 SkuPrices preview tables on a hub from Prices_final_v1_2 and Costs_final_v1_2 using SkuPrices_v1_5.kql. + +.DESCRIPTION +1. Creates or updates every function in the KQL file. +2. Builds SkuPrices_v1_5_runs_table (runs of unchanged prices) for each price type and the wide layout. +3. Builds SkuPrices_v1_5 (rows layout) one month and price type at a time, and SkuPricesWide_v1_5 (columns layout) one month at a time. +4. Builds SkuPriceIdv2_map and CostsWithSkuPriceIdv2 (Costs copy with SkuPriceIdv2) one month at a time. +Small steps keep each query within memory on small clusters. +#> +[CmdletBinding(SupportsShouldProcess)] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev', + [string] $KqlPath = "$PSScriptRoot/SkuPrices_v1_5.kql", + [switch] $SkipCosts +) + +$ErrorActionPreference = 'Stop' + +$ctx = Get-AzContext -Name $Context +if (-not $ctx) { throw "Az context '$Context' not found" } +Set-AzContext -Context $ctx -Scope Process -WhatIf:$false | Out-Null # This process only; leaves the saved default alone +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } + +function Invoke-Kusto { + param([string] $Csl, [string] $Endpoint = 'mgmt') + $body = @{ db = $Database; csl = $Csl; properties = @{ Options = @{ servertimeout = '01:00:00' } } } | ConvertTo-Json -Depth 5 + (Invoke-RestMethod -Uri "$Cluster/v1/rest/$Endpoint" -Method Post -Headers $headers -Body $body -TimeoutSec 3600).Tables[0] +} + +# Run one command, timing it; the first command for a table replaces it, the rest append +function Invoke-Step([string] $Table, [string] $Query, [ref] $First, [string] $Label) { + $csl = if ($First.Value) { ".set-or-replace $Table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| $Query" } else { ".append $Table <| $Query" } + if ($PSCmdlet.ShouldProcess($Table, $csl)) { + $sw = [Diagnostics.Stopwatch]::StartNew() + Invoke-Kusto $csl | Out-Null + Write-Host "$Label done in $([int]$sw.Elapsed.TotalSeconds)s" + } + $First.Value = $false +} + +# 1. Functions from the KQL file (every .create-or-alter function block, in order) +$raw = Get-Content -Path $KqlPath -Raw +$pos = 0 +while (($start = $raw.IndexOf('.create-or-alter function', $pos)) -ge 0) { + $depth = 0; $end = -1 + for ($i = $raw.IndexOf('{', $start); $i -lt $raw.Length; $i++) { + if ($raw[$i] -eq '{') { $depth++ } + elseif ($raw[$i] -eq '}') { $depth--; if ($depth -eq 0) { $end = $i + 1; break } } + } + if ($end -lt 0) { throw "Unbalanced function body in $KqlPath" } + $fn = $raw.Substring($start, $end - $start) + $name = [regex]::Match($fn, '\)\s*(\w+)\s*\(').Groups[1].Value + if ($PSCmdlet.ShouldProcess($name, '.create-or-alter function')) { Invoke-Kusto $fn | Out-Null; Write-Host "Updated $name" } + $pos = $end +} + +$types = 'List', 'Base', 'Contracted', 'Effective' + +# 2. Runs of unchanged prices, per price type and for the wide layout +$first = $true +$parts = 8 # Hash partitions of keys, to stay within memory +foreach ($type in $types + 'Wide') { for ($p = 0; $p -lt $parts; $p++) { Invoke-Step 'SkuPrices_v1_5_runs_table' "SkuPrices_v1_5_runs_build('$type', $p, $parts)" ([ref]$first) "Runs $type $p" } } + +# 3. Rows and columns layouts, one month (and price type) at a time +$months = (Invoke-Kusto 'Prices_final_v1_2 | summarize by M = startofmonth(x_IngestionTime) | order by M asc' 'query').Rows | ForEach-Object { ([datetime]$_[0]).ToString('yyyy-MM-dd') } +$firstRows = $true; $firstWide = $true +foreach ($m in $months) { + foreach ($type in $types) { Invoke-Step 'SkuPrices_v1_5' "SkuPrices_v1_5_build('$type', datetime($m))" ([ref]$firstRows) "Rows $m $type" } + Invoke-Step 'SkuPricesWide_v1_5' "SkuPricesWide_v1_5_build(datetime($m))" ([ref]$firstWide) "Wide $m" +} + +# 4. SkuPriceIdv2 map, then a Costs copy with SkuPriceIdv2, one month at a time +if (-not $SkipCosts) { + $first = $true + Invoke-Step 'SkuPriceIdv2_map' 'SkuPriceIdv2_map_build()' ([ref]$first) 'Map' + $costMonths = (Invoke-Kusto 'Costs_final_v1_2 | summarize by M = startofmonth(ChargePeriodStart) | order by M asc' 'query').Rows | ForEach-Object { ([datetime]$_[0]).ToString('yyyy-MM-dd') } + $first = $true + foreach ($m in $costMonths) { Invoke-Step 'CostsWithSkuPriceIdv2' "CostsWithSkuPriceIdv2_build(datetime($m))" ([ref]$first) "Costs $m" } +} + +# 5. Verify +foreach ($t in 'SkuPrices_v1_5', 'SkuPricesWide_v1_5') { + $c = (Invoke-Kusto "$t | summarize Rows = count(), Current = countif(isnull(SkuPriceEffectiveEnd))" 'query').Rows[0] + $columns = (Invoke-Kusto "$t | getschema | project ColumnName" 'query').Rows | ForEach-Object { $_[0] } + $sorted = [string[]] $columns.Clone(); [Array]::Sort($sorted, [StringComparer]::Ordinal) + Write-Host "${t}: $($c[0]) rows ($($c[1]) current); $($columns.Count) columns; alphabetical: $(-not (Compare-Object $columns $sorted -SyncWindow 0))" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md new file mode 100644 index 000000000..62f43f1ba --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md @@ -0,0 +1,96 @@ +# FOCUS 1.5 SkuPrices (preview, draft) + +Draft FinOps hubs implementation of the FOCUS 1.5 **SKU Price** dataset ([FOCUS PR 2424](https://github.com/FinOps-Open-Cost-and-Usage-Spec/FOCUS_Spec/pull/2424)), built from `Prices_final_v1_2`, with a comparison layout and scenarios to choose between them. Not wired into `app.bicep` or `.build.config` yet. Run manually against a hub. + +## Files + +| File | Purpose | +|---|---| +| `SkuPrices_v1_5.kql` | Functions + build commands for `SkuPrices_v1_5`, `SkuPricesWide_v1_5`, `CostsWithSkuPriceIdv2`, and helper tables (`SkuPrices_v1_5_runs_table`, `SkuPriceIdv2_map`) | +| `Build-SkuPrices.ps1` | Creates the functions and builds all tables in small steps (key partitions, months, price types) | +| `Test-SkuPrices.ps1` | Validation: duplicates, overlaps, current rows vs. latest sheet, rows vs. runs, price match, required columns, ranges | +| `SkuPrices_v1_5_scenarios.kql` | 14 scenarios, each with the most efficient query found for each layout (plus today's monthly table for cost joins) | +| `Test-SkuPriceScenarios.ps1` | Runs each query 3x, compares results, scores simplicity, time, CPU, memory, rows scanned, x_ usage, correctness | +| `Render-SkuPriceReport.ps1` + `SkuPrices_v1_5_report.template.html` | Renders the side-by-side HTML report from scenario results and live table sizes | + +Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-dev` Az context. Override with `-Cluster`, `-Database`, `-Context`. + +```powershell +./Build-SkuPrices.ps1 # ~45 min on ftk-dev +./Test-SkuPrices.ps1 +./Test-SkuPriceScenarios.ps1 # writes results JSON to the temp folder +./Render-SkuPriceReport.ps1 +``` + +## Layouts + +All three hold the same 14 months of prices. + +| | Today `Prices_final_v1_2` | Rows `SkuPrices_v1_5` | Columns `SkuPricesWide_v1_5` | +|---|---|---|---| +| Grain | Price per month | Price type per run of unchanged months | Price per contract per run of unchanged months | +| Prices | Columns | `UnitPrice` + `x_UnitPriceType` (List, Base, Contracted, Effective) | `ListUnitPrice`, `ContractedUnitPrice`, `x_BaseUnitPrice`, `x_EffectiveUnitPrice` | +| Dates | Month | Effective start/end per price type | One start/end per row (new row when any price changes) | +| List price scope | n/a | Public: no contract, global eligibility | Scoped to the contract with the rest of the row | +| Rows | 18.78M | 4.17M (3.89M current) | 1.79M (1.63M current) | +| Extent | 2.34 GB | 0.64 GB | 0.33 GB | + +Rows by type (all / current): List 1,348,812 / 1,236,379; Contracted 1,326,353 / 1,236,379 (98% = List); Base 1,056,500 / 1,023,491 (94% = List); Effective 438,723 / 388,900. + +## Mapping decisions + +* `SkuPriceId` = `SkuPriceIdv2` (unique per price); source `SkuPriceId` kept as `x_SkuPriceIdv1` +* `ContractId` = `/providers/microsoft.billing/billingaccounts/{x_BillingAccountId}` (EA enrollment / MCA billing account, not the profile) +* Savings plan rows carry only the Effective price (list/base/contracted on those rows are copies of the on-demand meter's prices) +* Effective dates are month-level runs of unchanged prices across monthly loads; current prices have no end +* `PricingRegionId` = region name (`x_SkuRegion`); `PricingServiceName` = meter category; `SkuPriceCreated`/`LastUpdated` = first/last ingestion +* `PurchaseDurationType` from `x_SkuTerm`; `PurchasePaymentModel` empty (not in price sheet) +* Non-SKU Price source columns get an `x_` prefix; `CommitmentDiscountCategory` kept as a filter label; columns sorted alphabetically + +## Cost and Usage join + +Costs `SkuPriceId` doesn't match the price sheet's, and Costs lacks `x_SkuProductId`/`x_SkuMeterType`, so `SkuPriceIdv2` can't be rebuilt from Costs columns. `CostsWithSkuPriceIdv2` is a copy of `Costs_final_v1_2` with `SkuPriceIdv2` looked up from prices by price type + meter + offer + term: + +| Cost type | Price used | Rows matched | Cost matched | +|---|---|---|---| +| On-demand usage | On-demand | 100% | 100% | +| Reservation-covered usage | On-demand (no reservation usage price) | 90.2% | 74.7% | +| Savings-plan-covered usage | Savings plan for the term | 83.1% (68 rows have no meter) | 100% | +| Reservation purchase | Reservation for the term (only when unique) | 16/16 | n/a | +| Adjustment | None | 0% | 0% | + +Effective-dated prices have no range lookup in KQL. Scenarios spread each price over the months it was in effect and look up on `SkuPriceId` + `ContractId` + `PricingCurrency` + month: no range join and no duplicated cost rows. Today's monthly table needs no spreading. + +## Scenarios + +| Id | Scenario | Simpler | Faster | CPU | Memory | Rows scanned | FOCUS-only | Correct | +|---|---|---|---|---|---|---|---|---| +| S01 | List price on a date | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S02 | Negotiated discount by service | Columns | Columns | Columns | Columns | Columns | Columns | Tie | +| S03 | Base price drift from list | Columns | Columns | Columns | Columns | Columns | Tie | Tie | +| S04 | Reservation price vs. on-demand | Tie | Columns | Columns | Tie | Columns | Columns | Tie | +| S05 | Savings plan rate vs. on-demand | Tie | Tie | Columns | Tie | Columns | Tie | Tie | +| S06 | Price changes since July | Rows | Tie | Tie | Rows | Columns | Rows | Tie | +| S07 | Cheapest region for a VM size | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S08 | Verify billed unit prices (+Monthly) | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S09 | Savings plan what-if for on-demand usage (+Monthly) | Tie | Monthly | Monthly | Tie | Columns | Tie | Tie | +| S10 | All prices for one SKU | Tie | Tie | Tie | Tie | Columns | Rows | Tie | +| S11 | Estimate a planned workload | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S12 | Public prices visible to each account | Tie | Rows | Rows | Tie | Columns | Columns | Rows | +| S13 | Usage rates vs. purchase fees | Rows | Tie | Tie | Tie | Columns | Rows | Tie | +| S14 | Backfill missing prices on cost data (+Monthly) | Monthly | Tie | Tie | Tie | Columns | Columns | Tie | +| | **Wins (Rows / Columns / Monthly)** | 2 / 2 / 1 | 1 / 3 / 1 | 1 / 4 / 1 | 1 / 2 / 0 | 0 / 14 / 0 | 3 / 8 / 0 | 1 / 0 / 0 | + +Scores: winner per measure, or Tie within 15% of the runner-up. Simpler = operators + 2x joins + lets; Faster/CPU/Memory/Rows scanned = median of 3 server-measured runs; FOCUS-only = fewer distinct x_ columns referenced; Correct = which layout returns the right answer when they differ (S12: columns hide the public list price from accounts outside the agreement). + +Covers 9 of 13 FOCUS PR 2595 queries plus 5 more (S03, S05, S06, S10, S14). Not testable with this data: announced price changes, public vs. negotiated at a quantity, tier resolution, next tier (no future prices or tiers in EA price sheets), negotiated savings plan discount, multiple currencies, consumption currency. Cost scenarios use a price only when its block unit matches the charge's `PricingUnit`. + +## Gaps / TODO + +* [ ] `PricingUnit` and `x_PricingBlockSize` empty: `PricingUnits` open data table has 0 rows on ftk-dev, so `UnitPrice` is per block (e.g., per 10 hours); scenarios parse the block size from `x_PricingUnitDescription` +* [ ] Savings plan list price ("list effective") is lost on ingestion: `Prices_transform_v1_2` nulls the SP `MarketPrice` and backfills on-demand prices; needed to measure negotiated SP discounts +* [ ] Decide whether to skip Base/Contracted rows equal to List (lookups fall back to List) +* [ ] MCA untested (collapses billing profiles into the billing account; check for profile-specific prices); latest month is global, not per account +* [ ] Decide final layout once FOCUS settles the price type column (planned for 1.6) +* [ ] Add `SkuPriceIdv2` to `Costs_transform_v1_2` (replaces `CostsWithSkuPriceIdv2`); find a reservation-covered usage price +* [ ] Wire into `IngestionSetup_v1_5.kql` / `HubSetup_v1_5.kql`, `app.bicep`, `.build.config`; add `SkuPrices()` hub function, docs, changelog diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Render-SkuPriceReport.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Render-SkuPriceReport.ps1 new file mode 100644 index 000000000..af726c335 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Render-SkuPriceReport.ps1 @@ -0,0 +1,56 @@ +<# +.SYNOPSIS +Renders the SKU Price layout comparison report (HTML) from scenario results and live table sizes. +#> +[CmdletBinding()] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev', + [string] $Template = "$PSScriptRoot/SkuPrices_v1_5_report.template.html", + [string] $Results = (Join-Path ([IO.Path]::GetTempPath()) 'SkuPrices_v1_5_scenarios.results.json'), + [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) "sku-price-layouts.html") +) + +$ErrorActionPreference = 'Stop' +Set-AzContext -Context (Get-AzContext -Name $Context) -Scope Process | Out-Null +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } +function Invoke-Kusto([string] $Csl, [string] $Endpoint = 'query') { + $body = @{ db = $Database; csl = $Csl } | ConvertTo-Json + , @((Invoke-RestMethod -Uri "$Cluster/v1/rest/$Endpoint" -Method Post -Headers $headers -Body $body -TimeoutSec 600).Tables[0].Rows) +} + +function Get-Size([string] $Table, [string] $Filter = '') { + $r = Invoke-Kusto ".show table $Table extents $Filter | summarize Rows = sum(RowCount), Extent = sum(ExtentSize), Compressed = sum(CompressedSize)" 'mgmt' + $cols = (Invoke-Kusto "$Table | getschema | count")[0][0] + [pscustomobject]@{ Rows = [double]$r[0][0]; Extent = [double]$r[0][1]; Compressed = [double]$r[0][2]; Columns = $cols } +} +function M($v) { '{0:0.00}M' -f ($v / 1e6) } +function G($v) { '{0:0.00}' -f ($v / 1GB) } +function X($a, $b) { '{0:0.0}×' -f ($a / $b) } + +$p = Get-Size 'Prices_final_v1_2' +$r = Get-Size 'SkuPrices_v1_5' +$c = Get-Size 'SkuPricesWide_v1_5' +$rCur = (Invoke-Kusto 'SkuPrices_v1_5 | where isnull(SkuPriceEffectiveEnd) | count')[0][0] +$cCur = (Invoke-Kusto 'SkuPricesWide_v1_5 | where isnull(SkuPriceEffectiveEnd) | count')[0][0] + +function Card($cls, $label, $table, $s, $ratio, [string[]] $bullets) { + @" +
$label $table
+
$(M $s.Rows)Rows
$(G $s.Extent)GB extent
$(G $s.Compressed)GB compressed
+
$ratio
+"@ +} +$tables = (Card 'mon' 'Today' 'Prices_final_v1_2' $p "$($p.Columns) columns. 14 monthly price sheets." @( + 'One row per price per month', 'All price types as columns', 'Month-level dates; joins match on month')) + + (Card 'rows' 'Rows' 'SkuPrices_v1_5' $r "$($r.Columns) columns. $(X $r.Rows $p.Rows) rows, $(X $r.Extent $p.Extent) extent of today. $(M $rCur) current." @( + 'One row per price type (x_UnitPriceType) per run of unchanged months', 'List rows are public: no contract, global eligibility', 'Effective start and end per price type')) + + (Card 'cols' 'Columns' 'SkuPricesWide_v1_5' $c "$($c.Columns) columns. $(X $c.Rows $p.Rows) rows, $(X $c.Extent $p.Extent) extent of today. $(M $cCur) current." @( + 'One row per price per contract per run; a new row when any price changes', 'List price repeats per contract and is scoped to it', 'One effective start and end for all prices on the row')) + +$json = ((Get-Content $Results -Raw | ConvertFrom-Json) | ConvertTo-Json -Depth 8 -Compress).Replace(' x_SkuPriceIdv1). +// Built one price type and one month at a time to stay within memory on small clusters. +// Effective dates are month-level: Prices_final_v1_2 holds one price sheet per month, not the provider's change dates. + +// 1. Source rows for one price type (rows layout) +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Prices_final_v1_2 rows for one price type with UnitPrice, ContractId, and the run key') +SkuPrices_v1_5_source(priceType: string) { + Prices_final_v1_2 + | extend UnitPrice = case( + priceType == 'List', ListUnitPrice, + priceType == 'Base', x_BaseUnitPrice, + priceType == 'Contracted', ContractedUnitPrice, + priceType == 'Effective', x_EffectiveUnitPrice, + real(null)) + | where isnotnull(UnitPrice) + // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices + | where priceType == 'Effective' or x_SkuPriceType != 'SavingsPlan' + // List prices are public: no contract. Everything else belongs to the agreement: EA enrollment or MCA billing account (not the billing profile) + | extend ContractId = iff(priceType == 'List', '', tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId))) + | extend tmp_k = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)), tmp_M = startofmonth(x_IngestionTime), tmp_V = hash_xxhash64(tostring(UnitPrice)) +} + +// 2. Source rows for the wide layout (all price types as columns) +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Prices_final_v1_2 rows for the wide layout with ContractId and the run key') +SkuPricesWide_v1_5_source() { + Prices_final_v1_2 + // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices + | extend tmp_IsSP = x_SkuPriceType == 'SavingsPlan' + | extend + ListUnitPrice = iff(tmp_IsSP, real(null), ListUnitPrice), + ContractedUnitPrice = iff(tmp_IsSP, real(null), ContractedUnitPrice), + x_BaseUnitPrice = iff(tmp_IsSP, real(null), x_BaseUnitPrice), + x_ContractedUnitPriceDiscount = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscountPercent) + // Every row carries a contracted price, so every row belongs to the agreement + | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) + | extend tmp_k = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)), tmp_M = startofmonth(x_IngestionTime), + tmp_V = hash_xxhash64(strcat(ListUnitPrice, '|', x_BaseUnitPrice, '|', ContractedUnitPrice, '|', x_EffectiveUnitPrice)) +} + +// 3. Runs: consecutive months with the same value for a key become one run with an effective start and end +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Runs of unchanged values: k (key), M (month), V (value hash), Start (effective start in that month)') +SkuPrices_v1_5_runs(T: (k: long, M: datetime, V: long, Start: datetime)) { + let hist = T | summarize hint.shufflekey = k V = take_any(V), Start = min(Start) by k, M; + let latestMonth = toscalar(hist | summarize max(M)); + // A run starts in a month when the value differs from the previous month or the key wasn't there the previous month + let runStarts = hist + | join kind=leftouter hint.shufflekey = k (hist | project k, M = datetime_add('month', 1, M), PrevV = V) on k, M + | where isnull(PrevV) or PrevV != V + | project k, RunStart = M; + hist + | join kind=inner hint.shufflekey = k runStarts on k + | where RunStart <= M + | summarize hint.shufflekey = k RunStart = max(RunStart) by k, M, Start + | summarize hint.shufflekey = k LastMonth = max(M), EffectiveStart = min(Start) by k, RunStart + // Runs still in the latest month are current: no end. Others end when the next month starts. + | extend EffectiveEnd = iff(LastMonth == latestMonth, datetime(null), datetime_add('month', 1, LastMonth)) +} + +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Runs for one price type (List, Base, Contracted, Effective) or Wide, for one hash partition of keys') +SkuPrices_v1_5_runs_build(priceType: string, part: int, parts: int) { + union + (SkuPrices_v1_5_source(priceType) | where priceType != 'Wide' and abs(tmp_k % parts) == part | project k = tmp_k, M = tmp_M, V = tmp_V, Start = x_EffectivePeriodStart), + (SkuPricesWide_v1_5_source() | where priceType == 'Wide' and abs(tmp_k % parts) == part | project k = tmp_k, M = tmp_M, V = tmp_V, Start = x_EffectivePeriodStart) + | invoke SkuPrices_v1_5_runs() + | extend x_UnitPriceType = priceType +} + +// 4. Rows layout: one price type, runs that end in one month +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows for one price type and the runs that end in one month (reads SkuPrices_v1_5_runs_table)') +SkuPrices_v1_5_build(priceType: string, month: datetime) { + let runs = SkuPrices_v1_5_runs_table | where x_UnitPriceType == priceType and LastMonth == month | project tmp_k = k, tmp_Start = EffectiveStart, tmp_End = EffectiveEnd; + let src = SkuPrices_v1_5_source(priceType) + | where tmp_M == month + | lookup kind=inner runs on tmp_k + | extend x_UnitPriceType = priceType + // One row per key (collapses list prices repeated across billing accounts) + | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, PricingCurrency + | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency); + // First time each price was seen + let created = Prices_final_v1_2 | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; + // Tier max = next tier min for the same meter and price (only meters with more than one tier) + let tierMax = src + | summarize Tiers = make_set(x_SkuTier) by tmp_TierGroup + | where array_length(Tiers) > 1 + | extend Tiers = array_sort_asc(Tiers) + | mv-expand with_itemindex = tmp_i x_SkuTier = Tiers to typeof(real) + | extend QuantityTierMaximum = todouble(Tiers[tmp_i + 1]) + | project tmp_TierGroup, x_SkuTier, QuantityTierMaximum; + src + | lookup kind=leftouter tierMax on tmp_TierGroup, x_SkuTier + | lookup kind=leftouter created on SkuPriceIdv2 + | extend x_SkuPriceIdv1 = SkuPriceId + | extend + PricingCurrencyCategory = 'Payable', + PricingRegionId = x_SkuRegion, // Region name, not ID; best available + PricingServiceName = x_SkuMeterCategory, // Meter category; price sheet has no service name + PurchaseDurationType = iff(ChargeCategory != 'Purchase' or isnull(x_SkuTerm), '', + case(x_SkuTerm == 12, '1 Year', x_SkuTerm % 12 == 0, strcat(x_SkuTerm / 12, ' Years'), x_SkuTerm == 1, '1 Month', strcat(x_SkuTerm, ' Months'))), + PurchasePaymentModel = '', // Not in the price sheet + QuantityTierMinimum = coalesce(x_SkuTier, real(0)), + ServiceProviderName = 'Microsoft', + SkuPriceDescription = x_SkuDescription, + SkuPriceEffectiveEnd = tmp_End, // Month after the last month with this price; empty when current + SkuPriceEffectiveStart = tmp_Start, // First month with this price + SkuPriceEligibility = iff(priceType == 'List', + dynamic({'IsGlobalScope': true}), + bag_pack('IsGlobalScope', false, 'InclusionOperator', 'Or', 'Inclusions', + pack_array( + // Matches the account itself (EA) and every billing profile under it (MCA) + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'In', 'Values', pack_array(ContractId)), + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'StartsWith', 'Values', pack_array(strcat(ContractId, '/')))))), + SkuPriceId = SkuPriceIdv2, + SkuPriceLastUpdated = x_IngestionTime, // Last ingestion of this price; provider date not available + // Discount vs. list for this row's price type (replaces the 6 wide discount columns) + x_UnitPriceDiscount = iff(priceType == 'List', real(null), ListUnitPrice - UnitPrice), + x_UnitPriceDiscountPercent = iff(priceType == 'List' or ListUnitPrice == 0, real(null), 100.0 * (ListUnitPrice - UnitPrice) / ListUnitPrice) + // List rows are collapsed across billing accounts and MCA rows across billing profiles, so clear values that differ between them + | extend tmp_IsList = priceType == 'List', tmp_IsMca = x_BillingProfileId != x_BillingAccountId + | extend + BillingAccountName = iff(tmp_IsList or tmp_IsMca, '', BillingAccountName), // MCA: profile name + x_BillingAccountId = iff(tmp_IsList, '', x_BillingAccountId), + x_BillingProfileId = iff(tmp_IsList or tmp_IsMca, '', x_BillingProfileId) + | project-away tmp_*, BillingAccountId, // EA: same as ContractId; MCA: profile-level, collapsed into ContractId + ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice, // In UnitPrice + x_UnitPriceType + x_ContractedUnitPriceDiscount, x_ContractedUnitPriceDiscountPercent, x_EffectiveUnitPriceDiscount, x_EffectiveUnitPriceDiscountPercent, + x_TotalUnitPriceDiscount, x_TotalUnitPriceDiscountPercent, // Replaced by x_UnitPriceDiscount(Percent) + x_EffectivePeriodStart, x_EffectivePeriodEnd // Replaced by SkuPriceEffectiveStart/End + // Source columns that are not SKU Price columns become custom columns + | project-rename + x_BillingAccountName = BillingAccountName, + x_BillingCurrency = BillingCurrency, + x_CommitmentDiscountType = CommitmentDiscountType, + x_CommitmentDiscountUnit = CommitmentDiscountUnit, + x_PricingCategory = PricingCategory, + x_SkuMeter = SkuMeter, + x_SkuPriceIdv2 = SkuPriceIdv2 + // All columns in alphabetical order (FOCUS columns sort before x_ columns) + | project-reorder * asc +} + +// 5. Columns layout (comparison): runs that end in one month +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows with price types as columns for the runs that end in one month (comparison layout)') +SkuPricesWide_v1_5_build(month: datetime) { + let runs = SkuPrices_v1_5_runs_table | where x_UnitPriceType == 'Wide' and LastMonth == month | project tmp_k = k, tmp_Start = EffectiveStart, tmp_End = EffectiveEnd; + let src = SkuPricesWide_v1_5_source() + | where tmp_M == month + | lookup kind=inner runs on tmp_k + | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, PricingCurrency + | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency); + let created = Prices_final_v1_2 | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; + let tierMax = src + | summarize Tiers = make_set(x_SkuTier) by tmp_TierGroup + | where array_length(Tiers) > 1 + | extend Tiers = array_sort_asc(Tiers) + | mv-expand with_itemindex = tmp_i x_SkuTier = Tiers to typeof(real) + | extend QuantityTierMaximum = todouble(Tiers[tmp_i + 1]) + | project tmp_TierGroup, x_SkuTier, QuantityTierMaximum; + src + | lookup kind=leftouter tierMax on tmp_TierGroup, x_SkuTier + | lookup kind=leftouter created on SkuPriceIdv2 + | extend x_SkuPriceIdv1 = SkuPriceId + | extend + PricingCurrencyCategory = 'Payable', + PricingRegionId = x_SkuRegion, + PricingServiceName = x_SkuMeterCategory, + PurchaseDurationType = iff(ChargeCategory != 'Purchase' or isnull(x_SkuTerm), '', + case(x_SkuTerm == 12, '1 Year', x_SkuTerm % 12 == 0, strcat(x_SkuTerm / 12, ' Years'), x_SkuTerm == 1, '1 Month', strcat(x_SkuTerm, ' Months'))), + PurchasePaymentModel = '', + QuantityTierMinimum = coalesce(x_SkuTier, real(0)), + ServiceProviderName = 'Microsoft', + SkuPriceDescription = x_SkuDescription, + SkuPriceEffectiveEnd = tmp_End, + SkuPriceEffectiveStart = tmp_Start, + // One eligibility for the whole row, even though the list price on it is public + SkuPriceEligibility = bag_pack('IsGlobalScope', false, 'InclusionOperator', 'Or', 'Inclusions', + pack_array( + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'In', 'Values', pack_array(ContractId)), + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'StartsWith', 'Values', pack_array(strcat(ContractId, '/'))))), + SkuPriceId = SkuPriceIdv2, + SkuPriceLastUpdated = x_IngestionTime + | extend tmp_IsMca = x_BillingProfileId != x_BillingAccountId + | extend + BillingAccountName = iff(tmp_IsMca, '', BillingAccountName), + x_BillingProfileId = iff(tmp_IsMca, '', x_BillingProfileId) + | project-away tmp_*, BillingAccountId, x_EffectivePeriodStart, x_EffectivePeriodEnd + | project-rename + x_BillingAccountName = BillingAccountName, + x_BillingCurrency = BillingCurrency, + x_CommitmentDiscountType = CommitmentDiscountType, + x_CommitmentDiscountUnit = CommitmentDiscountUnit, + x_PricingCategory = PricingCategory, + x_SkuMeter = SkuMeter, + x_SkuPriceIdv2 = SkuPriceIdv2 + | project-reorder * asc +} + +// 6. Build (Build-SkuPrices.ps1 runs these): runs per type, then each table one month (and type) at a time +.set-or-replace SkuPrices_v1_5_runs_table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_runs_build('List', 0, 8) + +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Base', 0, 8) + +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Contracted', 0, 8) + +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Effective', 0, 8) + +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Wide', 0, 8) + +// ...each partition 0-7 for each price type and Wide + +.set-or-replace SkuPrices_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_build('List', datetime(2025-09-01)) + +.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Base', datetime(2025-09-01)) + +// ...each price type for each month in Prices_final_v1_2 + +.set-or-replace SkuPricesWide_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPricesWide_v1_5_build(datetime(2025-09-01)) + +.append SkuPricesWide_v1_5 <| SkuPricesWide_v1_5_build(datetime(2025-10-01)) + +// ...each month in Prices_final_v1_2 + +// 7. Costs copy with SkuPriceIdv2, so Cost and Usage can join to SkuPrices on SkuPriceId +// Costs has no x_SkuProductId/x_SkuMeterType, so the ID is looked up from prices by price type + meter + offer + term +// On-demand usage -> on-demand price +// Reservation-covered usage -> on-demand price (Microsoft has no usage price for reservations) +// Savings-plan-covered usage -> savings plan price for the term +// Reservation purchase -> reservation price for the term, only when one ID matches +// Everything else -> empty +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Hash of price type + charge category + meter + offer + term, used to look up SkuPriceIdv2') +SkuPriceIdv2_key(priceType: string, chargeCategory: string, meterId: string, offerId: string, term: int) { + hash_xxhash64(strcat(priceType, '|', chargeCategory, '|', meterId, '|', offerId, '|', coalesce(term, 0))) +} + +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'SkuPriceIdv2 per lookup key, only where one ID matches') +SkuPriceIdv2_map_build() { + Prices_final_v1_2 + | distinct x_SkuPriceType, ChargeCategory, x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SkuPriceIdv2 + | summarize hint.shufflekey = x_SkuMeterId Ids = make_set(SkuPriceIdv2, 2) by k = SkuPriceIdv2_key(x_SkuPriceType, ChargeCategory, x_SkuMeterId, x_SkuOfferId, x_SkuTerm), x_SkuMeterId + | where array_length(Ids) == 1 + | project k, SkuPriceIdv2 = tostring(Ids[0]) +} + +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'Costs_final_v1_2 for one month with SkuPriceIdv2 (reads SkuPriceIdv2_map)') +CostsWithSkuPriceIdv2_build(month: datetime) { + let costs = Costs_final_v1_2 + | where ChargePeriodStart >= month and ChargePeriodStart < datetime_add('month', 1, month) + | extend tmp_PriceType = case( + ChargeCategory == 'Usage' and CommitmentDiscountCategory == 'Spend', 'SavingsPlan', + ChargeCategory == 'Usage', 'Consumption', + ChargeCategory == 'Purchase' and CommitmentDiscountType == 'Reservation', 'ReservedInstance', + '') + | extend tmp_k = SkuPriceIdv2_key(tmp_PriceType, ChargeCategory, x_SkuMeterId, + iff(tmp_PriceType == 'ReservedInstance', '', x_SkuOfferId), + iff(tmp_PriceType in ('SavingsPlan', 'ReservedInstance'), x_SkuTerm, int(null))); + // Only the keys this month needs, to keep the lookup small + let keys = costs | distinct tmp_k; + costs + | lookup kind=leftouter (SkuPriceIdv2_map | where k in (keys) | project tmp_k = k, SkuPriceIdv2) on tmp_k + | project-away tmp_* +} + +// Build the map, then Costs one month at a time (Build-SkuPrices.ps1 loops over every month in Costs_final_v1_2) +.set-or-replace SkuPriceIdv2_map with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPriceIdv2_map_build() + +.set-or-replace CostsWithSkuPriceIdv2 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| CostsWithSkuPriceIdv2_build(datetime(2025-09-01)) + +.append CostsWithSkuPriceIdv2 <| CostsWithSkuPriceIdv2_build(datetime(2025-10-01)) + +// ...one .append per remaining month +// Validation: Test-SkuPrices.ps1. Scenarios: SkuPrices_v1_5_scenarios.kql + Test-SkuPriceScenarios.ps1. diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html new file mode 100644 index 000000000..2e83897e9 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html @@ -0,0 +1,228 @@ +SKU Price Layouts + + + + +
+
+

FOCUS 1.5 SKU Price · FinOps hubs preview · ftk-dev

+

One row per price type, or one row per price?

+

The same Microsoft EA prices (14 months) loaded two ways and queried for 14 scenarios, 9 from FOCUS PR 2595. Cost-join scenarios also show today's monthly Prices table. Each query uses the cheapest pattern found for its layout.

+
+ +
+

Tables

+
__TABLES__
+
+ +
+

Scoreboard

+
+
+ + + +
ScenarioSimplerFasterCPUMemoryRows scannedFOCUS-onlyCorrect
+
+ Simpler: fewer operators + 2× joins + lets + Faster / CPU / Memory / Rows scanned: median of 3 server-measured runs + FOCUS-only: fewer distinct x_ columns referenced + Correct: same answer everywhere, or which layout is right + Within 15% (or a tiny absolute gap) of the runner-up is a tie. Values under each pill: Rows · Columns (· Monthly). +
+
+
+ +
+

Scenarios

+
+
+ + +
+ + + diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql new file mode 100644 index 000000000..8c286048a --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql @@ -0,0 +1,479 @@ +// FOCUS 1.5 SKU Price scenarios: row-per-price-type vs. column-per-price-type (vs. today's monthly table for cost joins) +// +// Rows = SkuPrices_v1_5 one row per SkuPriceId x price type x run of unchanged months (x_UnitPriceType: List, Base, Contracted, Effective) +// Columns = SkuPricesWide_v1_5 one row per SkuPriceId x run of unchanged months, with ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice +// Monthly = Prices_final_v1_2 today's FinOps hubs table: one row per price per month (only in cost-join scenarios) +// +// Both SKU Price tables hold the same 14 months of history from Prices_final_v1_2 (built by SkuPrices_v1_5.kql). +// Current prices have an empty SkuPriceEffectiveEnd. +// Cost and Usage joins use CostsWithSkuPriceIdv2 (Costs_final_v1_2 + SkuPriceIdv2) on SkuPriceId + ContractId + PricingCurrency + month. +// Price sheet prices are per block (e.g., "100 Hours"); cost scenarios convert to per unit only when the unit matches PricingUnit. +// Queries use FOCUS columns where they exist; x_ columns only where FOCUS has no equivalent (meter, offer, term, price type). +// Each query uses the cheapest pattern found for its layout. +// +// Format: "// ## ", "// Question:", "// Source:", "// Expect: same|different ()", optional "// Check: " +// (appended to every side when testing, for queries that return full rows), then "// ### Rows", "// ### Columns", and optional "// ### Monthly". + +// ## S01 List price on a date +// Question: What was the public list price for this SKU price on March 15, 2026? +// Source: PR 2595 "Find the List Prices in Force at a Point in Time" +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' and x_UnitPriceType == 'List' +| where SkuPriceEffectiveStart <= datetime(2026-03-15) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-03-15)) +| project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice +// ### Columns +// The list price repeats on every contract's row, so it needs distinct +SkuPricesWide_v1_5 +| where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' +| where SkuPriceEffectiveStart <= datetime(2026-03-15) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-03-15)) +| distinct SkuPriceId, PricingCurrency, ListUnitPrice + +// ## S02 Negotiated discount by service +// Question: How much does the contract lower current usage rates, by service? +// Source: PR 2595 "Measure What Negotiation Reduces the Rate By"; appendix standard_and_negotiated_prices.md +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' +| project SkuPriceId, PricingCurrency, PricingServiceName, ContractedUnitPrice = UnitPrice +| lookup kind=inner ( + SkuPrices_v1_5 + | where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'List' and UnitPrice > 0 + | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice + ) on SkuPriceId, PricingCurrency +| summarize Prices = count(), Discounted = countif(ContractedUnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice), 4) by PricingServiceName +| order by Discounted desc, PricingServiceName asc +| take 10 +// ### Columns +SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and ListUnitPrice > 0 and isnotnull(ContractedUnitPrice) +| summarize Prices = count(), Discounted = countif(ContractedUnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice), 4) by PricingServiceName +| order by Discounted desc, PricingServiceName asc +| take 10 + +// ## S03 Base price drift from list +// Question: Where has the current public list price moved away from the price fixed in the agreement? +// Source: PR 2595 ("Base" price type); research gap: PR 2424 has no column to tell Base from Contracted +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'Base' +| project SkuPriceId, PricingCurrency, PricingServiceName, BaseUnitPrice = UnitPrice +| lookup kind=inner ( + SkuPrices_v1_5 + | where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'List' + | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice + ) on SkuPriceId, PricingCurrency +| summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > BaseUnitPrice), ListBelowBase = countif(ListUnitPrice < BaseUnitPrice), MaxDriftPercent = round(max(iff(BaseUnitPrice > 0, 100.0 * (ListUnitPrice - BaseUnitPrice) / BaseUnitPrice, real(null))), 4) by PricingServiceName +| order by ListAboveBase + ListBelowBase desc, PricingServiceName asc +| take 10 +// ### Columns +SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) and isnotnull(x_BaseUnitPrice) and isnotnull(ListUnitPrice) +| summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > x_BaseUnitPrice), ListBelowBase = countif(ListUnitPrice < x_BaseUnitPrice), MaxDriftPercent = round(max(iff(x_BaseUnitPrice > 0, 100.0 * (ListUnitPrice - x_BaseUnitPrice) / x_BaseUnitPrice, real(null))), 4) by PricingServiceName +| order by ListAboveBase + ListBelowBase desc, PricingServiceName asc +| take 10 + +// ## S04 Reservation price vs. on-demand +// Question: For each reservation term, what is the hourly rate and the savings vs. the current on-demand contracted rate? +// Source: PR 2595 "Evaluate the Purchase Terms Offered for a SKU"; appendix commitment_pricing.md +// Expect: same +// ### Rows +let reservations = materialize(SkuPrices_v1_5 + | where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Purchase' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 + | project x_SkuMeterId, x_SkuTerm, HourlyRate = UnitPrice / (x_SkuTerm * 730.0)); +let meters = reservations | distinct x_SkuMeterId; +reservations +| lookup kind=inner ( + SkuPrices_v1_5 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and UnitPrice > 0 + | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId + ) on x_SkuMeterId +| summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc +// ### Columns +let reservations = materialize(SkuPricesWide_v1_5 + | where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Purchase' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 and isnotnull(ContractedUnitPrice) + | project x_SkuMeterId, x_SkuTerm, HourlyRate = ContractedUnitPrice / (x_SkuTerm * 730.0)); +let meters = reservations | distinct x_SkuMeterId; +reservations +| lookup kind=inner ( + SkuPricesWide_v1_5 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and ContractedUnitPrice > 0 + | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId + ) on x_SkuMeterId +| summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc + +// ## S05 Savings plan rate vs. on-demand +// Question: For each savings plan term, how much lower is the current savings plan rate than the on-demand contracted rate? +// Source: issue #2706 (commitment unit price on covered usage); scope_and_evolution.md +// Expect: same +// ### Rows +let savingsPlans = materialize(SkuPrices_v1_5 + | where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'Effective' and x_SkuPriceType == 'SavingsPlan' + | project x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SavingsPlanRate = UnitPrice); +let meters = savingsPlans | distinct x_SkuMeterId; +savingsPlans +| lookup kind=inner ( + SkuPrices_v1_5 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and UnitPrice > 0 + | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId, x_SkuOfferId + ) on x_SkuMeterId, x_SkuOfferId +| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - SavingsPlanRate) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc +// ### Columns +let savingsPlans = materialize(SkuPricesWide_v1_5 + | where isnull(SkuPriceEffectiveEnd) and x_SkuPriceType == 'SavingsPlan' and isnotnull(x_EffectiveUnitPrice) + | project x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SavingsPlanRate = x_EffectiveUnitPrice); +let meters = savingsPlans | distinct x_SkuMeterId; +savingsPlans +| lookup kind=inner ( + SkuPricesWide_v1_5 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and ContractedUnitPrice > 0 + | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId, x_SkuOfferId + ) on x_SkuMeterId, x_SkuOfferId +| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - SavingsPlanRate) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc + +// ## S06 Price changes since July +// Question: Which production prices changed since July 2026, by price type, and did they go up or down? +// Source: appendix price_effectivity_and_currency.md; working-draft "Fluctuation Tracking of Unit Prices" +// Expect: same +// ### Rows +// A change is a price that starts in or after July and replaces an earlier price for the same key (new SKUs excluded) +SkuPrices_v1_5 +| where SkuPriceEffectiveStart >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' +| project SkuPriceId, ContractId, PricingCurrency, x_UnitPriceType, Month = startofmonth(SkuPriceEffectiveStart), UnitPrice +| lookup kind=inner ( + SkuPrices_v1_5 + | where SkuPriceEffectiveEnd >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' + | project SkuPriceId, ContractId, PricingCurrency, x_UnitPriceType, Month = SkuPriceEffectiveEnd, PreviousPrice = UnitPrice + ) on SkuPriceId, ContractId, PricingCurrency, x_UnitPriceType, Month +| summarize Changed = count(), Up = countif(UnitPrice > PreviousPrice), Down = countif(UnitPrice < PreviousPrice) by PriceType = x_UnitPriceType +| order by PriceType asc +// ### Columns +// A row changes when any price on it changes, so each price column has to be compared with the previous row +SkuPricesWide_v1_5 +| where SkuPriceEffectiveStart >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' +| project SkuPriceId, ContractId, PricingCurrency, Month = startofmonth(SkuPriceEffectiveStart), ListUnitPrice, x_BaseUnitPrice, ContractedUnitPrice, x_EffectiveUnitPrice +| lookup kind=inner ( + SkuPricesWide_v1_5 + | where SkuPriceEffectiveEnd >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' + | project SkuPriceId, ContractId, PricingCurrency, Month = SkuPriceEffectiveEnd, PrevList = ListUnitPrice, PrevBase = x_BaseUnitPrice, PrevContracted = ContractedUnitPrice, PrevEffective = x_EffectiveUnitPrice + ) on SkuPriceId, ContractId, PricingCurrency, Month +| mv-expand kind = array p = pack_array( + pack_array('Base', x_BaseUnitPrice, PrevBase), + pack_array('Contracted', ContractedUnitPrice, PrevContracted), + pack_array('Effective', x_EffectiveUnitPrice, PrevEffective), + pack_array('List', ListUnitPrice, PrevList)) +| extend PriceType = tostring(p[0]), UnitPrice = todouble(p[1]), PreviousPrice = todouble(p[2]) +| where isnotnull(UnitPrice) and isnotnull(PreviousPrice) and UnitPrice != PreviousPrice +| summarize Changed = count(), Up = countif(UnitPrice > PreviousPrice), Down = countif(UnitPrice < PreviousPrice) by PriceType +| order by PriceType asc + +// ## S07 Cheapest region for a VM size +// Question: Which regions have the lowest current list price for each Dv5 VM size? +// Source: PR 2595 "Compare List Prices Across Regions" +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'List' and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' +| summarize Price = min(UnitPrice) by x_SkuMeter, PricingRegionId +| as hint.materialized = true T +| lookup kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter +| where Price == MinPrice +| summarize CheapestRegions = strcat_array(array_sort_asc(make_list(PricingRegionId)), ", ") by x_SkuMeter, MinPrice, MaxPrice, Regions +| order by x_SkuMeter asc +// ### Columns +SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' and isnotnull(ListUnitPrice) +| summarize Price = min(ListUnitPrice) by x_SkuMeter, PricingRegionId +| as hint.materialized = true T +| lookup kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter +| where Price == MinPrice +| summarize CheapestRegions = strcat_array(array_sort_asc(make_list(PricingRegionId)), ", ") by x_SkuMeter, MinPrice, MaxPrice, Regions +| order by x_SkuMeter asc + +// ## S08 Verify billed unit prices +// Question: Do the list and contracted unit prices on September's on-demand charges match the prices in effect in September? +// Source: PR 2595 "Compare Recorded Unit Prices with Published Rates"; dataset.md Relationships +// Expect: same +// ### Rows +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) + | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +// All charges are in September, so one "in effect on Sep 1" filter covers them +let prices = SkuPrices_v1_5 | where SkuPriceId in (ids) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)); +costs +| lookup kind=leftouter (prices | where x_UnitPriceType == 'List' | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Contracted' | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize Charges = count(), Priced = countif(isnotnull(PriceList * PerUnit)), ListMatches = countif(abs(ListUnitPrice - PriceList * PerUnit) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted * PerUnit) < 1e-6) +// ### Columns +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) + | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter ( + SkuPricesWide_v1_5 + | where SkuPriceId in (ids) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)) + | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription + ) on SkuPriceIdv2, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize Charges = count(), Priced = countif(isnotnull(PriceList * PerUnit)), ListMatches = countif(abs(ListUnitPrice - PriceList * PerUnit) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted * PerUnit) < 1e-6) +// ### Monthly +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) + | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter ( + Prices_final_v1_2 + | where SkuPriceIdv2 in (ids) and x_EffectivePeriodStart >= datetime(2026-09-01) and x_EffectivePeriodStart < datetime(2026-10-01) + | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription + ) on SkuPriceIdv2, x_BillingAccountId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize Charges = count(), Priced = countif(isnotnull(PriceList * PerUnit)), ListMatches = countif(abs(ListUnitPrice - PriceList * PerUnit) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted * PerUnit) < 1e-6) + +// ## S09 Savings plan what-if for on-demand usage +// Question: What would September's on-demand VM usage cost at list, contracted, and 1-year savings plan rates? +// Source: PR 2595 "Project the Effect of Moving Consumption to a Contracted Rate"; issue #2706 +// Expect: same +// ### Rows +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' + // No column links an on-demand price to its savings plan price; the IDs differ only by prefix + | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); +let ids = costs | distinct SkuPriceIdv2; +let spIds = costs | distinct SavingsPlanSkuPriceId; +let prices = SkuPrices_v1_5 | where (SkuPriceId in (ids) or SkuPriceId in (spIds)) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)); +costs +| lookup kind=leftouter (prices | where x_UnitPriceType == 'List' | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Contracted' | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Effective' | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = UnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize AtList = round(sum(PricingQuantity * PriceList * PerUnit), 2), AtContracted = round(sum(PricingQuantity * PriceContracted * PerUnit), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted) * PerUnit), 2), Billed = round(sum(EffectiveCost), 2) +// ### Columns +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' + // No column links an on-demand price to its savings plan price; the IDs differ only by prefix + | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); +let ids = costs | distinct SkuPriceIdv2; +let spIds = costs | distinct SavingsPlanSkuPriceId; +let prices = SkuPricesWide_v1_5 | where (SkuPriceId in (ids) or SkuPriceId in (spIds)) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)); +costs +| lookup kind=leftouter (prices | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (prices | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize AtList = round(sum(PricingQuantity * PriceList * PerUnit), 2), AtContracted = round(sum(PricingQuantity * PriceContracted * PerUnit), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted) * PerUnit), 2), Billed = round(sum(EffectiveCost), 2) +// ### Monthly +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' + // No column links an on-demand price to its savings plan price; the IDs differ only by prefix + | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), x_BillingAccountId, PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); +let ids = costs | distinct SkuPriceIdv2; +let spIds = costs | distinct SavingsPlanSkuPriceId; +let prices = Prices_final_v1_2 | where (SkuPriceIdv2 in (ids) or SkuPriceIdv2 in (spIds)) and x_EffectivePeriodStart >= datetime(2026-09-01) and x_EffectivePeriodStart < datetime(2026-10-01); +costs +| lookup kind=leftouter (prices | where x_SkuPriceType != 'SavingsPlan' | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, x_BillingAccountId, PricingCurrency +| lookup kind=leftouter (prices | where x_SkuPriceType == 'SavingsPlan' | project SavingsPlanSkuPriceId = SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice) on SavingsPlanSkuPriceId, x_BillingAccountId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize AtList = round(sum(PricingQuantity * PriceList * PerUnit), 2), AtContracted = round(sum(PricingQuantity * PriceContracted * PerUnit), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted) * PerUnit), 2), Billed = round(sum(EffectiveCost), 2) + +// ## S10 All prices for one SKU +// Question: Show every current price type for one VM size in one region side by side. +// Source: PR 2595 catalog discovery; FOCUS Cost and Usage ListUnitPrice/ContractedUnitPrice pairing +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' and isnull(SkuPriceEffectiveEnd) +| summarize + ListUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'List'), + BaseUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'Base'), + ContractedUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'Contracted'), + EffectiveUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'Effective') + by SkuPriceId, PricingCurrency +| order by SkuPriceId asc +// ### Columns +SkuPricesWide_v1_5 +| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' and isnull(SkuPriceEffectiveEnd) +| project SkuPriceId, PricingCurrency, ListUnitPrice, BaseUnitPrice = x_BaseUnitPrice, ContractedUnitPrice, EffectiveUnitPrice = x_EffectiveUnitPrice +| order by SkuPriceId asc + +// ## S11 Estimate a planned workload +// Question: What would 2 D2 v5, 4 D4 v5, and 1 E8 v5 VMs in US East cost for a month at current list and contracted rates? +// Source: PR 2595 "Estimate the Cost of a Planned Workload" +// Expect: same +// ### Rows +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; +let plan = datatable(SkuPriceId: string, Quantity: real) [ + 'ODU_DZH318Z08M9X_008W_1 Compute Hour_MS-AZR-0017P', 1460, + 'ODU_DZH318Z08M9X_007S_1 Compute Hour_MS-AZR-0017P', 2920, + 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; +SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType in ('List', 'Contracted') +| lookup kind=inner plan on SkuPriceId +| extend Cost = Quantity * UnitPrice / block(x_PricingUnitDescription) +| summarize AtList = round(sumif(Cost, x_UnitPriceType == 'List'), 2), AtContracted = round(sumif(Cost, x_UnitPriceType == 'Contracted'), 2) +// ### Columns +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; +let plan = datatable(SkuPriceId: string, Quantity: real) [ + 'ODU_DZH318Z08M9X_008W_1 Compute Hour_MS-AZR-0017P', 1460, + 'ODU_DZH318Z08M9X_007S_1 Compute Hour_MS-AZR-0017P', 2920, + 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; +SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) +| lookup kind=inner plan on SkuPriceId +| summarize AtList = round(sum(Quantity * ListUnitPrice / block(x_PricingUnitDescription)), 2), AtContracted = round(sum(Quantity * ContractedUnitPrice / block(x_PricingUnitDescription)), 2) + +// ## S12 Public prices visible to each account +// Question: How many current public list prices can the agreement's billing account see, and how many can an account outside the agreement see? +// Source: PR 2595 "Identify the Prices a Billing Account is Eligible For"; sku_price_eligibility_examples.md +// Expect: different (columns scope the whole row, including the public list price, to the agreement, so other accounts see none) +// ### Rows +let accounts = datatable(Account: string, BillingAccountId: string) [ + 'Agreement account', '/providers/microsoft.billing/billingaccounts/8611537', + 'Other account', '/providers/microsoft.billing/billingaccounts/1234567' ]; +SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'List' +// Evaluate each distinct eligibility object once instead of once per price +| summarize Prices = count() by Eligibility = tostring(SkuPriceEligibility) +| extend e = parse_json(Eligibility), tmp = 1 +| join kind=inner (accounts | extend tmp = 1) on tmp +| mv-apply rule = e.Inclusions on ( + summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( + (rule.Operator == 'In' and set_has_element(rule.Values, BillingAccountId)) or + (rule.Operator == 'StartsWith' and BillingAccountId startswith tostring(rule.Values[0]))))) +| summarize ListPrices = sumif(Prices, tobool(e.IsGlobalScope) or Included > 0) by Account +| order by Account asc +// ### Columns +let accounts = datatable(Account: string, BillingAccountId: string) [ + 'Agreement account', '/providers/microsoft.billing/billingaccounts/8611537', + 'Other account', '/providers/microsoft.billing/billingaccounts/1234567' ]; +SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) and isnotnull(ListUnitPrice) +// Evaluate each distinct eligibility object once instead of once per price +| summarize Prices = count() by Eligibility = tostring(SkuPriceEligibility) +| extend e = parse_json(Eligibility), tmp = 1 +| join kind=inner (accounts | extend tmp = 1) on tmp +| mv-apply rule = e.Inclusions on ( + summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( + (rule.Operator == 'In' and set_has_element(rule.Values, BillingAccountId)) or + (rule.Operator == 'StartsWith' and BillingAccountId startswith tostring(rule.Values[0]))))) +| summarize ListPrices = sumif(Prices, tobool(e.IsGlobalScope) or Included > 0) by Account +| order by Account asc + +// ## S13 Usage rates vs. purchase fees +// Question: How many current prices of each type are usage rates vs. purchase fees? +// Source: PR 2595 "Separate Usage Rates from Purchase Fees and Credit Values" +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) +| summarize Prices = count() by ChargeCategory, PriceType = x_UnitPriceType +| order by ChargeCategory asc, PriceType asc +// ### Columns +SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) +| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) by ChargeCategory +| mv-expand kind = array p = bag_pack('List', List, 'Base', Base, 'Contracted', Contracted, 'Effective', Effective) +| project ChargeCategory, PriceType = tostring(p[0]), Prices = tolong(p[1]) +| where Prices > 0 +| order by ChargeCategory asc, PriceType asc + +// ## S14 Backfill missing prices on cost data +// Question: Fill missing list and contracted unit prices and costs on every cost row from the prices in effect that month. +// Source: FinOps hubs Costs transform (price backfill); dataset.md Relationships (SkuPriceId + ContractId + effective dates) +// Expect: same +// Check: | summarize Rows = count(), MissingList = countif(isnull(ListUnitPrice) or ListUnitPrice == 0), MissingContracted = countif(isnull(ContractedUnitPrice) or ContractedUnitPrice == 0), ListCost = round(sum(ListCost), 2), ContractedCost = round(sum(ContractedCost), 2) +// ### Rows +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), Month = startofmonth(ChargePeriodStart), + MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0); +let ids = costs | where isnotempty(SkuPriceIdv2) and (MissingList or MissingContracted) | distinct SkuPriceIdv2; +// Spread each price over the months it was in effect, so a charge matches its month exactly (no range join) +let prices = SkuPrices_v1_5 + | where SkuPriceId in (ids) and x_UnitPriceType in ('List', 'Contracted') + | extend Months = datetime_diff('month', coalesce(SkuPriceEffectiveEnd, datetime_add('month', 1, startofmonth(now()))), startofmonth(SkuPriceEffectiveStart)) + | mv-expand i = range(0, Months - 1, 1) to typeof(int) + | extend Month = datetime_add('month', i, startofmonth(SkuPriceEffectiveStart)); +costs +| lookup kind=leftouter (prices | where x_UnitPriceType == 'List' | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, Month, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency, Month +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Contracted' | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, Month, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency, Month +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend + ListUnitPrice = iff(MissingList and PriceList * PerUnit > 0, PriceList * PerUnit, ListUnitPrice), + ContractedUnitPrice = iff(MissingContracted and PriceContracted * PerUnit > 0, PriceContracted * PerUnit, ContractedUnitPrice) +| extend + ListCost = iff(MissingList and ListUnitPrice > 0, PricingQuantity * ListUnitPrice, ListCost), + ContractedCost = iff(MissingContracted and ContractedUnitPrice > 0, PricingQuantity * ContractedUnitPrice, ContractedCost) +| project-away ContractId, Month, MissingList, MissingContracted, PriceList, PriceContracted, PriceUnit, PerUnit +// ### Columns +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), Month = startofmonth(ChargePeriodStart), + MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0); +let ids = costs | where isnotempty(SkuPriceIdv2) and (MissingList or MissingContracted) | distinct SkuPriceIdv2; +// Spread each price over the months it was in effect, so a charge matches its month exactly (no range join) +let prices = SkuPricesWide_v1_5 + | where SkuPriceId in (ids) + | extend Months = datetime_diff('month', coalesce(SkuPriceEffectiveEnd, datetime_add('month', 1, startofmonth(now()))), startofmonth(SkuPriceEffectiveStart)) + | mv-expand i = range(0, Months - 1, 1) to typeof(int) + | extend Month = datetime_add('month', i, startofmonth(SkuPriceEffectiveStart)); +costs +| lookup kind=leftouter (prices | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, Month, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency, Month +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend + ListUnitPrice = iff(MissingList and PriceList * PerUnit > 0, PriceList * PerUnit, ListUnitPrice), + ContractedUnitPrice = iff(MissingContracted and PriceContracted * PerUnit > 0, PriceContracted * PerUnit, ContractedUnitPrice) +| extend + ListCost = iff(MissingList and ListUnitPrice > 0, PricingQuantity * ListUnitPrice, ListCost), + ContractedCost = iff(MissingContracted and ContractedUnitPrice > 0, PricingQuantity * ContractedUnitPrice, ContractedCost) +| project-away ContractId, Month, MissingList, MissingContracted, PriceList, PriceContracted, PriceUnit, PerUnit +// ### Monthly +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | extend Month = startofmonth(ChargePeriodStart), + MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0); +let ids = costs | where isnotempty(SkuPriceIdv2) and (MissingList or MissingContracted) | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter ( + Prices_final_v1_2 + | where SkuPriceIdv2 in (ids) + | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, Month = startofmonth(x_EffectivePeriodStart), PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription + ) on SkuPriceIdv2, x_BillingAccountId, PricingCurrency, Month +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend + ListUnitPrice = iff(MissingList and PriceList * PerUnit > 0, PriceList * PerUnit, ListUnitPrice), + ContractedUnitPrice = iff(MissingContracted and PriceContracted * PerUnit > 0, PriceContracted * PerUnit, ContractedUnitPrice) +| extend + ListCost = iff(MissingList and ListUnitPrice > 0, PricingQuantity * ListUnitPrice, ListCost), + ContractedCost = iff(MissingContracted and ContractedUnitPrice > 0, PricingQuantity * ContractedUnitPrice, ContractedCost) +| project-away Month, MissingList, MissingContracted, PriceList, PriceContracted, PriceUnit, PerUnit diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 new file mode 100644 index 000000000..57c67ee03 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 @@ -0,0 +1,147 @@ +<# +.SYNOPSIS +Runs each scenario in SkuPrices_v1_5_scenarios.kql against both layouts, compares results, and scores them. + +.DESCRIPTION +Parses "// ## " scenarios with "// ### Rows" and "// ### Columns" queries. Runs each query -Runs times +(alternating sides, results cache off), and records the median of the server's execution time, CPU, peak memory, and rows +scanned (from QueryResourceConsumption). Also measures query size (lines, operators, joins, lets, x_ column references) and +whether both sides return the same answer. Writes everything to JSON for reporting. +#> +[CmdletBinding()] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev', + [string] $Path = "$PSScriptRoot/SkuPrices_v1_5_scenarios.kql", + [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) 'SkuPrices_v1_5_scenarios.results.json'), + [int] $Runs = 3, + [string[]] $Only +) + +$ErrorActionPreference = 'Stop' +Set-AzContext -Context (Get-AzContext -Name $Context) -Scope Process | Out-Null +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)" } + +function ConvertTo-Seconds([string] $ts) { if (-not $ts) { return 0.0 }; [TimeSpan]::Parse($ts).TotalSeconds } + +# Run a query on the v2 endpoint; return result rows plus server statistics +function Invoke-Kusto([string] $Csl) { + $body = @{ db = $Database; csl = $Csl; properties = @{ Options = @{ servertimeout = '00:20:00'; query_results_cache_max_age = '00:00:00' } } } | ConvertTo-Json -Depth 5 + $resp = Invoke-WebRequest -Uri "$Cluster/v2/rest/query" -Method Post -Headers $headers -ContentType 'application/json' -Body $body -TimeoutSec 1200 -SkipHttpErrorCheck + if ($resp.StatusCode -ne 200) { throw "HTTP $($resp.StatusCode): $($resp.Content.Substring(0, [Math]::Min(300, $resp.Content.Length)))" } + $frames = $resp.Content | ConvertFrom-Json -Depth 50 + $primary = $frames | Where-Object { $_.FrameType -eq 'DataTable' -and $_.TableKind -eq 'PrimaryResult' } | Select-Object -First 1 + $errors = $frames | Where-Object { $_.FrameType -eq 'DataTable' -and $_.TableKind -eq 'PrimaryResult' } | ForEach-Object { $_.Rows } | Where-Object { $_ -isnot [array] -and $_.OneApiErrors } + if ($errors) { throw ($errors[0].OneApiErrors[0].error.'@message') } + $info = $frames | Where-Object { $_.TableKind -eq 'QueryCompletionInformation' } | Select-Object -First 1 + $stats = $null + if ($info) { + $cols = $info.Columns.ColumnName + foreach ($row in $info.Rows) { if ($row[$cols.IndexOf('EventTypeName')] -eq 'QueryResourceConsumption') { $stats = $row[$cols.IndexOf('Payload')] | ConvertFrom-Json -Depth 50 } } + } + [pscustomobject]@{ + Columns = @($primary.Columns.ColumnName) + Rows = @($primary.Rows | Where-Object { $_ -is [array] }) + Ms = if ($stats) { [math]::Round($stats.ExecutionTime * 1000) } else { $null } + CpuMs = if ($stats) { [math]::Round((ConvertTo-Seconds $stats.resource_usage.cpu.'total cpu') * 1000) } else { $null } + MemoryMB = if ($stats) { [math]::Round($stats.resource_usage.memory.peak_per_node / 1MB, 1) } else { $null } + RowsScanned = if ($stats) { [long]$stats.input_dataset_statistics.rows.scanned } else { $null } + } +} + +function Get-Median([double[]] $v) { $s = $v | Sort-Object; $s[[int][math]::Floor(($s.Count - 1) / 2)] } + +# Normalize a result so equivalent outputs compare equal (sorted rows, rounded numbers) +function Get-Signature($Rows) { + ($Rows | ForEach-Object { ($_ | ForEach-Object { if ($_ -is [double] -or $_ -is [decimal] -or $_ -is [long] -or $_ -is [int]) { [math]::Round([double]$_, 4) } else { "$_" } }) -join '|' } | Sort-Object) -join "`n" +} + +# Query size measures (comments ignored) +function Get-Shape([string] $q) { + $code = ($q -split "`n" | ForEach-Object { ($_ -replace '//.*$', '').TrimEnd() } | Where-Object { $_.Trim() }) -join "`n" + [pscustomobject]@{ + Lines = @($code -split "`n").Count + Operators = ([regex]::Matches($code, '\|\s*[a-z-]+')).Count + Joins = ([regex]::Matches($code, '\b(join|lookup)\b')).Count + Lets = ([regex]::Matches($code, '(?m)^\s*let\b')).Count + XColumns = @([regex]::Matches($code, '\bx_[A-Za-z0-9]+\b') | ForEach-Object Value | Select-Object -Unique).Count + } +} + +# Parse scenarios +$text = Get-Content -Path $Path -Raw +$scenarios = foreach ($block in ($text -split '(?m)^// ## ') | Select-Object -Skip 1) { + $lines = $block -split "`n" + $id, $name = $lines[0].Trim() -split ' ', 2 + $meta = @{} + foreach ($l in $lines) { if ($l -match '^// (Question|Source|Expect|Check): (.+)$') { $meta[$Matches[1]] = $Matches[2].Trim() } } + $q = @{} + foreach ($part in ($block -split '(?m)^(?=// ### (?:Rows|Columns|Monthly)\s*$)') | Select-Object -Skip 1) { + if ($part -match '^// ### (Rows|Columns|Monthly)') { $q[$Matches[1]] = ($part -replace '^// ### \w+\s*\n', '').Trim() } + } + [pscustomobject]@{ Id = $id; Name = $name.Trim(); Question = $meta.Question; Source = $meta.Source; Expect = $meta.Expect; Check = $meta.Check; Queries = $q } +} +if ($Only) { $scenarios = $scenarios | Where-Object Id -in $Only } + +# Winner among sides: lowest value wins; within 15% (or a tiny absolute gap) of the runner-up is a tie +function Get-Winner([hashtable] $values, [double] $minGap = 0) { + $v = $values.GetEnumerator() | Where-Object { $null -ne $_.Value } | Sort-Object Value + if (@($v).Count -lt 2) { return 'n/a' } + $best = $v[0]; $next = $v[1]; $hi = [math]::Max([double]$best.Value, [double]$next.Value) + if ($hi -eq 0 -or [math]::Abs($next.Value - $best.Value) -le [math]::Max($minGap, 0.15 * $hi)) { 'Tie' } else { $best.Key } +} + +$results = foreach ($s in $scenarios) { + $sideNames = @('Rows', 'Columns', 'Monthly' | Where-Object { $s.Queries[$_] }) + $runsBySide = @{}; $err = @{} + foreach ($side in $sideNames) { $runsBySide[$side] = @() } + for ($i = 0; $i -lt $Runs; $i++) { + $order = if ($i % 2) { [array]::Reverse(($o = @($sideNames))); $o } else { $sideNames } + foreach ($side in $order) { + if ($err[$side]) { continue } + $csl = $s.Queries[$side] + $(if ($s.Check) { "`n" + $s.Check } else { '' }) + try { $runsBySide[$side] += Invoke-Kusto $csl } catch { $err[$side] = $_.Exception.Message } + } + } + $sides = [ordered]@{} + foreach ($side in $sideNames) { + $shape = Get-Shape $s.Queries[$side]; $rs = $runsBySide[$side] + $sides[$side] = if ($err[$side]) { [pscustomobject]@{ Error = $err[$side] } } else { + [pscustomobject]@{ + Ms = Get-Median ($rs.Ms); CpuMs = Get-Median ($rs.CpuMs); MemoryMB = Get-Median ($rs.MemoryMB); RowsScanned = Get-Median ($rs.RowsScanned) + ResultRows = $rs[0].Rows.Count; Columns = $rs[0].Columns + Data = @($rs[0].Rows | Select-Object -First 15 | ForEach-Object { , @($_) }) + Signature = Get-Signature $rs[0].Rows + Lines = $shape.Lines; Operators = $shape.Operators; Joins = $shape.Joins; Lets = $shape.Lets; XColumns = $shape.XColumns + } + } + } + $ok = @($sides.Values | Where-Object { -not $_.Error }) + $hasError = $ok.Count -lt $sides.Count + $match = -not $hasError -and @($ok.Signature | Select-Object -Unique).Count -eq 1 + $expectSame = $s.Expect -like 'same*' + $status = if ($hasError) { 'ERROR' } elseif ($match -eq $expectSame) { 'PASS' } else { 'FAIL' } + $score = if ($status -ne 'ERROR') { + $m = { param($f) $h = @{}; foreach ($k in $sides.Keys) { $h[$k] = & $f $sides[$k] }; $h } + [ordered]@{ + Simpler = Get-Winner (& $m { param($x) $x.Operators + 2 * $x.Joins + $x.Lets }) 1 + Faster = Get-Winner (& $m { param($x) $x.Ms }) 50 + Cpu = Get-Winner (& $m { param($x) $x.CpuMs }) 50 + Memory = Get-Winner (& $m { param($x) $x.MemoryMB }) 5 + Scanned = Get-Winner (& $m { param($x) $x.RowsScanned }) + Portable = Get-Winner (& $m { param($x) $x.XColumns }) + Correct = if ($match) { 'Tie' } elseif ($s.Expect -match 'different') { 'Rows' } else { 'n/a' } + } + } + $line = ($sides.Keys | ForEach-Object { $x = $sides[$_]; "{0} {1}ms cpu {2}ms {3}MB {4}ops" -f $_, $x.Ms, $x.CpuMs, $x.MemoryMB, $x.Operators }) -join ' | ' + Write-Host ("{0,-5} {1} {2,-40} {3} match={4}" -f $status, $s.Id, $s.Name, $line, $match) + if ($status -eq 'ERROR') { $sides.Keys | ForEach-Object { if ($sides[$_].Error) { Write-Host " ${_}: $($sides[$_].Error)" } } } + $out = [ordered]@{ Id = $s.Id; Name = $s.Name; Question = $s.Question; Source = $s.Source; Expect = $s.Expect; Check = $s.Check; Status = $status; Match = $match; Score = $score; Sides = @($sideNames) } + foreach ($side in $sideNames) { $out["${side}Query"] = $s.Queries[$side]; $out[$side] = $sides[$side] | Select-Object * -ExcludeProperty Signature } + [pscustomobject]$out +} + +$results | ConvertTo-Json -Depth 8 | Set-Content -Path $OutFile +Write-Host "`nPassed: $(@($results | Where-Object Status -eq 'PASS').Count) of $($results.Count). Results: $OutFile" diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 new file mode 100644 index 000000000..06b9ce0cb --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 @@ -0,0 +1,128 @@ +<# +.SYNOPSIS +Validates SkuPrices_v1_5 on ftk-dev against the source price sheet and the PR 2424 SKU Price requirements. +#> +[CmdletBinding()] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev' +) + +$ErrorActionPreference = 'Stop' +Set-AzContext -Context (Get-AzContext -Name $Context) -Scope Process | Out-Null +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } + +function Invoke-Kusto([string] $Csl) { + $body = @{ db = $Database; csl = $Csl; properties = @{ Options = @{ servertimeout = '00:30:00' } } } | ConvertTo-Json -Depth 5 + $t = (Invoke-RestMethod -Uri "$Cluster/v1/rest/query" -Method Post -Headers $headers -Body $body -TimeoutSec 1800).Tables[0] + $names = $t.Columns | ForEach-Object ColumnName + $t.Rows | ForEach-Object { $r = $_; $o = [ordered]@{}; for ($i = 0; $i -lt $names.Count; $i++) { $o[$names[$i]] = $r[$i] }; [pscustomobject]$o } +} + +# Latest price sheet per billing account, same as the build +$sheet = @' +let latestSheet = Prices_final_v1_2 | summarize tmp_LatestDay = max(startofday(x_IngestionTime)) by BillingAccountId; +let sheet = Prices_final_v1_2 | lookup kind=inner latestSheet on BillingAccountId | where startofday(x_IngestionTime) == tmp_LatestDay; +'@ + +# Each check returns one row; Pass = expected +$checks = [ordered]@{ + 'Dupes: extended key (+x_UnitPriceType)' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize n = count() by ServiceProviderName, SkuPriceId, ContractId, SkuPriceEffectiveStart, PricingCurrency, x_UnitPriceType | where n > 1 | count' } + 'Dupes: FOCUS 1.5 key (info: price type collisions)' = @{ Expect = $null; Csl = 'SkuPrices_v1_5 | summarize n = count() by ServiceProviderName, SkuPriceId, ContractId, SkuPriceEffectiveStart, PricingCurrency | where n > 1 | count' } + 'Overlapping effective periods (extended key)' = @{ Expect = 0; Csl = @' +SkuPrices_v1_5 +| project k = strcat(ServiceProviderName, '|', SkuPriceId, '|', ContractId, '|', PricingCurrency, '|', x_UnitPriceType), SkuPriceEffectiveStart, SkuPriceEffectiveEnd +| sort by k asc, SkuPriceEffectiveStart asc +| where next(k) == k and (isnull(SkuPriceEffectiveEnd) or next(SkuPriceEffectiveStart) < SkuPriceEffectiveEnd) +| count +'@ } + 'Source: same key, different prices (hidden by take_any)' = @{ Expect = 0; Csl = $sheet + @' +sheet +| summarize List = dcount(ListUnitPrice), Base = dcount(x_BaseUnitPrice), Contracted = dcount(ContractedUnitPrice), Effective = dcount(x_EffectiveUnitPrice), n = count() + by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency +| where List > 1 or Base > 1 or Contracted > 1 or Effective > 1 +| count +'@ } + 'Source: duplicate rows on key (info)' = @{ Expect = $null; Csl = $sheet + 'sheet | summarize n = count() by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | where n > 1 | summarize Keys = count(), ExtraRows = sum(n - 1)' } + 'Current rows: List vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ListUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'List' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current rows: Base vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_BaseUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Base' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current rows: Contracted vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ContractedUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current rows: Effective vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_EffectiveUnitPrice) | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current UnitPrice + discount match the latest sheet' = @{ Expect = 0; Csl = $sheet + @' +let s = sheet | summarize take_any(ListUnitPrice, x_BaseUnitPrice, ContractedUnitPrice, x_EffectiveUnitPrice) by SkuPriceIdv2, PricingCurrency; +SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) +| lookup kind=leftouter s on $left.x_SkuPriceIdv2 == $right.SkuPriceIdv2, PricingCurrency +| extend Src = case(x_UnitPriceType == 'List', ListUnitPrice, x_UnitPriceType == 'Base', x_BaseUnitPrice, x_UnitPriceType == 'Contracted', ContractedUnitPrice, x_EffectiveUnitPrice) +| where isnull(Src) or UnitPrice != Src or (x_UnitPriceType != 'List' and abs(x_UnitPriceDiscount - (ListUnitPrice - UnitPrice)) > 1e-9) +| count +'@ } + 'Mandatory columns null/empty' = @{ Expect = 0; Csl = @' +SkuPrices_v1_5 +| where isempty(ChargeCategory) or isempty(PricingCurrency) or isempty(PricingCurrencyCategory) or isempty(PricingServiceName) or isempty(PricingUnit) + or isempty(ServiceProviderName) or isempty(SkuId) or isempty(SkuPriceDescription) or isempty(SkuPriceId) or isnull(SkuPriceCreated) + or isnull(SkuPriceLastUpdated) or isnull(UnitPrice) or isnull(SkuPriceEligibility) or isnull(QuantityTierMinimum) +| count +'@ } + 'Mandatory column gaps by column (info)' = @{ Expect = $null; Csl = @' +SkuPrices_v1_5 +| summarize ChargeCategory = countif(isempty(ChargeCategory)), PricingServiceName = countif(isempty(PricingServiceName)), PricingUnit = countif(isempty(PricingUnit)), + SkuId = countif(isempty(SkuId)), SkuPriceDescription = countif(isempty(SkuPriceDescription)), PricingRegionId = countif(isempty(PricingRegionId)), SkuPriceEffectiveEnd = countif(isnull(SkuPriceEffectiveEnd)), SkuPriceEffectiveStart = countif(isnull(SkuPriceEffectiveStart)) +'@ } + 'ContractId empty iff List' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where (x_UnitPriceType == 'List') != isempty(ContractId) | count" } + 'Negative UnitPrice' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | where UnitPrice < 0 | count' } + 'ChargeCategory not Usage/Purchase/Credit' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where ChargeCategory !in ('Usage', 'Purchase', 'Credit') | count" } + 'PurchaseDurationType set on non-Purchase' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where ChargeCategory != 'Purchase' and isnotempty(PurchaseDurationType) | count" } + 'EffectiveEnd <= EffectiveStart' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | where isnotnull(SkuPriceEffectiveEnd) and isnotnull(SkuPriceEffectiveStart) and SkuPriceEffectiveEnd <= SkuPriceEffectiveStart | count' } + 'Tier max <= tier min' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | where isnotnull(QuantityTierMaximum) and QuantityTierMaximum <= QuantityTierMinimum | count' } + 'SkuPriceId with >1 SkuId/PricingUnit/ChargeCategory' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize s = dcount(SkuId), u = dcount(PricingUnit), c = dcount(ChargeCategory) by SkuPriceId | where s > 1 or u > 1 or c > 1 | count' } + 'Effective start distribution (info)' = @{ Expect = $null; Csl = "SkuPrices_v1_5 | summarize Rows = count() by Start = startofmonth(SkuPriceEffectiveStart), x_UnitPriceType | summarize Rows = sum(Rows), Types = make_set(x_UnitPriceType) by Start | order by Start asc" } + 'Price changed inside its run, or run started late (sample of 2000 Contracted)' = @{ Expect = 0; Csl = @' +let s = SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | sample 2000 | project SkuPriceIdv2 = x_SkuPriceIdv2, PricingCurrency, Start = startofmonth(SkuPriceEffectiveStart), End = coalesce(SkuPriceEffectiveEnd, datetime(2100-01-01)), Cur = UnitPrice; +Prices_final_v1_2 +| where x_SkuPriceType != 'SavingsPlan' +| extend M = startofmonth(x_IngestionTime) +| join kind=inner s on SkuPriceIdv2, PricingCurrency +| summarize Changed = countif(M >= Start and M < End and ContractedUnitPrice != Cur), PriorSame = countif(M == datetime_add('month', -1, Start) and ContractedUnitPrice == Cur) by SkuPriceIdv2, PricingCurrency, Start +| where Changed > 0 or PriorSame > 0 +| count +'@ } + 'Rows per type match runs table' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType | join kind=fullouter (SkuPrices_v1_5_runs_table | where x_UnitPriceType != 'Wide' | summarize Runs = count() by x_UnitPriceType) on x_UnitPriceType | summarize Diff = sum(abs(coalesce(Rows, 0) - coalesce(Runs, 0)))" } + 'Wide: dupes on SkuPriceId + ContractId + currency + start' = @{ Expect = 0; Csl = 'SkuPricesWide_v1_5 | summarize n = count() by SkuPriceId, ContractId, PricingCurrency, SkuPriceEffectiveStart | where n > 1 | count' } + 'Wide: overlapping effective periods' = @{ Expect = 0; Csl = @' +SkuPricesWide_v1_5 +| project k = strcat(SkuPriceId, '|', ContractId, '|', PricingCurrency), SkuPriceEffectiveStart, SkuPriceEffectiveEnd +| sort by k asc, SkuPriceEffectiveStart asc +| where next(k) == k and (isnull(SkuPriceEffectiveEnd) or next(SkuPriceEffectiveStart) < SkuPriceEffectiveEnd) +| count +'@ } + 'Wide: rows match runs table' = @{ Expect = 0; Csl = "SkuPricesWide_v1_5 | count | extend Runs = toscalar(SkuPrices_v1_5_runs_table | where x_UnitPriceType == 'Wide' | count) | project Diff = Count - Runs, Rows = Count, Runs" } + 'Wide: current rows vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | summarize by BillingAccountId, SkuPriceIdv2, PricingCurrency | count); SkuPricesWide_v1_5 | where isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Savings plan rows not Effective' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where x_SkuPriceType == 'SavingsPlan' and x_UnitPriceType != 'Effective' | count" } + 'SkuPriceId with >1 tier minimum' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize t = dcount(QuantityTierMinimum) by SkuPriceId | where t > 1 | count' } + 'Price types by CommitmentDiscountCategory (info)' = @{ Expect = $null; Csl = "SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType, CommitmentDiscountCategory, ChargeCategory | order by x_UnitPriceType asc, Rows desc" } + 'Base/Contracted equal to List (info)' = @{ Expect = $null; Csl = @' +SkuPrices_v1_5 +| where x_UnitPriceType in ('Base', 'Contracted') and isnull(SkuPriceEffectiveEnd) +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and isnull(SkuPriceEffectiveEnd) | project SkuPriceId, PricingCurrency, ListPrice = UnitPrice) on SkuPriceId, PricingCurrency +| summarize Rows = count(), SameAsList = countif(UnitPrice == ListPrice), NoList = countif(isnull(ListPrice)) by x_UnitPriceType +'@ } +} + +$failed = 0 +foreach ($name in $checks.Keys) { + $c = $checks[$name] + try { $rows = @(Invoke-Kusto $c.Csl) } catch { Write-Host "ERROR $name -> $($_.ErrorDetails.Message ?? $_.Exception.Message)"; $failed++; continue } + if ($null -eq $c.Expect) { + Write-Host "INFO $name"; $rows | Format-Table -AutoSize | Out-String -Width 200 | Write-Host + } + else { + $first = $rows[0].PSObject.Properties.Value | Select-Object -First 1 + $ok = [long]$first -eq $c.Expect + if (-not $ok) { $failed++ } + Write-Host ("{0} {1} -> {2}" -f ($(if ($ok) { 'PASS' } else { 'FAIL' })), $name, (($rows[0].PSObject.Properties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ', ')) + } +} +Write-Host "`nFailed: $failed"