From 04a6c038f6d9c9d4579b6cbc3bb895f1b4420396 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Wed, 7 Oct 2026 21:47:41 -0700 Subject: [PATCH 1/8] feat(hubs): draft FOCUS 1.5 SkuPrices preview Adds a preview (unwired) FOCUS 1.5 SKU Price dataset built from Prices_final_v1_2, a one-row-per-price comparison table, validation, and 10 side-by-side scenarios. Co-Authored-By: Claude Opus 5.5 --- .../SkuPrices_v1_5/Build-SkuPrices.ps1 | 86 +++++++ .../preview/SkuPrices_v1_5/README.md | 82 ++++++ .../preview/SkuPrices_v1_5/SkuPrices_v1_5.kql | 227 ++++++++++++++++ .../SkuPrices_v1_5_scenarios.kql | 243 ++++++++++++++++++ .../SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 | 76 ++++++ .../preview/SkuPrices_v1_5/Test-SkuPrices.ps1 | 115 +++++++++ 6 files changed, 829 insertions(+) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 new file mode 100644 index 000000000..e31302a66 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 @@ -0,0 +1,86 @@ +<# +.SYNOPSIS +Rebuilds SkuPrices_v1_5 on ftk-dev from Prices_final_v1_2 using SkuPrices_v1_5.kql. + +.DESCRIPTION +Creates or updates SkuPrices_v1_5_build from the KQL file, then builds the table one price type at a time +(List replaces the table and its schema; Base, Contracted, and Effective append) to stay within memory. +#> +[CmdletBinding(SupportsShouldProcess)] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev', + [string] $KqlPath = "$PSScriptRoot/SkuPrices_v1_5.kql" +) + +$ErrorActionPreference = 'Stop' + +$ctx = Get-AzContext -Name $Context +if (-not $ctx) { throw "Az context '$Context' not found" } +Set-AzContext -Context $ctx -Scope Process -WhatIf:$false | Out-Null # This process only; leaves the saved default alone +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } + +function Invoke-Kusto { + param([string] $Csl, [string] $Endpoint = 'mgmt') + $body = @{ db = $Database; csl = $Csl; properties = @{ Options = @{ servertimeout = '01:00:00' } } } | ConvertTo-Json -Depth 5 + (Invoke-RestMethod -Uri "$Cluster/v1/rest/$Endpoint" -Method Post -Headers $headers -Body $body -TimeoutSec 3600).Tables[0] +} + +# 1. Functions from the KQL file (every .create-or-alter function block, in order) +$raw = Get-Content -Path $KqlPath -Raw +$pos = 0 +while (($start = $raw.IndexOf('.create-or-alter function', $pos)) -ge 0) { + $depth = 0; $end = -1 + for ($i = $raw.IndexOf('{', $start); $i -lt $raw.Length; $i++) { + if ($raw[$i] -eq '{') { $depth++ } + elseif ($raw[$i] -eq '}') { $depth--; if ($depth -eq 0) { $end = $i + 1; break } } + } + if ($end -lt 0) { throw "Unbalanced function body in $KqlPath" } + $fn = $raw.Substring($start, $end - $start) + $name = [regex]::Match($fn, '\)\s*(\w+)\s*\(').Groups[1].Value + if ($PSCmdlet.ShouldProcess($name, '.create-or-alter function')) { Invoke-Kusto $fn | Out-Null; Write-Host "Updated $name" } + $pos = $end +} + +# 2. Effective starts table, one price type at a time +foreach ($type in 'List', 'Base', 'Contracted', 'Effective') { + $csl = if ($type -eq 'List') { ".set-or-replace SkuPrices_v1_5_starts_table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_starts('$type')" } + else { ".append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('$type')" } + if ($PSCmdlet.ShouldProcess('SkuPrices_v1_5_starts_table', $csl)) { + $sw = [Diagnostics.Stopwatch]::StartNew() + Invoke-Kusto $csl | Out-Null + Write-Host "Starts $type done in $([int]$sw.Elapsed.TotalSeconds)s" + } +} + +# 3. Build the table one price type at a time +$steps = [ordered]@{ + List = ".set-or-replace SkuPrices_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_build('List')" + Base = ".append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Base')" + Contracted = ".append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Contracted')" + Effective = ".append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Effective')" +} +foreach ($type in $steps.Keys) { + if ($PSCmdlet.ShouldProcess('SkuPrices_v1_5', $steps[$type])) { + $sw = [Diagnostics.Stopwatch]::StartNew() + Invoke-Kusto $steps[$type] | Out-Null + Write-Host "$type done in $([int]$sw.Elapsed.TotalSeconds)s" + } +} + +# 4. Comparison table (one row per price, price types as columns) +$csl = ".set-or-replace SkuPricesWide_v1_5 with (folder = FOCUS 1.5 preview, recreate_schema = true) <| SkuPricesWide_v1_5_build()" +if ($PSCmdlet.ShouldProcess(SkuPricesWide_v1_5, $csl)) { + $sw = [Diagnostics.Stopwatch]::StartNew() + Invoke-Kusto $csl | Out-Null + Write-Host "Wide done in $([int]$sw.Elapsed.TotalSeconds)s" +} + +# 5. Verify +$counts = Invoke-Kusto 'SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType' 'query' +$counts.Rows | ForEach-Object { Write-Host "$($_[0]): $($_[1])" } +$columns = (Invoke-Kusto 'SkuPrices_v1_5 | getschema | project ColumnName' 'query').Rows | ForEach-Object { $_[0] } +$sorted = [string[]] $columns.Clone(); [Array]::Sort($sorted, [StringComparer]::Ordinal) +Write-Host "Columns: $($columns.Count); alphabetical: $(-not (Compare-Object $columns $sorted -SyncWindow 0))" diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md new file mode 100644 index 000000000..5b19479d6 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md @@ -0,0 +1,82 @@ +# FOCUS 1.5 SkuPrices (preview, draft) + +Draft FinOps hubs implementation of the FOCUS 1.5 **SKU Price** dataset ([FOCUS PR 2424](https://github.com/FinOps-Open-Cost-and-Usage-Spec/FOCUS_Spec/pull/2424)), built from `Prices_final_v1_2`. Not wired into `app.bicep` or `.build.config` yet. Run manually against a hub. + +## Files + +| File | Purpose | +|---|---| +| `SkuPrices_v1_5.kql` | Functions + build commands for `SkuPrices_v1_5`, `SkuPricesWide_v1_5`, and the `SkuPrices_v1_5_starts_table` helper | +| `Build-SkuPrices.ps1` | Creates the functions and builds all tables, one price type at a time | +| `Test-SkuPrices.ps1` | Validation: duplicates, row counts vs. source, price/discount match, required columns, ranges | +| `SkuPrices_v1_5_scenarios.kql` | 10 scenarios, each with a query for both layouts | +| `Test-SkuPriceScenarios.ps1` | Runs both queries per scenario, compares results, times them, writes JSON | + +Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-dev` Az context. Override with `-Cluster`, `-Database`, `-Context`. + +```powershell +./Build-SkuPrices.ps1 # ~10 min on ftk-dev +./Test-SkuPrices.ps1 +./Test-SkuPriceScenarios.ps1 +``` + +## Layouts + +| | `SkuPrices_v1_5` (rows) | `SkuPricesWide_v1_5` (columns) | +|---|---|---| +| Grain | SkuPriceId x price type x contract x currency | SkuPriceId x contract x currency | +| Price | `UnitPrice` + `x_UnitPriceType` (List, Base, Contracted, Effective) | `ListUnitPrice`, `ContractedUnitPrice`, `x_BaseUnitPrice`, `x_EffectiveUnitPrice` | +| ContractId | Empty for List; agreement for others | Agreement on every row | +| Eligibility | Global for List; agreement (+ MCA profiles) for others | Agreement on every row (list price mis-scoped) | +| Effective start | Per price type | Latest change of any price on the row | + +## Mapping decisions + +* `SkuPriceId` = `SkuPriceIdv2` (unique per price); source `SkuPriceId` kept as `x_SkuPriceIdv1` +* `ContractId` = `/providers/microsoft.billing/billingaccounts/{x_BillingAccountId}` (EA enrollment / MCA billing account, not the profile) +* Savings plan rows carry only the Effective price (list/base/contracted on those rows are copies of the on-demand meter's prices) +* `SkuPriceEffectiveStart` = first month of the current unchanged price across all monthly loads; `SkuPriceEffectiveEnd` = null (current price) +* `PricingRegionId` = region name (`x_SkuRegion`); `PricingServiceName` = meter category; `SkuPriceCreated`/`LastUpdated` = first/last ingestion +* `PurchaseDurationType` from `x_SkuTerm`; `PurchasePaymentModel` empty (not in price sheet) +* Non-SKU Price source columns get an `x_` prefix; `CommitmentDiscountCategory` kept as a filter label; columns sorted alphabetically + +## Results on ftk-dev (EA, Oct 2026 price sheet, 14 months of history) + +| | Rows | Extent size | +|---|---|---| +| `Prices_final_v1_2` (14 monthly copies) | 18,778,328 | 2.51 GB | +| `Prices_final_v1_2` (latest month) | 1,625,302 | 217 MB | +| `SkuPrices_v1_5` (current prices) | 3,885,217 | 656 MB | +| `SkuPrices_v1_5` with full history (est.) | ~4.17M | ~0.70 GB | +| ...skipping Base/Contracted rows equal to List (est.) | ~1.88M | ~0.32 GB | + +* Rows by type: List 1,236,402; Contracted 1,236,402 (98.0% = List); Base 1,023,513 (94.0% = List); Effective 388,900 +* Validation: 18/19 checks pass; `PricingUnit` empty (see gaps) +* Scenarios: 10/10 return the same answer in both layouts, except S06 (expected: wide rows can't tell which price changed) + +## Scenarios + +| Id | Scenario | Rows: lines / joins | Columns: lines / joins | +|---|---|---|---| +| S01 | List price for a SKU | 4 / 0 | 3 / 0 | +| S02 | Negotiated discount by service | 11 / 1 | 6 / 0 | +| S03 | Base price drift from list | 10 / 1 | 5 / 0 | +| S04 | Reservation price vs. on-demand | 11 / 1 | 11 / 1 | +| S05 | Savings plan rate vs. on-demand | 10 / 1 | 10 / 1 | +| S06 | Recent price changes by price type | 4 / 0 | 7 / 0 (less precise) | +| S07 | Cheapest region for a VM size | 9 / 1 | 9 / 1 | +| S08 | Verify billed unit prices | 8 / 1 | 8 / 1 | +| S09 | Savings plan what-if for on-demand usage | 12 / 1 | 12 / 1 | +| S10 | All prices for one SKU | 9 / 0 | 4 / 0 | + +Not testable with this data: negotiated savings plan discount, quantity tiers (EA has no tier minimums), multiple currencies, consumption currency, price as of a past date. + +## Gaps / TODO + +* [ ] `PricingUnit` and `x_PricingBlockSize` empty: `PricingUnits` open data table has 0 rows on ftk-dev, so `UnitPrice` is per block (e.g., per 10 hours), not per unit +* [ ] Savings plan list price ("list effective") is lost on ingestion: `Prices_transform_v1_2` nulls the SP `MarketPrice` and backfills on-demand prices; needed to measure negotiated SP discounts +* [ ] Store replaced prices (full history), not only current prices +* [ ] Decide whether to skip Base/Contracted rows equal to List (lookups fall back to List) +* [ ] MCA untested (collapses billing profiles into the billing account; check for profile-specific prices) +* [ ] Decide final layout once FOCUS settles the price type column (planned for 1.6) +* [ ] Wire into `IngestionSetup_v1_5.kql` / `HubSetup_v1_5.kql`, `app.bicep`, `.build.config`; add `SkuPrices()` hub function, docs, changelog diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql new file mode 100644 index 000000000..9c927b4c3 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql @@ -0,0 +1,227 @@ +// SkuPrices_v1_5 (PREVIEW): FOCUS 1.5 SKU Price dataset (FOCUS PR 2424 draft) built from Prices_final_v1_2 +// Run in the Ingestion database. Build-SkuPrices.ps1 runs everything below in order. +// +// Tables: +// SkuPrices_v1_5 One row per SkuPriceId x price type x contract x currency (x_UnitPriceType: List, Base, Contracted, Effective) +// SkuPricesWide_v1_5 Comparison layout: one row per SkuPriceId x contract x currency, price types as columns +// SkuPrices_v1_5_starts_table Build helper: first month of each price's current unchanged value +// +// Custom columns cover FOCUS 1.5 gaps: x_UnitPriceType (no price type column), x_SkuTerm (no term on usage prices). +// Source columns that are not SKU Price columns get an x_ prefix (source SkuPriceId -> x_SkuPriceIdv1). +// Built one price type at a time to stay within memory on small clusters. + +// 1. Effective start function: first month of the current unchanged run of each price across all monthly loads +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Original effective start per SKU price for one price type') +SkuPrices_v1_5_starts(priceType: string) { + // Narrow history keyed by a 64-bit hash of SkuPriceId + ContractId + PricingCurrency (keeps memory low) + let hist = Prices_final_v1_2 + | extend UnitPrice = case( + priceType == 'List', ListUnitPrice, + priceType == 'Base', x_BaseUnitPrice, + priceType == 'Contracted', ContractedUnitPrice, + priceType == 'Effective', x_EffectiveUnitPrice, + real(null)) + | where isnotnull(UnitPrice) + | where priceType == 'Effective' or x_SkuPriceType != 'SavingsPlan' + | extend ContractId = iff(priceType == 'List', '', tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId))) + | project k = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)), M = startofmonth(x_IngestionTime), Start = x_EffectivePeriodStart, UnitPrice + | summarize hint.shufflekey = k UnitPrice = take_any(UnitPrice), Start = min(Start) by k, M; + // Current price = price in the most recent month for each key + let cur = hist | summarize hint.shufflekey = k arg_max(M, UnitPrice) by k | project k, Cur = UnitPrice; + // Last month the price differed from the current price + let lastDiff = hist + | join kind=inner hint.shufflekey = k cur on k + | where UnitPrice != Cur + | summarize hint.shufflekey = k LastDiff = max(M) by k; + hist + | join kind=leftouter hint.shufflekey = k lastDiff on k + | where isnull(LastDiff) or M > LastDiff + | summarize hint.shufflekey = k OriginalStart = min(Start) by k + | extend x_UnitPriceType = priceType +} + +// 2. Build function (reads SkuPrices_v1_5_starts_table, built first) +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows for one price type (List, Base, Contracted, Effective)') +SkuPrices_v1_5_build(priceType: string) { + // Latest price sheet per billing account (by day, since one ingestion spans minutes) + let latestSheet = Prices_final_v1_2 + | summarize tmp_LatestDay = max(startofday(x_IngestionTime)) by BillingAccountId; + // First time each price was seen (narrow scan) + let created = Prices_final_v1_2 + | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; + let src = Prices_final_v1_2 + | lookup kind=inner latestSheet on BillingAccountId + | where startofday(x_IngestionTime) == tmp_LatestDay + | extend UnitPrice = case( + priceType == 'List', ListUnitPrice, + priceType == 'Base', x_BaseUnitPrice, + priceType == 'Contracted', ContractedUnitPrice, + priceType == 'Effective', x_EffectiveUnitPrice, + real(null)) + | where isnotnull(UnitPrice) + // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices + | where priceType == 'Effective' or x_SkuPriceType != 'SavingsPlan' + | extend x_UnitPriceType = priceType + // List prices are public: no contract. Everything else belongs to the agreement: EA enrollment or MCA billing account (not the billing profile) + | extend ContractId = iff(priceType == 'List', '', tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId))) + // One row per key (collapses list prices repeated across billing accounts) + | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, x_EffectivePeriodStart, PricingCurrency + | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency, '|', x_EffectivePeriodStart); + // Tier max = next tier min for the same meter and price (only meters with more than one tier) + let tierMax = src + | summarize Tiers = make_set(x_SkuTier) by tmp_TierGroup + | where array_length(Tiers) > 1 + | extend Tiers = array_sort_asc(Tiers) + | mv-expand with_itemindex = tmp_i x_SkuTier = Tiers to typeof(real) + | extend QuantityTierMaximum = todouble(Tiers[tmp_i + 1]) + | project tmp_TierGroup, x_SkuTier, QuantityTierMaximum; + src + | lookup kind=leftouter tierMax on tmp_TierGroup, x_SkuTier + | lookup kind=leftouter created on SkuPriceIdv2 + | extend tmp_k = hash_xxhash64(strcat(SkuPriceIdv2, "|", ContractId, "|", PricingCurrency)) + | lookup kind=leftouter (SkuPrices_v1_5_starts_table | where x_UnitPriceType == priceType | project tmp_k = k, OriginalStart) on tmp_k + | extend x_SkuPriceIdv1 = SkuPriceId + | extend + PricingCurrencyCategory = 'Payable', + PricingRegionId = x_SkuRegion, // Region name, not ID; best available + PricingServiceName = x_SkuMeterCategory, // Meter category; price sheet has no service name + PurchaseDurationType = iff(ChargeCategory != 'Purchase' or isnull(x_SkuTerm), '', + case(x_SkuTerm == 12, '1 Year', x_SkuTerm % 12 == 0, strcat(x_SkuTerm / 12, ' Years'), x_SkuTerm == 1, '1 Month', strcat(x_SkuTerm, ' Months'))), + PurchasePaymentModel = '', // Not in the price sheet + QuantityTierMinimum = coalesce(x_SkuTier, real(0)), + ServiceProviderName = 'Microsoft', + SkuPriceDescription = x_SkuDescription, + SkuPriceEffectiveEnd = datetime(null), // Current price; no known end (monthly sheet end is not a price end) + SkuPriceEffectiveStart = coalesce(OriginalStart, x_EffectivePeriodStart), // First month of the current unchanged price + SkuPriceEligibility = iff(priceType == 'List', + dynamic({'IsGlobalScope': true}), + bag_pack('IsGlobalScope', false, 'InclusionOperator', 'Or', 'Inclusions', + pack_array( + // Matches the account itself (EA) and every billing profile under it (MCA) + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'In', 'Values', pack_array(ContractId)), + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'StartsWith', 'Values', pack_array(strcat(ContractId, '/')))))), + SkuPriceId = SkuPriceIdv2, + SkuPriceLastUpdated = x_IngestionTime, // Last ingestion; provider date not available + // Discount vs. list for this row's price type (replaces the 6 wide discount columns) + x_UnitPriceDiscount = iff(priceType == 'List', real(null), ListUnitPrice - UnitPrice), + x_UnitPriceDiscountPercent = iff(priceType == 'List' or ListUnitPrice == 0, real(null), 100.0 * (ListUnitPrice - UnitPrice) / ListUnitPrice) + // List rows are collapsed across billing accounts and MCA rows across billing profiles, so clear values that differ between them + | extend tmp_IsList = priceType == 'List', tmp_IsMca = x_BillingProfileId != x_BillingAccountId + | extend + BillingAccountName = iff(tmp_IsList or tmp_IsMca, '', BillingAccountName), // MCA: profile name + x_BillingAccountId = iff(tmp_IsList, '', x_BillingAccountId), + x_BillingProfileId = iff(tmp_IsList or tmp_IsMca, '', x_BillingProfileId) + | project-away tmp_*, OriginalStart, BillingAccountId, // EA: same as ContractId; MCA: profile-level, collapsed into ContractId + ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice, // In UnitPrice + x_UnitPriceType + x_ContractedUnitPriceDiscount, x_ContractedUnitPriceDiscountPercent, x_EffectiveUnitPriceDiscount, x_EffectiveUnitPriceDiscountPercent, + x_TotalUnitPriceDiscount, x_TotalUnitPriceDiscountPercent, // Replaced by x_UnitPriceDiscount(Percent) + x_EffectivePeriodStart, x_EffectivePeriodEnd // Same as SkuPriceEffectiveStart/End + // Source columns that are not SKU Price columns become custom columns + | project-rename + x_BillingAccountName = BillingAccountName, + x_BillingCurrency = BillingCurrency, + x_CommitmentDiscountType = CommitmentDiscountType, + x_CommitmentDiscountUnit = CommitmentDiscountUnit, + x_PricingCategory = PricingCategory, + x_SkuMeter = SkuMeter, + x_SkuPriceIdv2 = SkuPriceIdv2 + // All columns in alphabetical order (FOCUS columns sort before x_ columns) + | project-reorder * asc +} + +// 3. Comparison function: same prices in the one-row-per-price layout (all price types as columns) +// Same source, keys, and effective-start logic as SkuPrices_v1_5, so scenario queries can be compared side by side +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows with one row per price and price types as columns (comparison layout)') +SkuPricesWide_v1_5_build() { + let latestSheet = Prices_final_v1_2 + | summarize tmp_LatestDay = max(startofday(x_IngestionTime)) by BillingAccountId; + let created = Prices_final_v1_2 + | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; + let starts = SkuPrices_v1_5_starts_table | project k, x_UnitPriceType, OriginalStart; + let src = Prices_final_v1_2 + | lookup kind=inner latestSheet on BillingAccountId + | where startofday(x_IngestionTime) == tmp_LatestDay + // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices + | extend tmp_IsSP = x_SkuPriceType == 'SavingsPlan' + | extend + ListUnitPrice = iff(tmp_IsSP, real(null), ListUnitPrice), + ContractedUnitPrice = iff(tmp_IsSP, real(null), ContractedUnitPrice), + x_BaseUnitPrice = iff(tmp_IsSP, real(null), x_BaseUnitPrice), + x_ContractedUnitPriceDiscount = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscountPercent) + // Every row carries a contracted price, so every row belongs to the agreement + | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) + | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, x_EffectivePeriodStart, PricingCurrency + | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency, '|', x_EffectivePeriodStart); + let tierMax = src + | summarize Tiers = make_set(x_SkuTier) by tmp_TierGroup + | where array_length(Tiers) > 1 + | extend Tiers = array_sort_asc(Tiers) + | mv-expand with_itemindex = tmp_i x_SkuTier = Tiers to typeof(real) + | extend QuantityTierMaximum = todouble(Tiers[tmp_i + 1]) + | project tmp_TierGroup, x_SkuTier, QuantityTierMaximum; + src + | lookup kind=leftouter tierMax on tmp_TierGroup, x_SkuTier + | lookup kind=leftouter created on SkuPriceIdv2 + // A row changes when any of its prices change, so its effective start is the latest start across its price types + | extend tmp_kList = hash_xxhash64(strcat(SkuPriceIdv2, '|', '', '|', PricingCurrency)), tmp_kAcct = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)) + | lookup kind=leftouter (starts | where x_UnitPriceType == 'List' | project tmp_kList = k, tmp_StartList = OriginalStart) on tmp_kList + | lookup kind=leftouter (starts | where x_UnitPriceType == 'Base' | project tmp_kAcct = k, tmp_StartBase = OriginalStart) on tmp_kAcct + | lookup kind=leftouter (starts | where x_UnitPriceType == 'Contracted' | project tmp_kAcct = k, tmp_StartContracted = OriginalStart) on tmp_kAcct + | lookup kind=leftouter (starts | where x_UnitPriceType == 'Effective' | project tmp_kAcct = k, tmp_StartEffective = OriginalStart) on tmp_kAcct + | extend x_SkuPriceIdv1 = SkuPriceId + | extend + PricingCurrencyCategory = 'Payable', + PricingRegionId = x_SkuRegion, + PricingServiceName = x_SkuMeterCategory, + PurchaseDurationType = iff(ChargeCategory != 'Purchase' or isnull(x_SkuTerm), '', + case(x_SkuTerm == 12, '1 Year', x_SkuTerm % 12 == 0, strcat(x_SkuTerm / 12, ' Years'), x_SkuTerm == 1, '1 Month', strcat(x_SkuTerm, ' Months'))), + PurchasePaymentModel = '', + QuantityTierMinimum = coalesce(x_SkuTier, real(0)), + ServiceProviderName = 'Microsoft', + SkuPriceDescription = x_SkuDescription, + SkuPriceEffectiveEnd = datetime(null), + SkuPriceEffectiveStart = coalesce(max_of(tmp_StartList, tmp_StartBase, tmp_StartContracted, tmp_StartEffective), x_EffectivePeriodStart), + // One eligibility for the whole row, even though the list price on it is public + SkuPriceEligibility = bag_pack('IsGlobalScope', false, 'InclusionOperator', 'Or', 'Inclusions', + pack_array( + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'In', 'Values', pack_array(ContractId)), + bag_pack('Dimension', 'BillingAccountId', 'Operator', 'StartsWith', 'Values', pack_array(strcat(ContractId, '/'))))), + SkuPriceId = SkuPriceIdv2, + SkuPriceLastUpdated = x_IngestionTime + | extend tmp_IsMca = x_BillingProfileId != x_BillingAccountId + | extend + BillingAccountName = iff(tmp_IsMca, '', BillingAccountName), + x_BillingProfileId = iff(tmp_IsMca, '', x_BillingProfileId) + | project-away tmp_*, BillingAccountId, x_EffectivePeriodStart, x_EffectivePeriodEnd + | project-rename + x_BillingAccountName = BillingAccountName, + x_BillingCurrency = BillingCurrency, + x_CommitmentDiscountType = CommitmentDiscountType, + x_CommitmentDiscountUnit = CommitmentDiscountUnit, + x_PricingCategory = PricingCategory, + x_SkuMeter = SkuMeter, + x_SkuPriceIdv2 = SkuPriceIdv2 + | project-reorder * asc +} + +// 4. Build the tables in this order (Build-SkuPrices.ps1 runs these), one price type at a time to stay within memory +.set-or-replace SkuPrices_v1_5_starts_table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_starts('List') + +.append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('Base') + +.append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('Contracted') + +.append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('Effective') + +.set-or-replace SkuPrices_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_build('List') + +.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Base') + +.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Contracted') + +.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Effective') + +.set-or-replace SkuPricesWide_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPricesWide_v1_5_build() + +// Validation: Test-SkuPrices.ps1. Scenarios: SkuPrices_v1_5_scenarios.kql + Test-SkuPriceScenarios.ps1. diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql new file mode 100644 index 000000000..1a577a11a --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql @@ -0,0 +1,243 @@ +// FOCUS 1.5 SKU Price scenarios: row-per-price-type vs. column-per-price-type +// +// Rows = SkuPrices_v1_5 one row per SkuPriceId x price type (x_UnitPriceType: List, Base, Contracted, Effective) +// Columns = SkuPricesWide_v1_5 one row per SkuPriceId with ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice +// +// Both tables are built from the same Prices_final_v1_2 data by SkuPrices_v1_5.kql. +// Queries use FOCUS columns where they exist; x_ columns only where FOCUS has no equivalent (meter, offer, term, price type). +// Test-SkuPriceScenarios.ps1 runs each pair and compares results. +// +// Format: "// ## ", "// Question: ...", "// Expect: same|different ()", then "// ### Rows" and "// ### Columns" queries. + +// ## S01 List price for a SKU +// Question: What is the public list price for this SKU price? +// Source: PR 2595 "Find the List Prices in Force at a Point in Time" +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' +| where x_UnitPriceType == 'List' +| project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice +// ### Columns +SkuPricesWide_v1_5 +| where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' +| distinct SkuPriceId, PricingCurrency, ListUnitPrice + +// ## S02 Negotiated discount by service +// Question: How much does the contract lower usage rates, by service? +// Source: PR 2595 "Measure What Negotiation Reduces the Rate By"; appendix standard_and_negotiated_prices.md +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where x_UnitPriceType == 'Contracted' and ChargeCategory == 'Usage' +| join kind=inner ( + SkuPrices_v1_5 + | where x_UnitPriceType == 'List' + | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice + ) on SkuPriceId, PricingCurrency +| where ListUnitPrice > 0 +| summarize Prices = count(), Discounted = countif(UnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - UnitPrice) / ListUnitPrice), 4) by PricingServiceName +| order by Discounted desc, PricingServiceName asc +| take 10 +// ### Columns +SkuPricesWide_v1_5 +| where ChargeCategory == 'Usage' and isnotnull(ContractedUnitPrice) +| where ListUnitPrice > 0 +| summarize Prices = count(), Discounted = countif(ContractedUnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice), 4) by PricingServiceName +| order by Discounted desc, PricingServiceName asc +| take 10 + +// ## S03 Base price drift from list +// Question: Where has the public list price moved away from the price fixed in the agreement? +// Source: PR 2595 ("Base" price type); research gap: PR 2424 has no column to tell Base from Contracted +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where x_UnitPriceType == 'Base' +| join kind=inner ( + SkuPrices_v1_5 + | where x_UnitPriceType == 'List' + | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice + ) on SkuPriceId, PricingCurrency +| summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > UnitPrice), ListBelowBase = countif(ListUnitPrice < UnitPrice), MaxDriftPercent = round(max(iff(UnitPrice > 0, 100.0 * (ListUnitPrice - UnitPrice) / UnitPrice, real(null))), 4) by PricingServiceName +| order by ListAboveBase + ListBelowBase desc, PricingServiceName asc +| take 10 +// ### Columns +SkuPricesWide_v1_5 +| where isnotnull(x_BaseUnitPrice) and isnotnull(ListUnitPrice) +| summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > x_BaseUnitPrice), ListBelowBase = countif(ListUnitPrice < x_BaseUnitPrice), MaxDriftPercent = round(max(iff(x_BaseUnitPrice > 0, 100.0 * (ListUnitPrice - x_BaseUnitPrice) / x_BaseUnitPrice, real(null))), 4) by PricingServiceName +| order by ListAboveBase + ListBelowBase desc, PricingServiceName asc +| take 10 + +// ## S04 Reservation price vs. on-demand +// Question: For each reservation term, what is the hourly rate and the savings vs. the on-demand contracted rate? +// Source: PR 2595 "Evaluate the Purchase Terms Offered for a SKU"; appendix commitment_pricing.md +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where ChargeCategory == 'Purchase' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 +| extend HourlyRate = UnitPrice / (x_SkuTerm * 730.0) +| join kind=inner ( + SkuPrices_v1_5 + | where ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' + | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId + ) on x_SkuMeterId +| where OnDemandRate > 0 +| summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc +// ### Columns +SkuPricesWide_v1_5 +| where ChargeCategory == 'Purchase' and x_SkuPriceType == 'ReservedInstance' and isnotnull(ContractedUnitPrice) and x_SkuTerm >= 12 +| extend HourlyRate = ContractedUnitPrice / (x_SkuTerm * 730.0) +| join kind=inner ( + SkuPricesWide_v1_5 + | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' + | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId + ) on x_SkuMeterId +| where OnDemandRate > 0 +| summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc + +// ## S05 Savings plan rate vs. on-demand +// Question: For each savings plan term, how much lower is the savings plan rate than the on-demand contracted rate? +// Source: issue #2706 (commitment unit price on covered usage); scope_and_evolution.md +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where x_UnitPriceType == 'Effective' and x_SkuPriceType == 'SavingsPlan' +| join kind=inner ( + SkuPrices_v1_5 + | where ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' + | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId, x_SkuOfferId + ) on x_SkuMeterId, x_SkuOfferId +| where OnDemandRate > 0 +| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - UnitPrice) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc +// ### Columns +SkuPricesWide_v1_5 +| where x_SkuPriceType == 'SavingsPlan' and isnotnull(x_EffectiveUnitPrice) +| join kind=inner ( + SkuPricesWide_v1_5 + | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' + | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId, x_SkuOfferId + ) on x_SkuMeterId, x_SkuOfferId +| where OnDemandRate > 0 +| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - x_EffectiveUnitPrice) / OnDemandRate), 2) by Term = x_SkuTerm +| order by Term asc + +// ## S06 Recent price changes by price type +// Question: Which list, base, contracted, and savings plan prices changed since July 2026? +// Source: appendix price_effectivity_and_currency.md; working-draft "Fluctuation Tracking of Unit Prices" +// Expect: different (columns share one effective start per row, so a change to any price marks every price on the row as changed) +// ### Rows +SkuPrices_v1_5 +| where SkuPriceEffectiveStart >= datetime(2026-07-01) +| summarize Changed = count() by PriceType = x_UnitPriceType +| order by PriceType asc +// ### Columns +SkuPricesWide_v1_5 +| where SkuPriceEffectiveStart >= datetime(2026-07-01) +| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) +| evaluate narrow() +| project PriceType = Column, Changed = tolong(Value) +| where Changed > 0 +| order by PriceType asc + +// ## S07 Cheapest region for a VM size +// Question: Which regions have the lowest list price for each Dv5 VM size? +// Source: PR 2595 "Compare List Prices Across Regions" +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where x_UnitPriceType == 'List' and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' +| where x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' +| summarize Price = min(UnitPrice) by x_SkuMeter, PricingRegionId +| as hint.materialized = true T +| join kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter +| where Price == MinPrice +| summarize CheapestRegions = strcat_array(array_sort_asc(make_list(PricingRegionId)), ", ") by x_SkuMeter, MinPrice, MaxPrice, Regions +| order by x_SkuMeter asc +// ### Columns +SkuPricesWide_v1_5 +| where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and isnotnull(ListUnitPrice) +| where x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' +| summarize Price = min(ListUnitPrice) by x_SkuMeter, PricingRegionId +| as hint.materialized = true T +| join kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter +| where Price == MinPrice +| summarize CheapestRegions = strcat_array(array_sort_asc(make_list(PricingRegionId)), ", ") by x_SkuMeter, MinPrice, MaxPrice, Regions +| order by x_SkuMeter asc + +// ## S08 Verify billed unit prices +// Question: Do the list and contracted unit prices on last month's on-demand charges match the price list? +// Source: dataset.md Relationships; PR 2595 "Compare Recorded Unit Prices with Published Rates" +// Expect: same +// ### Rows +let prices = SkuPrices_v1_5 + | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_UnitPriceType in ('List', 'Contracted') + | summarize PriceList = take_anyif(UnitPrice, x_UnitPriceType == 'List'), PriceContracted = take_anyif(UnitPrice, x_UnitPriceType == 'Contracted') by x_SkuMeterId, x_SkuOfferId, PricingCurrency; +Costs_final_v1_2 +| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) +| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) +| lookup kind=leftouter prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) +// ### Columns +let prices = SkuPricesWide_v1_5 + | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' + | summarize PriceList = take_any(ListUnitPrice), PriceContracted = take_any(ContractedUnitPrice) by x_SkuMeterId, x_SkuOfferId, PricingCurrency; +Costs_final_v1_2 +| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) +| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) +| lookup kind=leftouter prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) + +// ## S09 Savings plan what-if for on-demand usage +// Question: What would last month's on-demand VM usage cost at list, contracted, and 1-year savings plan rates? +// Source: PR 2595 "Project the Effect of Moving Consumption to a Contracted Rate"; issue #2706 +// Expect: same +// ### Rows +let prices = SkuPrices_v1_5 + | where ChargeCategory == 'Usage' and (x_SkuPriceType == 'Consumption' or (x_SkuPriceType == 'SavingsPlan' and x_SkuTerm == 12)) + | summarize + PriceList = take_anyif(UnitPrice, x_SkuPriceType == 'Consumption' and x_UnitPriceType == 'List'), + PriceContracted = take_anyif(UnitPrice, x_SkuPriceType == 'Consumption' and x_UnitPriceType == 'Contracted'), + PriceSavingsPlan = take_anyif(UnitPrice, x_SkuPriceType == 'SavingsPlan' and x_UnitPriceType == 'Effective') + by x_SkuMeterId, x_SkuOfferId, PricingCurrency; +Costs_final_v1_2 +| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) +| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' +| lookup kind=inner prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) +// ### Columns +let prices = SkuPricesWide_v1_5 + | where ChargeCategory == 'Usage' and (x_SkuPriceType == 'Consumption' or (x_SkuPriceType == 'SavingsPlan' and x_SkuTerm == 12)) + | summarize + PriceList = take_anyif(ListUnitPrice, x_SkuPriceType == 'Consumption'), + PriceContracted = take_anyif(ContractedUnitPrice, x_SkuPriceType == 'Consumption'), + PriceSavingsPlan = take_anyif(x_EffectiveUnitPrice, x_SkuPriceType == 'SavingsPlan') + by x_SkuMeterId, x_SkuOfferId, PricingCurrency; +Costs_final_v1_2 +| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) +| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' +| lookup kind=inner prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) + +// ## S10 All prices for one SKU +// Question: Show every price type for one VM size in one region side by side. +// Source: PR 2595 catalog discovery; FOCUS Cost and Usage ListUnitPrice/ContractedUnitPrice pairing +// Expect: same +// ### Rows +SkuPrices_v1_5 +| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' +| summarize + ListUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'List'), + BaseUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'Base'), + ContractedUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'Contracted'), + EffectiveUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'Effective') + by SkuPriceId, PricingCurrency +| order by SkuPriceId asc +// ### Columns +SkuPricesWide_v1_5 +| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' +| project SkuPriceId, PricingCurrency, ListUnitPrice, BaseUnitPrice = x_BaseUnitPrice, ContractedUnitPrice, EffectiveUnitPrice = x_EffectiveUnitPrice +| order by SkuPriceId asc diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 new file mode 100644 index 000000000..87d6acc8b --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 @@ -0,0 +1,76 @@ +<# +.SYNOPSIS +Runs each scenario in SkuPrices_v1_5_scenarios.kql against both layouts and compares results. + +.DESCRIPTION +Parses "// ## " scenarios with "// ### Rows" and "// ### Columns" queries, runs both on ftk-dev, +and reports duration, result rows, query size, and whether the two results match. Writes results to JSON for reporting. +#> +[CmdletBinding()] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev', + [string] $Path = "$PSScriptRoot/SkuPrices_v1_5_scenarios.kql", + [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) "SkuPrices_v1_5_scenarios.results.json") +) + +$ErrorActionPreference = 'Stop' +Set-AzContext -Context (Get-AzContext -Name $Context) -Scope Process | Out-Null +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } + +function Invoke-Kusto([string] $Csl) { + $body = @{ db = $Database; csl = $Csl; properties = @{ Options = @{ servertimeout = '00:20:00'; query_results_cache_max_age = '00:00:00' } } } | ConvertTo-Json -Depth 5 + $sw = [Diagnostics.Stopwatch]::StartNew() + $t = (Invoke-RestMethod -Uri "$Cluster/v1/rest/query" -Method Post -Headers $headers -Body $body -TimeoutSec 1200).Tables[0] + [pscustomobject]@{ Ms = $sw.ElapsedMilliseconds; Columns = @($t.Columns | ForEach-Object ColumnName); Rows = @($t.Rows) } +} + +# Normalize a result so equivalent outputs compare equal (sorted rows, rounded numbers) +function Get-Signature($Result) { + $lines = $Result.Rows | ForEach-Object { ($_ | ForEach-Object { if ($_ -is [double] -or $_ -is [decimal]) { [math]::Round([double]$_, 6) } else { "$_" } }) -join '|' } | Sort-Object + ($lines -join "`n") +} + +# Parse scenarios +$text = Get-Content -Path $Path -Raw +$scenarios = foreach ($block in ($text -split '(?m)^// ## ') | Select-Object -Skip 1) { + $lines = $block -split "`n" + $id, $name = $lines[0].Trim() -split ' ', 2 + $meta = @{} + foreach ($l in $lines) { if ($l -match '^// (Question|Source|Expect): (.+)$') { $meta[$Matches[1]] = $Matches[2].Trim() } } + $rowsQuery = ($block -split '(?m)^// ### Rows\s*$')[1] -split '(?m)^// ### Columns\s*$' | Select-Object -First 1 + $colsQuery = ($block -split '(?m)^// ### Columns\s*$')[1] + [pscustomobject]@{ Id = $id; Name = $name.Trim(); Question = $meta.Question; Source = $meta.Source; Expect = $meta.Expect; RowsQuery = $rowsQuery.Trim(); ColumnsQuery = $colsQuery.Trim() } +} + +$results = foreach ($s in $scenarios) { + $r = @{} + foreach ($side in 'Rows', 'Columns') { + $q = $s."${side}Query" + try { + $res = Invoke-Kusto $q + $r[$side] = [pscustomobject]@{ + Ms = $res.Ms; ResultRows = $res.Rows.Count; Columns = $res.Columns + Data = @($res.Rows | Select-Object -First 15 | ForEach-Object { , @($_) }) + Signature = Get-Signature $res; Error = $null + Lines = @($q -split "`n" | Where-Object { $_.Trim() }).Count + Operators = ([regex]::Matches($q, '(?m)^\s*\|')).Count + Joins = ([regex]::Matches($q, '\b(join|lookup)\b')).Count + } + } + catch { $r[$side] = [pscustomobject]@{ Error = ($_.ErrorDetails.Message ?? $_.Exception.Message) } } + } + $match = -not $r.Rows.Error -and -not $r.Columns.Error -and $r.Rows.Signature -eq $r.Columns.Signature + $expectSame = $s.Expect -like 'same*' + $status = if ($r.Rows.Error -or $r.Columns.Error) { 'ERROR' } elseif ($match -eq $expectSame) { 'PASS' } else { 'FAIL' } + Write-Host ("{0,-5} {1} {2,-40} rows {3,6}ms/{4,3}L/{5}J cols {6,6}ms/{7,3}L/{8}J match={9}" -f $status, $s.Id, $s.Name, $r.Rows.Ms, $r.Rows.Lines, $r.Rows.Joins, $r.Columns.Ms, $r.Columns.Lines, $r.Columns.Joins, $match) + if ($status -eq 'ERROR') { Write-Host " Rows: $($r.Rows.Error)`n Columns: $($r.Columns.Error)" } + [pscustomobject]@{ Id = $s.Id; Name = $s.Name; Question = $s.Question; Source = $s.Source; Expect = $s.Expect; Status = $status; Match = $match + RowsQuery = $s.RowsQuery; ColumnsQuery = $s.ColumnsQuery + Rows = $r.Rows | Select-Object * -ExcludeProperty Signature; Columns = $r.Columns | Select-Object * -ExcludeProperty Signature } +} + +$results | ConvertTo-Json -Depth 8 | Set-Content -Path $OutFile +Write-Host "`nPassed: $(@($results | Where-Object Status -eq 'PASS').Count) of $($results.Count). Results: $OutFile" diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 new file mode 100644 index 000000000..1f3c1d42e --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 @@ -0,0 +1,115 @@ +<# +.SYNOPSIS +Validates SkuPrices_v1_5 on ftk-dev against the source price sheet and the PR 2424 SKU Price requirements. +#> +[CmdletBinding()] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev' +) + +$ErrorActionPreference = 'Stop' +Set-AzContext -Context (Get-AzContext -Name $Context) -Scope Process | Out-Null +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } + +function Invoke-Kusto([string] $Csl) { + $body = @{ db = $Database; csl = $Csl; properties = @{ Options = @{ servertimeout = '00:30:00' } } } | ConvertTo-Json -Depth 5 + $t = (Invoke-RestMethod -Uri "$Cluster/v1/rest/query" -Method Post -Headers $headers -Body $body -TimeoutSec 1800).Tables[0] + $names = $t.Columns | ForEach-Object ColumnName + $t.Rows | ForEach-Object { $r = $_; $o = [ordered]@{}; for ($i = 0; $i -lt $names.Count; $i++) { $o[$names[$i]] = $r[$i] }; [pscustomobject]$o } +} + +# Latest price sheet per billing account, same as the build +$sheet = @' +let latestSheet = Prices_final_v1_2 | summarize tmp_LatestDay = max(startofday(x_IngestionTime)) by BillingAccountId; +let sheet = Prices_final_v1_2 | lookup kind=inner latestSheet on BillingAccountId | where startofday(x_IngestionTime) == tmp_LatestDay; +'@ + +# Each check returns one row; Pass = expected +$checks = [ordered]@{ + 'Dupes: extended key (+x_UnitPriceType)' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize n = count() by ServiceProviderName, SkuPriceId, ContractId, SkuPriceEffectiveStart, PricingCurrency, x_UnitPriceType | where n > 1 | count' } + 'Dupes: FOCUS 1.5 key (info: price type collisions)' = @{ Expect = $null; Csl = 'SkuPrices_v1_5 | summarize n = count() by ServiceProviderName, SkuPriceId, ContractId, SkuPriceEffectiveStart, PricingCurrency | where n > 1 | count' } + 'Overlapping effective periods (extended key)' = @{ Expect = 0; Csl = @' +SkuPrices_v1_5 +| project k = strcat(ServiceProviderName, '|', SkuPriceId, '|', ContractId, '|', PricingCurrency, '|', x_UnitPriceType), SkuPriceEffectiveStart, SkuPriceEffectiveEnd +| sort by k asc, SkuPriceEffectiveStart asc +| where next(k) == k and (isnull(SkuPriceEffectiveEnd) or next(SkuPriceEffectiveStart) < SkuPriceEffectiveEnd) +| count +'@ } + 'Source: same key, different prices (hidden by take_any)' = @{ Expect = 0; Csl = $sheet + @' +sheet +| summarize List = dcount(ListUnitPrice), Base = dcount(x_BaseUnitPrice), Contracted = dcount(ContractedUnitPrice), Effective = dcount(x_EffectiveUnitPrice), n = count() + by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency +| where List > 1 or Base > 1 or Contracted > 1 or Effective > 1 +| count +'@ } + 'Source: duplicate rows on key (info)' = @{ Expect = $null; Csl = $sheet + 'sheet | summarize n = count() by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | where n > 1 | summarize Keys = count(), ExtraRows = sum(n - 1)' } + 'Row count: List vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ListUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'List' | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Row count: Base vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_BaseUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Base' | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Row count: Contracted vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ContractedUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Row count: Effective vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_EffectiveUnitPrice) | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' | summarize Diff = count() - s, Rows = count(), Source = s" } + 'UnitPrice + discount match the source sheet' = @{ Expect = 0; Csl = $sheet + @' +let s = sheet | summarize take_any(ListUnitPrice, x_BaseUnitPrice, ContractedUnitPrice, x_EffectiveUnitPrice) by SkuPriceIdv2, PricingCurrency; +SkuPrices_v1_5 +| lookup kind=leftouter s on $left.x_SkuPriceIdv2 == $right.SkuPriceIdv2, PricingCurrency +| extend Src = case(x_UnitPriceType == 'List', ListUnitPrice, x_UnitPriceType == 'Base', x_BaseUnitPrice, x_UnitPriceType == 'Contracted', ContractedUnitPrice, x_EffectiveUnitPrice) +| where isnull(Src) or UnitPrice != Src or (x_UnitPriceType != 'List' and abs(x_UnitPriceDiscount - (ListUnitPrice - UnitPrice)) > 1e-9) +| count +'@ } + 'Mandatory columns null/empty' = @{ Expect = 0; Csl = @' +SkuPrices_v1_5 +| where isempty(ChargeCategory) or isempty(PricingCurrency) or isempty(PricingCurrencyCategory) or isempty(PricingServiceName) or isempty(PricingUnit) + or isempty(ServiceProviderName) or isempty(SkuId) or isempty(SkuPriceDescription) or isempty(SkuPriceId) or isnull(SkuPriceCreated) + or isnull(SkuPriceLastUpdated) or isnull(UnitPrice) or isnull(SkuPriceEligibility) or isnull(QuantityTierMinimum) +| count +'@ } + 'Mandatory column gaps by column (info)' = @{ Expect = $null; Csl = @' +SkuPrices_v1_5 +| summarize ChargeCategory = countif(isempty(ChargeCategory)), PricingServiceName = countif(isempty(PricingServiceName)), PricingUnit = countif(isempty(PricingUnit)), + SkuId = countif(isempty(SkuId)), SkuPriceDescription = countif(isempty(SkuPriceDescription)), PricingRegionId = countif(isempty(PricingRegionId)), SkuPriceEffectiveEnd = countif(isnull(SkuPriceEffectiveEnd)), SkuPriceEffectiveStart = countif(isnull(SkuPriceEffectiveStart)) +'@ } + 'ContractId empty iff List' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where (x_UnitPriceType == 'List') != isempty(ContractId) | count" } + 'Negative UnitPrice' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | where UnitPrice < 0 | count' } + 'ChargeCategory not Usage/Purchase/Credit' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where ChargeCategory !in ('Usage', 'Purchase', 'Credit') | count" } + 'PurchaseDurationType set on non-Purchase' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where ChargeCategory != 'Purchase' and isnotempty(PurchaseDurationType) | count" } + 'EffectiveEnd <= EffectiveStart' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | where isnotnull(SkuPriceEffectiveEnd) and isnotnull(SkuPriceEffectiveStart) and SkuPriceEffectiveEnd <= SkuPriceEffectiveStart | count' } + 'Tier max <= tier min' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | where isnotnull(QuantityTierMaximum) and QuantityTierMaximum <= QuantityTierMinimum | count' } + 'SkuPriceId with >1 SkuId/PricingUnit/ChargeCategory' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize s = dcount(SkuId), u = dcount(PricingUnit), c = dcount(ChargeCategory) by SkuPriceId | where s > 1 or u > 1 or c > 1 | count' } + 'Effective start distribution (info)' = @{ Expect = $null; Csl = "SkuPrices_v1_5 | summarize Rows = count() by Start = startofmonth(SkuPriceEffectiveStart), x_UnitPriceType | summarize Rows = sum(Rows), Types = make_set(x_UnitPriceType) by Start | order by Start asc" } + 'Price changed after its effective start (sample of 2000 Contracted)' = @{ Expect = 0; Csl = @' +let s = SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | sample 2000 | project SkuPriceIdv2 = x_SkuPriceIdv2, PricingCurrency, Start = SkuPriceEffectiveStart, Cur = UnitPrice; +Prices_final_v1_2 +| where x_SkuPriceType != 'SavingsPlan' +| lookup kind=inner s on SkuPriceIdv2, PricingCurrency +| summarize Changed = countif(x_EffectivePeriodStart >= Start and ContractedUnitPrice != Cur), PriorSame = countif(startofmonth(x_EffectivePeriodStart) == datetime_add('month', -1, Start) and ContractedUnitPrice == Cur) by SkuPriceIdv2, PricingCurrency +| where Changed > 0 or PriorSame > 0 +| count +'@ } + 'Savings plan rows not Effective' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where x_SkuPriceType == 'SavingsPlan' and x_UnitPriceType != 'Effective' | count" } + 'SkuPriceId with >1 tier minimum' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize t = dcount(QuantityTierMinimum) by SkuPriceId | where t > 1 | count' } + 'Price types by CommitmentDiscountCategory (info)' = @{ Expect = $null; Csl = "SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType, CommitmentDiscountCategory, ChargeCategory | order by x_UnitPriceType asc, Rows desc" } + 'Base/Contracted equal to List (info)' = @{ Expect = $null; Csl = @' +SkuPrices_v1_5 +| where x_UnitPriceType in ('Base', 'Contracted') +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' | project SkuPriceId, SkuPriceEffectiveStart, PricingCurrency, ListPrice = UnitPrice) on SkuPriceId, SkuPriceEffectiveStart, PricingCurrency +| summarize Rows = count(), SameAsList = countif(UnitPrice == ListPrice), NoList = countif(isnull(ListPrice)) by x_UnitPriceType +'@ } +} + +$failed = 0 +foreach ($name in $checks.Keys) { + $c = $checks[$name] + try { $rows = @(Invoke-Kusto $c.Csl) } catch { Write-Host "ERROR $name -> $($_.ErrorDetails.Message ?? $_.Exception.Message)"; $failed++; continue } + if ($null -eq $c.Expect) { + Write-Host "INFO $name"; $rows | Format-Table -AutoSize | Out-String -Width 200 | Write-Host + } + else { + $first = $rows[0].PSObject.Properties.Value | Select-Object -First 1 + $ok = [long]$first -eq $c.Expect + if (-not $ok) { $failed++ } + Write-Host ("{0} {1} -> {2}" -f ($(if ($ok) { 'PASS' } else { 'FAIL' })), $name, (($rows[0].PSObject.Properties | ForEach-Object { "$($_.Name)=$($_.Value)" }) -join ', ')) + } +} +Write-Host "`nFailed: $failed" From 6f6398bc962a0ebf6e7b698caa15b798abe39f50 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 8 Oct 2026 02:49:08 -0700 Subject: [PATCH 2/8] docs(hubs): add SkuPricesWide_v1_5 size to SkuPrices preview README Co-Authored-By: Claude Opus 5.5 --- .../Analytics/preview/SkuPrices_v1_5/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md index 5b19479d6..6b1eb4bf5 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md @@ -47,6 +47,7 @@ Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-d | `Prices_final_v1_2` (14 monthly copies) | 18,778,328 | 2.51 GB | | `Prices_final_v1_2` (latest month) | 1,625,302 | 217 MB | | `SkuPrices_v1_5` (current prices) | 3,885,217 | 656 MB | +| `SkuPricesWide_v1_5` (current prices, columns layout) | 1,625,302 | 329 MB | | `SkuPrices_v1_5` with full history (est.) | ~4.17M | ~0.70 GB | | ...skipping Base/Contracted rows equal to List (est.) | ~1.88M | ~0.32 GB | From f97c1921468867405851a86f3ecf602c2c978f51 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 8 Oct 2026 12:45:02 -0700 Subject: [PATCH 3/8] feat(hubs): join SkuPrices preview to Costs via SkuPriceIdv2 Adds CostsWithSkuPriceIdv2 (Costs copy with SkuPriceIdv2 looked up by price type, meter, offer, and term), real SkuPriceId + ContractId joins in S08/S09, 3 more PR 2595 scenarios (S11-S13), block-size handling, and fixes stripped quotes in Build-SkuPrices.ps1. Co-Authored-By: Claude Opus 5.5 --- .../SkuPrices_v1_5/Build-SkuPrices.ps1 | 18 ++- .../preview/SkuPrices_v1_5/README.md | 33 +++-- .../preview/SkuPrices_v1_5/SkuPrices_v1_5.kql | 48 +++++++ .../SkuPrices_v1_5_scenarios.kql | 136 ++++++++++++++---- 4 files changed, 196 insertions(+), 39 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 index e31302a66..1423fd5b0 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 @@ -71,14 +71,26 @@ foreach ($type in $steps.Keys) { } # 4. Comparison table (one row per price, price types as columns) -$csl = ".set-or-replace SkuPricesWide_v1_5 with (folder = FOCUS 1.5 preview, recreate_schema = true) <| SkuPricesWide_v1_5_build()" -if ($PSCmdlet.ShouldProcess(SkuPricesWide_v1_5, $csl)) { +$csl = ".set-or-replace SkuPricesWide_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPricesWide_v1_5_build()" +if ($PSCmdlet.ShouldProcess('SkuPricesWide_v1_5', $csl)) { $sw = [Diagnostics.Stopwatch]::StartNew() Invoke-Kusto $csl | Out-Null Write-Host "Wide done in $([int]$sw.Elapsed.TotalSeconds)s" } -# 5. Verify +# 5. SkuPriceIdv2 map, then a Costs copy with SkuPriceIdv2, one month at a time +$csl = ".set-or-replace SkuPriceIdv2_map with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPriceIdv2_map_build()" +if ($PSCmdlet.ShouldProcess('SkuPriceIdv2_map', $csl)) { Invoke-Kusto $csl | Out-Null; Write-Host 'Map done' } +$months = (Invoke-Kusto 'Costs_final_v1_2 | summarize by M = startofmonth(ChargePeriodStart) | order by M asc' 'query').Rows | ForEach-Object { ([datetime]$_[0]).ToString('yyyy-MM-dd') } +$first = $true +foreach ($m in $months) { + $csl = if ($first) { ".set-or-replace CostsWithSkuPriceIdv2 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| CostsWithSkuPriceIdv2_build(datetime($m))" } + else { ".append CostsWithSkuPriceIdv2 <| CostsWithSkuPriceIdv2_build(datetime($m))" } + if ($PSCmdlet.ShouldProcess('CostsWithSkuPriceIdv2', $csl)) { Invoke-Kusto $csl | Out-Null; Write-Host "Costs $m done" } + $first = $false +} + +# 6. Verify $counts = Invoke-Kusto 'SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType' 'query' $counts.Rows | ForEach-Object { Write-Host "$($_[0]): $($_[1])" } $columns = (Invoke-Kusto 'SkuPrices_v1_5 | getschema | project ColumnName' 'query').Rows | ForEach-Object { $_[0] } diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md index 6b1eb4bf5..7fca7cd78 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md @@ -6,10 +6,10 @@ Draft FinOps hubs implementation of the FOCUS 1.5 **SKU Price** dataset ([FOCUS | File | Purpose | |---|---| -| `SkuPrices_v1_5.kql` | Functions + build commands for `SkuPrices_v1_5`, `SkuPricesWide_v1_5`, and the `SkuPrices_v1_5_starts_table` helper | +| `SkuPrices_v1_5.kql` | Functions + build commands for `SkuPrices_v1_5`, `SkuPricesWide_v1_5`, `CostsWithSkuPriceIdv2`, and helper tables (`SkuPrices_v1_5_starts_table`, `SkuPriceIdv2_map`) | | `Build-SkuPrices.ps1` | Creates the functions and builds all tables, one price type at a time | | `Test-SkuPrices.ps1` | Validation: duplicates, row counts vs. source, price/discount match, required columns, ranges | -| `SkuPrices_v1_5_scenarios.kql` | 10 scenarios, each with a query for both layouts | +| `SkuPrices_v1_5_scenarios.kql` | 13 scenarios, each with a query for both layouts | | `Test-SkuPriceScenarios.ps1` | Runs both queries per scenario, compares results, times them, writes JSON | Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-dev` Az context. Override with `-Cluster`, `-Database`, `-Context`. @@ -40,6 +40,19 @@ Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-d * `PurchaseDurationType` from `x_SkuTerm`; `PurchasePaymentModel` empty (not in price sheet) * Non-SKU Price source columns get an `x_` prefix; `CommitmentDiscountCategory` kept as a filter label; columns sorted alphabetically +## Cost and Usage join + +Costs `SkuPriceId` doesn't match the price sheet's, and Costs lacks `x_SkuProductId`/`x_SkuMeterType`, so `SkuPriceIdv2` can't be rebuilt from Costs columns. `CostsWithSkuPriceIdv2` is a copy of `Costs_final_v1_2` with `SkuPriceIdv2` looked up from prices by price type + meter + offer + term: + +| Cost type | Price used | Rows matched | Cost matched | +|---|---|---|---| +| On-demand usage | On-demand | 100% | 100% | +| Reservation-covered usage | On-demand (no reservation usage price) | 90.2% | 74.7% | +| Savings-plan-covered usage | Savings plan for the term | 83.1% (68 rows have no meter) | 100% | +| Reservation purchase | Reservation for the term (only when unique) | 16/16 | n/a | +| Adjustment | None | 0% | 0% | + +Scenarios join `CostsWithSkuPriceIdv2` to prices on `SkuPriceId` + `ContractId` + `PricingCurrency`. Next step: add `SkuPriceIdv2` to `Costs_transform_v1_2` itself. ## Results on ftk-dev (EA, Oct 2026 price sheet, 14 months of history) | | Rows | Extent size | @@ -53,7 +66,7 @@ Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-d * Rows by type: List 1,236,402; Contracted 1,236,402 (98.0% = List); Base 1,023,513 (94.0% = List); Effective 388,900 * Validation: 18/19 checks pass; `PricingUnit` empty (see gaps) -* Scenarios: 10/10 return the same answer in both layouts, except S06 (expected: wide rows can't tell which price changed) +* Scenarios: 13/13 pass; all match except S06 and S12, which differ as expected ## Scenarios @@ -64,20 +77,24 @@ Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-d | S03 | Base price drift from list | 10 / 1 | 5 / 0 | | S04 | Reservation price vs. on-demand | 11 / 1 | 11 / 1 | | S05 | Savings plan rate vs. on-demand | 10 / 1 | 10 / 1 | -| S06 | Recent price changes by price type | 4 / 0 | 7 / 0 (less precise) | +| S06 | Recent price changes by price type | 4 / 0 | 7 / 0 (differs: can't tell which price changed) | | S07 | Cheapest region for a VM size | 9 / 1 | 9 / 1 | -| S08 | Verify billed unit prices | 8 / 1 | 8 / 1 | -| S09 | Savings plan what-if for on-demand usage | 12 / 1 | 12 / 1 | +| S08 | Verify billed unit prices | 14 / 2 | 9 / 1 | +| S09 | Savings plan what-if for on-demand usage | 14 / 3 | 12 / 2 | | S10 | All prices for one SKU | 9 / 0 | 4 / 0 | +| S11 | Estimate a planned workload | 10 / 1 | 8 / 1 | +| S12 | Prices a billing account is eligible for | 9 / 0 | 12 / 0 (differs: list price hidden from other accounts) | +| S13 | Usage rates vs. purchase fees | 3 / 0 | 6 / 0 | -Not testable with this data: negotiated savings plan discount, quantity tiers (EA has no tier minimums), multiple currencies, consumption currency, price as of a past date. +Covers 9 of 13 FOCUS PR 2595 queries plus 4 more (S03, S05, S06, S10). Not testable with this data: announced price changes, public vs. negotiated at a quantity, tier resolution, next tier (no future prices or tiers in EA price sheets), negotiated savings plan discount, multiple currencies, consumption currency. S12 (eligibility JSON) takes ~4 minutes in both layouts. ## Gaps / TODO -* [ ] `PricingUnit` and `x_PricingBlockSize` empty: `PricingUnits` open data table has 0 rows on ftk-dev, so `UnitPrice` is per block (e.g., per 10 hours), not per unit +* [ ] `PricingUnit` and `x_PricingBlockSize` empty: `PricingUnits` open data table has 0 rows on ftk-dev, so `UnitPrice` is per block (e.g., per 10 hours); scenarios parse the block size from `x_PricingUnitDescription` * [ ] Savings plan list price ("list effective") is lost on ingestion: `Prices_transform_v1_2` nulls the SP `MarketPrice` and backfills on-demand prices; needed to measure negotiated SP discounts * [ ] Store replaced prices (full history), not only current prices * [ ] Decide whether to skip Base/Contracted rows equal to List (lookups fall back to List) * [ ] MCA untested (collapses billing profiles into the billing account; check for profile-specific prices) * [ ] Decide final layout once FOCUS settles the price type column (planned for 1.6) +* [ ] Add `SkuPriceIdv2` to `Costs_transform_v1_2` (replaces `CostsWithSkuPriceIdv2`); find a reservation-covered usage price * [ ] Wire into `IngestionSetup_v1_5.kql` / `HubSetup_v1_5.kql`, `app.bicep`, `.build.config`; add `SkuPrices()` hub function, docs, changelog diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql index 9c927b4c3..b7431be01 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5.kql @@ -224,4 +224,52 @@ SkuPricesWide_v1_5_build() { .set-or-replace SkuPricesWide_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPricesWide_v1_5_build() +// 5. Costs copy with SkuPriceIdv2, so Cost and Usage can join to SkuPrices on SkuPriceId +// Costs has no x_SkuProductId/x_SkuMeterType, so the ID is looked up from prices by price type + meter + offer + term +// On-demand usage -> on-demand price +// Reservation-covered usage -> on-demand price (Microsoft has no usage price for reservations) +// Savings-plan-covered usage -> savings plan price for the term +// Reservation purchase -> reservation price for the term, only when one ID matches +// Everything else -> empty +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Hash of price type + charge category + meter + offer + term, used to look up SkuPriceIdv2') +SkuPriceIdv2_key(priceType: string, chargeCategory: string, meterId: string, offerId: string, term: int) { + hash_xxhash64(strcat(priceType, '|', chargeCategory, '|', meterId, '|', offerId, '|', coalesce(term, 0))) +} + +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'SkuPriceIdv2 per lookup key, only where one ID matches') +SkuPriceIdv2_map_build() { + Prices_final_v1_2 + | distinct x_SkuPriceType, ChargeCategory, x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SkuPriceIdv2 + | summarize hint.shufflekey = x_SkuMeterId Ids = make_set(SkuPriceIdv2, 2) by k = SkuPriceIdv2_key(x_SkuPriceType, ChargeCategory, x_SkuMeterId, x_SkuOfferId, x_SkuTerm), x_SkuMeterId + | where array_length(Ids) == 1 + | project k, SkuPriceIdv2 = tostring(Ids[0]) +} + +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'Costs_final_v1_2 for one month with SkuPriceIdv2 (reads SkuPriceIdv2_map)') +CostsWithSkuPriceIdv2_build(month: datetime) { + let costs = Costs_final_v1_2 + | where ChargePeriodStart >= month and ChargePeriodStart < datetime_add('month', 1, month) + | extend tmp_PriceType = case( + ChargeCategory == 'Usage' and CommitmentDiscountCategory == 'Spend', 'SavingsPlan', + ChargeCategory == 'Usage', 'Consumption', + ChargeCategory == 'Purchase' and CommitmentDiscountType == 'Reservation', 'ReservedInstance', + '') + | extend tmp_k = SkuPriceIdv2_key(tmp_PriceType, ChargeCategory, x_SkuMeterId, + iff(tmp_PriceType == 'ReservedInstance', '', x_SkuOfferId), + iff(tmp_PriceType in ('SavingsPlan', 'ReservedInstance'), x_SkuTerm, int(null))); + // Only the keys this month needs, to keep the lookup small + let keys = costs | distinct tmp_k; + costs + | lookup kind=leftouter (SkuPriceIdv2_map | where k in (keys) | project tmp_k = k, SkuPriceIdv2) on tmp_k + | project-away tmp_* +} + +// Build the map, then Costs one month at a time (Build-SkuPrices.ps1 loops over every month in Costs_final_v1_2) +.set-or-replace SkuPriceIdv2_map with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPriceIdv2_map_build() + +.set-or-replace CostsWithSkuPriceIdv2 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| CostsWithSkuPriceIdv2_build(datetime(2025-09-01)) + +.append CostsWithSkuPriceIdv2 <| CostsWithSkuPriceIdv2_build(datetime(2025-10-01)) + +// ...one .append per remaining month // Validation: Test-SkuPrices.ps1. Scenarios: SkuPrices_v1_5_scenarios.kql + Test-SkuPriceScenarios.ps1. diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql index 1a577a11a..a318b7eb2 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql @@ -5,6 +5,8 @@ // // Both tables are built from the same Prices_final_v1_2 data by SkuPrices_v1_5.kql. // Queries use FOCUS columns where they exist; x_ columns only where FOCUS has no equivalent (meter, offer, term, price type). +// Cost and Usage joins use CostsWithSkuPriceIdv2 (Costs_final_v1_2 + SkuPriceIdv2) on SkuPriceId + ContractId + PricingCurrency. +// Prices are divided by the block size in x_PricingUnitDescription (e.g., "100 Hours") where per-unit prices matter. // Test-SkuPriceScenarios.ps1 runs each pair and compares results. // // Format: "// ## ", "// Question: ...", "// Expect: same|different ()", then "// ### Rows" and "// ### Columns" queries. @@ -170,25 +172,32 @@ SkuPricesWide_v1_5 // ## S08 Verify billed unit prices // Question: Do the list and contracted unit prices on last month's on-demand charges match the price list? -// Source: dataset.md Relationships; PR 2595 "Compare Recorded Unit Prices with Published Rates" +// Source: PR 2595 "Compare Recorded Unit Prices with Published Rates"; dataset.md Relationships // Expect: same // ### Rows -let prices = SkuPrices_v1_5 - | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_UnitPriceType in ('List', 'Contracted') - | summarize PriceList = take_anyif(UnitPrice, x_UnitPriceType == 'List'), PriceContracted = take_anyif(UnitPrice, x_UnitPriceType == 'Contracted') by x_SkuMeterId, x_SkuOfferId, PricingCurrency; -Costs_final_v1_2 +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; +let list = SkuPrices_v1_5 + | where x_UnitPriceType == 'List' + | project SkuPriceId, PricingCurrency, PriceList = UnitPrice / block(x_PricingUnitDescription); +let contracted = SkuPrices_v1_5 + | where x_UnitPriceType == 'Contracted' + | project SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice / block(x_PricingUnitDescription); +CostsWithSkuPriceIdv2 | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) -| lookup kind=leftouter prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) +| lookup kind=leftouter list on $left.SkuPriceIdv2 == $right.SkuPriceId, PricingCurrency +| lookup kind=leftouter contracted on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency | summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) // ### Columns +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; let prices = SkuPricesWide_v1_5 - | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' - | summarize PriceList = take_any(ListUnitPrice), PriceContracted = take_any(ContractedUnitPrice) by x_SkuMeterId, x_SkuOfferId, PricingCurrency; -Costs_final_v1_2 + | project SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice / block(x_PricingUnitDescription), PriceContracted = ContractedUnitPrice / block(x_PricingUnitDescription); +CostsWithSkuPriceIdv2 | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) -| lookup kind=leftouter prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) +| lookup kind=leftouter prices on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency | summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) // ## S09 Savings plan what-if for on-demand usage @@ -196,30 +205,32 @@ Costs_final_v1_2 // Source: PR 2595 "Project the Effect of Moving Consumption to a Contracted Rate"; issue #2706 // Expect: same // ### Rows -let prices = SkuPrices_v1_5 - | where ChargeCategory == 'Usage' and (x_SkuPriceType == 'Consumption' or (x_SkuPriceType == 'SavingsPlan' and x_SkuTerm == 12)) - | summarize - PriceList = take_anyif(UnitPrice, x_SkuPriceType == 'Consumption' and x_UnitPriceType == 'List'), - PriceContracted = take_anyif(UnitPrice, x_SkuPriceType == 'Consumption' and x_UnitPriceType == 'Contracted'), - PriceSavingsPlan = take_anyif(UnitPrice, x_SkuPriceType == 'SavingsPlan' and x_UnitPriceType == 'Effective') - by x_SkuMeterId, x_SkuOfferId, PricingCurrency; -Costs_final_v1_2 +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; +let list = SkuPrices_v1_5 | where x_UnitPriceType == 'List' | project SkuPriceId, PricingCurrency, PriceList = UnitPrice / block(x_PricingUnitDescription); +let contracted = SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | project SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice / block(x_PricingUnitDescription); +let savingsPlan = SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' | project SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = UnitPrice / block(x_PricingUnitDescription); +CostsWithSkuPriceIdv2 | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' -| lookup kind=inner prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) +// No column links an on-demand price to its savings plan price; the IDs differ only by prefix +| extend SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)) +| lookup kind=leftouter list on $left.SkuPriceIdv2 == $right.SkuPriceId, PricingCurrency +| lookup kind=leftouter contracted on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency +| lookup kind=leftouter savingsPlan on $left.SavingsPlanSkuPriceId == $right.SkuPriceId, ContractId, PricingCurrency | summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) // ### Columns -let prices = SkuPricesWide_v1_5 - | where ChargeCategory == 'Usage' and (x_SkuPriceType == 'Consumption' or (x_SkuPriceType == 'SavingsPlan' and x_SkuTerm == 12)) - | summarize - PriceList = take_anyif(ListUnitPrice, x_SkuPriceType == 'Consumption'), - PriceContracted = take_anyif(ContractedUnitPrice, x_SkuPriceType == 'Consumption'), - PriceSavingsPlan = take_anyif(x_EffectiveUnitPrice, x_SkuPriceType == 'SavingsPlan') - by x_SkuMeterId, x_SkuOfferId, PricingCurrency; -Costs_final_v1_2 +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; +let prices = SkuPricesWide_v1_5 | project SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice / block(x_PricingUnitDescription), PriceContracted = ContractedUnitPrice / block(x_PricingUnitDescription); +let savingsPlan = SkuPricesWide_v1_5 | project SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice / block(x_PricingUnitDescription); +CostsWithSkuPriceIdv2 | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' -| lookup kind=inner prices on x_SkuMeterId, x_SkuOfferId, PricingCurrency +| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) +// No column links an on-demand price to its savings plan price; the IDs differ only by prefix +| extend SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)) +| lookup kind=leftouter prices on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency +| lookup kind=leftouter savingsPlan on $left.SavingsPlanSkuPriceId == $right.SkuPriceId, ContractId, PricingCurrency | summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) // ## S10 All prices for one SKU @@ -241,3 +252,72 @@ SkuPricesWide_v1_5 | where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' | project SkuPriceId, PricingCurrency, ListUnitPrice, BaseUnitPrice = x_BaseUnitPrice, ContractedUnitPrice, EffectiveUnitPrice = x_EffectiveUnitPrice | order by SkuPriceId asc + +// ## S11 Estimate a planned workload +// Question: What would 2 D2 v5, 4 D4 v5, and 1 E8 v5 VMs in US East cost for a month at list and contracted rates? +// Source: PR 2595 "Estimate the Cost of a Planned Workload" +// Expect: same +// ### Rows +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; +let plan = datatable(SkuPriceId: string, Quantity: real) [ + 'ODU_DZH318Z08M9X_008W_1 Compute Hour_MS-AZR-0017P', 1460, + 'ODU_DZH318Z08M9X_007S_1 Compute Hour_MS-AZR-0017P', 2920, + 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; +SkuPrices_v1_5 +| where x_UnitPriceType in ('List', 'Contracted') +| join kind=inner plan on SkuPriceId +| extend Cost = Quantity * UnitPrice / block(x_PricingUnitDescription) +| summarize AtList = round(sumif(Cost, x_UnitPriceType == 'List'), 2), AtContracted = round(sumif(Cost, x_UnitPriceType == 'Contracted'), 2) +// ### Columns +let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; +let plan = datatable(SkuPriceId: string, Quantity: real) [ + 'ODU_DZH318Z08M9X_008W_1 Compute Hour_MS-AZR-0017P', 1460, + 'ODU_DZH318Z08M9X_007S_1 Compute Hour_MS-AZR-0017P', 2920, + 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; +SkuPricesWide_v1_5 +| join kind=inner plan on SkuPriceId +| summarize AtList = round(sum(Quantity * ListUnitPrice / block(x_PricingUnitDescription)), 2), AtContracted = round(sum(Quantity * ContractedUnitPrice / block(x_PricingUnitDescription)), 2) + +// ## S12 Prices a billing account is eligible for +// Question: How many prices of each type can a billing account with no agreement on this price list use? +// Source: PR 2595 "Identify the Prices a Billing Account is Eligible For"; sku_price_eligibility_examples.md +// Expect: different (columns scope the whole row to the agreement, so the public list price is hidden from other accounts) +// ### Rows +let account = '/providers/microsoft.billing/billingaccounts/1234567'; +SkuPrices_v1_5 +| mv-apply rule = SkuPriceEligibility.Inclusions on ( + summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( + (rule.Operator == 'In' and set_has_element(rule.Values, account)) or + (rule.Operator == 'StartsWith' and account startswith tostring(rule.Values[0]))))) +| where tobool(SkuPriceEligibility.IsGlobalScope) or Included > 0 +| summarize Prices = count() by PriceType = x_UnitPriceType +| order by PriceType asc +// ### Columns +let account = '/providers/microsoft.billing/billingaccounts/1234567'; +SkuPricesWide_v1_5 +| mv-apply rule = SkuPriceEligibility.Inclusions on ( + summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( + (rule.Operator == 'In' and set_has_element(rule.Values, account)) or + (rule.Operator == 'StartsWith' and account startswith tostring(rule.Values[0]))))) +| where tobool(SkuPriceEligibility.IsGlobalScope) or Included > 0 +| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) +| evaluate narrow() +| project PriceType = Column, Prices = tolong(Value) +| where Prices > 0 +| order by PriceType asc + +// ## S13 Usage rates vs. purchase fees +// Question: How many prices of each type are usage rates vs. purchase fees? +// Source: PR 2595 "Separate Usage Rates from Purchase Fees and Credit Values" +// Expect: same +// ### Rows +SkuPrices_v1_5 +| summarize Prices = count() by ChargeCategory, PriceType = x_UnitPriceType +| order by ChargeCategory asc, PriceType asc +// ### Columns +SkuPricesWide_v1_5 +| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) by ChargeCategory +| mv-expand kind = array p = bag_pack('List', List, 'Base', Base, 'Contracted', Contracted, 'Effective', Effective) +| project ChargeCategory, PriceType = tostring(p[0]), Prices = tolong(p[1]) +| where Prices > 0 +| order by ChargeCategory asc, PriceType asc From 417dd2826b30147d73ef569cf39fcb50fe66f5c6 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 8 Oct 2026 13:05:08 -0700 Subject: [PATCH 4/8] feat(hubs): score SkuPrices layouts across 14 optimized scenarios Adds S14 price backfill, unit-aware cost joins, server-measured CPU/memory/rows scanned (median of 3), and per-measure winners. Optimizes queries (lookup, semi-joins, distinct eligibility evaluation). Co-Authored-By: Claude Opus 5.5 --- .../preview/SkuPrices_v1_5/README.md | 44 ++-- .../SkuPrices_v1_5_scenarios.kql | 249 +++++++++++------- .../SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 | 118 ++++++--- 3 files changed, 265 insertions(+), 146 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md index 7fca7cd78..5e4c08d4d 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md @@ -9,8 +9,8 @@ Draft FinOps hubs implementation of the FOCUS 1.5 **SKU Price** dataset ([FOCUS | `SkuPrices_v1_5.kql` | Functions + build commands for `SkuPrices_v1_5`, `SkuPricesWide_v1_5`, `CostsWithSkuPriceIdv2`, and helper tables (`SkuPrices_v1_5_starts_table`, `SkuPriceIdv2_map`) | | `Build-SkuPrices.ps1` | Creates the functions and builds all tables, one price type at a time | | `Test-SkuPrices.ps1` | Validation: duplicates, row counts vs. source, price/discount match, required columns, ranges | -| `SkuPrices_v1_5_scenarios.kql` | 13 scenarios, each with a query for both layouts | -| `Test-SkuPriceScenarios.ps1` | Runs both queries per scenario, compares results, times them, writes JSON | +| `SkuPrices_v1_5_scenarios.kql` | 14 scenarios, each with the most efficient query found for both layouts | +| `Test-SkuPriceScenarios.ps1` | Runs both queries per scenario (3x), compares results, scores simplicity, time, CPU, memory, rows scanned, x_ usage | Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-dev` Az context. Override with `-Cluster`, `-Database`, `-Context`. @@ -66,27 +66,31 @@ Scenarios join `CostsWithSkuPriceIdv2` to prices on `SkuPriceId` + `ContractId` * Rows by type: List 1,236,402; Contracted 1,236,402 (98.0% = List); Base 1,023,513 (94.0% = List); Effective 388,900 * Validation: 18/19 checks pass; `PricingUnit` empty (see gaps) -* Scenarios: 13/13 pass; all match except S06 and S12, which differ as expected +* Scenarios: 14/14 pass; all match except S06 and S12, which differ as expected (Rows is correct) ## Scenarios -| Id | Scenario | Rows: lines / joins | Columns: lines / joins | -|---|---|---|---| -| S01 | List price for a SKU | 4 / 0 | 3 / 0 | -| S02 | Negotiated discount by service | 11 / 1 | 6 / 0 | -| S03 | Base price drift from list | 10 / 1 | 5 / 0 | -| S04 | Reservation price vs. on-demand | 11 / 1 | 11 / 1 | -| S05 | Savings plan rate vs. on-demand | 10 / 1 | 10 / 1 | -| S06 | Recent price changes by price type | 4 / 0 | 7 / 0 (differs: can't tell which price changed) | -| S07 | Cheapest region for a VM size | 9 / 1 | 9 / 1 | -| S08 | Verify billed unit prices | 14 / 2 | 9 / 1 | -| S09 | Savings plan what-if for on-demand usage | 14 / 3 | 12 / 2 | -| S10 | All prices for one SKU | 9 / 0 | 4 / 0 | -| S11 | Estimate a planned workload | 10 / 1 | 8 / 1 | -| S12 | Prices a billing account is eligible for | 9 / 0 | 12 / 0 (differs: list price hidden from other accounts) | -| S13 | Usage rates vs. purchase fees | 3 / 0 | 6 / 0 | - -Covers 9 of 13 FOCUS PR 2595 queries plus 4 more (S03, S05, S06, S10). Not testable with this data: announced price changes, public vs. negotiated at a quantity, tier resolution, next tier (no future prices or tiers in EA price sheets), negotiated savings plan discount, multiple currencies, consumption currency. S12 (eligibility JSON) takes ~4 minutes in both layouts. +| Id | Scenario | Simpler | Faster | CPU | Memory | Rows scanned | FOCUS-only | Correct | +|---|---|---|---|---|---|---|---|---| +| S01 | List price for a SKU | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S02 | Negotiated discount by service | Columns | Columns | Columns | Columns | Columns | Columns | Tie | +| S03 | Base price drift from list | Columns | Columns | Columns | Columns | Columns | Tie | Tie | +| S04 | Reservation price vs. on-demand | Tie | Columns | Columns | Columns | Columns | Tie | Tie | +| S05 | Savings plan rate vs. on-demand | Tie | Columns | Tie | Tie | Columns | Tie | Tie | +| S06 | Recent price changes by price type | Rows | Tie | Tie | Tie | Columns | Rows | Rows | +| S07 | Cheapest region for a VM size | Tie | Tie | Tie | Tie | Columns | Tie | Tie | +| S08 | Verify billed unit prices | Columns | Tie | Tie | Columns | Columns | Columns | Tie | +| S09 | Savings plan what-if for on-demand usage | Columns | Columns | Tie | Columns | Columns | Tie | Tie | +| S10 | All prices for one SKU | Tie | Tie | Tie | Tie | Columns | Rows | Tie | +| S11 | Estimate a planned workload | Columns | Tie | Tie | Columns | Columns | Columns | Tie | +| S12 | Prices a billing account is eligible for | Rows | Tie | Columns | Tie | Columns | Rows | Rows | +| S13 | Usage rates vs. purchase fees | Rows | Tie | Tie | Tie | Columns | Rows | Tie | +| S14 | Backfill missing prices on cost data | Columns | Columns | Tie | Columns | Columns | Columns | Tie | +| | **Wins (Rows / Columns)** | 3 / 6 | 0 / 6 | 0 / 4 | 0 / 7 | 0 / 14 | 4 / 5 | 2 / 0 | + +Scores: winner per measure (Rows, Columns, or Tie within 15%). Simpler = operators + 2x joins + lets; Faster/CPU/Memory/Rows scanned = median of 3 server-measured runs; FOCUS-only = fewer distinct x_ columns referenced; Correct = which layout returns the right answer when they differ. + +Covers 9 of 13 FOCUS PR 2595 queries plus 5 more (S03, S05, S06, S10, S14 price backfill). Not testable with this data: announced price changes, public vs. negotiated at a quantity, tier resolution, next tier (no future prices or tiers in EA price sheets), negotiated savings plan discount, multiple currencies, consumption currency. Efficiency patterns: filter and project early, lookup with a small right side, limit prices to IDs/meters the query needs, evaluate eligibility once per distinct JSON value (S12: 4.5 min to 1.9 s). Cost scenarios use a price only when its block unit matches the charge's PricingUnit; S09 at list = 99.96% of billed. ## Gaps / TODO diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql index a318b7eb2..e2283b600 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql @@ -4,10 +4,12 @@ // Columns = SkuPricesWide_v1_5 one row per SkuPriceId with ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice // // Both tables are built from the same Prices_final_v1_2 data by SkuPrices_v1_5.kql. -// Queries use FOCUS columns where they exist; x_ columns only where FOCUS has no equivalent (meter, offer, term, price type). // Cost and Usage joins use CostsWithSkuPriceIdv2 (Costs_final_v1_2 + SkuPriceIdv2) on SkuPriceId + ContractId + PricingCurrency. // Prices are divided by the block size in x_PricingUnitDescription (e.g., "100 Hours") where per-unit prices matter. -// Test-SkuPriceScenarios.ps1 runs each pair and compares results. +// Queries use FOCUS columns where they exist; x_ columns only where FOCUS has no equivalent (meter, offer, term, price type). +// Each query uses the cheapest pattern found for its layout: filter and project early, lookup with a small right side, +// limit prices to the IDs a cost query needs, and evaluate eligibility once per distinct JSON value. +// Test-SkuPriceScenarios.ps1 runs each pair, compares results, and scores them. // // Format: "// ## ", "// Question: ...", "// Expect: same|different ()", then "// ### Rows" and "// ### Columns" queries. @@ -17,10 +19,10 @@ // Expect: same // ### Rows SkuPrices_v1_5 -| where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' -| where x_UnitPriceType == 'List' +| where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' and x_UnitPriceType == 'List' | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice // ### Columns +// The list price repeats on every contract's row, so it needs distinct SkuPricesWide_v1_5 | where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' | distinct SkuPriceId, PricingCurrency, ListUnitPrice @@ -31,20 +33,19 @@ SkuPricesWide_v1_5 // Expect: same // ### Rows SkuPrices_v1_5 -| where x_UnitPriceType == 'Contracted' and ChargeCategory == 'Usage' -| join kind=inner ( +| where ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' +| project SkuPriceId, PricingCurrency, PricingServiceName, ContractedUnitPrice = UnitPrice +| lookup kind=inner ( SkuPrices_v1_5 - | where x_UnitPriceType == 'List' + | where ChargeCategory == 'Usage' and x_UnitPriceType == 'List' and UnitPrice > 0 | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice ) on SkuPriceId, PricingCurrency -| where ListUnitPrice > 0 -| summarize Prices = count(), Discounted = countif(UnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - UnitPrice) / ListUnitPrice), 4) by PricingServiceName +| summarize Prices = count(), Discounted = countif(ContractedUnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice), 4) by PricingServiceName | order by Discounted desc, PricingServiceName asc | take 10 // ### Columns SkuPricesWide_v1_5 -| where ChargeCategory == 'Usage' and isnotnull(ContractedUnitPrice) -| where ListUnitPrice > 0 +| where ChargeCategory == 'Usage' and ListUnitPrice > 0 and isnotnull(ContractedUnitPrice) | summarize Prices = count(), Discounted = countif(ContractedUnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice), 4) by PricingServiceName | order by Discounted desc, PricingServiceName asc | take 10 @@ -56,12 +57,13 @@ SkuPricesWide_v1_5 // ### Rows SkuPrices_v1_5 | where x_UnitPriceType == 'Base' -| join kind=inner ( +| project SkuPriceId, PricingCurrency, PricingServiceName, BaseUnitPrice = UnitPrice +| lookup kind=inner ( SkuPrices_v1_5 | where x_UnitPriceType == 'List' | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice ) on SkuPriceId, PricingCurrency -| summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > UnitPrice), ListBelowBase = countif(ListUnitPrice < UnitPrice), MaxDriftPercent = round(max(iff(UnitPrice > 0, 100.0 * (ListUnitPrice - UnitPrice) / UnitPrice, real(null))), 4) by PricingServiceName +| summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > BaseUnitPrice), ListBelowBase = countif(ListUnitPrice < BaseUnitPrice), MaxDriftPercent = round(max(iff(BaseUnitPrice > 0, 100.0 * (ListUnitPrice - BaseUnitPrice) / BaseUnitPrice, real(null))), 4) by PricingServiceName | order by ListAboveBase + ListBelowBase desc, PricingServiceName asc | take 10 // ### Columns @@ -76,27 +78,29 @@ SkuPricesWide_v1_5 // Source: PR 2595 "Evaluate the Purchase Terms Offered for a SKU"; appendix commitment_pricing.md // Expect: same // ### Rows -SkuPrices_v1_5 -| where ChargeCategory == 'Purchase' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 -| extend HourlyRate = UnitPrice / (x_SkuTerm * 730.0) -| join kind=inner ( +let reservations = materialize(SkuPrices_v1_5 + | where ChargeCategory == 'Purchase' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 + | project x_SkuMeterId, x_SkuTerm, HourlyRate = UnitPrice / (x_SkuTerm * 730.0)); +let meters = reservations | distinct x_SkuMeterId; +reservations +| lookup kind=inner ( SkuPrices_v1_5 - | where ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' + | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and UnitPrice > 0 | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId ) on x_SkuMeterId -| where OnDemandRate > 0 | summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc // ### Columns -SkuPricesWide_v1_5 -| where ChargeCategory == 'Purchase' and x_SkuPriceType == 'ReservedInstance' and isnotnull(ContractedUnitPrice) and x_SkuTerm >= 12 -| extend HourlyRate = ContractedUnitPrice / (x_SkuTerm * 730.0) -| join kind=inner ( +let reservations = materialize(SkuPricesWide_v1_5 + | where ChargeCategory == 'Purchase' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 and isnotnull(ContractedUnitPrice) + | project x_SkuMeterId, x_SkuTerm, HourlyRate = ContractedUnitPrice / (x_SkuTerm * 730.0)); +let meters = reservations | distinct x_SkuMeterId; +reservations +| lookup kind=inner ( SkuPricesWide_v1_5 - | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' + | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and ContractedUnitPrice > 0 | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId ) on x_SkuMeterId -| where OnDemandRate > 0 | summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc @@ -105,26 +109,30 @@ SkuPricesWide_v1_5 // Source: issue #2706 (commitment unit price on covered usage); scope_and_evolution.md // Expect: same // ### Rows -SkuPrices_v1_5 -| where x_UnitPriceType == 'Effective' and x_SkuPriceType == 'SavingsPlan' -| join kind=inner ( +let savingsPlans = materialize(SkuPrices_v1_5 + | where x_UnitPriceType == 'Effective' and x_SkuPriceType == 'SavingsPlan' + | project x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SavingsPlanRate = UnitPrice); +let meters = savingsPlans | distinct x_SkuMeterId; +savingsPlans +| lookup kind=inner ( SkuPrices_v1_5 - | where ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' + | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and UnitPrice > 0 | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId, x_SkuOfferId ) on x_SkuMeterId, x_SkuOfferId -| where OnDemandRate > 0 -| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - UnitPrice) / OnDemandRate), 2) by Term = x_SkuTerm +| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - SavingsPlanRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc // ### Columns -SkuPricesWide_v1_5 -| where x_SkuPriceType == 'SavingsPlan' and isnotnull(x_EffectiveUnitPrice) -| join kind=inner ( +let savingsPlans = materialize(SkuPricesWide_v1_5 + | where x_SkuPriceType == 'SavingsPlan' and isnotnull(x_EffectiveUnitPrice) + | project x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SavingsPlanRate = x_EffectiveUnitPrice); +let meters = savingsPlans | distinct x_SkuMeterId; +savingsPlans +| lookup kind=inner ( SkuPricesWide_v1_5 - | where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' + | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and ContractedUnitPrice > 0 | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId, x_SkuOfferId ) on x_SkuMeterId, x_SkuOfferId -| where OnDemandRate > 0 -| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - x_EffectiveUnitPrice) / OnDemandRate), 2) by Term = x_SkuTerm +| summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - SavingsPlanRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc // ## S06 Recent price changes by price type @@ -151,21 +159,19 @@ SkuPricesWide_v1_5 // Expect: same // ### Rows SkuPrices_v1_5 -| where x_UnitPriceType == 'List' and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' -| where x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' +| where x_UnitPriceType == 'List' and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' | summarize Price = min(UnitPrice) by x_SkuMeter, PricingRegionId | as hint.materialized = true T -| join kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter +| lookup kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter | where Price == MinPrice | summarize CheapestRegions = strcat_array(array_sort_asc(make_list(PricingRegionId)), ", ") by x_SkuMeter, MinPrice, MaxPrice, Regions | order by x_SkuMeter asc // ### Columns SkuPricesWide_v1_5 -| where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and isnotnull(ListUnitPrice) -| where x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' +| where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' and isnotnull(ListUnitPrice) | summarize Price = min(ListUnitPrice) by x_SkuMeter, PricingRegionId | as hint.materialized = true T -| join kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter +| lookup kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter | where Price == MinPrice | summarize CheapestRegions = strcat_array(array_sort_asc(make_list(PricingRegionId)), ", ") by x_SkuMeter, MinPrice, MaxPrice, Regions | order by x_SkuMeter asc @@ -175,29 +181,31 @@ SkuPricesWide_v1_5 // Source: PR 2595 "Compare Recorded Unit Prices with Published Rates"; dataset.md Relationships // Expect: same // ### Rows -let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; -let list = SkuPrices_v1_5 - | where x_UnitPriceType == 'List' - | project SkuPriceId, PricingCurrency, PriceList = UnitPrice / block(x_PricingUnitDescription); -let contracted = SkuPrices_v1_5 - | where x_UnitPriceType == 'Contracted' - | project SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice / block(x_PricingUnitDescription); -CostsWithSkuPriceIdv2 -| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) -| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) -| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) -| lookup kind=leftouter list on $left.SkuPriceIdv2 == $right.SkuPriceId, PricingCurrency -| lookup kind=leftouter contracted on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) + | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit | summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) // ### Columns -let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; -let prices = SkuPricesWide_v1_5 - | project SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice / block(x_PricingUnitDescription), PriceContracted = ContractedUnitPrice / block(x_PricingUnitDescription); -CostsWithSkuPriceIdv2 -| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) -| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) -| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) -| lookup kind=leftouter prices on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) + | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit | summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) // ## S09 Savings plan what-if for on-demand usage @@ -205,32 +213,37 @@ CostsWithSkuPriceIdv2 // Source: PR 2595 "Project the Effect of Moving Consumption to a Contracted Rate"; issue #2706 // Expect: same // ### Rows -let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; -let list = SkuPrices_v1_5 | where x_UnitPriceType == 'List' | project SkuPriceId, PricingCurrency, PriceList = UnitPrice / block(x_PricingUnitDescription); -let contracted = SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | project SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice / block(x_PricingUnitDescription); -let savingsPlan = SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' | project SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = UnitPrice / block(x_PricingUnitDescription); -CostsWithSkuPriceIdv2 -| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) -| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' -| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) -// No column links an on-demand price to its savings plan price; the IDs differ only by prefix -| extend SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)) -| lookup kind=leftouter list on $left.SkuPriceIdv2 == $right.SkuPriceId, PricingCurrency -| lookup kind=leftouter contracted on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency -| lookup kind=leftouter savingsPlan on $left.SavingsPlanSkuPriceId == $right.SkuPriceId, ContractId, PricingCurrency +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' + // No column links an on-demand price to its savings plan price; the IDs differ only by prefix + | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); +let ids = costs | distinct SkuPriceIdv2; +let spIds = costs | distinct SavingsPlanSkuPriceId; +costs +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' and SkuPriceId in (spIds) | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = UnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit, PriceSavingsPlan = PriceSavingsPlan * PerUnit | summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) // ### Columns -let block = (d: string) { coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0) }; -let prices = SkuPricesWide_v1_5 | project SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice / block(x_PricingUnitDescription), PriceContracted = ContractedUnitPrice / block(x_PricingUnitDescription); -let savingsPlan = SkuPricesWide_v1_5 | project SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice / block(x_PricingUnitDescription); -CostsWithSkuPriceIdv2 -| where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) -| where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' -| extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) -// No column links an on-demand price to its savings plan price; the IDs differ only by prefix -| extend SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)) -| lookup kind=leftouter prices on $left.SkuPriceIdv2 == $right.SkuPriceId, ContractId, PricingCurrency -| lookup kind=leftouter savingsPlan on $left.SavingsPlanSkuPriceId == $right.SkuPriceId, ContractId, PricingCurrency +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' + // No column links an on-demand price to its savings plan price; the IDs differ only by prefix + | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); +let ids = costs | distinct SkuPriceIdv2; +let spIds = costs | distinct SavingsPlanSkuPriceId; +costs +| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (spIds) | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit, PriceSavingsPlan = PriceSavingsPlan * PerUnit | summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) // ## S10 All prices for one SKU @@ -265,7 +278,7 @@ let plan = datatable(SkuPriceId: string, Quantity: real) [ 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; SkuPrices_v1_5 | where x_UnitPriceType in ('List', 'Contracted') -| join kind=inner plan on SkuPriceId +| lookup kind=inner plan on SkuPriceId | extend Cost = Quantity * UnitPrice / block(x_PricingUnitDescription) | summarize AtList = round(sumif(Cost, x_UnitPriceType == 'List'), 2), AtContracted = round(sumif(Cost, x_UnitPriceType == 'Contracted'), 2) // ### Columns @@ -275,7 +288,7 @@ let plan = datatable(SkuPriceId: string, Quantity: real) [ 'ODU_DZH318Z08M9X_007S_1 Compute Hour_MS-AZR-0017P', 2920, 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; SkuPricesWide_v1_5 -| join kind=inner plan on SkuPriceId +| lookup kind=inner plan on SkuPriceId | summarize AtList = round(sum(Quantity * ListUnitPrice / block(x_PricingUnitDescription)), 2), AtContracted = round(sum(Quantity * ContractedUnitPrice / block(x_PricingUnitDescription)), 2) // ## S12 Prices a billing account is eligible for @@ -285,22 +298,28 @@ SkuPricesWide_v1_5 // ### Rows let account = '/providers/microsoft.billing/billingaccounts/1234567'; SkuPrices_v1_5 -| mv-apply rule = SkuPriceEligibility.Inclusions on ( +// Evaluate each distinct eligibility object once instead of once per price +| summarize Prices = count() by PriceType = x_UnitPriceType, Eligibility = tostring(SkuPriceEligibility) +| extend e = parse_json(Eligibility) +| mv-apply rule = e.Inclusions on ( summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( (rule.Operator == 'In' and set_has_element(rule.Values, account)) or (rule.Operator == 'StartsWith' and account startswith tostring(rule.Values[0]))))) -| where tobool(SkuPriceEligibility.IsGlobalScope) or Included > 0 -| summarize Prices = count() by PriceType = x_UnitPriceType +| where tobool(e.IsGlobalScope) or Included > 0 +| summarize Prices = sum(Prices) by PriceType | order by PriceType asc // ### Columns let account = '/providers/microsoft.billing/billingaccounts/1234567'; SkuPricesWide_v1_5 -| mv-apply rule = SkuPriceEligibility.Inclusions on ( +// Evaluate each distinct eligibility object once instead of once per price +| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) by Eligibility = tostring(SkuPriceEligibility) +| extend e = parse_json(Eligibility) +| mv-apply rule = e.Inclusions on ( summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( (rule.Operator == 'In' and set_has_element(rule.Values, account)) or (rule.Operator == 'StartsWith' and account startswith tostring(rule.Values[0]))))) -| where tobool(SkuPriceEligibility.IsGlobalScope) or Included > 0 -| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) +| where tobool(e.IsGlobalScope) or Included > 0 +| summarize List = sum(List), Base = sum(Base), Contracted = sum(Contracted), Effective = sum(Effective) | evaluate narrow() | project PriceType = Column, Prices = tolong(Value) | where Prices > 0 @@ -321,3 +340,41 @@ SkuPricesWide_v1_5 | project ChargeCategory, PriceType = tostring(p[0]), Prices = tolong(p[1]) | where Prices > 0 | order by ChargeCategory asc, PriceType asc + +// ## S14 Backfill missing prices on cost data +// Question: Which cost rows are missing a list or contracted unit price, and how much list and contracted cost does the price list fill in? +// Source: FinOps hubs Costs transform (price backfill); dataset.md Relationships (SkuPriceId + ContractId + effective dates) +// Expect: same +// ### Rows +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where isnotempty(SkuPriceIdv2) and (isnull(ListUnitPrice) or ListUnitPrice == 0 or isnull(ContractedUnitPrice) or ContractedUnitPrice == 0) + | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ChargePeriodStart, PricingQuantity, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription, ListStart = SkuPriceEffectiveStart) on SkuPriceIdv2, PricingCurrency +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice, ContractedStart = SkuPriceEffectiveStart) on SkuPriceIdv2, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit +// Only use a price that was in effect when the charge happened +| extend PriceList = iff(ChargePeriodStart >= ListStart, PriceList, real(null)), PriceContracted = iff(ChargePeriodStart >= ContractedStart, PriceContracted, real(null)) +| extend MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0 +| summarize MissingList = countif(MissingList), FilledList = countif(MissingList and PriceList > 0), ListCostAdded = round(sumif(PricingQuantity * PriceList, MissingList), 2), + MissingContracted = countif(MissingContracted), FilledContracted = countif(MissingContracted and PriceContracted > 0), ContractedCostAdded = round(sumif(PricingQuantity * PriceContracted, MissingContracted), 2) +// ### Columns +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where isnotempty(SkuPriceIdv2) and (isnull(ListUnitPrice) or ListUnitPrice == 0 or isnull(ContractedUnitPrice) or ContractedUnitPrice == 0) + | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ChargePeriodStart, PricingQuantity, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription, PriceStart = SkuPriceEffectiveStart) on SkuPriceIdv2, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit +// Only use a price that was in effect when the charge happened; the row has one start for all of its prices +| extend PriceList = iff(ChargePeriodStart >= PriceStart, PriceList, real(null)), PriceContracted = iff(ChargePeriodStart >= PriceStart, PriceContracted, real(null)) +| extend MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0 +| summarize MissingList = countif(MissingList), FilledList = countif(MissingList and PriceList > 0), ListCostAdded = round(sumif(PricingQuantity * PriceList, MissingList), 2), + MissingContracted = countif(MissingContracted), FilledContracted = countif(MissingContracted and PriceContracted > 0), ContractedCostAdded = round(sumif(PricingQuantity * PriceContracted, MissingContracted), 2) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 index 87d6acc8b..1c88dfc88 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 @@ -1,10 +1,12 @@ <# .SYNOPSIS -Runs each scenario in SkuPrices_v1_5_scenarios.kql against both layouts and compares results. +Runs each scenario in SkuPrices_v1_5_scenarios.kql against both layouts, compares results, and scores them. .DESCRIPTION -Parses "// ## " scenarios with "// ### Rows" and "// ### Columns" queries, runs both on ftk-dev, -and reports duration, result rows, query size, and whether the two results match. Writes results to JSON for reporting. +Parses "// ## " scenarios with "// ### Rows" and "// ### Columns" queries. Runs each query -Runs times +(alternating sides, results cache off), and records the median of the server's execution time, CPU, peak memory, and rows +scanned (from QueryResourceConsumption). Also measures query size (lines, operators, joins, lets, x_ column references) and +whether both sides return the same answer. Writes everything to JSON for reporting. #> [CmdletBinding()] param( @@ -12,25 +14,60 @@ param( [string] $Database = 'Ingestion', [string] $Context = 'fh-dev', [string] $Path = "$PSScriptRoot/SkuPrices_v1_5_scenarios.kql", - [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) "SkuPrices_v1_5_scenarios.results.json") + [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) "SkuPrices_v1_5_scenarios.results.json"), + [int] $Runs = 3, + [string[]] $Only ) $ErrorActionPreference = 'Stop' Set-AzContext -Context (Get-AzContext -Name $Context) -Scope Process | Out-Null $secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token -$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)" } +function ConvertTo-Seconds([string] $ts) { if (-not $ts) { return 0.0 }; [TimeSpan]::Parse($ts).TotalSeconds } + +# Run a query on the v2 endpoint; return result rows plus server statistics function Invoke-Kusto([string] $Csl) { $body = @{ db = $Database; csl = $Csl; properties = @{ Options = @{ servertimeout = '00:20:00'; query_results_cache_max_age = '00:00:00' } } } | ConvertTo-Json -Depth 5 - $sw = [Diagnostics.Stopwatch]::StartNew() - $t = (Invoke-RestMethod -Uri "$Cluster/v1/rest/query" -Method Post -Headers $headers -Body $body -TimeoutSec 1200).Tables[0] - [pscustomobject]@{ Ms = $sw.ElapsedMilliseconds; Columns = @($t.Columns | ForEach-Object ColumnName); Rows = @($t.Rows) } + $resp = Invoke-WebRequest -Uri "$Cluster/v2/rest/query" -Method Post -Headers $headers -ContentType 'application/json' -Body $body -TimeoutSec 1200 -SkipHttpErrorCheck + if ($resp.StatusCode -ne 200) { throw "HTTP $($resp.StatusCode): $($resp.Content.Substring(0, [Math]::Min(300, $resp.Content.Length)))" } + $frames = $resp.Content | ConvertFrom-Json -Depth 50 + $primary = $frames | Where-Object { $_.FrameType -eq 'DataTable' -and $_.TableKind -eq 'PrimaryResult' } | Select-Object -First 1 + $errors = $frames | Where-Object { $_.FrameType -eq 'DataTable' -and $_.TableKind -eq 'PrimaryResult' } | ForEach-Object { $_.Rows } | Where-Object { $_ -isnot [array] -and $_.OneApiErrors } + if ($errors) { throw ($errors[0].OneApiErrors[0].error.'@message') } + $info = $frames | Where-Object { $_.TableKind -eq 'QueryCompletionInformation' } | Select-Object -First 1 + $stats = $null + if ($info) { + $cols = $info.Columns.ColumnName + foreach ($row in $info.Rows) { if ($row[$cols.IndexOf('EventTypeName')] -eq 'QueryResourceConsumption') { $stats = $row[$cols.IndexOf('Payload')] | ConvertFrom-Json -Depth 50 } } + } + [pscustomobject]@{ + Columns = @($primary.Columns.ColumnName) + Rows = @($primary.Rows | Where-Object { $_ -is [array] }) + Ms = if ($stats) { [math]::Round($stats.ExecutionTime * 1000) } else { $null } + CpuMs = if ($stats) { [math]::Round((ConvertTo-Seconds $stats.resource_usage.cpu.'total cpu') * 1000) } else { $null } + MemoryMB = if ($stats) { [math]::Round($stats.resource_usage.memory.peak_per_node / 1MB, 1) } else { $null } + RowsScanned = if ($stats) { [long]$stats.input_dataset_statistics.rows.scanned } else { $null } + } } +function Get-Median([double[]] $v) { $s = $v | Sort-Object; $s[[int][math]::Floor(($s.Count - 1) / 2)] } + # Normalize a result so equivalent outputs compare equal (sorted rows, rounded numbers) -function Get-Signature($Result) { - $lines = $Result.Rows | ForEach-Object { ($_ | ForEach-Object { if ($_ -is [double] -or $_ -is [decimal]) { [math]::Round([double]$_, 6) } else { "$_" } }) -join '|' } | Sort-Object - ($lines -join "`n") +function Get-Signature($Rows) { + ($Rows | ForEach-Object { ($_ | ForEach-Object { if ($_ -is [double] -or $_ -is [decimal] -or $_ -is [long] -or $_ -is [int]) { [math]::Round([double]$_, 4) } else { "$_" } }) -join '|' } | Sort-Object) -join "`n" +} + +# Query size measures (comments ignored) +function Get-Shape([string] $q) { + $code = ($q -split "`n" | ForEach-Object { ($_ -replace '//.*$', '').TrimEnd() } | Where-Object { $_.Trim() }) -join "`n" + [pscustomobject]@{ + Lines = @($code -split "`n").Count + Operators = ([regex]::Matches($code, '\|\s*[a-z-]+')).Count + Joins = ([regex]::Matches($code, '\b(join|lookup)\b')).Count + Lets = ([regex]::Matches($code, '(?m)^\s*let\b')).Count + XColumns = @([regex]::Matches($code, '\bx_[A-Za-z0-9]+\b') | ForEach-Object Value | Select-Object -Unique).Count + } } # Parse scenarios @@ -40,36 +77,57 @@ $scenarios = foreach ($block in ($text -split '(?m)^// ## ') | Select-Object -Sk $id, $name = $lines[0].Trim() -split ' ', 2 $meta = @{} foreach ($l in $lines) { if ($l -match '^// (Question|Source|Expect): (.+)$') { $meta[$Matches[1]] = $Matches[2].Trim() } } - $rowsQuery = ($block -split '(?m)^// ### Rows\s*$')[1] -split '(?m)^// ### Columns\s*$' | Select-Object -First 1 + $rowsQuery = (($block -split '(?m)^// ### Rows\s*$')[1] -split '(?m)^// ### Columns\s*$')[0] $colsQuery = ($block -split '(?m)^// ### Columns\s*$')[1] [pscustomobject]@{ Id = $id; Name = $name.Trim(); Question = $meta.Question; Source = $meta.Source; Expect = $meta.Expect; RowsQuery = $rowsQuery.Trim(); ColumnsQuery = $colsQuery.Trim() } } +if ($Only) { $scenarios = $scenarios | Where-Object Id -in $Only } $results = foreach ($s in $scenarios) { - $r = @{} + $runsBySide = @{ Rows = @(); Columns = @() } + $err = @{} + for ($i = 0; $i -lt $Runs; $i++) { + foreach ($side in $(if ($i % 2) { 'Columns', 'Rows' } else { 'Rows', 'Columns' })) { + if ($err[$side]) { continue } + try { $runsBySide[$side] += Invoke-Kusto $s."${side}Query" } catch { $err[$side] = $_.Exception.Message } + } + } + $sides = @{} foreach ($side in 'Rows', 'Columns') { - $q = $s."${side}Query" - try { - $res = Invoke-Kusto $q - $r[$side] = [pscustomobject]@{ - Ms = $res.Ms; ResultRows = $res.Rows.Count; Columns = $res.Columns - Data = @($res.Rows | Select-Object -First 15 | ForEach-Object { , @($_) }) - Signature = Get-Signature $res; Error = $null - Lines = @($q -split "`n" | Where-Object { $_.Trim() }).Count - Operators = ([regex]::Matches($q, '(?m)^\s*\|')).Count - Joins = ([regex]::Matches($q, '\b(join|lookup)\b')).Count + $q = $s."${side}Query"; $shape = Get-Shape $q; $rs = $runsBySide[$side] + $sides[$side] = if ($err[$side]) { [pscustomobject]@{ Error = $err[$side] } } else { + [pscustomobject]@{ + Ms = Get-Median ($rs.Ms); CpuMs = Get-Median ($rs.CpuMs); MemoryMB = Get-Median ($rs.MemoryMB); RowsScanned = Get-Median ($rs.RowsScanned) + ResultRows = $rs[0].Rows.Count; Columns = $rs[0].Columns + Data = @($rs[0].Rows | Select-Object -First 15 | ForEach-Object { , @($_) }) + Signature = Get-Signature $rs[0].Rows + Lines = $shape.Lines; Operators = $shape.Operators; Joins = $shape.Joins; Lets = $shape.Lets; XColumns = $shape.XColumns } } - catch { $r[$side] = [pscustomobject]@{ Error = ($_.ErrorDetails.Message ?? $_.Exception.Message) } } } - $match = -not $r.Rows.Error -and -not $r.Columns.Error -and $r.Rows.Signature -eq $r.Columns.Signature + $R = $sides.Rows; $C = $sides.Columns + $match = -not $R.Error -and -not $C.Error -and $R.Signature -eq $C.Signature $expectSame = $s.Expect -like 'same*' - $status = if ($r.Rows.Error -or $r.Columns.Error) { 'ERROR' } elseif ($match -eq $expectSame) { 'PASS' } else { 'FAIL' } - Write-Host ("{0,-5} {1} {2,-40} rows {3,6}ms/{4,3}L/{5}J cols {6,6}ms/{7,3}L/{8}J match={9}" -f $status, $s.Id, $s.Name, $r.Rows.Ms, $r.Rows.Lines, $r.Rows.Joins, $r.Columns.Ms, $r.Columns.Lines, $r.Columns.Joins, $match) - if ($status -eq 'ERROR') { Write-Host " Rows: $($r.Rows.Error)`n Columns: $($r.Columns.Error)" } - [pscustomobject]@{ Id = $s.Id; Name = $s.Name; Question = $s.Question; Source = $s.Source; Expect = $s.Expect; Status = $status; Match = $match + $status = if ($R.Error -or $C.Error) { 'ERROR' } elseif ($match -eq $expectSame) { 'PASS' } else { 'FAIL' } + + # Winners: lower is better; within 15% (or tiny absolute gap) is a tie + function Win($r, $c, $minGap = 0) { if ($null -eq $r -or $null -eq $c) { return 'n/a' }; $hi = [math]::Max($r, $c); if ($hi -eq 0 -or [math]::Abs($r - $c) -le [math]::Max($minGap, 0.15 * $hi)) { 'Tie' } elseif ($r -lt $c) { 'Rows' } else { 'Columns' } } + $score = if ($status -ne 'ERROR') { + [ordered]@{ + Simpler = Win ($R.Operators + 2 * $R.Joins + $R.Lets) ($C.Operators + 2 * $C.Joins + $C.Lets) 1 + Faster = Win $R.Ms $C.Ms 50 + Cpu = Win $R.CpuMs $C.CpuMs 50 + Memory = Win $R.MemoryMB $C.MemoryMB 5 + Scanned = Win $R.RowsScanned $C.RowsScanned + Portable = Win $R.XColumns $C.XColumns + Correct = if ($match) { 'Tie' } elseif ($s.Expect -match 'different') { 'Rows' } else { 'n/a' } + } + } + Write-Host ("{0,-5} {1} {2,-42} rows {3,6}ms cpu {4,6}ms {5,6}MB {6,4}ops cols {7,6}ms cpu {8,6}ms {9,6}MB {10,4}ops match={11}" -f $status, $s.Id, $s.Name, $R.Ms, $R.CpuMs, $R.MemoryMB, $R.Operators, $C.Ms, $C.CpuMs, $C.MemoryMB, $C.Operators, $match) + if ($status -eq 'ERROR') { Write-Host " Rows: $($R.Error)`n Columns: $($C.Error)" } + [pscustomobject]@{ Id = $s.Id; Name = $s.Name; Question = $s.Question; Source = $s.Source; Expect = $s.Expect; Status = $status; Match = $match; Score = $score RowsQuery = $s.RowsQuery; ColumnsQuery = $s.ColumnsQuery - Rows = $r.Rows | Select-Object * -ExcludeProperty Signature; Columns = $r.Columns | Select-Object * -ExcludeProperty Signature } + Rows = $R | Select-Object * -ExcludeProperty Signature; Columns = $C | Select-Object * -ExcludeProperty Signature } } $results | ConvertTo-Json -Depth 8 | Set-Content -Path $OutFile From 95ccc5ce84efe558509d68bf5c3fbb41b992ccca Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Thu, 8 Oct 2026 14:56:17 -0700 Subject: [PATCH 5/8] feat(hubs): full-history SkuPrices layouts and 3-way scenario report Builds both layouts with 14 months of effective-dated history (runs of unchanged prices), adds today's monthly table to cost-join scenarios, real backfill in S14, clearer S06/S12, report renderer + template. Co-Authored-By: Claude Opus 5.5 --- .../SkuPrices_v1_5/Build-SkuPrices.ps1 | 94 +++-- .../preview/SkuPrices_v1_5/README.md | 84 +++-- .../SkuPrices_v1_5/Render-SkuPriceReport.ps1 | 56 +++ .../preview/SkuPrices_v1_5/SkuPrices_v1_5.kql | 224 ++++++------ .../SkuPrices_v1_5_report.template.html | 228 +++++++++++++ .../SkuPrices_v1_5_scenarios.kql | 321 ++++++++++++------ .../SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 | 73 ++-- .../preview/SkuPrices_v1_5/Test-SkuPrices.ps1 | 35 +- 8 files changed, 758 insertions(+), 357 deletions(-) create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Render-SkuPriceReport.ps1 create mode 100644 src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 index 1423fd5b0..cd7c72e1c 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Build-SkuPrices.ps1 @@ -1,17 +1,21 @@ <# .SYNOPSIS -Rebuilds SkuPrices_v1_5 on ftk-dev from Prices_final_v1_2 using SkuPrices_v1_5.kql. +Rebuilds the FOCUS 1.5 SkuPrices preview tables on a hub from Prices_final_v1_2 and Costs_final_v1_2 using SkuPrices_v1_5.kql. .DESCRIPTION -Creates or updates SkuPrices_v1_5_build from the KQL file, then builds the table one price type at a time -(List replaces the table and its schema; Base, Contracted, and Effective append) to stay within memory. +1. Creates or updates every function in the KQL file. +2. Builds SkuPrices_v1_5_runs_table (runs of unchanged prices) for each price type and the wide layout. +3. Builds SkuPrices_v1_5 (rows layout) one month and price type at a time, and SkuPricesWide_v1_5 (columns layout) one month at a time. +4. Builds SkuPriceIdv2_map and CostsWithSkuPriceIdv2 (Costs copy with SkuPriceIdv2) one month at a time. +Small steps keep each query within memory on small clusters. #> [CmdletBinding(SupportsShouldProcess)] param( [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', [string] $Database = 'Ingestion', [string] $Context = 'fh-dev', - [string] $KqlPath = "$PSScriptRoot/SkuPrices_v1_5.kql" + [string] $KqlPath = "$PSScriptRoot/SkuPrices_v1_5.kql", + [switch] $SkipCosts ) $ErrorActionPreference = 'Stop' @@ -28,6 +32,17 @@ function Invoke-Kusto { (Invoke-RestMethod -Uri "$Cluster/v1/rest/$Endpoint" -Method Post -Headers $headers -Body $body -TimeoutSec 3600).Tables[0] } +# Run one command, timing it; the first command for a table replaces it, the rest append +function Invoke-Step([string] $Table, [string] $Query, [ref] $First, [string] $Label) { + $csl = if ($First.Value) { ".set-or-replace $Table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| $Query" } else { ".append $Table <| $Query" } + if ($PSCmdlet.ShouldProcess($Table, $csl)) { + $sw = [Diagnostics.Stopwatch]::StartNew() + Invoke-Kusto $csl | Out-Null + Write-Host "$Label done in $([int]$sw.Elapsed.TotalSeconds)s" + } + $First.Value = $false +} + # 1. Functions from the KQL file (every .create-or-alter function block, in order) $raw = Get-Content -Path $KqlPath -Raw $pos = 0 @@ -44,55 +59,34 @@ while (($start = $raw.IndexOf('.create-or-alter function', $pos)) -ge 0) { $pos = $end } -# 2. Effective starts table, one price type at a time -foreach ($type in 'List', 'Base', 'Contracted', 'Effective') { - $csl = if ($type -eq 'List') { ".set-or-replace SkuPrices_v1_5_starts_table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_starts('$type')" } - else { ".append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('$type')" } - if ($PSCmdlet.ShouldProcess('SkuPrices_v1_5_starts_table', $csl)) { - $sw = [Diagnostics.Stopwatch]::StartNew() - Invoke-Kusto $csl | Out-Null - Write-Host "Starts $type done in $([int]$sw.Elapsed.TotalSeconds)s" - } -} +$types = 'List', 'Base', 'Contracted', 'Effective' -# 3. Build the table one price type at a time -$steps = [ordered]@{ - List = ".set-or-replace SkuPrices_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_build('List')" - Base = ".append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Base')" - Contracted = ".append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Contracted')" - Effective = ".append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Effective')" -} -foreach ($type in $steps.Keys) { - if ($PSCmdlet.ShouldProcess('SkuPrices_v1_5', $steps[$type])) { - $sw = [Diagnostics.Stopwatch]::StartNew() - Invoke-Kusto $steps[$type] | Out-Null - Write-Host "$type done in $([int]$sw.Elapsed.TotalSeconds)s" - } -} +# 2. Runs of unchanged prices, per price type and for the wide layout +$first = $true +$parts = 8 # Hash partitions of keys, to stay within memory +foreach ($type in $types + 'Wide') { for ($p = 0; $p -lt $parts; $p++) { Invoke-Step 'SkuPrices_v1_5_runs_table' "SkuPrices_v1_5_runs_build('$type', $p, $parts)" ([ref]$first) "Runs $type $p" } } -# 4. Comparison table (one row per price, price types as columns) -$csl = ".set-or-replace SkuPricesWide_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPricesWide_v1_5_build()" -if ($PSCmdlet.ShouldProcess('SkuPricesWide_v1_5', $csl)) { - $sw = [Diagnostics.Stopwatch]::StartNew() - Invoke-Kusto $csl | Out-Null - Write-Host "Wide done in $([int]$sw.Elapsed.TotalSeconds)s" +# 3. Rows and columns layouts, one month (and price type) at a time +$months = (Invoke-Kusto 'Prices_final_v1_2 | summarize by M = startofmonth(x_IngestionTime) | order by M asc' 'query').Rows | ForEach-Object { ([datetime]$_[0]).ToString('yyyy-MM-dd') } +$firstRows = $true; $firstWide = $true +foreach ($m in $months) { + foreach ($type in $types) { Invoke-Step 'SkuPrices_v1_5' "SkuPrices_v1_5_build('$type', datetime($m))" ([ref]$firstRows) "Rows $m $type" } + Invoke-Step 'SkuPricesWide_v1_5' "SkuPricesWide_v1_5_build(datetime($m))" ([ref]$firstWide) "Wide $m" } -# 5. SkuPriceIdv2 map, then a Costs copy with SkuPriceIdv2, one month at a time -$csl = ".set-or-replace SkuPriceIdv2_map with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPriceIdv2_map_build()" -if ($PSCmdlet.ShouldProcess('SkuPriceIdv2_map', $csl)) { Invoke-Kusto $csl | Out-Null; Write-Host 'Map done' } -$months = (Invoke-Kusto 'Costs_final_v1_2 | summarize by M = startofmonth(ChargePeriodStart) | order by M asc' 'query').Rows | ForEach-Object { ([datetime]$_[0]).ToString('yyyy-MM-dd') } -$first = $true -foreach ($m in $months) { - $csl = if ($first) { ".set-or-replace CostsWithSkuPriceIdv2 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| CostsWithSkuPriceIdv2_build(datetime($m))" } - else { ".append CostsWithSkuPriceIdv2 <| CostsWithSkuPriceIdv2_build(datetime($m))" } - if ($PSCmdlet.ShouldProcess('CostsWithSkuPriceIdv2', $csl)) { Invoke-Kusto $csl | Out-Null; Write-Host "Costs $m done" } - $first = $false +# 4. SkuPriceIdv2 map, then a Costs copy with SkuPriceIdv2, one month at a time +if (-not $SkipCosts) { + $first = $true + Invoke-Step 'SkuPriceIdv2_map' 'SkuPriceIdv2_map_build()' ([ref]$first) 'Map' + $costMonths = (Invoke-Kusto 'Costs_final_v1_2 | summarize by M = startofmonth(ChargePeriodStart) | order by M asc' 'query').Rows | ForEach-Object { ([datetime]$_[0]).ToString('yyyy-MM-dd') } + $first = $true + foreach ($m in $costMonths) { Invoke-Step 'CostsWithSkuPriceIdv2' "CostsWithSkuPriceIdv2_build(datetime($m))" ([ref]$first) "Costs $m" } } -# 6. Verify -$counts = Invoke-Kusto 'SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType' 'query' -$counts.Rows | ForEach-Object { Write-Host "$($_[0]): $($_[1])" } -$columns = (Invoke-Kusto 'SkuPrices_v1_5 | getschema | project ColumnName' 'query').Rows | ForEach-Object { $_[0] } -$sorted = [string[]] $columns.Clone(); [Array]::Sort($sorted, [StringComparer]::Ordinal) -Write-Host "Columns: $($columns.Count); alphabetical: $(-not (Compare-Object $columns $sorted -SyncWindow 0))" +# 5. Verify +foreach ($t in 'SkuPrices_v1_5', 'SkuPricesWide_v1_5') { + $c = (Invoke-Kusto "$t | summarize Rows = count(), Current = countif(isnull(SkuPriceEffectiveEnd))" 'query').Rows[0] + $columns = (Invoke-Kusto "$t | getschema | project ColumnName" 'query').Rows | ForEach-Object { $_[0] } + $sorted = [string[]] $columns.Clone(); [Array]::Sort($sorted, [StringComparer]::Ordinal) + Write-Host "${t}: $($c[0]) rows ($($c[1]) current); $($columns.Count) columns; alphabetical: $(-not (Compare-Object $columns $sorted -SyncWindow 0))" +} diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md index 5e4c08d4d..62f43f1ba 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/README.md @@ -1,41 +1,48 @@ # FOCUS 1.5 SkuPrices (preview, draft) -Draft FinOps hubs implementation of the FOCUS 1.5 **SKU Price** dataset ([FOCUS PR 2424](https://github.com/FinOps-Open-Cost-and-Usage-Spec/FOCUS_Spec/pull/2424)), built from `Prices_final_v1_2`. Not wired into `app.bicep` or `.build.config` yet. Run manually against a hub. +Draft FinOps hubs implementation of the FOCUS 1.5 **SKU Price** dataset ([FOCUS PR 2424](https://github.com/FinOps-Open-Cost-and-Usage-Spec/FOCUS_Spec/pull/2424)), built from `Prices_final_v1_2`, with a comparison layout and scenarios to choose between them. Not wired into `app.bicep` or `.build.config` yet. Run manually against a hub. ## Files | File | Purpose | |---|---| -| `SkuPrices_v1_5.kql` | Functions + build commands for `SkuPrices_v1_5`, `SkuPricesWide_v1_5`, `CostsWithSkuPriceIdv2`, and helper tables (`SkuPrices_v1_5_starts_table`, `SkuPriceIdv2_map`) | -| `Build-SkuPrices.ps1` | Creates the functions and builds all tables, one price type at a time | -| `Test-SkuPrices.ps1` | Validation: duplicates, row counts vs. source, price/discount match, required columns, ranges | -| `SkuPrices_v1_5_scenarios.kql` | 14 scenarios, each with the most efficient query found for both layouts | -| `Test-SkuPriceScenarios.ps1` | Runs both queries per scenario (3x), compares results, scores simplicity, time, CPU, memory, rows scanned, x_ usage | +| `SkuPrices_v1_5.kql` | Functions + build commands for `SkuPrices_v1_5`, `SkuPricesWide_v1_5`, `CostsWithSkuPriceIdv2`, and helper tables (`SkuPrices_v1_5_runs_table`, `SkuPriceIdv2_map`) | +| `Build-SkuPrices.ps1` | Creates the functions and builds all tables in small steps (key partitions, months, price types) | +| `Test-SkuPrices.ps1` | Validation: duplicates, overlaps, current rows vs. latest sheet, rows vs. runs, price match, required columns, ranges | +| `SkuPrices_v1_5_scenarios.kql` | 14 scenarios, each with the most efficient query found for each layout (plus today's monthly table for cost joins) | +| `Test-SkuPriceScenarios.ps1` | Runs each query 3x, compares results, scores simplicity, time, CPU, memory, rows scanned, x_ usage, correctness | +| `Render-SkuPriceReport.ps1` + `SkuPrices_v1_5_report.template.html` | Renders the side-by-side HTML report from scenario results and live table sizes | Scripts default to the `ftk-dev.westus` cluster, `Ingestion` database, and `fh-dev` Az context. Override with `-Cluster`, `-Database`, `-Context`. ```powershell -./Build-SkuPrices.ps1 # ~10 min on ftk-dev +./Build-SkuPrices.ps1 # ~45 min on ftk-dev ./Test-SkuPrices.ps1 -./Test-SkuPriceScenarios.ps1 +./Test-SkuPriceScenarios.ps1 # writes results JSON to the temp folder +./Render-SkuPriceReport.ps1 ``` ## Layouts -| | `SkuPrices_v1_5` (rows) | `SkuPricesWide_v1_5` (columns) | -|---|---|---| -| Grain | SkuPriceId x price type x contract x currency | SkuPriceId x contract x currency | -| Price | `UnitPrice` + `x_UnitPriceType` (List, Base, Contracted, Effective) | `ListUnitPrice`, `ContractedUnitPrice`, `x_BaseUnitPrice`, `x_EffectiveUnitPrice` | -| ContractId | Empty for List; agreement for others | Agreement on every row | -| Eligibility | Global for List; agreement (+ MCA profiles) for others | Agreement on every row (list price mis-scoped) | -| Effective start | Per price type | Latest change of any price on the row | +All three hold the same 14 months of prices. + +| | Today `Prices_final_v1_2` | Rows `SkuPrices_v1_5` | Columns `SkuPricesWide_v1_5` | +|---|---|---|---| +| Grain | Price per month | Price type per run of unchanged months | Price per contract per run of unchanged months | +| Prices | Columns | `UnitPrice` + `x_UnitPriceType` (List, Base, Contracted, Effective) | `ListUnitPrice`, `ContractedUnitPrice`, `x_BaseUnitPrice`, `x_EffectiveUnitPrice` | +| Dates | Month | Effective start/end per price type | One start/end per row (new row when any price changes) | +| List price scope | n/a | Public: no contract, global eligibility | Scoped to the contract with the rest of the row | +| Rows | 18.78M | 4.17M (3.89M current) | 1.79M (1.63M current) | +| Extent | 2.34 GB | 0.64 GB | 0.33 GB | + +Rows by type (all / current): List 1,348,812 / 1,236,379; Contracted 1,326,353 / 1,236,379 (98% = List); Base 1,056,500 / 1,023,491 (94% = List); Effective 438,723 / 388,900. ## Mapping decisions * `SkuPriceId` = `SkuPriceIdv2` (unique per price); source `SkuPriceId` kept as `x_SkuPriceIdv1` * `ContractId` = `/providers/microsoft.billing/billingaccounts/{x_BillingAccountId}` (EA enrollment / MCA billing account, not the profile) * Savings plan rows carry only the Effective price (list/base/contracted on those rows are copies of the on-demand meter's prices) -* `SkuPriceEffectiveStart` = first month of the current unchanged price across all monthly loads; `SkuPriceEffectiveEnd` = null (current price) +* Effective dates are month-level runs of unchanged prices across monthly loads; current prices have no end * `PricingRegionId` = region name (`x_SkuRegion`); `PricingServiceName` = meter category; `SkuPriceCreated`/`LastUpdated` = first/last ingestion * `PurchaseDurationType` from `x_SkuTerm`; `PurchasePaymentModel` empty (not in price sheet) * Non-SKU Price source columns get an `x_` prefix; `CommitmentDiscountCategory` kept as a filter label; columns sorted alphabetically @@ -52,53 +59,38 @@ Costs `SkuPriceId` doesn't match the price sheet's, and Costs lacks `x_SkuProduc | Reservation purchase | Reservation for the term (only when unique) | 16/16 | n/a | | Adjustment | None | 0% | 0% | -Scenarios join `CostsWithSkuPriceIdv2` to prices on `SkuPriceId` + `ContractId` + `PricingCurrency`. Next step: add `SkuPriceIdv2` to `Costs_transform_v1_2` itself. -## Results on ftk-dev (EA, Oct 2026 price sheet, 14 months of history) - -| | Rows | Extent size | -|---|---|---| -| `Prices_final_v1_2` (14 monthly copies) | 18,778,328 | 2.51 GB | -| `Prices_final_v1_2` (latest month) | 1,625,302 | 217 MB | -| `SkuPrices_v1_5` (current prices) | 3,885,217 | 656 MB | -| `SkuPricesWide_v1_5` (current prices, columns layout) | 1,625,302 | 329 MB | -| `SkuPrices_v1_5` with full history (est.) | ~4.17M | ~0.70 GB | -| ...skipping Base/Contracted rows equal to List (est.) | ~1.88M | ~0.32 GB | - -* Rows by type: List 1,236,402; Contracted 1,236,402 (98.0% = List); Base 1,023,513 (94.0% = List); Effective 388,900 -* Validation: 18/19 checks pass; `PricingUnit` empty (see gaps) -* Scenarios: 14/14 pass; all match except S06 and S12, which differ as expected (Rows is correct) +Effective-dated prices have no range lookup in KQL. Scenarios spread each price over the months it was in effect and look up on `SkuPriceId` + `ContractId` + `PricingCurrency` + month: no range join and no duplicated cost rows. Today's monthly table needs no spreading. ## Scenarios | Id | Scenario | Simpler | Faster | CPU | Memory | Rows scanned | FOCUS-only | Correct | |---|---|---|---|---|---|---|---|---| -| S01 | List price for a SKU | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S01 | List price on a date | Tie | Tie | Tie | Tie | Columns | Columns | Tie | | S02 | Negotiated discount by service | Columns | Columns | Columns | Columns | Columns | Columns | Tie | | S03 | Base price drift from list | Columns | Columns | Columns | Columns | Columns | Tie | Tie | -| S04 | Reservation price vs. on-demand | Tie | Columns | Columns | Columns | Columns | Tie | Tie | -| S05 | Savings plan rate vs. on-demand | Tie | Columns | Tie | Tie | Columns | Tie | Tie | -| S06 | Recent price changes by price type | Rows | Tie | Tie | Tie | Columns | Rows | Rows | -| S07 | Cheapest region for a VM size | Tie | Tie | Tie | Tie | Columns | Tie | Tie | -| S08 | Verify billed unit prices | Columns | Tie | Tie | Columns | Columns | Columns | Tie | -| S09 | Savings plan what-if for on-demand usage | Columns | Columns | Tie | Columns | Columns | Tie | Tie | +| S04 | Reservation price vs. on-demand | Tie | Columns | Columns | Tie | Columns | Columns | Tie | +| S05 | Savings plan rate vs. on-demand | Tie | Tie | Columns | Tie | Columns | Tie | Tie | +| S06 | Price changes since July | Rows | Tie | Tie | Rows | Columns | Rows | Tie | +| S07 | Cheapest region for a VM size | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S08 | Verify billed unit prices (+Monthly) | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S09 | Savings plan what-if for on-demand usage (+Monthly) | Tie | Monthly | Monthly | Tie | Columns | Tie | Tie | | S10 | All prices for one SKU | Tie | Tie | Tie | Tie | Columns | Rows | Tie | -| S11 | Estimate a planned workload | Columns | Tie | Tie | Columns | Columns | Columns | Tie | -| S12 | Prices a billing account is eligible for | Rows | Tie | Columns | Tie | Columns | Rows | Rows | +| S11 | Estimate a planned workload | Tie | Tie | Tie | Tie | Columns | Columns | Tie | +| S12 | Public prices visible to each account | Tie | Rows | Rows | Tie | Columns | Columns | Rows | | S13 | Usage rates vs. purchase fees | Rows | Tie | Tie | Tie | Columns | Rows | Tie | -| S14 | Backfill missing prices on cost data | Columns | Columns | Tie | Columns | Columns | Columns | Tie | -| | **Wins (Rows / Columns)** | 3 / 6 | 0 / 6 | 0 / 4 | 0 / 7 | 0 / 14 | 4 / 5 | 2 / 0 | +| S14 | Backfill missing prices on cost data (+Monthly) | Monthly | Tie | Tie | Tie | Columns | Columns | Tie | +| | **Wins (Rows / Columns / Monthly)** | 2 / 2 / 1 | 1 / 3 / 1 | 1 / 4 / 1 | 1 / 2 / 0 | 0 / 14 / 0 | 3 / 8 / 0 | 1 / 0 / 0 | -Scores: winner per measure (Rows, Columns, or Tie within 15%). Simpler = operators + 2x joins + lets; Faster/CPU/Memory/Rows scanned = median of 3 server-measured runs; FOCUS-only = fewer distinct x_ columns referenced; Correct = which layout returns the right answer when they differ. +Scores: winner per measure, or Tie within 15% of the runner-up. Simpler = operators + 2x joins + lets; Faster/CPU/Memory/Rows scanned = median of 3 server-measured runs; FOCUS-only = fewer distinct x_ columns referenced; Correct = which layout returns the right answer when they differ (S12: columns hide the public list price from accounts outside the agreement). -Covers 9 of 13 FOCUS PR 2595 queries plus 5 more (S03, S05, S06, S10, S14 price backfill). Not testable with this data: announced price changes, public vs. negotiated at a quantity, tier resolution, next tier (no future prices or tiers in EA price sheets), negotiated savings plan discount, multiple currencies, consumption currency. Efficiency patterns: filter and project early, lookup with a small right side, limit prices to IDs/meters the query needs, evaluate eligibility once per distinct JSON value (S12: 4.5 min to 1.9 s). Cost scenarios use a price only when its block unit matches the charge's PricingUnit; S09 at list = 99.96% of billed. +Covers 9 of 13 FOCUS PR 2595 queries plus 5 more (S03, S05, S06, S10, S14). Not testable with this data: announced price changes, public vs. negotiated at a quantity, tier resolution, next tier (no future prices or tiers in EA price sheets), negotiated savings plan discount, multiple currencies, consumption currency. Cost scenarios use a price only when its block unit matches the charge's `PricingUnit`. ## Gaps / TODO * [ ] `PricingUnit` and `x_PricingBlockSize` empty: `PricingUnits` open data table has 0 rows on ftk-dev, so `UnitPrice` is per block (e.g., per 10 hours); scenarios parse the block size from `x_PricingUnitDescription` * [ ] Savings plan list price ("list effective") is lost on ingestion: `Prices_transform_v1_2` nulls the SP `MarketPrice` and backfills on-demand prices; needed to measure negotiated SP discounts -* [ ] Store replaced prices (full history), not only current prices * [ ] Decide whether to skip Base/Contracted rows equal to List (lookups fall back to List) -* [ ] MCA untested (collapses billing profiles into the billing account; check for profile-specific prices) +* [ ] MCA untested (collapses billing profiles into the billing account; check for profile-specific prices); latest month is global, not per account * [ ] Decide final layout once FOCUS settles the price type column (planned for 1.6) * [ ] Add `SkuPriceIdv2` to `Costs_transform_v1_2` (replaces `CostsWithSkuPriceIdv2`); find a reservation-covered usage price * [ ] Wire into `IngestionSetup_v1_5.kql` / `HubSetup_v1_5.kql`, `app.bicep`, `.build.config`; add `SkuPrices()` hub function, docs, changelog diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Render-SkuPriceReport.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Render-SkuPriceReport.ps1 new file mode 100644 index 000000000..af726c335 --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Render-SkuPriceReport.ps1 @@ -0,0 +1,56 @@ +<# +.SYNOPSIS +Renders the SKU Price layout comparison report (HTML) from scenario results and live table sizes. +#> +[CmdletBinding()] +param( + [string] $Cluster = 'https://ftk-dev.westus.kusto.windows.net', + [string] $Database = 'Ingestion', + [string] $Context = 'fh-dev', + [string] $Template = "$PSScriptRoot/SkuPrices_v1_5_report.template.html", + [string] $Results = (Join-Path ([IO.Path]::GetTempPath()) 'SkuPrices_v1_5_scenarios.results.json'), + [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) "sku-price-layouts.html") +) + +$ErrorActionPreference = 'Stop' +Set-AzContext -Context (Get-AzContext -Name $Context) -Scope Process | Out-Null +$secure = (Get-AzAccessToken -ResourceUrl $Cluster -AsSecureString).Token +$headers = @{ Authorization = "Bearer $([System.Net.NetworkCredential]::new('', $secure).Password)"; 'Content-Type' = 'application/json' } +function Invoke-Kusto([string] $Csl, [string] $Endpoint = 'query') { + $body = @{ db = $Database; csl = $Csl } | ConvertTo-Json + , @((Invoke-RestMethod -Uri "$Cluster/v1/rest/$Endpoint" -Method Post -Headers $headers -Body $body -TimeoutSec 600).Tables[0].Rows) +} + +function Get-Size([string] $Table, [string] $Filter = '') { + $r = Invoke-Kusto ".show table $Table extents $Filter | summarize Rows = sum(RowCount), Extent = sum(ExtentSize), Compressed = sum(CompressedSize)" 'mgmt' + $cols = (Invoke-Kusto "$Table | getschema | count")[0][0] + [pscustomobject]@{ Rows = [double]$r[0][0]; Extent = [double]$r[0][1]; Compressed = [double]$r[0][2]; Columns = $cols } +} +function M($v) { '{0:0.00}M' -f ($v / 1e6) } +function G($v) { '{0:0.00}' -f ($v / 1GB) } +function X($a, $b) { '{0:0.0}×' -f ($a / $b) } + +$p = Get-Size 'Prices_final_v1_2' +$r = Get-Size 'SkuPrices_v1_5' +$c = Get-Size 'SkuPricesWide_v1_5' +$rCur = (Invoke-Kusto 'SkuPrices_v1_5 | where isnull(SkuPriceEffectiveEnd) | count')[0][0] +$cCur = (Invoke-Kusto 'SkuPricesWide_v1_5 | where isnull(SkuPriceEffectiveEnd) | count')[0][0] + +function Card($cls, $label, $table, $s, $ratio, [string[]] $bullets) { + @" +
$label $table
+
$(M $s.Rows)Rows
$(G $s.Extent)GB extent
$(G $s.Compressed)GB compressed
+
$ratio
    $(($bullets | ForEach-Object { "
  • $_
    • " }) -join '')
+"@ +} +$tables = (Card 'mon' 'Today' 'Prices_final_v1_2' $p "$($p.Columns) columns. 14 monthly price sheets." @( + 'One row per price per month', 'All price types as columns', 'Month-level dates; joins match on month')) + + (Card 'rows' 'Rows' 'SkuPrices_v1_5' $r "$($r.Columns) columns. $(X $r.Rows $p.Rows) rows, $(X $r.Extent $p.Extent) extent of today. $(M $rCur) current." @( + 'One row per price type (x_UnitPriceType) per run of unchanged months', 'List rows are public: no contract, global eligibility', 'Effective start and end per price type')) + + (Card 'cols' 'Columns' 'SkuPricesWide_v1_5' $c "$($c.Columns) columns. $(X $c.Rows $p.Rows) rows, $(X $c.Extent $p.Extent) extent of today. $(M $cCur) current." @( + 'One row per price per contract per run; a new row when any price changes', 'List price repeats per contract and is scoped to it', 'One effective start and end for all prices on the row')) + +$json = ((Get-Content $Results -Raw | ConvertFrom-Json) | ConvertTo-Json -Depth 8 -Compress).Replace(' x_SkuPriceIdv1). -// Built one price type at a time to stay within memory on small clusters. +// Built one price type and one month at a time to stay within memory on small clusters. +// Effective dates are month-level: Prices_final_v1_2 holds one price sheet per month, not the provider's change dates. -// 1. Effective start function: first month of the current unchanged run of each price across all monthly loads -.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Original effective start per SKU price for one price type') -SkuPrices_v1_5_starts(priceType: string) { - // Narrow history keyed by a 64-bit hash of SkuPriceId + ContractId + PricingCurrency (keeps memory low) - let hist = Prices_final_v1_2 - | extend UnitPrice = case( - priceType == 'List', ListUnitPrice, - priceType == 'Base', x_BaseUnitPrice, - priceType == 'Contracted', ContractedUnitPrice, - priceType == 'Effective', x_EffectiveUnitPrice, - real(null)) - | where isnotnull(UnitPrice) - | where priceType == 'Effective' or x_SkuPriceType != 'SavingsPlan' - | extend ContractId = iff(priceType == 'List', '', tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId))) - | project k = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)), M = startofmonth(x_IngestionTime), Start = x_EffectivePeriodStart, UnitPrice - | summarize hint.shufflekey = k UnitPrice = take_any(UnitPrice), Start = min(Start) by k, M; - // Current price = price in the most recent month for each key - let cur = hist | summarize hint.shufflekey = k arg_max(M, UnitPrice) by k | project k, Cur = UnitPrice; - // Last month the price differed from the current price - let lastDiff = hist - | join kind=inner hint.shufflekey = k cur on k - | where UnitPrice != Cur - | summarize hint.shufflekey = k LastDiff = max(M) by k; +// 1. Source rows for one price type (rows layout) +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Prices_final_v1_2 rows for one price type with UnitPrice, ContractId, and the run key') +SkuPrices_v1_5_source(priceType: string) { + Prices_final_v1_2 + | extend UnitPrice = case( + priceType == 'List', ListUnitPrice, + priceType == 'Base', x_BaseUnitPrice, + priceType == 'Contracted', ContractedUnitPrice, + priceType == 'Effective', x_EffectiveUnitPrice, + real(null)) + | where isnotnull(UnitPrice) + // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices + | where priceType == 'Effective' or x_SkuPriceType != 'SavingsPlan' + // List prices are public: no contract. Everything else belongs to the agreement: EA enrollment or MCA billing account (not the billing profile) + | extend ContractId = iff(priceType == 'List', '', tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId))) + | extend tmp_k = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)), tmp_M = startofmonth(x_IngestionTime), tmp_V = hash_xxhash64(tostring(UnitPrice)) +} + +// 2. Source rows for the wide layout (all price types as columns) +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Prices_final_v1_2 rows for the wide layout with ContractId and the run key') +SkuPricesWide_v1_5_source() { + Prices_final_v1_2 + // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices + | extend tmp_IsSP = x_SkuPriceType == 'SavingsPlan' + | extend + ListUnitPrice = iff(tmp_IsSP, real(null), ListUnitPrice), + ContractedUnitPrice = iff(tmp_IsSP, real(null), ContractedUnitPrice), + x_BaseUnitPrice = iff(tmp_IsSP, real(null), x_BaseUnitPrice), + x_ContractedUnitPriceDiscount = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscount), + x_ContractedUnitPriceDiscountPercent = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscountPercent) + // Every row carries a contracted price, so every row belongs to the agreement + | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) + | extend tmp_k = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)), tmp_M = startofmonth(x_IngestionTime), + tmp_V = hash_xxhash64(strcat(ListUnitPrice, '|', x_BaseUnitPrice, '|', ContractedUnitPrice, '|', x_EffectiveUnitPrice)) +} + +// 3. Runs: consecutive months with the same value for a key become one run with an effective start and end +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Runs of unchanged values: k (key), M (month), V (value hash), Start (effective start in that month)') +SkuPrices_v1_5_runs(T: (k: long, M: datetime, V: long, Start: datetime)) { + let hist = T | summarize hint.shufflekey = k V = take_any(V), Start = min(Start) by k, M; + let latestMonth = toscalar(hist | summarize max(M)); + // A run starts in a month when the value differs from the previous month or the key wasn't there the previous month + let runStarts = hist + | join kind=leftouter hint.shufflekey = k (hist | project k, M = datetime_add('month', 1, M), PrevV = V) on k, M + | where isnull(PrevV) or PrevV != V + | project k, RunStart = M; hist - | join kind=leftouter hint.shufflekey = k lastDiff on k - | where isnull(LastDiff) or M > LastDiff - | summarize hint.shufflekey = k OriginalStart = min(Start) by k + | join kind=inner hint.shufflekey = k runStarts on k + | where RunStart <= M + | summarize hint.shufflekey = k RunStart = max(RunStart) by k, M, Start + | summarize hint.shufflekey = k LastMonth = max(M), EffectiveStart = min(Start) by k, RunStart + // Runs still in the latest month are current: no end. Others end when the next month starts. + | extend EffectiveEnd = iff(LastMonth == latestMonth, datetime(null), datetime_add('month', 1, LastMonth)) +} + +.create-or-alter function with (folder = 'FOCUS 1.5 preview', docstring = 'Runs for one price type (List, Base, Contracted, Effective) or Wide, for one hash partition of keys') +SkuPrices_v1_5_runs_build(priceType: string, part: int, parts: int) { + union + (SkuPrices_v1_5_source(priceType) | where priceType != 'Wide' and abs(tmp_k % parts) == part | project k = tmp_k, M = tmp_M, V = tmp_V, Start = x_EffectivePeriodStart), + (SkuPricesWide_v1_5_source() | where priceType == 'Wide' and abs(tmp_k % parts) == part | project k = tmp_k, M = tmp_M, V = tmp_V, Start = x_EffectivePeriodStart) + | invoke SkuPrices_v1_5_runs() | extend x_UnitPriceType = priceType } -// 2. Build function (reads SkuPrices_v1_5_starts_table, built first) -.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows for one price type (List, Base, Contracted, Effective)') -SkuPrices_v1_5_build(priceType: string) { - // Latest price sheet per billing account (by day, since one ingestion spans minutes) - let latestSheet = Prices_final_v1_2 - | summarize tmp_LatestDay = max(startofday(x_IngestionTime)) by BillingAccountId; - // First time each price was seen (narrow scan) - let created = Prices_final_v1_2 - | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; - let src = Prices_final_v1_2 - | lookup kind=inner latestSheet on BillingAccountId - | where startofday(x_IngestionTime) == tmp_LatestDay - | extend UnitPrice = case( - priceType == 'List', ListUnitPrice, - priceType == 'Base', x_BaseUnitPrice, - priceType == 'Contracted', ContractedUnitPrice, - priceType == 'Effective', x_EffectiveUnitPrice, - real(null)) - | where isnotnull(UnitPrice) - // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices - | where priceType == 'Effective' or x_SkuPriceType != 'SavingsPlan' +// 4. Rows layout: one price type, runs that end in one month +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows for one price type and the runs that end in one month (reads SkuPrices_v1_5_runs_table)') +SkuPrices_v1_5_build(priceType: string, month: datetime) { + let runs = SkuPrices_v1_5_runs_table | where x_UnitPriceType == priceType and LastMonth == month | project tmp_k = k, tmp_Start = EffectiveStart, tmp_End = EffectiveEnd; + let src = SkuPrices_v1_5_source(priceType) + | where tmp_M == month + | lookup kind=inner runs on tmp_k | extend x_UnitPriceType = priceType - // List prices are public: no contract. Everything else belongs to the agreement: EA enrollment or MCA billing account (not the billing profile) - | extend ContractId = iff(priceType == 'List', '', tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId))) // One row per key (collapses list prices repeated across billing accounts) - | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, x_EffectivePeriodStart, PricingCurrency - | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency, '|', x_EffectivePeriodStart); + | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, PricingCurrency + | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency); + // First time each price was seen + let created = Prices_final_v1_2 | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; // Tier max = next tier min for the same meter and price (only meters with more than one tier) let tierMax = src | summarize Tiers = make_set(x_SkuTier) by tmp_TierGroup @@ -78,8 +100,6 @@ SkuPrices_v1_5_build(priceType: string) { src | lookup kind=leftouter tierMax on tmp_TierGroup, x_SkuTier | lookup kind=leftouter created on SkuPriceIdv2 - | extend tmp_k = hash_xxhash64(strcat(SkuPriceIdv2, "|", ContractId, "|", PricingCurrency)) - | lookup kind=leftouter (SkuPrices_v1_5_starts_table | where x_UnitPriceType == priceType | project tmp_k = k, OriginalStart) on tmp_k | extend x_SkuPriceIdv1 = SkuPriceId | extend PricingCurrencyCategory = 'Payable', @@ -91,8 +111,8 @@ SkuPrices_v1_5_build(priceType: string) { QuantityTierMinimum = coalesce(x_SkuTier, real(0)), ServiceProviderName = 'Microsoft', SkuPriceDescription = x_SkuDescription, - SkuPriceEffectiveEnd = datetime(null), // Current price; no known end (monthly sheet end is not a price end) - SkuPriceEffectiveStart = coalesce(OriginalStart, x_EffectivePeriodStart), // First month of the current unchanged price + SkuPriceEffectiveEnd = tmp_End, // Month after the last month with this price; empty when current + SkuPriceEffectiveStart = tmp_Start, // First month with this price SkuPriceEligibility = iff(priceType == 'List', dynamic({'IsGlobalScope': true}), bag_pack('IsGlobalScope', false, 'InclusionOperator', 'Or', 'Inclusions', @@ -101,21 +121,21 @@ SkuPrices_v1_5_build(priceType: string) { bag_pack('Dimension', 'BillingAccountId', 'Operator', 'In', 'Values', pack_array(ContractId)), bag_pack('Dimension', 'BillingAccountId', 'Operator', 'StartsWith', 'Values', pack_array(strcat(ContractId, '/')))))), SkuPriceId = SkuPriceIdv2, - SkuPriceLastUpdated = x_IngestionTime, // Last ingestion; provider date not available + SkuPriceLastUpdated = x_IngestionTime, // Last ingestion of this price; provider date not available // Discount vs. list for this row's price type (replaces the 6 wide discount columns) x_UnitPriceDiscount = iff(priceType == 'List', real(null), ListUnitPrice - UnitPrice), x_UnitPriceDiscountPercent = iff(priceType == 'List' or ListUnitPrice == 0, real(null), 100.0 * (ListUnitPrice - UnitPrice) / ListUnitPrice) // List rows are collapsed across billing accounts and MCA rows across billing profiles, so clear values that differ between them | extend tmp_IsList = priceType == 'List', tmp_IsMca = x_BillingProfileId != x_BillingAccountId | extend - BillingAccountName = iff(tmp_IsList or tmp_IsMca, '', BillingAccountName), // MCA: profile name - x_BillingAccountId = iff(tmp_IsList, '', x_BillingAccountId), - x_BillingProfileId = iff(tmp_IsList or tmp_IsMca, '', x_BillingProfileId) - | project-away tmp_*, OriginalStart, BillingAccountId, // EA: same as ContractId; MCA: profile-level, collapsed into ContractId + BillingAccountName = iff(tmp_IsList or tmp_IsMca, '', BillingAccountName), // MCA: profile name + x_BillingAccountId = iff(tmp_IsList, '', x_BillingAccountId), + x_BillingProfileId = iff(tmp_IsList or tmp_IsMca, '', x_BillingProfileId) + | project-away tmp_*, BillingAccountId, // EA: same as ContractId; MCA: profile-level, collapsed into ContractId ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice, // In UnitPrice + x_UnitPriceType x_ContractedUnitPriceDiscount, x_ContractedUnitPriceDiscountPercent, x_EffectiveUnitPriceDiscount, x_EffectiveUnitPriceDiscountPercent, x_TotalUnitPriceDiscount, x_TotalUnitPriceDiscountPercent, // Replaced by x_UnitPriceDiscount(Percent) - x_EffectivePeriodStart, x_EffectivePeriodEnd // Same as SkuPriceEffectiveStart/End + x_EffectivePeriodStart, x_EffectivePeriodEnd // Replaced by SkuPriceEffectiveStart/End // Source columns that are not SKU Price columns become custom columns | project-rename x_BillingAccountName = BillingAccountName, @@ -129,30 +149,16 @@ SkuPrices_v1_5_build(priceType: string) { | project-reorder * asc } -// 3. Comparison function: same prices in the one-row-per-price layout (all price types as columns) -// Same source, keys, and effective-start logic as SkuPrices_v1_5, so scenario queries can be compared side by side -.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows with one row per price and price types as columns (comparison layout)') -SkuPricesWide_v1_5_build() { - let latestSheet = Prices_final_v1_2 - | summarize tmp_LatestDay = max(startofday(x_IngestionTime)) by BillingAccountId; - let created = Prices_final_v1_2 - | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; - let starts = SkuPrices_v1_5_starts_table | project k, x_UnitPriceType, OriginalStart; - let src = Prices_final_v1_2 - | lookup kind=inner latestSheet on BillingAccountId - | where startofday(x_IngestionTime) == tmp_LatestDay - // Savings plan rows carry only the SP rate; their list/base/contracted prices are copies of the on-demand meter's prices - | extend tmp_IsSP = x_SkuPriceType == 'SavingsPlan' - | extend - ListUnitPrice = iff(tmp_IsSP, real(null), ListUnitPrice), - ContractedUnitPrice = iff(tmp_IsSP, real(null), ContractedUnitPrice), - x_BaseUnitPrice = iff(tmp_IsSP, real(null), x_BaseUnitPrice), - x_ContractedUnitPriceDiscount = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscount), - x_ContractedUnitPriceDiscountPercent = iff(tmp_IsSP, real(null), x_ContractedUnitPriceDiscountPercent) - // Every row carries a contracted price, so every row belongs to the agreement - | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)) - | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, x_EffectivePeriodStart, PricingCurrency - | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency, '|', x_EffectivePeriodStart); +// 5. Columns layout (comparison): runs that end in one month +.create-or-alter function with (folder = 'FOCUS 1.5 preview', skipvalidation = 'true', docstring = 'FOCUS 1.5 SKU Price rows with price types as columns for the runs that end in one month (comparison layout)') +SkuPricesWide_v1_5_build(month: datetime) { + let runs = SkuPrices_v1_5_runs_table | where x_UnitPriceType == 'Wide' and LastMonth == month | project tmp_k = k, tmp_Start = EffectiveStart, tmp_End = EffectiveEnd; + let src = SkuPricesWide_v1_5_source() + | where tmp_M == month + | lookup kind=inner runs on tmp_k + | summarize hint.shufflekey = SkuPriceIdv2 take_any(*) by SkuPriceIdv2, ContractId, PricingCurrency + | extend tmp_TierGroup = strcat(x_SkuMeterId, '|', x_SkuPriceType, '|', x_SkuTerm, '|', x_SkuOfferId, '|', ContractId, '|', PricingCurrency); + let created = Prices_final_v1_2 | summarize hint.shufflekey = SkuPriceIdv2 SkuPriceCreated = min(x_IngestionTime) by SkuPriceIdv2; let tierMax = src | summarize Tiers = make_set(x_SkuTier) by tmp_TierGroup | where array_length(Tiers) > 1 @@ -163,12 +169,6 @@ SkuPricesWide_v1_5_build() { src | lookup kind=leftouter tierMax on tmp_TierGroup, x_SkuTier | lookup kind=leftouter created on SkuPriceIdv2 - // A row changes when any of its prices change, so its effective start is the latest start across its price types - | extend tmp_kList = hash_xxhash64(strcat(SkuPriceIdv2, '|', '', '|', PricingCurrency)), tmp_kAcct = hash_xxhash64(strcat(SkuPriceIdv2, '|', ContractId, '|', PricingCurrency)) - | lookup kind=leftouter (starts | where x_UnitPriceType == 'List' | project tmp_kList = k, tmp_StartList = OriginalStart) on tmp_kList - | lookup kind=leftouter (starts | where x_UnitPriceType == 'Base' | project tmp_kAcct = k, tmp_StartBase = OriginalStart) on tmp_kAcct - | lookup kind=leftouter (starts | where x_UnitPriceType == 'Contracted' | project tmp_kAcct = k, tmp_StartContracted = OriginalStart) on tmp_kAcct - | lookup kind=leftouter (starts | where x_UnitPriceType == 'Effective' | project tmp_kAcct = k, tmp_StartEffective = OriginalStart) on tmp_kAcct | extend x_SkuPriceIdv1 = SkuPriceId | extend PricingCurrencyCategory = 'Payable', @@ -180,8 +180,8 @@ SkuPricesWide_v1_5_build() { QuantityTierMinimum = coalesce(x_SkuTier, real(0)), ServiceProviderName = 'Microsoft', SkuPriceDescription = x_SkuDescription, - SkuPriceEffectiveEnd = datetime(null), - SkuPriceEffectiveStart = coalesce(max_of(tmp_StartList, tmp_StartBase, tmp_StartContracted, tmp_StartEffective), x_EffectivePeriodStart), + SkuPriceEffectiveEnd = tmp_End, + SkuPriceEffectiveStart = tmp_Start, // One eligibility for the whole row, even though the list price on it is public SkuPriceEligibility = bag_pack('IsGlobalScope', false, 'InclusionOperator', 'Or', 'Inclusions', pack_array( @@ -205,26 +205,32 @@ SkuPricesWide_v1_5_build() { | project-reorder * asc } -// 4. Build the tables in this order (Build-SkuPrices.ps1 runs these), one price type at a time to stay within memory -.set-or-replace SkuPrices_v1_5_starts_table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_starts('List') +// 6. Build (Build-SkuPrices.ps1 runs these): runs per type, then each table one month (and type) at a time +.set-or-replace SkuPrices_v1_5_runs_table with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_runs_build('List', 0, 8) + +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Base', 0, 8) + +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Contracted', 0, 8) + +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Effective', 0, 8) -.append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('Base') +.append SkuPrices_v1_5_runs_table <| SkuPrices_v1_5_runs_build('Wide', 0, 8) -.append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('Contracted') +// ...each partition 0-7 for each price type and Wide -.append SkuPrices_v1_5_starts_table <| SkuPrices_v1_5_starts('Effective') +.set-or-replace SkuPrices_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_build('List', datetime(2025-09-01)) -.set-or-replace SkuPrices_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPrices_v1_5_build('List') +.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Base', datetime(2025-09-01)) -.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Base') +// ...each price type for each month in Prices_final_v1_2 -.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Contracted') +.set-or-replace SkuPricesWide_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPricesWide_v1_5_build(datetime(2025-09-01)) -.append SkuPrices_v1_5 <| SkuPrices_v1_5_build('Effective') +.append SkuPricesWide_v1_5 <| SkuPricesWide_v1_5_build(datetime(2025-10-01)) -.set-or-replace SkuPricesWide_v1_5 with (folder = 'FOCUS 1.5 preview', recreate_schema = true) <| SkuPricesWide_v1_5_build() +// ...each month in Prices_final_v1_2 -// 5. Costs copy with SkuPriceIdv2, so Cost and Usage can join to SkuPrices on SkuPriceId +// 7. Costs copy with SkuPriceIdv2, so Cost and Usage can join to SkuPrices on SkuPriceId // Costs has no x_SkuProductId/x_SkuMeterType, so the ID is looked up from prices by price type + meter + offer + term // On-demand usage -> on-demand price // Reservation-covered usage -> on-demand price (Microsoft has no usage price for reservations) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html new file mode 100644 index 000000000..60e1c48de --- /dev/null +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html @@ -0,0 +1,228 @@ +SKU Price Layouts + + + + +
+
+

FOCUS 1.5 SKU Price · FinOps hubs preview · ftk-dev

+

One row per price type, or one row per price?

+

The same Microsoft EA prices (14 months) loaded two ways and queried for 14 scenarios, 9 from FOCUS PR 2595. Cost-join scenarios also show today's monthly Prices table. Each query uses the cheapest pattern found for its layout.

+
+ +
+

Tables

+
__TABLES__
+
+ +
+

Scoreboard

+
+
+ + + +
ScenarioSimplerFasterCPUMemoryRows scannedFOCUS-onlyCorrect
+
+ Simpler: fewer operators + 2× joins + lets + Faster / CPU / Memory / Rows scanned: median of 3 server-measured runs + FOCUS-only: fewer distinct x_ columns referenced + Correct: same answer everywhere, or which layout is right + Within 15% (or a tiny absolute gap) of the runner-up is a tie. Values under each pill: Rows · Columns (· Monthly). +
+
+
+ +
+

Scenarios

+
+
+ +
+

History. Both SKU Price tables hold all 14 months: one row per run of months with an unchanged price. Rows start a new row when that price type changes; Columns start a new row when any price on the row changes. Current prices have no effective end.

+

Cost and Usage join. Costs SkuPriceId doesn't match the price sheet, so cost scenarios use CostsWithSkuPriceIdv2: a copy of Costs with SkuPriceIdv2 looked up by price type, meter, offer, and term. Effective-dated prices are spread over the months they were in effect so each charge matches its month exactly, with no range join and no duplicated cost rows. Today's monthly table needs no spreading.

+

Units. PricingUnit is empty on ftk-dev (no PricingUnits open data), so price sheet prices are per block (e.g., per 100 hours). Cost scenarios divide by the block size and use a price only when its unit matches the charge's PricingUnit.

+

Not testable with this data: the 4 PR 2595 queries on announced price changes and quantity tiers, negotiated savings plan discount (savings plan list price is dropped on ingestion), multiple currencies, consumption currency.

+

Source: draft PR microsoft/finops-toolkit#2339 (Analytics/preview/SkuPrices_v1_5). Generated __GENERATED__.

+
+
+ + + diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql index e2283b600..8c286048a 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_scenarios.kql @@ -1,43 +1,46 @@ -// FOCUS 1.5 SKU Price scenarios: row-per-price-type vs. column-per-price-type +// FOCUS 1.5 SKU Price scenarios: row-per-price-type vs. column-per-price-type (vs. today's monthly table for cost joins) // -// Rows = SkuPrices_v1_5 one row per SkuPriceId x price type (x_UnitPriceType: List, Base, Contracted, Effective) -// Columns = SkuPricesWide_v1_5 one row per SkuPriceId with ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice +// Rows = SkuPrices_v1_5 one row per SkuPriceId x price type x run of unchanged months (x_UnitPriceType: List, Base, Contracted, Effective) +// Columns = SkuPricesWide_v1_5 one row per SkuPriceId x run of unchanged months, with ListUnitPrice, ContractedUnitPrice, x_BaseUnitPrice, x_EffectiveUnitPrice +// Monthly = Prices_final_v1_2 today's FinOps hubs table: one row per price per month (only in cost-join scenarios) // -// Both tables are built from the same Prices_final_v1_2 data by SkuPrices_v1_5.kql. -// Cost and Usage joins use CostsWithSkuPriceIdv2 (Costs_final_v1_2 + SkuPriceIdv2) on SkuPriceId + ContractId + PricingCurrency. -// Prices are divided by the block size in x_PricingUnitDescription (e.g., "100 Hours") where per-unit prices matter. +// Both SKU Price tables hold the same 14 months of history from Prices_final_v1_2 (built by SkuPrices_v1_5.kql). +// Current prices have an empty SkuPriceEffectiveEnd. +// Cost and Usage joins use CostsWithSkuPriceIdv2 (Costs_final_v1_2 + SkuPriceIdv2) on SkuPriceId + ContractId + PricingCurrency + month. +// Price sheet prices are per block (e.g., "100 Hours"); cost scenarios convert to per unit only when the unit matches PricingUnit. // Queries use FOCUS columns where they exist; x_ columns only where FOCUS has no equivalent (meter, offer, term, price type). -// Each query uses the cheapest pattern found for its layout: filter and project early, lookup with a small right side, -// limit prices to the IDs a cost query needs, and evaluate eligibility once per distinct JSON value. -// Test-SkuPriceScenarios.ps1 runs each pair, compares results, and scores them. +// Each query uses the cheapest pattern found for its layout. // -// Format: "// ## ", "// Question: ...", "// Expect: same|different ()", then "// ### Rows" and "// ### Columns" queries. +// Format: "// ## ", "// Question:", "// Source:", "// Expect: same|different ()", optional "// Check: " +// (appended to every side when testing, for queries that return full rows), then "// ### Rows", "// ### Columns", and optional "// ### Monthly". -// ## S01 List price for a SKU -// Question: What is the public list price for this SKU price? +// ## S01 List price on a date +// Question: What was the public list price for this SKU price on March 15, 2026? // Source: PR 2595 "Find the List Prices in Force at a Point in Time" // Expect: same // ### Rows SkuPrices_v1_5 | where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' and x_UnitPriceType == 'List' +| where SkuPriceEffectiveStart <= datetime(2026-03-15) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-03-15)) | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice // ### Columns // The list price repeats on every contract's row, so it needs distinct SkuPricesWide_v1_5 | where SkuPriceId == 'ODU_DZH318Z0F6ZF_002N_1 Compute Hour_MS-AZR-0017P' +| where SkuPriceEffectiveStart <= datetime(2026-03-15) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-03-15)) | distinct SkuPriceId, PricingCurrency, ListUnitPrice // ## S02 Negotiated discount by service -// Question: How much does the contract lower usage rates, by service? +// Question: How much does the contract lower current usage rates, by service? // Source: PR 2595 "Measure What Negotiation Reduces the Rate By"; appendix standard_and_negotiated_prices.md // Expect: same // ### Rows SkuPrices_v1_5 -| where ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' +| where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' | project SkuPriceId, PricingCurrency, PricingServiceName, ContractedUnitPrice = UnitPrice | lookup kind=inner ( SkuPrices_v1_5 - | where ChargeCategory == 'Usage' and x_UnitPriceType == 'List' and UnitPrice > 0 + | where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'List' and UnitPrice > 0 | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice ) on SkuPriceId, PricingCurrency | summarize Prices = count(), Discounted = countif(ContractedUnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice), 4) by PricingServiceName @@ -45,22 +48,22 @@ SkuPrices_v1_5 | take 10 // ### Columns SkuPricesWide_v1_5 -| where ChargeCategory == 'Usage' and ListUnitPrice > 0 and isnotnull(ContractedUnitPrice) +| where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and ListUnitPrice > 0 and isnotnull(ContractedUnitPrice) | summarize Prices = count(), Discounted = countif(ContractedUnitPrice < ListUnitPrice), AvgDiscountPercent = round(avg(100.0 * (ListUnitPrice - ContractedUnitPrice) / ListUnitPrice), 4) by PricingServiceName | order by Discounted desc, PricingServiceName asc | take 10 // ## S03 Base price drift from list -// Question: Where has the public list price moved away from the price fixed in the agreement? +// Question: Where has the current public list price moved away from the price fixed in the agreement? // Source: PR 2595 ("Base" price type); research gap: PR 2424 has no column to tell Base from Contracted // Expect: same // ### Rows SkuPrices_v1_5 -| where x_UnitPriceType == 'Base' +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'Base' | project SkuPriceId, PricingCurrency, PricingServiceName, BaseUnitPrice = UnitPrice | lookup kind=inner ( SkuPrices_v1_5 - | where x_UnitPriceType == 'List' + | where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'List' | project SkuPriceId, PricingCurrency, ListUnitPrice = UnitPrice ) on SkuPriceId, PricingCurrency | summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > BaseUnitPrice), ListBelowBase = countif(ListUnitPrice < BaseUnitPrice), MaxDriftPercent = round(max(iff(BaseUnitPrice > 0, 100.0 * (ListUnitPrice - BaseUnitPrice) / BaseUnitPrice, real(null))), 4) by PricingServiceName @@ -68,98 +71,116 @@ SkuPrices_v1_5 | take 10 // ### Columns SkuPricesWide_v1_5 -| where isnotnull(x_BaseUnitPrice) and isnotnull(ListUnitPrice) +| where isnull(SkuPriceEffectiveEnd) and isnotnull(x_BaseUnitPrice) and isnotnull(ListUnitPrice) | summarize Prices = count(), ListAboveBase = countif(ListUnitPrice > x_BaseUnitPrice), ListBelowBase = countif(ListUnitPrice < x_BaseUnitPrice), MaxDriftPercent = round(max(iff(x_BaseUnitPrice > 0, 100.0 * (ListUnitPrice - x_BaseUnitPrice) / x_BaseUnitPrice, real(null))), 4) by PricingServiceName | order by ListAboveBase + ListBelowBase desc, PricingServiceName asc | take 10 // ## S04 Reservation price vs. on-demand -// Question: For each reservation term, what is the hourly rate and the savings vs. the on-demand contracted rate? +// Question: For each reservation term, what is the hourly rate and the savings vs. the current on-demand contracted rate? // Source: PR 2595 "Evaluate the Purchase Terms Offered for a SKU"; appendix commitment_pricing.md // Expect: same // ### Rows let reservations = materialize(SkuPrices_v1_5 - | where ChargeCategory == 'Purchase' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 + | where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Purchase' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 | project x_SkuMeterId, x_SkuTerm, HourlyRate = UnitPrice / (x_SkuTerm * 730.0)); let meters = reservations | distinct x_SkuMeterId; reservations | lookup kind=inner ( SkuPrices_v1_5 - | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and UnitPrice > 0 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and UnitPrice > 0 | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId ) on x_SkuMeterId | summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc // ### Columns let reservations = materialize(SkuPricesWide_v1_5 - | where ChargeCategory == 'Purchase' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 and isnotnull(ContractedUnitPrice) + | where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Purchase' and x_SkuPriceType == 'ReservedInstance' and x_SkuTerm >= 12 and isnotnull(ContractedUnitPrice) | project x_SkuMeterId, x_SkuTerm, HourlyRate = ContractedUnitPrice / (x_SkuTerm * 730.0)); let meters = reservations | distinct x_SkuMeterId; reservations | lookup kind=inner ( SkuPricesWide_v1_5 - | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and ContractedUnitPrice > 0 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuOfferId == 'MS-AZR-0017P' and ContractedUnitPrice > 0 | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId ) on x_SkuMeterId | summarize Reservations = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - HourlyRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc // ## S05 Savings plan rate vs. on-demand -// Question: For each savings plan term, how much lower is the savings plan rate than the on-demand contracted rate? +// Question: For each savings plan term, how much lower is the current savings plan rate than the on-demand contracted rate? // Source: issue #2706 (commitment unit price on covered usage); scope_and_evolution.md // Expect: same // ### Rows let savingsPlans = materialize(SkuPrices_v1_5 - | where x_UnitPriceType == 'Effective' and x_SkuPriceType == 'SavingsPlan' + | where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'Effective' and x_SkuPriceType == 'SavingsPlan' | project x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SavingsPlanRate = UnitPrice); let meters = savingsPlans | distinct x_SkuMeterId; savingsPlans | lookup kind=inner ( SkuPrices_v1_5 - | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and UnitPrice > 0 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_UnitPriceType == 'Contracted' and x_SkuPriceType == 'Consumption' and UnitPrice > 0 | summarize OnDemandRate = min(UnitPrice) by x_SkuMeterId, x_SkuOfferId ) on x_SkuMeterId, x_SkuOfferId | summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - SavingsPlanRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc // ### Columns let savingsPlans = materialize(SkuPricesWide_v1_5 - | where x_SkuPriceType == 'SavingsPlan' and isnotnull(x_EffectiveUnitPrice) + | where isnull(SkuPriceEffectiveEnd) and x_SkuPriceType == 'SavingsPlan' and isnotnull(x_EffectiveUnitPrice) | project x_SkuMeterId, x_SkuOfferId, x_SkuTerm, SavingsPlanRate = x_EffectiveUnitPrice); let meters = savingsPlans | distinct x_SkuMeterId; savingsPlans | lookup kind=inner ( SkuPricesWide_v1_5 - | where x_SkuMeterId in (meters) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and ContractedUnitPrice > 0 + | where x_SkuMeterId in (meters) and isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and ContractedUnitPrice > 0 | summarize OnDemandRate = min(ContractedUnitPrice) by x_SkuMeterId, x_SkuOfferId ) on x_SkuMeterId, x_SkuOfferId | summarize Prices = count(), AvgSavingsPercent = round(avg(100.0 * (OnDemandRate - SavingsPlanRate) / OnDemandRate), 2) by Term = x_SkuTerm | order by Term asc -// ## S06 Recent price changes by price type -// Question: Which list, base, contracted, and savings plan prices changed since July 2026? +// ## S06 Price changes since July +// Question: Which production prices changed since July 2026, by price type, and did they go up or down? // Source: appendix price_effectivity_and_currency.md; working-draft "Fluctuation Tracking of Unit Prices" -// Expect: different (columns share one effective start per row, so a change to any price marks every price on the row as changed) +// Expect: same // ### Rows +// A change is a price that starts in or after July and replaces an earlier price for the same key (new SKUs excluded) SkuPrices_v1_5 -| where SkuPriceEffectiveStart >= datetime(2026-07-01) -| summarize Changed = count() by PriceType = x_UnitPriceType +| where SkuPriceEffectiveStart >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' +| project SkuPriceId, ContractId, PricingCurrency, x_UnitPriceType, Month = startofmonth(SkuPriceEffectiveStart), UnitPrice +| lookup kind=inner ( + SkuPrices_v1_5 + | where SkuPriceEffectiveEnd >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' + | project SkuPriceId, ContractId, PricingCurrency, x_UnitPriceType, Month = SkuPriceEffectiveEnd, PreviousPrice = UnitPrice + ) on SkuPriceId, ContractId, PricingCurrency, x_UnitPriceType, Month +| summarize Changed = count(), Up = countif(UnitPrice > PreviousPrice), Down = countif(UnitPrice < PreviousPrice) by PriceType = x_UnitPriceType | order by PriceType asc // ### Columns +// A row changes when any price on it changes, so each price column has to be compared with the previous row SkuPricesWide_v1_5 -| where SkuPriceEffectiveStart >= datetime(2026-07-01) -| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) -| evaluate narrow() -| project PriceType = Column, Changed = tolong(Value) -| where Changed > 0 +| where SkuPriceEffectiveStart >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' +| project SkuPriceId, ContractId, PricingCurrency, Month = startofmonth(SkuPriceEffectiveStart), ListUnitPrice, x_BaseUnitPrice, ContractedUnitPrice, x_EffectiveUnitPrice +| lookup kind=inner ( + SkuPricesWide_v1_5 + | where SkuPriceEffectiveEnd >= datetime(2026-07-01) and x_SkuOfferId != 'MS-AZR-0148P' + | project SkuPriceId, ContractId, PricingCurrency, Month = SkuPriceEffectiveEnd, PrevList = ListUnitPrice, PrevBase = x_BaseUnitPrice, PrevContracted = ContractedUnitPrice, PrevEffective = x_EffectiveUnitPrice + ) on SkuPriceId, ContractId, PricingCurrency, Month +| mv-expand kind = array p = pack_array( + pack_array('Base', x_BaseUnitPrice, PrevBase), + pack_array('Contracted', ContractedUnitPrice, PrevContracted), + pack_array('Effective', x_EffectiveUnitPrice, PrevEffective), + pack_array('List', ListUnitPrice, PrevList)) +| extend PriceType = tostring(p[0]), UnitPrice = todouble(p[1]), PreviousPrice = todouble(p[2]) +| where isnotnull(UnitPrice) and isnotnull(PreviousPrice) and UnitPrice != PreviousPrice +| summarize Changed = count(), Up = countif(UnitPrice > PreviousPrice), Down = countif(UnitPrice < PreviousPrice) by PriceType | order by PriceType asc // ## S07 Cheapest region for a VM size -// Question: Which regions have the lowest list price for each Dv5 VM size? +// Question: Which regions have the lowest current list price for each Dv5 VM size? // Source: PR 2595 "Compare List Prices Across Regions" // Expect: same // ### Rows SkuPrices_v1_5 -| where x_UnitPriceType == 'List' and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'List' and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' | summarize Price = min(UnitPrice) by x_SkuMeter, PricingRegionId | as hint.materialized = true T | lookup kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter @@ -168,7 +189,7 @@ SkuPrices_v1_5 | order by x_SkuMeter asc // ### Columns SkuPricesWide_v1_5 -| where ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' and isnotnull(ListUnitPrice) +| where isnull(SkuPriceEffectiveEnd) and ChargeCategory == 'Usage' and x_SkuPriceType == 'Consumption' and x_SkuMeterSubcategory == 'Dv5 Series' and x_SkuOfferId == 'MS-AZR-0017P' and isnotnull(ListUnitPrice) | summarize Price = min(ListUnitPrice) by x_SkuMeter, PricingRegionId | as hint.materialized = true T | lookup kind=inner (T | summarize MinPrice = min(Price), MaxPrice = max(Price), Regions = count() by x_SkuMeter) on x_SkuMeter @@ -177,7 +198,7 @@ SkuPricesWide_v1_5 | order by x_SkuMeter asc // ## S08 Verify billed unit prices -// Question: Do the list and contracted unit prices on last month's on-demand charges match the price list? +// Question: Do the list and contracted unit prices on September's on-demand charges match the prices in effect in September? // Source: PR 2595 "Compare Recorded Unit Prices with Published Rates"; dataset.md Relationships // Expect: same // ### Rows @@ -188,12 +209,13 @@ let costs = materialize(CostsWithSkuPriceIdv2 | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); let ids = costs | distinct SkuPriceIdv2; +// All charges are in September, so one "in effect on Sep 1" filter covers them +let prices = SkuPrices_v1_5 | where SkuPriceId in (ids) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)); costs -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'List' | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Contracted' | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency | extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) -| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit -| summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) +| summarize Charges = count(), Priced = countif(isnotnull(PriceList * PerUnit)), ListMatches = countif(abs(ListUnitPrice - PriceList * PerUnit) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted * PerUnit) < 1e-6) // ### Columns // Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; @@ -203,13 +225,32 @@ let costs = materialize(CostsWithSkuPriceIdv2 | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); let ids = costs | distinct SkuPriceIdv2; costs -| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter ( + SkuPricesWide_v1_5 + | where SkuPriceId in (ids) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)) + | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription + ) on SkuPriceIdv2, ContractId, PricingCurrency | extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) -| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit -| summarize Charges = count(), Priced = countif(isnotnull(PriceList)), ListMatches = countif(abs(ListUnitPrice - PriceList) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted) < 1e-6) +| summarize Charges = count(), Priced = countif(isnotnull(PriceList * PerUnit)), ListMatches = countif(abs(ListUnitPrice - PriceList * PerUnit) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted * PerUnit) < 1e-6) +// ### Monthly +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) + | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PricingUnit, ListUnitPrice, ContractedUnitPrice); +let ids = costs | distinct SkuPriceIdv2; +costs +| lookup kind=leftouter ( + Prices_final_v1_2 + | where SkuPriceIdv2 in (ids) and x_EffectivePeriodStart >= datetime(2026-09-01) and x_EffectivePeriodStart < datetime(2026-10-01) + | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription + ) on SkuPriceIdv2, x_BillingAccountId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize Charges = count(), Priced = countif(isnotnull(PriceList * PerUnit)), ListMatches = countif(abs(ListUnitPrice - PriceList * PerUnit) < 1e-6), ContractedMatches = countif(abs(ContractedUnitPrice - PriceContracted * PerUnit) < 1e-6) // ## S09 Savings plan what-if for on-demand usage -// Question: What would last month's on-demand VM usage cost at list, contracted, and 1-year savings plan rates? +// Question: What would September's on-demand VM usage cost at list, contracted, and 1-year savings plan rates? // Source: PR 2595 "Project the Effect of Moving Consumption to a Contracted Rate"; issue #2706 // Expect: same // ### Rows @@ -222,13 +263,13 @@ let costs = materialize(CostsWithSkuPriceIdv2 | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); let ids = costs | distinct SkuPriceIdv2; let spIds = costs | distinct SavingsPlanSkuPriceId; +let prices = SkuPrices_v1_5 | where (SkuPriceId in (ids) or SkuPriceId in (spIds)) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)); costs -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' and SkuPriceId in (spIds) | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = UnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'List' | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Contracted' | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Effective' | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = UnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency | extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) -| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit, PriceSavingsPlan = PriceSavingsPlan * PerUnit -| summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) +| summarize AtList = round(sum(PricingQuantity * PriceList * PerUnit), 2), AtContracted = round(sum(PricingQuantity * PriceContracted * PerUnit), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted) * PerUnit), 2), Billed = round(sum(EffectiveCost), 2) // ### Columns // Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; @@ -239,20 +280,36 @@ let costs = materialize(CostsWithSkuPriceIdv2 | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); let ids = costs | distinct SkuPriceIdv2; let spIds = costs | distinct SavingsPlanSkuPriceId; +let prices = SkuPricesWide_v1_5 | where (SkuPriceId in (ids) or SkuPriceId in (spIds)) and SkuPriceEffectiveStart <= datetime(2026-09-01) and (isnull(SkuPriceEffectiveEnd) or SkuPriceEffectiveEnd > datetime(2026-09-01)); +costs +| lookup kind=leftouter (prices | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (prices | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| summarize AtList = round(sum(PricingQuantity * PriceList * PerUnit), 2), AtContracted = round(sum(PricingQuantity * PriceContracted * PerUnit), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted) * PerUnit), 2), Billed = round(sum(EffectiveCost), 2) +// ### Monthly +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | where ChargePeriodStart >= datetime(2026-09-01) and ChargePeriodStart < datetime(2026-10-01) + | where ChargeCategory == 'Usage' and isempty(CommitmentDiscountCategory) and ServiceName == 'Virtual Machines' + // No column links an on-demand price to its savings plan price; the IDs differ only by prefix + | project SkuPriceIdv2, SavingsPlanSkuPriceId = strcat('SPU12', substring(SkuPriceIdv2, 3)), x_BillingAccountId, PricingCurrency, PricingUnit, PricingQuantity, EffectiveCost); +let ids = costs | distinct SkuPriceIdv2; +let spIds = costs | distinct SavingsPlanSkuPriceId; +let prices = Prices_final_v1_2 | where (SkuPriceIdv2 in (ids) or SkuPriceIdv2 in (spIds)) and x_EffectivePeriodStart >= datetime(2026-09-01) and x_EffectivePeriodStart < datetime(2026-10-01); costs -| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency -| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (spIds) | project SavingsPlanSkuPriceId = SkuPriceId, ContractId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice) on SavingsPlanSkuPriceId, ContractId, PricingCurrency +| lookup kind=leftouter (prices | where x_SkuPriceType != 'SavingsPlan' | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, x_BillingAccountId, PricingCurrency +| lookup kind=leftouter (prices | where x_SkuPriceType == 'SavingsPlan' | project SavingsPlanSkuPriceId = SkuPriceIdv2, x_BillingAccountId, PricingCurrency, PriceSavingsPlan = x_EffectiveUnitPrice) on SavingsPlanSkuPriceId, x_BillingAccountId, PricingCurrency | extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) -| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit, PriceSavingsPlan = PriceSavingsPlan * PerUnit -| summarize AtList = round(sum(PricingQuantity * PriceList), 2), AtContracted = round(sum(PricingQuantity * PriceContracted), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted)), 2), Billed = round(sum(EffectiveCost), 2) +| summarize AtList = round(sum(PricingQuantity * PriceList * PerUnit), 2), AtContracted = round(sum(PricingQuantity * PriceContracted * PerUnit), 2), AtSavingsPlan = round(sum(PricingQuantity * coalesce(PriceSavingsPlan, PriceContracted) * PerUnit), 2), Billed = round(sum(EffectiveCost), 2) // ## S10 All prices for one SKU -// Question: Show every price type for one VM size in one region side by side. +// Question: Show every current price type for one VM size in one region side by side. // Source: PR 2595 catalog discovery; FOCUS Cost and Usage ListUnitPrice/ContractedUnitPrice pairing // Expect: same // ### Rows SkuPrices_v1_5 -| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' +| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' and isnull(SkuPriceEffectiveEnd) | summarize ListUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'List'), BaseUnitPrice = take_anyif(UnitPrice, x_UnitPriceType == 'Base'), @@ -262,12 +319,12 @@ SkuPrices_v1_5 | order by SkuPriceId asc // ### Columns SkuPricesWide_v1_5 -| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' +| where x_SkuMeterId == '0404074d-2846-5fe5-bd12-2c8e858c3f30' and ChargeCategory == 'Usage' and isnull(SkuPriceEffectiveEnd) | project SkuPriceId, PricingCurrency, ListUnitPrice, BaseUnitPrice = x_BaseUnitPrice, ContractedUnitPrice, EffectiveUnitPrice = x_EffectiveUnitPrice | order by SkuPriceId asc // ## S11 Estimate a planned workload -// Question: What would 2 D2 v5, 4 D4 v5, and 1 E8 v5 VMs in US East cost for a month at list and contracted rates? +// Question: What would 2 D2 v5, 4 D4 v5, and 1 E8 v5 VMs in US East cost for a month at current list and contracted rates? // Source: PR 2595 "Estimate the Cost of a Planned Workload" // Expect: same // ### Rows @@ -277,7 +334,7 @@ let plan = datatable(SkuPriceId: string, Quantity: real) [ 'ODU_DZH318Z08M9X_007S_1 Compute Hour_MS-AZR-0017P', 2920, 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; SkuPrices_v1_5 -| where x_UnitPriceType in ('List', 'Contracted') +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType in ('List', 'Contracted') | lookup kind=inner plan on SkuPriceId | extend Cost = Quantity * UnitPrice / block(x_PricingUnitDescription) | summarize AtList = round(sumif(Cost, x_UnitPriceType == 'List'), 2), AtContracted = round(sumif(Cost, x_UnitPriceType == 'Contracted'), 2) @@ -288,53 +345,59 @@ let plan = datatable(SkuPriceId: string, Quantity: real) [ 'ODU_DZH318Z08M9X_007S_1 Compute Hour_MS-AZR-0017P', 2920, 'ODU_DZH318Z08M9J_003G_1 Compute Hour_MS-AZR-0017P', 730 ]; SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) | lookup kind=inner plan on SkuPriceId | summarize AtList = round(sum(Quantity * ListUnitPrice / block(x_PricingUnitDescription)), 2), AtContracted = round(sum(Quantity * ContractedUnitPrice / block(x_PricingUnitDescription)), 2) -// ## S12 Prices a billing account is eligible for -// Question: How many prices of each type can a billing account with no agreement on this price list use? +// ## S12 Public prices visible to each account +// Question: How many current public list prices can the agreement's billing account see, and how many can an account outside the agreement see? // Source: PR 2595 "Identify the Prices a Billing Account is Eligible For"; sku_price_eligibility_examples.md -// Expect: different (columns scope the whole row to the agreement, so the public list price is hidden from other accounts) +// Expect: different (columns scope the whole row, including the public list price, to the agreement, so other accounts see none) // ### Rows -let account = '/providers/microsoft.billing/billingaccounts/1234567'; +let accounts = datatable(Account: string, BillingAccountId: string) [ + 'Agreement account', '/providers/microsoft.billing/billingaccounts/8611537', + 'Other account', '/providers/microsoft.billing/billingaccounts/1234567' ]; SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) and x_UnitPriceType == 'List' // Evaluate each distinct eligibility object once instead of once per price -| summarize Prices = count() by PriceType = x_UnitPriceType, Eligibility = tostring(SkuPriceEligibility) -| extend e = parse_json(Eligibility) +| summarize Prices = count() by Eligibility = tostring(SkuPriceEligibility) +| extend e = parse_json(Eligibility), tmp = 1 +| join kind=inner (accounts | extend tmp = 1) on tmp | mv-apply rule = e.Inclusions on ( summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( - (rule.Operator == 'In' and set_has_element(rule.Values, account)) or - (rule.Operator == 'StartsWith' and account startswith tostring(rule.Values[0]))))) -| where tobool(e.IsGlobalScope) or Included > 0 -| summarize Prices = sum(Prices) by PriceType -| order by PriceType asc + (rule.Operator == 'In' and set_has_element(rule.Values, BillingAccountId)) or + (rule.Operator == 'StartsWith' and BillingAccountId startswith tostring(rule.Values[0]))))) +| summarize ListPrices = sumif(Prices, tobool(e.IsGlobalScope) or Included > 0) by Account +| order by Account asc // ### Columns -let account = '/providers/microsoft.billing/billingaccounts/1234567'; +let accounts = datatable(Account: string, BillingAccountId: string) [ + 'Agreement account', '/providers/microsoft.billing/billingaccounts/8611537', + 'Other account', '/providers/microsoft.billing/billingaccounts/1234567' ]; SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) and isnotnull(ListUnitPrice) // Evaluate each distinct eligibility object once instead of once per price -| summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) by Eligibility = tostring(SkuPriceEligibility) -| extend e = parse_json(Eligibility) +| summarize Prices = count() by Eligibility = tostring(SkuPriceEligibility) +| extend e = parse_json(Eligibility), tmp = 1 +| join kind=inner (accounts | extend tmp = 1) on tmp | mv-apply rule = e.Inclusions on ( summarize Included = countif(rule.Dimension == 'BillingAccountId' and ( - (rule.Operator == 'In' and set_has_element(rule.Values, account)) or - (rule.Operator == 'StartsWith' and account startswith tostring(rule.Values[0]))))) -| where tobool(e.IsGlobalScope) or Included > 0 -| summarize List = sum(List), Base = sum(Base), Contracted = sum(Contracted), Effective = sum(Effective) -| evaluate narrow() -| project PriceType = Column, Prices = tolong(Value) -| where Prices > 0 -| order by PriceType asc + (rule.Operator == 'In' and set_has_element(rule.Values, BillingAccountId)) or + (rule.Operator == 'StartsWith' and BillingAccountId startswith tostring(rule.Values[0]))))) +| summarize ListPrices = sumif(Prices, tobool(e.IsGlobalScope) or Included > 0) by Account +| order by Account asc // ## S13 Usage rates vs. purchase fees -// Question: How many prices of each type are usage rates vs. purchase fees? +// Question: How many current prices of each type are usage rates vs. purchase fees? // Source: PR 2595 "Separate Usage Rates from Purchase Fees and Credit Values" // Expect: same // ### Rows SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) | summarize Prices = count() by ChargeCategory, PriceType = x_UnitPriceType | order by ChargeCategory asc, PriceType asc // ### Columns SkuPricesWide_v1_5 +| where isnull(SkuPriceEffectiveEnd) | summarize List = countif(isnotnull(ListUnitPrice)), Base = countif(isnotnull(x_BaseUnitPrice)), Contracted = countif(isnotnull(ContractedUnitPrice)), Effective = countif(isnotnull(x_EffectiveUnitPrice)) by ChargeCategory | mv-expand kind = array p = bag_pack('List', List, 'Base', Base, 'Contracted', Contracted, 'Effective', Effective) | project ChargeCategory, PriceType = tostring(p[0]), Prices = tolong(p[1]) @@ -342,39 +405,75 @@ SkuPricesWide_v1_5 | order by ChargeCategory asc, PriceType asc // ## S14 Backfill missing prices on cost data -// Question: Which cost rows are missing a list or contracted unit price, and how much list and contracted cost does the price list fill in? +// Question: Fill missing list and contracted unit prices and costs on every cost row from the prices in effect that month. // Source: FinOps hubs Costs transform (price backfill); dataset.md Relationships (SkuPriceId + ContractId + effective dates) // Expect: same +// Check: | summarize Rows = count(), MissingList = countif(isnull(ListUnitPrice) or ListUnitPrice == 0), MissingContracted = countif(isnull(ContractedUnitPrice) or ContractedUnitPrice == 0), ListCost = round(sum(ListCost), 2), ContractedCost = round(sum(ContractedCost), 2) // ### Rows // Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; let costs = materialize(CostsWithSkuPriceIdv2 - | where isnotempty(SkuPriceIdv2) and (isnull(ListUnitPrice) or ListUnitPrice == 0 or isnull(ContractedUnitPrice) or ContractedUnitPrice == 0) - | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ChargePeriodStart, PricingQuantity, ListUnitPrice, ContractedUnitPrice); -let ids = costs | distinct SkuPriceIdv2; + | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), Month = startofmonth(ChargePeriodStart), + MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0); +let ids = costs | where isnotempty(SkuPriceIdv2) and (MissingList or MissingContracted) | distinct SkuPriceIdv2; +// Spread each price over the months it was in effect, so a charge matches its month exactly (no range join) +let prices = SkuPrices_v1_5 + | where SkuPriceId in (ids) and x_UnitPriceType in ('List', 'Contracted') + | extend Months = datetime_diff('month', coalesce(SkuPriceEffectiveEnd, datetime_add('month', 1, startofmonth(now()))), startofmonth(SkuPriceEffectiveStart)) + | mv-expand i = range(0, Months - 1, 1) to typeof(int) + | extend Month = datetime_add('month', i, startofmonth(SkuPriceEffectiveStart)); costs -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription, ListStart = SkuPriceEffectiveStart) on SkuPriceIdv2, PricingCurrency -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceContracted = UnitPrice, ContractedStart = SkuPriceEffectiveStart) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter (prices | where x_UnitPriceType == 'List' | project SkuPriceIdv2 = SkuPriceId, PricingCurrency, Month, PriceList = UnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, PricingCurrency, Month +| lookup kind=leftouter (prices | where x_UnitPriceType == 'Contracted' | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, Month, PriceContracted = UnitPrice) on SkuPriceIdv2, ContractId, PricingCurrency, Month | extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) -| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit -// Only use a price that was in effect when the charge happened -| extend PriceList = iff(ChargePeriodStart >= ListStart, PriceList, real(null)), PriceContracted = iff(ChargePeriodStart >= ContractedStart, PriceContracted, real(null)) -| extend MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0 -| summarize MissingList = countif(MissingList), FilledList = countif(MissingList and PriceList > 0), ListCostAdded = round(sumif(PricingQuantity * PriceList, MissingList), 2), - MissingContracted = countif(MissingContracted), FilledContracted = countif(MissingContracted and PriceContracted > 0), ContractedCostAdded = round(sumif(PricingQuantity * PriceContracted, MissingContracted), 2) +| extend + ListUnitPrice = iff(MissingList and PriceList * PerUnit > 0, PriceList * PerUnit, ListUnitPrice), + ContractedUnitPrice = iff(MissingContracted and PriceContracted * PerUnit > 0, PriceContracted * PerUnit, ContractedUnitPrice) +| extend + ListCost = iff(MissingList and ListUnitPrice > 0, PricingQuantity * ListUnitPrice, ListCost), + ContractedCost = iff(MissingContracted and ContractedUnitPrice > 0, PricingQuantity * ContractedUnitPrice, ContractedCost) +| project-away ContractId, Month, MissingList, MissingContracted, PriceList, PriceContracted, PriceUnit, PerUnit // ### Columns // Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; let costs = materialize(CostsWithSkuPriceIdv2 - | where isnotempty(SkuPriceIdv2) and (isnull(ListUnitPrice) or ListUnitPrice == 0 or isnull(ContractedUnitPrice) or ContractedUnitPrice == 0) - | project SkuPriceIdv2, ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), PricingCurrency, PricingUnit, ChargePeriodStart, PricingQuantity, ListUnitPrice, ContractedUnitPrice); -let ids = costs | distinct SkuPriceIdv2; + | extend ContractId = tolower(strcat('/providers/microsoft.billing/billingaccounts/', x_BillingAccountId)), Month = startofmonth(ChargePeriodStart), + MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0); +let ids = costs | where isnotempty(SkuPriceIdv2) and (MissingList or MissingContracted) | distinct SkuPriceIdv2; +// Spread each price over the months it was in effect, so a charge matches its month exactly (no range join) +let prices = SkuPricesWide_v1_5 + | where SkuPriceId in (ids) + | extend Months = datetime_diff('month', coalesce(SkuPriceEffectiveEnd, datetime_add('month', 1, startofmonth(now()))), startofmonth(SkuPriceEffectiveStart)) + | mv-expand i = range(0, Months - 1, 1) to typeof(int) + | extend Month = datetime_add('month', i, startofmonth(SkuPriceEffectiveStart)); +costs +| lookup kind=leftouter (prices | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, Month, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription) on SkuPriceIdv2, ContractId, PricingCurrency, Month +| extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) +| extend + ListUnitPrice = iff(MissingList and PriceList * PerUnit > 0, PriceList * PerUnit, ListUnitPrice), + ContractedUnitPrice = iff(MissingContracted and PriceContracted * PerUnit > 0, PriceContracted * PerUnit, ContractedUnitPrice) +| extend + ListCost = iff(MissingList and ListUnitPrice > 0, PricingQuantity * ListUnitPrice, ListCost), + ContractedCost = iff(MissingContracted and ContractedUnitPrice > 0, PricingQuantity * ContractedUnitPrice, ContractedCost) +| project-away ContractId, Month, MissingList, MissingContracted, PriceList, PriceContracted, PriceUnit, PerUnit +// ### Monthly +// Price sheet prices are per block (e.g., "100 Hours"); use them only when the unit matches the charge's PricingUnit +let block = (d: string, unit: string) { let u = extract(@'^[\d.]+\s*(.*)$', 1, d); let priceUnit = iff(isempty(u) or u startswith '/', strcat('Units', u), u); iff(tolower(trim_end('s', priceUnit)) == tolower(trim_end('s', unit)), coalesce(todouble(extract(@'^([\d.]+)', 1, d)), 1.0), real(null)) }; +let costs = materialize(CostsWithSkuPriceIdv2 + | extend Month = startofmonth(ChargePeriodStart), + MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0); +let ids = costs | where isnotempty(SkuPriceIdv2) and (MissingList or MissingContracted) | distinct SkuPriceIdv2; costs -| lookup kind=leftouter (SkuPricesWide_v1_5 | where SkuPriceId in (ids) | project SkuPriceIdv2 = SkuPriceId, ContractId, PricingCurrency, PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription, PriceStart = SkuPriceEffectiveStart) on SkuPriceIdv2, ContractId, PricingCurrency +| lookup kind=leftouter ( + Prices_final_v1_2 + | where SkuPriceIdv2 in (ids) + | project SkuPriceIdv2, x_BillingAccountId, PricingCurrency, Month = startofmonth(x_EffectivePeriodStart), PriceList = ListUnitPrice, PriceContracted = ContractedUnitPrice, PriceUnit = x_PricingUnitDescription + ) on SkuPriceIdv2, x_BillingAccountId, PricingCurrency, Month | extend PerUnit = 1.0 / block(PriceUnit, PricingUnit) -| extend PriceList = PriceList * PerUnit, PriceContracted = PriceContracted * PerUnit -// Only use a price that was in effect when the charge happened; the row has one start for all of its prices -| extend PriceList = iff(ChargePeriodStart >= PriceStart, PriceList, real(null)), PriceContracted = iff(ChargePeriodStart >= PriceStart, PriceContracted, real(null)) -| extend MissingList = isnull(ListUnitPrice) or ListUnitPrice == 0, MissingContracted = isnull(ContractedUnitPrice) or ContractedUnitPrice == 0 -| summarize MissingList = countif(MissingList), FilledList = countif(MissingList and PriceList > 0), ListCostAdded = round(sumif(PricingQuantity * PriceList, MissingList), 2), - MissingContracted = countif(MissingContracted), FilledContracted = countif(MissingContracted and PriceContracted > 0), ContractedCostAdded = round(sumif(PricingQuantity * PriceContracted, MissingContracted), 2) +| extend + ListUnitPrice = iff(MissingList and PriceList * PerUnit > 0, PriceList * PerUnit, ListUnitPrice), + ContractedUnitPrice = iff(MissingContracted and PriceContracted * PerUnit > 0, PriceContracted * PerUnit, ContractedUnitPrice) +| extend + ListCost = iff(MissingList and ListUnitPrice > 0, PricingQuantity * ListUnitPrice, ListCost), + ContractedCost = iff(MissingContracted and ContractedUnitPrice > 0, PricingQuantity * ContractedUnitPrice, ContractedCost) +| project-away Month, MissingList, MissingContracted, PriceList, PriceContracted, PriceUnit, PerUnit diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 index 1c88dfc88..57c67ee03 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPriceScenarios.ps1 @@ -14,7 +14,7 @@ param( [string] $Database = 'Ingestion', [string] $Context = 'fh-dev', [string] $Path = "$PSScriptRoot/SkuPrices_v1_5_scenarios.kql", - [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) "SkuPrices_v1_5_scenarios.results.json"), + [string] $OutFile = (Join-Path ([IO.Path]::GetTempPath()) 'SkuPrices_v1_5_scenarios.results.json'), [int] $Runs = 3, [string[]] $Only ) @@ -76,25 +76,38 @@ $scenarios = foreach ($block in ($text -split '(?m)^// ## ') | Select-Object -Sk $lines = $block -split "`n" $id, $name = $lines[0].Trim() -split ' ', 2 $meta = @{} - foreach ($l in $lines) { if ($l -match '^// (Question|Source|Expect): (.+)$') { $meta[$Matches[1]] = $Matches[2].Trim() } } - $rowsQuery = (($block -split '(?m)^// ### Rows\s*$')[1] -split '(?m)^// ### Columns\s*$')[0] - $colsQuery = ($block -split '(?m)^// ### Columns\s*$')[1] - [pscustomobject]@{ Id = $id; Name = $name.Trim(); Question = $meta.Question; Source = $meta.Source; Expect = $meta.Expect; RowsQuery = $rowsQuery.Trim(); ColumnsQuery = $colsQuery.Trim() } + foreach ($l in $lines) { if ($l -match '^// (Question|Source|Expect|Check): (.+)$') { $meta[$Matches[1]] = $Matches[2].Trim() } } + $q = @{} + foreach ($part in ($block -split '(?m)^(?=// ### (?:Rows|Columns|Monthly)\s*$)') | Select-Object -Skip 1) { + if ($part -match '^// ### (Rows|Columns|Monthly)') { $q[$Matches[1]] = ($part -replace '^// ### \w+\s*\n', '').Trim() } + } + [pscustomobject]@{ Id = $id; Name = $name.Trim(); Question = $meta.Question; Source = $meta.Source; Expect = $meta.Expect; Check = $meta.Check; Queries = $q } } if ($Only) { $scenarios = $scenarios | Where-Object Id -in $Only } +# Winner among sides: lowest value wins; within 15% (or a tiny absolute gap) of the runner-up is a tie +function Get-Winner([hashtable] $values, [double] $minGap = 0) { + $v = $values.GetEnumerator() | Where-Object { $null -ne $_.Value } | Sort-Object Value + if (@($v).Count -lt 2) { return 'n/a' } + $best = $v[0]; $next = $v[1]; $hi = [math]::Max([double]$best.Value, [double]$next.Value) + if ($hi -eq 0 -or [math]::Abs($next.Value - $best.Value) -le [math]::Max($minGap, 0.15 * $hi)) { 'Tie' } else { $best.Key } +} + $results = foreach ($s in $scenarios) { - $runsBySide = @{ Rows = @(); Columns = @() } - $err = @{} + $sideNames = @('Rows', 'Columns', 'Monthly' | Where-Object { $s.Queries[$_] }) + $runsBySide = @{}; $err = @{} + foreach ($side in $sideNames) { $runsBySide[$side] = @() } for ($i = 0; $i -lt $Runs; $i++) { - foreach ($side in $(if ($i % 2) { 'Columns', 'Rows' } else { 'Rows', 'Columns' })) { + $order = if ($i % 2) { [array]::Reverse(($o = @($sideNames))); $o } else { $sideNames } + foreach ($side in $order) { if ($err[$side]) { continue } - try { $runsBySide[$side] += Invoke-Kusto $s."${side}Query" } catch { $err[$side] = $_.Exception.Message } + $csl = $s.Queries[$side] + $(if ($s.Check) { "`n" + $s.Check } else { '' }) + try { $runsBySide[$side] += Invoke-Kusto $csl } catch { $err[$side] = $_.Exception.Message } } } - $sides = @{} - foreach ($side in 'Rows', 'Columns') { - $q = $s."${side}Query"; $shape = Get-Shape $q; $rs = $runsBySide[$side] + $sides = [ordered]@{} + foreach ($side in $sideNames) { + $shape = Get-Shape $s.Queries[$side]; $rs = $runsBySide[$side] $sides[$side] = if ($err[$side]) { [pscustomobject]@{ Error = $err[$side] } } else { [pscustomobject]@{ Ms = Get-Median ($rs.Ms); CpuMs = Get-Median ($rs.CpuMs); MemoryMB = Get-Median ($rs.MemoryMB); RowsScanned = Get-Median ($rs.RowsScanned) @@ -105,29 +118,29 @@ $results = foreach ($s in $scenarios) { } } } - $R = $sides.Rows; $C = $sides.Columns - $match = -not $R.Error -and -not $C.Error -and $R.Signature -eq $C.Signature + $ok = @($sides.Values | Where-Object { -not $_.Error }) + $hasError = $ok.Count -lt $sides.Count + $match = -not $hasError -and @($ok.Signature | Select-Object -Unique).Count -eq 1 $expectSame = $s.Expect -like 'same*' - $status = if ($R.Error -or $C.Error) { 'ERROR' } elseif ($match -eq $expectSame) { 'PASS' } else { 'FAIL' } - - # Winners: lower is better; within 15% (or tiny absolute gap) is a tie - function Win($r, $c, $minGap = 0) { if ($null -eq $r -or $null -eq $c) { return 'n/a' }; $hi = [math]::Max($r, $c); if ($hi -eq 0 -or [math]::Abs($r - $c) -le [math]::Max($minGap, 0.15 * $hi)) { 'Tie' } elseif ($r -lt $c) { 'Rows' } else { 'Columns' } } + $status = if ($hasError) { 'ERROR' } elseif ($match -eq $expectSame) { 'PASS' } else { 'FAIL' } $score = if ($status -ne 'ERROR') { + $m = { param($f) $h = @{}; foreach ($k in $sides.Keys) { $h[$k] = & $f $sides[$k] }; $h } [ordered]@{ - Simpler = Win ($R.Operators + 2 * $R.Joins + $R.Lets) ($C.Operators + 2 * $C.Joins + $C.Lets) 1 - Faster = Win $R.Ms $C.Ms 50 - Cpu = Win $R.CpuMs $C.CpuMs 50 - Memory = Win $R.MemoryMB $C.MemoryMB 5 - Scanned = Win $R.RowsScanned $C.RowsScanned - Portable = Win $R.XColumns $C.XColumns - Correct = if ($match) { 'Tie' } elseif ($s.Expect -match 'different') { 'Rows' } else { 'n/a' } + Simpler = Get-Winner (& $m { param($x) $x.Operators + 2 * $x.Joins + $x.Lets }) 1 + Faster = Get-Winner (& $m { param($x) $x.Ms }) 50 + Cpu = Get-Winner (& $m { param($x) $x.CpuMs }) 50 + Memory = Get-Winner (& $m { param($x) $x.MemoryMB }) 5 + Scanned = Get-Winner (& $m { param($x) $x.RowsScanned }) + Portable = Get-Winner (& $m { param($x) $x.XColumns }) + Correct = if ($match) { 'Tie' } elseif ($s.Expect -match 'different') { 'Rows' } else { 'n/a' } } } - Write-Host ("{0,-5} {1} {2,-42} rows {3,6}ms cpu {4,6}ms {5,6}MB {6,4}ops cols {7,6}ms cpu {8,6}ms {9,6}MB {10,4}ops match={11}" -f $status, $s.Id, $s.Name, $R.Ms, $R.CpuMs, $R.MemoryMB, $R.Operators, $C.Ms, $C.CpuMs, $C.MemoryMB, $C.Operators, $match) - if ($status -eq 'ERROR') { Write-Host " Rows: $($R.Error)`n Columns: $($C.Error)" } - [pscustomobject]@{ Id = $s.Id; Name = $s.Name; Question = $s.Question; Source = $s.Source; Expect = $s.Expect; Status = $status; Match = $match; Score = $score - RowsQuery = $s.RowsQuery; ColumnsQuery = $s.ColumnsQuery - Rows = $R | Select-Object * -ExcludeProperty Signature; Columns = $C | Select-Object * -ExcludeProperty Signature } + $line = ($sides.Keys | ForEach-Object { $x = $sides[$_]; "{0} {1}ms cpu {2}ms {3}MB {4}ops" -f $_, $x.Ms, $x.CpuMs, $x.MemoryMB, $x.Operators }) -join ' | ' + Write-Host ("{0,-5} {1} {2,-40} {3} match={4}" -f $status, $s.Id, $s.Name, $line, $match) + if ($status -eq 'ERROR') { $sides.Keys | ForEach-Object { if ($sides[$_].Error) { Write-Host " ${_}: $($sides[$_].Error)" } } } + $out = [ordered]@{ Id = $s.Id; Name = $s.Name; Question = $s.Question; Source = $s.Source; Expect = $s.Expect; Check = $s.Check; Status = $status; Match = $match; Score = $score; Sides = @($sideNames) } + foreach ($side in $sideNames) { $out["${side}Query"] = $s.Queries[$side]; $out[$side] = $sides[$side] | Select-Object * -ExcludeProperty Signature } + [pscustomobject]$out } $results | ConvertTo-Json -Depth 8 | Set-Content -Path $OutFile diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 index 1f3c1d42e..06b9ce0cb 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/Test-SkuPrices.ps1 @@ -46,13 +46,14 @@ sheet | count '@ } 'Source: duplicate rows on key (info)' = @{ Expect = $null; Csl = $sheet + 'sheet | summarize n = count() by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | where n > 1 | summarize Keys = count(), ExtraRows = sum(n - 1)' } - 'Row count: List vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ListUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'List' | summarize Diff = count() - s, Rows = count(), Source = s" } - 'Row count: Base vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_BaseUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Base' | summarize Diff = count() - s, Rows = count(), Source = s" } - 'Row count: Contracted vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ContractedUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | summarize Diff = count() - s, Rows = count(), Source = s" } - 'Row count: Effective vs source' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_EffectiveUnitPrice) | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' | summarize Diff = count() - s, Rows = count(), Source = s" } - 'UnitPrice + discount match the source sheet' = @{ Expect = 0; Csl = $sheet + @' + 'Current rows: List vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ListUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'List' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current rows: Base vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_BaseUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Base' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current rows: Contracted vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(ContractedUnitPrice) and x_SkuPriceType != 'SavingsPlan' | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current rows: Effective vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | where isnotnull(x_EffectiveUnitPrice) | summarize by BillingAccountId, SkuPriceIdv2, x_EffectivePeriodStart, PricingCurrency | count); SkuPrices_v1_5 | where x_UnitPriceType == 'Effective' and isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } + 'Current UnitPrice + discount match the latest sheet' = @{ Expect = 0; Csl = $sheet + @' let s = sheet | summarize take_any(ListUnitPrice, x_BaseUnitPrice, ContractedUnitPrice, x_EffectiveUnitPrice) by SkuPriceIdv2, PricingCurrency; SkuPrices_v1_5 +| where isnull(SkuPriceEffectiveEnd) | lookup kind=leftouter s on $left.x_SkuPriceIdv2 == $right.SkuPriceIdv2, PricingCurrency | extend Src = case(x_UnitPriceType == 'List', ListUnitPrice, x_UnitPriceType == 'Base', x_BaseUnitPrice, x_UnitPriceType == 'Contracted', ContractedUnitPrice, x_EffectiveUnitPrice) | where isnull(Src) or UnitPrice != Src or (x_UnitPriceType != 'List' and abs(x_UnitPriceDiscount - (ListUnitPrice - UnitPrice)) > 1e-9) @@ -78,22 +79,34 @@ SkuPrices_v1_5 'Tier max <= tier min' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | where isnotnull(QuantityTierMaximum) and QuantityTierMaximum <= QuantityTierMinimum | count' } 'SkuPriceId with >1 SkuId/PricingUnit/ChargeCategory' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize s = dcount(SkuId), u = dcount(PricingUnit), c = dcount(ChargeCategory) by SkuPriceId | where s > 1 or u > 1 or c > 1 | count' } 'Effective start distribution (info)' = @{ Expect = $null; Csl = "SkuPrices_v1_5 | summarize Rows = count() by Start = startofmonth(SkuPriceEffectiveStart), x_UnitPriceType | summarize Rows = sum(Rows), Types = make_set(x_UnitPriceType) by Start | order by Start asc" } - 'Price changed after its effective start (sample of 2000 Contracted)' = @{ Expect = 0; Csl = @' -let s = SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | sample 2000 | project SkuPriceIdv2 = x_SkuPriceIdv2, PricingCurrency, Start = SkuPriceEffectiveStart, Cur = UnitPrice; + 'Price changed inside its run, or run started late (sample of 2000 Contracted)' = @{ Expect = 0; Csl = @' +let s = SkuPrices_v1_5 | where x_UnitPriceType == 'Contracted' | sample 2000 | project SkuPriceIdv2 = x_SkuPriceIdv2, PricingCurrency, Start = startofmonth(SkuPriceEffectiveStart), End = coalesce(SkuPriceEffectiveEnd, datetime(2100-01-01)), Cur = UnitPrice; Prices_final_v1_2 | where x_SkuPriceType != 'SavingsPlan' -| lookup kind=inner s on SkuPriceIdv2, PricingCurrency -| summarize Changed = countif(x_EffectivePeriodStart >= Start and ContractedUnitPrice != Cur), PriorSame = countif(startofmonth(x_EffectivePeriodStart) == datetime_add('month', -1, Start) and ContractedUnitPrice == Cur) by SkuPriceIdv2, PricingCurrency +| extend M = startofmonth(x_IngestionTime) +| join kind=inner s on SkuPriceIdv2, PricingCurrency +| summarize Changed = countif(M >= Start and M < End and ContractedUnitPrice != Cur), PriorSame = countif(M == datetime_add('month', -1, Start) and ContractedUnitPrice == Cur) by SkuPriceIdv2, PricingCurrency, Start | where Changed > 0 or PriorSame > 0 | count '@ } + 'Rows per type match runs table' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType | join kind=fullouter (SkuPrices_v1_5_runs_table | where x_UnitPriceType != 'Wide' | summarize Runs = count() by x_UnitPriceType) on x_UnitPriceType | summarize Diff = sum(abs(coalesce(Rows, 0) - coalesce(Runs, 0)))" } + 'Wide: dupes on SkuPriceId + ContractId + currency + start' = @{ Expect = 0; Csl = 'SkuPricesWide_v1_5 | summarize n = count() by SkuPriceId, ContractId, PricingCurrency, SkuPriceEffectiveStart | where n > 1 | count' } + 'Wide: overlapping effective periods' = @{ Expect = 0; Csl = @' +SkuPricesWide_v1_5 +| project k = strcat(SkuPriceId, '|', ContractId, '|', PricingCurrency), SkuPriceEffectiveStart, SkuPriceEffectiveEnd +| sort by k asc, SkuPriceEffectiveStart asc +| where next(k) == k and (isnull(SkuPriceEffectiveEnd) or next(SkuPriceEffectiveStart) < SkuPriceEffectiveEnd) +| count +'@ } + 'Wide: rows match runs table' = @{ Expect = 0; Csl = "SkuPricesWide_v1_5 | count | extend Runs = toscalar(SkuPrices_v1_5_runs_table | where x_UnitPriceType == 'Wide' | count) | project Diff = Count - Runs, Rows = Count, Runs" } + 'Wide: current rows vs latest sheet' = @{ Expect = 0; Csl = $sheet + "let s = toscalar(sheet | summarize by BillingAccountId, SkuPriceIdv2, PricingCurrency | count); SkuPricesWide_v1_5 | where isnull(SkuPriceEffectiveEnd) | summarize Diff = count() - s, Rows = count(), Source = s" } 'Savings plan rows not Effective' = @{ Expect = 0; Csl = "SkuPrices_v1_5 | where x_SkuPriceType == 'SavingsPlan' and x_UnitPriceType != 'Effective' | count" } 'SkuPriceId with >1 tier minimum' = @{ Expect = 0; Csl = 'SkuPrices_v1_5 | summarize t = dcount(QuantityTierMinimum) by SkuPriceId | where t > 1 | count' } 'Price types by CommitmentDiscountCategory (info)' = @{ Expect = $null; Csl = "SkuPrices_v1_5 | summarize Rows = count() by x_UnitPriceType, CommitmentDiscountCategory, ChargeCategory | order by x_UnitPriceType asc, Rows desc" } 'Base/Contracted equal to List (info)' = @{ Expect = $null; Csl = @' SkuPrices_v1_5 -| where x_UnitPriceType in ('Base', 'Contracted') -| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' | project SkuPriceId, SkuPriceEffectiveStart, PricingCurrency, ListPrice = UnitPrice) on SkuPriceId, SkuPriceEffectiveStart, PricingCurrency +| where x_UnitPriceType in ('Base', 'Contracted') and isnull(SkuPriceEffectiveEnd) +| lookup kind=leftouter (SkuPrices_v1_5 | where x_UnitPriceType == 'List' and isnull(SkuPriceEffectiveEnd) | project SkuPriceId, PricingCurrency, ListPrice = UnitPrice) on SkuPriceId, PricingCurrency | summarize Rows = count(), SameAsList = countif(UnitPrice == ListPrice), NoList = countif(isnull(ListPrice)) by x_UnitPriceType '@ } } From ea5c55736c7d3d57a0021503d286a74a0446bbae Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Fri, 16 Oct 2026 15:11:18 -0700 Subject: [PATCH 6/8] fix(hubs): escape scenario ids in SkuPrices report (CodeQL DOM text as HTML) Co-Authored-By: Claude Sonnet 5.5 --- .../SkuPrices_v1_5/SkuPrices_v1_5_report.template.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html index 60e1c48de..595a76130 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html @@ -189,8 +189,8 @@

Scoreboard

return '

' + label + ' · differs

' + head + '' + body + '
'; } function sideHtml(k, q, s) { - return '
' + - '
' + k + ' ' + TABLE[k] + '' + + return '
' + + '
' + esc(k) + ' ' + esc(TABLE[k]) + '' + '
' + '
' + s.Operators + ' operators' + s.Joins + ' joins' + ms(s.Ms) + '' + ms(s.CpuMs) + ' CPU' + mb(s.MemoryMB) + '' + big(s.RowsScanned) + ' scanned
' + '
' + highlight(q) + '
'; @@ -198,7 +198,7 @@

Scoreboard

const board = document.querySelector('#scoreboard tbody'), foot = document.querySelector('#scoreboard tfoot'), main = document.getElementById('scenarios'); const keys = ['Simpler', 'Faster', 'Cpu', 'Memory', 'Scanned', 'Portable', 'Correct']; data.forEach(s => { - board.insertAdjacentHTML('beforeend', '' + s.Id + ' ' + esc(s.Name) + '' + + board.insertAdjacentHTML('beforeend', '' + esc(s.Id) + ' ' + esc(s.Name) + '' + cell(s, 'Simpler', cx) + cell(s, 'Faster', x => ms(x.Ms)) + cell(s, 'Cpu', x => ms(x.CpuMs)) + cell(s, 'Memory', x => mb(x.MemoryMB)) + cell(s, 'Scanned', x => big(x.RowsScanned)) + cell(s, 'Portable', x => x.XColumns) + cell(s, 'Correct') + ''); const same = (s.Expect || '').startsWith('same'); @@ -206,8 +206,8 @@

Scoreboard

const why = same ? '' : '

' + esc((s.Expect || '').replace(/^different\s*\(?/, '').replace(/\)$/, '')) + '

'; const check = s.Check ? '

Testing appends ' + esc(s.Check) + ' to every query so results can be compared.

' : ''; main.insertAdjacentHTML('beforeend', - '
' + - '' + s.Id + '' + esc(s.Name) + '' + p + '' + + '
' + + '' + esc(s.Id) + '' + esc(s.Name) + '' + p + '' + '

' + esc(s.Question) + '

Source: ' + esc(s.Source) + '

' + why + check + '
' + (s.Sides || ['Rows', 'Columns']).map(k => sideHtml(k, s[k + 'Query'], s[k])).join('') + '
' + resultHtml(s) + '
'); }); From 26afed7bd41362b7b2b09de8792aecd726cd1fe2 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Fri, 16 Oct 2026 15:22:07 -0700 Subject: [PATCH 7/8] fix(hubs): escape remaining interpolated values in SkuPrices report Co-Authored-By: Claude Sonnet 5.5 --- .../SkuPrices_v1_5/SkuPrices_v1_5_report.template.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html index 595a76130..2e83897e9 100644 --- a/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html +++ b/src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5/SkuPrices_v1_5_report.template.html @@ -159,9 +159,9 @@

Scoreboard

const big = v => v == null ? '—' : (v >= 1e6 ? (v / 1e6).toFixed(2) + 'M' : v >= 1e3 ? (v / 1e3).toFixed(0) + 'K' : v); const fmt = v => v === null || v === undefined ? '' : (typeof v === 'number' && !Number.isInteger(v) ? (+v.toFixed(6)).toLocaleString('en-US', { maximumFractionDigits: 6 }) : typeof v === 'number' ? v.toLocaleString('en-US') : v); const cx = s => s.Operators + 2 * s.Joins + s.Lets; - const pill = w => '' + w + ''; + const pill = w => '' + esc(w) + ''; const subs = (s, f) => (s.Sides || ['Rows', 'Columns']).map(k => f(s[k])).join(' · '); - function cell(s, key, f) { return '' + pill((s.Score || {})[key]) + (f ? '' + subs(s, f) + '' : '') + ''; } + function cell(s, key, f) { return '' + pill((s.Score || {})[key]) + (f ? '' + esc(subs(s, f)) + '' : '') + ''; } // One table when every layout returns the same answer; one row is shown as label/value pairs function resultHtml(s) { @@ -182,7 +182,7 @@

Scoreboard

const keyOf = r => String(r[0]); const keys = [...new Set(sides.flatMap(k => (s[k].Data || []).map(keyOf)))]; const valCols = first.Columns.slice(1); - let head = '' + esc(first.Columns[0]) + '' + valCols.map(c => sides.map(k => '' + esc(c) + ' · ' + k + '').join('')).join('') + ''; + let head = '' + esc(first.Columns[0]) + '' + valCols.map(c => sides.map(k => '' + esc(c) + ' · ' + esc(k) + '').join('')).join('') + ''; let body = keys.map(key => '' + esc(key) + '' + valCols.map((c, i) => sides.map(k => { const row = (s[k].Data || []).find(r => keyOf(r) === key); return '' + (row ? esc(fmt(row[i + 1])) : '—') + ''; }).join('')).join('') + '').join(''); @@ -192,7 +192,7 @@

Scoreboard

return '
' + '
' + esc(k) + ' ' + esc(TABLE[k]) + '' + '
' + - '
' + s.Operators + ' operators' + s.Joins + ' joins' + ms(s.Ms) + '' + ms(s.CpuMs) + ' CPU' + mb(s.MemoryMB) + '' + big(s.RowsScanned) + ' scanned
' + + '
' + esc(s.Operators) + ' operators' + esc(s.Joins) + ' joins' + esc(ms(s.Ms)) + '' + esc(ms(s.CpuMs)) + ' CPU' + esc(mb(s.MemoryMB)) + '' + esc(big(s.RowsScanned)) + ' scanned
' + '
' + highlight(q) + '
'; } const board = document.querySelector('#scoreboard tbody'), foot = document.querySelector('#scoreboard tfoot'), main = document.getElementById('scenarios'); From c6730964ed799853dbf6179f66fa963eaa447403 Mon Sep 17 00:00:00 2001 From: Michael Flanakin Date: Mon, 26 Oct 2026 00:07:37 -0700 Subject: [PATCH 8/8] docs(hubs): Add SkuPrices preview changelog and docs Co-Authored-By: Claude Opus 5.5 --- docs-mslearn/toolkit/changelog.md | 3 ++- docs-mslearn/toolkit/hubs/data-model.md | 11 ++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index 6ca4dd7c3..a45a464fd 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 09/11/2026 +ms.date: 10/26/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -29,6 +29,7 @@ The following section lists features and enhancements that are currently in deve - **Added** - Added VNet and private network modes, including opt-in NAT Gateway support for private mode; NAT Gateway incurs additional cost when enabled ([#2163](https://github.com/microsoft/finops-toolkit/pull/2163)). + - Added a draft FOCUS 1.5 SKU Price preview (`SkuPrices_v1_5` and `SkuPricesWide_v1_5`) with validation and scenario scripts under `Analytics/preview/SkuPrices_v1_5`. The preview is not deployed with FinOps hubs; run it manually against a hub ([#2339](https://github.com/microsoft/finops-toolkit/pull/2339)). - **Changed** - Clarified that the FinOps toolkit exclusively manages the FinOps hub virtual network and documented customer-managed private endpoints as the preferred private-access topology, with virtual network peering as a secondary option ([#2156](https://github.com/microsoft/finops-toolkit/issues/2156)). - Replaced redundant `tolower()` comparisons in hub KQL with case-insensitive operators (`has`, `=~`, `!~`) so the engine can use the term index instead of scanning every row ([#2213](https://github.com/microsoft/finops-toolkit/issues/2213)). diff --git a/docs-mslearn/toolkit/hubs/data-model.md b/docs-mslearn/toolkit/hubs/data-model.md index e06a19519..d8cd74438 100644 --- a/docs-mslearn/toolkit/hubs/data-model.md +++ b/docs-mslearn/toolkit/hubs/data-model.md @@ -3,7 +3,7 @@ title: FinOps hubs data model description: Learn about the tables and functions available in FinOps hubs to build your own queries, reports, and dashboards. author: flanakin ms.author: micflan -ms.date: 04/01/2026 +ms.date: 10/26/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -642,6 +642,15 @@ Columns in the **Prices** managed dataset include: | SkuMeter | String | New in FOCUS 1.2. Renamed from x_SkuMeterName. | | SkuPriceId | String | | | SkuPriceIdv2 | String | Source: Hubs add-on. | + +### FOCUS 1.5 SKU prices preview + +A draft of the FOCUS 1.5 SKU Price dataset is available in the `src/templates/finops-hub/modules/Microsoft.FinOpsHubs/Analytics/preview/SkuPrices_v1_5` folder of the [FinOps toolkit repository](https://github.com/microsoft/finops-toolkit). It isn't deployed with FinOps hubs and may change before release. The preview builds 2 tables from **Prices_final_v1_2** in the **Ingestion** database: + +- **SkuPrices_v1_5** has one row per SKU price and price type (`x_UnitPriceType` of List, Base, Contracted, or Effective) with effective start and end dates. +- **SkuPricesWide_v1_5** has the same prices with each price type in its own column. + +To try it, run `Build-SkuPrices.ps1` against your hub, then `Test-SkuPrices.ps1` to validate the results. See the README in that folder for mapping decisions and known gaps. | x_BaseUnitPrice | Real | Source: Microsoft. | | x_BillingAccountAgreement | String | Source: Hubs add-on. | | x_BillingAccountId | String | Source: Microsoft, MCA only. |