diff --git a/.github/workflows/base-installer-cd.yml b/.github/workflows/base-installer-cd.yml index d01a45034d..0d781c258d 100644 --- a/.github/workflows/base-installer-cd.yml +++ b/.github/workflows/base-installer-cd.yml @@ -161,7 +161,7 @@ jobs: id: build shell: powershell run: | - .\build.ps1 -BuildInstaller -Configuration Release -Verbosity detailed -BuildTests -MsBuildArgs @("/bl") *>&1 | Tee-Object -FilePath build.log + .\build.ps1 -BuildInstaller -Configuration Release -Verbosity detailed -BuildTests -MsBuildArgs @("/bl", "/p:FailOnRemovedSinceLastBase=${{ inputs.make_release == 'true' }}") *>&1 | Tee-Object -FilePath build.log - name: Run tests shell: powershell diff --git a/.github/workflows/patch-installer-cd.yml b/.github/workflows/patch-installer-cd.yml index 9541bc27fc..b72d7f327e 100644 --- a/.github/workflows/patch-installer-cd.yml +++ b/.github/workflows/patch-installer-cd.yml @@ -39,7 +39,7 @@ on: # on Jenkins-built bases. base_release: description: 'The github release for the base build artifacts (separate only for bootstrapping; should be removed after 9.3 is the stable)' - default: 'build-1452' # When updating this, update base_build_number and their fallbacks below, too. + default: 'build-1452' # Must match base_build_number and its fallbacks. base_build_number: description: 'The base build number' required: false @@ -61,6 +61,7 @@ jobs: CROWDIN_API_KEY: ${{ secrets.FLEX_CROWDIN_API }} LcmRootDir: ${{ github.workspace }}/Localizations/LCMRepo FILESTOSIGNLATER: ./signExternally + KEEP_INTERMEDIATE_MSI: true GH_TOKEN: ${{ github.token }} BASE_BUILD_NUMBER: ${{ inputs.base_build_number || '1452' }} name: Upload strings for l10n, Build, and run Tests @@ -258,6 +259,21 @@ jobs: $patchPath = $patch.FullName "patch_file=$patchPath" >> $env:GITHUB_OUTPUT + "patch_version=$(($patch.Name -split '_')[1])" >> $env:GITHUB_OUTPUT + "ledger_file=$($patchPath -replace '\.msp$', '_components.tsv')" >> $env:GITHUB_OUTPUT + "update_msi=$((Get-ChildItem PatchableInstaller/CreateUpdatePatch/Update -Filter '*.msi' | Select-Object -First 1).FullName)" >> $env:GITHUB_OUTPUT + "master_msi=$((Get-ChildItem PatchableInstaller/CreateUpdatePatch/Master -Filter '*.msi' | Select-Object -First 1).FullName)" >> $env:GITHUB_OUTPUT + + # Patch component identity must remain stable across the base and previous patch. + - name: Check patch keeps base and previous-patch components + shell: pwsh + run: | + .\scripts\Installer\Check-PatchComponentLedger.ps1 ` + -MasterMsi "${{ steps.find_patch.outputs.master_msi }}" ` + -UpdateMsi "${{ steps.find_patch.outputs.update_msi }}" ` + -BaseBuildNumber $env:BASE_BUILD_NUMBER ` + -PatchVersion "${{ steps.find_patch.outputs.patch_version }}" ` + -OutLedger "${{ steps.find_patch.outputs.ledger_file }}" - name: Sign Patch if: github.event_name != 'pull_request' @@ -287,9 +303,22 @@ jobs: if (-not (Test-Path $patchPath)) { throw "Patch file not found at $patchPath" } + $ledgerPath = "${{ steps.find_patch.outputs.ledger_file }}" + if (-not (Test-Path $ledgerPath)) { + throw "Ledger file not found at $ledgerPath" + } $patchFile = Split-Path $patchPath -Leaf $s3Key = "jobs/FieldWorks-Win-all-Release-Patch/$($env:FW_BUILD_NUMBER)/$patchFile" + $ledgerKey = "jobs/FieldWorks-Win-all-Release-Patch/$($env:FW_BUILD_NUMBER)/$(Split-Path $ledgerPath -Leaf)" + aws s3 cp $ledgerPath "s3://flex-updates/$ledgerKey" + if ($LASTEXITCODE -ne 0) { + throw "Ledger upload failed with exit code $LASTEXITCODE" + } + Write-Host "Uploaded to s3://flex-updates/$ledgerKey" aws s3 cp $patchPath "s3://flex-updates/$s3Key" + if ($LASTEXITCODE -ne 0) { + throw "Patch upload failed with exit code $LASTEXITCODE" + } Write-Host "Uploaded to s3://flex-updates/$s3Key" - name: Upload Build Logs diff --git a/Build/Installer.legacy.targets b/Build/Installer.legacy.targets index ce2a6acb03..12a45c4bdd 100644 --- a/Build/Installer.legacy.targets +++ b/Build/Installer.legacy.targets @@ -105,10 +105,8 @@ /> @@ -116,6 +114,7 @@ + + +Create an issue to remove the placeholder before the next base build, unless one already exists for the current base. +``` + +The `RemovedSinceLastBase` entry makes the build write a zero-byte stand-in at that path. +That file-backed component remains in the patch, so machines that already have the real file keep +working while the removal issue is completed. + +## Before publishing a base + +A scheduled base verification build warns while any `RemovedSinceLastBase` entries remain. +A base release build fails. Both messages list the stand-in paths. Remove each +`RemovedSinceLastBase` entry from `Build/Installer.legacy.targets`. CI builds start clean; +a dirty local build must also delete each corresponding zero-byte file from the build output. + +Complete the removal issue before creating the base. A new base establishes the file-backed component +set under MSI APPFOLDER that future patches must preserve. + +## Running the check locally + +```powershell +.\scripts\Installer\Check-PatchComponentLedger.ps1 -MasterMsi -UpdateMsi ` + -BaseBuildNumber 1452 -PatchVersion 9.3.12.2761 ` + -OutLedger out.tsv +``` diff --git a/FLExInstaller/AGENTS.md b/FLExInstaller/AGENTS.md index 91abd4a475..331add5fa3 100644 --- a/FLExInstaller/AGENTS.md +++ b/FLExInstaller/AGENTS.md @@ -13,10 +13,11 @@ Minimal installer guidance for agents. - Heat exclusions: **`PatchableInstallerHeatExclude.xml`** is copied to **`PatchableInstaller/BaseInstallerBuild/heat-exclude.xml`** before Heat (see **`Build/Installer.legacy.targets`** `CopyFilesToInstall`). - **`buildMsi.bat`** passes **`-fv`** to **`light.exe`** so **`MsiAssemblyName`** includes **fileVersion** (same intent as MSBuild **`SetMsiAssemblyNameFileVersion=true`**), which helps GAC servicing when **`AssemblyVersion`** is unchanged but the binary’s **file version** increases. - Newtonsoft.Json and similar authored components live in **`CustomComponents.wxi`** (overlays **`PatchableInstaller/Common`**), with definitions guarded by **``** so patch/update authoring omits them when only **`UPDATEBUILDDIR`** is set. Add matching **`ComponentRef`** entries in **`FLExInstaller/CustomFeatures.wxi`** inside the **same** **`...`** so patch builds do not emit dangling refs (**LGHT0094**). WiX 6 **`Framework.wxs`** uses the same pattern for **`Feature Complete`**. Do not use **`FeatureRef Id="Complete"`** from an include that appears before **`Framework.wxs`** defines `Complete` (Light **LGHT0095**). -- **Patch error `PYRO0305: The File '' was removed in the patch`:** a file present in the base/**Master** harvest is missing from the new **Update** harvest, and WiX 3 **`pyro.exe`** forbids removing files in a patch. Typical trigger: a code change stops emitting a file the base release still ships — e.g. reg-free COM manifests dropped by a "reduce COM usage" change (`ManagedLgIcuCollator.manifest`, `ManagedVwWindow.manifest`). - - **Stopgap fix (patch against the existing base):** add the dropped file to the **`RemovedSinceLastBase`** item list in the **`RescuePatching`** target of **`Build/Installer.legacy.targets`** (runs via `BuildProduct`). It writes a zero-byte placeholder into the build output (**`$(dir-outputBase)`**) so the file appears in both harvests and `pyro` treats it as *changed*, not *removed*. Mirror the existing entries (`ManagedVwWindow.manifest`, `SimpleRootSite.manifest`) and add **only** the basenames actually dropped — usually just the `*.manifest`, not a still-shipping `.dll`. - - **Permanent fix:** cut a new **Base** build so the file is absent from Master too, then delete the now-stale `RemovedSinceLastBase` entries (the target comment notes a base build should warn when these exist). - - **Do not** add it to **`PatchableInstallerHeatExclude.xml`** — that list is for artifacts that must never be harvested (build-output dedup / test-only files), not for reconciling files removed since the base. +- **Patch file-backed component removal:** the patch ledger check compares file-backed components under MSI APPFOLDER in the new **Update** MSI and **Master** MSI with the complete file-backed ledger from the immediately previous published MSP. The first published patch creates the initial ledger in S3; after a ledger-bearing patch exists, its matching ledger must be beside the immediately previous MSP. Ledgers are not stored in this repository. Later patch versions are ignored. + - A missing file-backed component from either source uses the same fix: add its output path, preserving the file's relative output path, to the **`RemovedSinceLastBase`** item list in the **`RescuePatching`** target of **`Build/Installer.legacy.targets`**. The target writes a zero-byte stand-in into **`$(dir-outputBase)`** so the file-backed component remains in the patch. + - Create an issue to remove the stand-in before the next base, unless one already exists for the current base. + - While any **`RemovedSinceLastBase`** entries remain, a scheduled base verification build warns and a base release build fails. Remove each entry before creating the base; a dirty local build must also delete the zero-byte file. +- **Do not** add a dropped file-backed component to **`PatchableInstallerHeatExclude.xml`**. That list is for artifacts that must never be harvested, not for preserving file-backed patch component identity. ## Constraints diff --git a/scripts/Installer/Check-PatchComponentLedger.ps1 b/scripts/Installer/Check-PatchComponentLedger.ps1 new file mode 100644 index 0000000000..4e1c9bf4d3 --- /dev/null +++ b/scripts/Installer/Check-PatchComponentLedger.ps1 @@ -0,0 +1,77 @@ +<# +.SYNOPSIS +Fails a patch build when the new Update MSI drops a file-backed component under MSI APPFOLDER +from the base or previous patch. + +.DESCRIPTION +Compares the new Update MSI with file-backed components under MSI APPFOLDER from the base +MSI and the complete file-backed ledger from the immediately previous published patch. +Writes the complete update-minus-base file-backed ledger for this patch. + +.PARAMETER MasterMsi +The base (Master) MSI rebuilt by the patch build. + +.PARAMETER UpdateMsi +The upgraded (Update) MSI the patch was diffed from. + +.PARAMETER PatchVersion +The new patch's product version, e.g. 9.3.12.2761. + +.PARAMETER OutLedger +Where to write this patch's file-backed component ledger. + +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$MasterMsi, + [Parameter(Mandatory = $true)][string]$UpdateMsi, + [Parameter(Mandatory = $true)][string]$BaseBuildNumber, + [Parameter(Mandatory = $true)][string]$PatchVersion, + [Parameter(Mandatory = $true)][string]$OutLedger +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'PatchComponentLedger.psm1') -Force + +$master = Get-MsiComponents -MsiPath $MasterMsi +$update = Get-MsiComponents -MsiPath $UpdateMsi + +$publishedPatchKeys = @(Get-PublishedPatchKeys -BaseBuildNumber $BaseBuildNumber -Wildcard '*.msp') +$publishedLedgerKeys = @(Get-PublishedPatchKeys -BaseBuildNumber $BaseBuildNumber -Wildcard '*_components.tsv') +$previous = Select-PreviousPublishedPatch ` + -PatchKeys $publishedPatchKeys ` + -LedgerKeys $publishedLedgerKeys ` + -BaseBuildNumber $BaseBuildNumber ` + -PatchVersion $PatchVersion + +$ledgerFiles = New-Object System.Collections.Generic.List[string] +if ($previous.LedgerKey) { + $downloads = Join-Path ([IO.Path]::GetTempPath()) "fw-patch-ledgers-b$BaseBuildNumber" + New-Item -ItemType Directory -Force -Path $downloads | Out-Null + $ledgerFiles.Add((Save-PublishedFile -Key $previous.LedgerKey -Directory $downloads)) +} + +$previousLedger = Read-ComponentLedger -LedgerFiles $ledgerFiles.ToArray() +$required = @{} +foreach ($entry in $master.Values) { $required[$entry.ComponentId] = $entry } +foreach ($entry in $previousLedger.Values) { + if (-not $required.ContainsKey($entry.ComponentId)) { $required[$entry.ComponentId] = $entry } +} +$dropped = @(Get-MissingComponents -Required $required -Available $update) +$newLedgerEntries = @(Get-UpdateMinusBaseLedgerEntries -Master $master -Update $update) +Write-ComponentLedger -Path $OutLedger -Entries $newLedgerEntries -Heading "Complete update-minus-base file-backed ledger under MSI APPFOLDER for patch $PatchVersion on base $BaseBuildNumber" +Write-Output "Patch $PatchVersion ledger contains $($newLedgerEntries.Count) update-minus-base file-backed components under MSI APPFOLDER; ledger written to $OutLedger" +Write-Output "Checking against $($required.Count) required file-backed components under MSI APPFOLDER" + +if ($dropped.Count -eq 0) { + Write-Output '[OK] The patch keeps every required file-backed component under MSI APPFOLDER.' + exit 0 +} + +$message = Format-DroppedComponentMessage -PatchVersion $PatchVersion -BaseBuildNumber $BaseBuildNumber -Dropped $dropped +Write-Output $message +if ($env:GITHUB_ACTIONS -eq 'true') { + Write-Output ('::error title=Patch file-backed component removal::' + ($message -replace "`r?`n", ' ')) +} +exit 1 diff --git a/scripts/Installer/PatchComponentLedger.Tests.ps1 b/scripts/Installer/PatchComponentLedger.Tests.ps1 new file mode 100644 index 0000000000..0a119fcfd1 --- /dev/null +++ b/scripts/Installer/PatchComponentLedger.Tests.ps1 @@ -0,0 +1,155 @@ +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +Import-Module (Join-Path $PSScriptRoot 'PatchComponentLedger.psm1') -Force + +$script:AssertionCount = 0 +$script:TestFiles = New-Object System.Collections.Generic.List[string] + +function Assert-True { + param([bool]$Condition, [string]$Message) + $script:AssertionCount++ + if (-not $Condition) { throw $Message } +} + +function Assert-Equal { + param($Actual, $Expected, [string]$Message) + $script:AssertionCount++ + if ($Actual -ne $Expected) { + throw "$Message Expected: $Expected Actual: $Actual" + } +} + +function Assert-Contains { + param([string]$Actual, [string]$Expected, [string]$Message) + $script:AssertionCount++ + if (-not $Actual.Contains($Expected)) { + throw "$Message Expected to find: $Expected Actual: $Actual" + } +} + +function Assert-Throws { + param([scriptblock]$Action, [string]$ExpectedText) + $script:AssertionCount++ + try { + & $Action + } + catch { + if ($_.Exception.Message.Contains($ExpectedText)) { return } + throw "Expected an error containing '$ExpectedText', got '$($_.Exception.Message)'." + } + throw "Expected an error containing '$ExpectedText'." +} + +function New-TestLedgerFile { + param([string[]]$Lines) + $path = [IO.Path]::GetTempFileName() + Set-Content -LiteralPath $path -Value $Lines -Encoding Ascii + $script:TestFiles.Add($path) + return $path +} + +try { + $patch2758 = 'jobs/FieldWorks-Win-all-Release-Patch/2758/FieldWorks_9.3.12.2758_b1452_x64.msp' + $patch2760 = 'jobs/FieldWorks-Win-all-Release-Patch/2760/FieldWorks_9.3.12.2760_b1452_x64.msp' + $patchOtherBase = 'jobs/FieldWorks-Win-all-Release-Patch/2759/FieldWorks_9.3.12.2759_b1453_x64.msp' + $ledger2760 = $patch2760 -replace '\.msp$', '_components.tsv' + $selection = Select-PreviousPublishedPatch -PatchKeys @($patch2758, $patchOtherBase, $patch2760) -LedgerKeys @($ledger2760) -BaseBuildNumber '1452' -PatchVersion '9.3.12.2761' + Assert-Equal "$($selection.PatchKey)|$($selection.LedgerKey)" "$patch2760|$ledger2760" 'The selector must choose the latest earlier patch and ledger on the requested base.' + + $unmatchedLedger = $patch2760 -replace '\.msp$', '_components.tsv' + $bootstrap = Select-PreviousPublishedPatch -PatchKeys @($patch2758) -LedgerKeys @($unmatchedLedger) -BaseBuildNumber '1452' -PatchVersion '9.3.12.2761' + Assert-Equal "$($bootstrap.PatchKey)|$($bootstrap.LedgerKey)" "$patch2758|" 'A ledger without its corresponding published patch must not be returned.' + + Assert-Throws { + Select-PreviousPublishedPatch -PatchKeys @($patch2758, $patch2760) -LedgerKeys @($patch2758 -replace '\.msp$', '_components.tsv') -BaseBuildNumber '1452' -PatchVersion '9.3.12.2761' + } 'Bootstrap has ended' + + $master = @{ + '{33333333-3333-3333-3333-333333333333}' = [pscustomobject]@{ + ComponentId = '{33333333-3333-3333-3333-333333333333}' + Component = 'cmpBase' + File = 'Base.dll' + Feature = 'Complete' + } + } + Assert-Equal (Get-MsiDirectoryName -DefaultDir 'short|Long Name:SourceDir') 'Long Name' 'Directory parsing must use the target long name before the source portion.' + Assert-Equal (Get-MsiDirectoryName -DefaultDir 'TargetDir:SourceDir') 'TargetDir' 'Directory parsing must retain a target name without a short name.' + Assert-Equal (Get-MsiDirectoryName -DefaultDir '.') '' 'Dot directories must be transparent.' + $directories = @{ + APPFOLDER = [pscustomobject]@{ Name = 'App'; Parent = '' } + Help = [pscustomobject]@{ Name = 'Help'; Parent = 'APPFOLDER' } + Nested = [pscustomobject]@{ Name = 'Nested'; Parent = 'Help' } + Dot = [pscustomobject]@{ Name = ''; Parent = 'Help' } + Outside = [pscustomobject]@{ Name = 'Outside'; Parent = '' } + } + Assert-Equal (Get-RelativeMsiFilePath -Directories $directories -DirectoryId 'APPFOLDER' -FileName 'Root.dll') 'Root.dll' 'Files directly under APPFOLDER must keep their basename.' + Assert-Equal (Get-RelativeMsiFilePath -Directories $directories -DirectoryId 'Nested' -FileName 'Nested.dll') 'Help/Nested/Nested.dll' 'Nested files must keep their path below APPFOLDER.' + Assert-Equal (Get-RelativeMsiFilePath -Directories $directories -DirectoryId 'Dot' -FileName 'Dot.dll') 'Help/Dot.dll' 'Dot directories must continue to their parent.' + Assert-Equal (Get-RelativeMsiFilePath -Directories $directories -DirectoryId 'Outside' -FileName 'Outside.dll') $null 'Files outside APPFOLDER must be excluded.' + + $update = @{ + '{33333333-3333-3333-3333-333333333333}' = $master['{33333333-3333-3333-3333-333333333333}'] + '{44444444-4444-4444-4444-444444444444}' = [pscustomobject]@{ + ComponentId = '{44444444-4444-4444-4444-444444444444}' + Component = 'cmpNew' + File = 'New.dll' + Feature = 'Complete' + } + '{55555555-5555-5555-5555-555555555555}' = [pscustomobject]@{ + ComponentId = '{55555555-5555-5555-5555-555555555555}' + Component = 'cmpOther' + File = 'Other.dll' + Feature = 'Complete' + } + } + $newLedgerEntries = @(Get-UpdateMinusBaseLedgerEntries -Master $master -Update $update) + Assert-Equal (($newLedgerEntries | Sort-Object File | ForEach-Object File) -join ',') 'New.dll,Other.dll' 'The generated ledger must contain every current update-minus-base component.' + $ledgerPath = New-TestLedgerFile + Write-ComponentLedger -Path $ledgerPath -Entries $newLedgerEntries -Heading 'test' + $writtenLines = Get-Content -LiteralPath $ledgerPath + Assert-Equal $writtenLines[1] ('# ComponentId' + [char]9 + 'Component' + [char]9 + 'File' + [char]9 + 'Feature') 'Ledgers must contain only the four component fields.' + $roundTrip = Read-ComponentLedger -LedgerFiles $ledgerPath + Assert-Equal (($roundTrip.Values | Sort-Object File | ForEach-Object File) -join ',') 'New.dll,Other.dll' 'The complete generated ledger must be readable.' + + $previous = [pscustomobject]@{ + ComponentId = '{66666666-6666-6666-6666-666666666666}' + Component = 'cmpPrevious' + File = 'Previous.dll' + Feature = 'Complete' + } + $required = @{} + foreach ($component in $master.Values) { $required[$component.ComponentId] = $component } + $required[$previous.ComponentId] = $previous + $missing = @(Get-MissingComponents -Required $required -Available $update) + Assert-Equal (($missing | Sort-Object File | ForEach-Object File) -join ',') 'Previous.dll' 'The comparison must detect the component missing from the previous-patch requirements.' + + $message = Format-DroppedComponentMessage -PatchVersion '9.3.12.2761' -BaseBuildNumber '1452' -Dropped @($previous) + Assert-Contains $message 'preserve the relative output path when one exists' 'The remediation must describe relative paths without making a universal exact-path claim.' + Assert-Contains $message '' 'The remediation must provide an actionable stand-in example.' + + $targetText = Get-Content -Raw (Join-Path $PSScriptRoot '..\..\Build\Installer.legacy.targets') + $releaseFlag = '$' + '(FailOnRemovedSinceLastBase)' + Assert-True ($targetText.Contains(' + param( + [Parameter(Mandatory = $true)][AllowEmptyString()][string]$DefaultDir + ) + $target = ($DefaultDir -split ':', 2)[0] + if ([string]::IsNullOrEmpty($target) -or $target -eq '.') { return '' } + return ($target -split '\|', 2)[-1] +} + +function Get-RelativeMsiFilePath { + <# + .SYNOPSIS + Resolves a file path relative to the MSI APPFOLDER directory. + #> + param( + [Parameter(Mandatory = $true)][hashtable]$Directories, + [Parameter(Mandatory = $true)][string]$DirectoryId, + [Parameter(Mandatory = $true)][AllowEmptyString()][string]$FileName + ) + if ([string]::IsNullOrWhiteSpace($FileName)) { return $null } + if ($DirectoryId -ieq 'APPFOLDER') { return $FileName } + + $segments = New-Object System.Collections.Generic.List[string] + $visited = @{} + $current = $DirectoryId + while ($current -and $current -ine 'APPFOLDER') { + if ($visited.ContainsKey($current) -or -not $Directories.ContainsKey($current)) { return $null } + $visited[$current] = $true + $directory = $Directories[$current] + if (-not [string]::IsNullOrWhiteSpace($directory.Name)) { $segments.Insert(0, $directory.Name) } + $current = $directory.Parent + } + if ($current -ine 'APPFOLDER') { return $null } + $segments.Add($FileName) + return ($segments -join '/') +} + +function Get-MsiComponents { + <# + .SYNOPSIS + Returns file-backed components under MSI APPFOLDER keyed by ComponentId, each + with their relative path and feature. + #> + param([Parameter(Mandatory = $true)][string]$MsiPath) + + $installer = New-Object -ComObject WindowsInstaller.Installer + # 0 = msiOpenDatabaseModeReadOnly + $db = $installer.GetType().InvokeMember('OpenDatabase', 'InvokeMethod', $null, $installer, @((Resolve-Path -LiteralPath $MsiPath).Path, 0)) + + $directories = @{} + foreach ($row in (Invoke-MsiQuery -Database $db -Sql 'SELECT `Directory`,`Directory_Parent`,`DefaultDir` FROM `Directory`' -Columns 3)) { + $name = Get-MsiDirectoryName -DefaultDir ([string]$row[2]) + $directories[$row[0]] = [pscustomobject]@{ Name = $name; Parent = $row[1] } + } + + $fileNames = @{} + foreach ($row in (Invoke-MsiQuery -Database $db -Sql 'SELECT `File`,`FileName` FROM `File`' -Columns 2)) { + # FileName is "SHORT|Long name" when a short name exists. + $fileNames[$row[0]] = ($row[1] -split '\|')[-1] + } + $features = @{} + foreach ($row in (Invoke-MsiQuery -Database $db -Sql 'SELECT `Feature_`,`Component_` FROM `FeatureComponents`' -Columns 2)) { + $features[$row[1]] = $row[0] + } + $components = @{} + foreach ($row in (Invoke-MsiQuery -Database $db -Sql 'SELECT `Component`,`ComponentId`,`Directory_`,`KeyPath` FROM `Component`' -Columns 4)) { + if ([string]::IsNullOrEmpty($row[1]) -or [string]::IsNullOrEmpty($row[2]) -or -not $fileNames.ContainsKey($row[3])) { continue } + $file = Get-RelativeMsiFilePath -Directories $directories -DirectoryId $row[2] -FileName $fileNames[$row[3]] + if ($null -eq $file) { continue } + $feature = '' + if ($features.ContainsKey($row[0])) { $feature = $features[$row[0]] } + $components[$row[1].ToUpperInvariant()] = [pscustomobject]@{ + ComponentId = $row[1].ToUpperInvariant() + Component = $row[0] + File = $file + Feature = $feature + } + } + [System.Runtime.InteropServices.Marshal]::ReleaseComObject($db) | Out-Null + return $components +} + +function Read-ComponentLedger { + <# + .SYNOPSIS + Reads a ledger of file-backed components under MSI APPFOLDER keyed by ComponentId. + #> + param([Parameter(Mandatory = $true)][AllowEmptyCollection()][string[]]$LedgerFiles) + + $ledger = @{} + foreach ($file in $LedgerFiles) { + if (-not (Test-Path -LiteralPath $file -PathType Leaf)) { + throw "Component ledger file not found: $file" + } + $lineNumber = 0 + foreach ($line in (Get-Content -LiteralPath $file)) { + $lineNumber++ + if ($line -match '^\s*(#|$)') { continue } + $cells = [regex]::Split($line, [char]9) + if ($cells.Count -ne $script:LedgerColumns.Count -or @($cells | Where-Object { [string]::IsNullOrWhiteSpace($_) }).Count -gt 0) { + throw "Malformed component ledger row in '$file' at line $lineNumber. Expected four non-empty tab-separated fields." + } + $id = $cells[0].ToUpperInvariant() + if ($ledger.ContainsKey($id)) { + throw "Duplicate component ID '$id' in component ledger '$file' at line $lineNumber." + } + $ledger[$id] = [pscustomobject]@{ + ComponentId = $id + Component = $cells[1] + File = $cells[2] + Feature = $cells[3] + } + } + } + return $ledger +} + +function Write-ComponentLedger { + <# + .SYNOPSIS + Writes file-backed components under MSI APPFOLDER to a tab-separated ledger. + #> + param( + [Parameter(Mandatory = $true)][string]$Path, + [Parameter(Mandatory = $true)][object[]]$Entries, + [string]$Heading + ) + $lines = New-Object System.Collections.Generic.List[string] + if ($Heading) { $lines.Add("# $Heading") } + $lines.Add('# ' + ($script:LedgerColumns -join "`t")) + foreach ($entry in ($Entries | Sort-Object File, ComponentId)) { + $lines.Add((($script:LedgerColumns | ForEach-Object { $entry.$_ }) -join "`t")) + } + # ASCII keeps the file byte-identical under Windows PowerShell 5.1 and PowerShell 7. + Set-Content -LiteralPath $Path -Value $lines -Encoding Ascii +} + +function Get-UpdateBucketKeys { + <# + .SYNOPSIS + Lists the update-bucket keys under a prefix whose names match a wildcard. + #> + param( + [Parameter(Mandatory = $true)][string]$Prefix, + [Parameter(Mandatory = $true)][string]$Like + ) + $keys = New-Object System.Collections.Generic.List[string] + $token = $null + do { + $url = "$script:UpdateBucket/?list-type=2&prefix=$Prefix" + if ($token) { $url += '&continuation-token=' + [uri]::EscapeDataString($token) } + [xml]$page = (Invoke-WebRequest -Uri $url -UseBasicParsing).Content + foreach ($item in @($page.ListBucketResult.Contents)) { + if ($null -ne $item -and $item.Key -like $Like) { $keys.Add($item.Key) } + } + $token = $null + if ($page.ListBucketResult.IsTruncated -eq 'true') { $token = $page.ListBucketResult.NextContinuationToken } + } while ($token) + return , $keys +} + +function Get-PublishedPatchKeys { + <# + .SYNOPSIS + Lists the patch-job keys published for one base whose names end with a wildcard. + #> + param( + [Parameter(Mandatory = $true)][string]$BaseBuildNumber, + [Parameter(Mandatory = $true)][string]$Wildcard + ) + return , (Get-UpdateBucketKeys -Prefix $script:PatchPrefix -Like "*_b${BaseBuildNumber}_$Wildcard") +} + +function Save-PublishedFile { + <# + .SYNOPSIS + Downloads one published file to a local directory. + #> + param( + [Parameter(Mandatory = $true)][string]$Key, + [Parameter(Mandatory = $true)][string]$Directory + ) + $target = Join-Path $Directory (Split-Path $Key -Leaf) + Invoke-WebRequest -Uri "$script:UpdateBucket/$Key" -OutFile $target -UseBasicParsing + return $target +} + +function Get-PatchVersionFromKey { + <# + .SYNOPSIS + Parses a patch version from a published object key. + #> + param([Parameter(Mandatory = $true)][string]$Key) + # FieldWorks__b_. + return [version]((Split-Path $Key -Leaf) -split '_')[1] +} + +function Select-PreviousPublishedPatch { + <# + .SYNOPSIS + Selects the previous patch and matching ledger for a base and version. + #> + param( + [Parameter(Mandatory = $true)][string[]]$PatchKeys, + [string[]]$LedgerKeys = @(), + [Parameter(Mandatory = $true)][string]$BaseBuildNumber, + [Parameter(Mandatory = $true)][string]$PatchVersion + ) + $baseMarker = "_b${BaseBuildNumber}_" + $targetVersion = [version]$PatchVersion + $eligiblePatches = New-Object System.Collections.Generic.List[object] + foreach ($key in $PatchKeys) { + if (-not $key.Contains($baseMarker)) { continue } + try { $version = Get-PatchVersionFromKey -Key $key } + catch { continue } + if ($version -lt $targetVersion) { + $eligiblePatches.Add([pscustomobject]@{ Key = $key; Version = $version }) + } + } + if ($eligiblePatches.Count -eq 0) { + return [pscustomobject]@{ + PatchKey = $null + LedgerKey = $null + } + } + $orderedPatches = @($eligiblePatches | Sort-Object Version) + $previous = $orderedPatches[$orderedPatches.Count - 1] + $expectedLedgerKey = $previous.Key -replace '\.msp$', '_components.tsv' + $matchingLedger = @($LedgerKeys | Where-Object { $_ -eq $expectedLedgerKey }) + if ($matchingLedger.Count -gt 0) { + $ledgerKey = $matchingLedger[0] + } + else { + # No ledger beside the previous patch is expected only before the first ledger-bearing + # patch. + foreach ($candidate in $orderedPatches) { + if ($candidate.Version -ge $previous.Version) { continue } + $candidateLedger = $candidate.Key -replace '\.msp$', '_components.tsv' + if (@($LedgerKeys | Where-Object { $_ -eq $candidateLedger }).Count -gt 0) { + throw "Bootstrap has ended: published patch $($previous.Key) has no matching ledger $expectedLedgerKey. Build a new base, or reconstruct that ledger by applying the published patch to its base MSI and publish it beside the patch, before building another patch." + } + } + $ledgerKey = $null + } + return [pscustomobject]@{ + PatchKey = $previous.Key + LedgerKey = $ledgerKey + } +} + +function Format-RemovedComponentRemediation { + <# + .SYNOPSIS + Formats remediation for file-backed components under MSI APPFOLDER missing from a patch. + #> + param([Parameter(Mandatory = $true)][object[]]$Dropped) + $lines = New-Object System.Collections.Generic.List[string] + $lines.Add('Remediation:') + $lines.Add('Add the output path for each missing file to RemovedSinceLastBase in Build/Installer.legacy.targets; preserve the relative output path when one exists:') + foreach ($entry in ($Dropped | Sort-Object File, ComponentId)) { + $path = '$(dir-outputBase)/' + $entry.File + $lines.Add((' ')) + } + $lines.Add('Create an issue to remove the placeholder before the next base build, unless one already exists for the current base.') + return ($lines -join [Environment]::NewLine) +} + +function Get-UpdateMinusBaseLedgerEntries { + <# + .SYNOPSIS + Returns update file-backed components under MSI APPFOLDER absent from the base MSI. + #> + param( + [Parameter(Mandatory = $true)][hashtable]$Master, + [Parameter(Mandatory = $true)][hashtable]$Update + ) + $entries = New-Object System.Collections.Generic.List[object] + foreach ($id in $Update.Keys) { + if ($Master.ContainsKey($id)) { continue } + $entries.Add($Update[$id]) + } + return $entries.ToArray() +} + +function Get-MissingComponents { + <# + .SYNOPSIS + Finds required file-backed components under MSI APPFOLDER absent from the + available set. + #> + param( + [Parameter(Mandatory = $true)][hashtable]$Required, + [Parameter(Mandatory = $true)][hashtable]$Available + ) + $missing = New-Object System.Collections.Generic.List[object] + foreach ($id in $Required.Keys) { + if (-not $Available.ContainsKey($id)) { $missing.Add($Required[$id]) } + } + return ($missing | Sort-Object File, ComponentId) +} + +function Format-DroppedComponentMessage { + <# + .SYNOPSIS + Formats a diagnostic for file-backed components under MSI APPFOLDER + dropped by a patch. + #> + param( + [Parameter(Mandatory = $true)][string]$PatchVersion, + [Parameter(Mandatory = $true)][string]$BaseBuildNumber, + [Parameter(Mandatory = $true)][object[]]$Dropped + ) + $lines = New-Object System.Collections.Generic.List[string] + foreach ($entry in $Dropped) { + $lines.Add("Patch $PatchVersion drops file-backed component $($entry.ComponentId)") + $lines.Add(" file: $($entry.File) (feature $($entry.Feature))") + $lines.Add(" base: $BaseBuildNumber") + } + $lines.Add((Format-RemovedComponentRemediation -Dropped $Dropped)) + return ($lines -join [Environment]::NewLine) +} + +Export-ModuleMember -Function Get-MsiDirectoryName, Get-RelativeMsiFilePath, Get-MsiComponents, Read-ComponentLedger, Write-ComponentLedger, Get-UpdateBucketKeys, Get-PublishedPatchKeys, Save-PublishedFile, Get-PatchVersionFromKey, Select-PreviousPublishedPatch, Get-UpdateMinusBaseLedgerEntries, Get-MissingComponents, Format-RemovedComponentRemediation, Format-DroppedComponentMessage