LT-22801: Keep patches from dropping shipped components

.github/workflows/base-installer-cd.yml

@@ -161,7 +161,7 @@ jobs:
         id: build
         shell: powershell
         run: |
-          .\build.ps1 -BuildInstaller -Configuration Release -Verbosity detailed -BuildTests -MsBuildArgs @("/bl") *>&1 | Tee-Object -FilePath build.log
+          .\build.ps1 -BuildInstaller -Configuration Release -Verbosity detailed -BuildTests -MsBuildArgs @("/bl", "/p:FailOnRemovedSinceLastBase=${{ inputs.make_release == 'true' }}") *>&1 | Tee-Object -FilePath build.log
 
       - name: Run tests
         shell: powershell

.github/workflows/patch-installer-cd.yml

@@ -39,7 +39,7 @@ on:
 # on Jenkins-built bases.
       base_release:
         description: 'The github release for the base build artifacts (separate only for bootstrapping; should be removed after 9.3 is the stable)'
-        default: 'build-1452' # When updating this, update base_build_number and their fallbacks below, too.
+        default: 'build-1452' # Must match base_build_number and its fallbacks.
       base_build_number:
         description: 'The base build number'
         required: false
@@ -61,6 +61,7 @@ jobs:
       CROWDIN_API_KEY: ${{ secrets.FLEX_CROWDIN_API }}
       LcmRootDir: ${{ github.workspace }}/Localizations/LCMRepo
       FILESTOSIGNLATER: ./signExternally
+      KEEP_INTERMEDIATE_MSI: true
       GH_TOKEN: ${{ github.token }}
       BASE_BUILD_NUMBER: ${{ inputs.base_build_number || '1452' }}
     name: Upload strings for l10n, Build, and run Tests
@@ -258,6 +259,21 @@ jobs:
 
             $patchPath = $patch.FullName
             "patch_file=$patchPath" >> $env:GITHUB_OUTPUT
+            "patch_version=$(($patch.Name -split '_')[1])" >> $env:GITHUB_OUTPUT
+            "ledger_file=$($patchPath -replace '\.msp$', '_components.tsv')" >> $env:GITHUB_OUTPUT
+            "update_msi=$((Get-ChildItem PatchableInstaller/CreateUpdatePatch/Update -Filter '*.msi' | Select-Object -First 1).FullName)" >> $env:GITHUB_OUTPUT
+            "master_msi=$((Get-ChildItem PatchableInstaller/CreateUpdatePatch/Master -Filter '*.msi' | Select-Object -First 1).FullName)" >> $env:GITHUB_OUTPUT
+
+      # Patch component identity must remain stable across the base and previous patch.
+      - name: Check patch keeps base and previous-patch components
+        shell: pwsh
+        run: |
+          .\scripts\Installer\Check-PatchComponentLedger.ps1 `
+            -MasterMsi "${{ steps.find_patch.outputs.master_msi }}" `
+            -UpdateMsi "${{ steps.find_patch.outputs.update_msi }}" `
+            -BaseBuildNumber $env:BASE_BUILD_NUMBER `
+            -PatchVersion "${{ steps.find_patch.outputs.patch_version }}" `
+            -OutLedger "${{ steps.find_patch.outputs.ledger_file }}"
 
       - name: Sign Patch
         if: github.event_name != 'pull_request'
@@ -287,9 +303,22 @@ jobs:
           if (-not (Test-Path $patchPath)) {
             throw "Patch file not found at $patchPath"
           }
+          $ledgerPath = "${{ steps.find_patch.outputs.ledger_file }}"
+          if (-not (Test-Path $ledgerPath)) {
+            throw "Ledger file not found at $ledgerPath"
+          }
           $patchFile = Split-Path $patchPath -Leaf
           $s3Key = "jobs/FieldWorks-Win-all-Release-Patch/$($env:FW_BUILD_NUMBER)/$patchFile"
+          $ledgerKey = "jobs/FieldWorks-Win-all-Release-Patch/$($env:FW_BUILD_NUMBER)/$(Split-Path $ledgerPath -Leaf)"
+          aws s3 cp $ledgerPath "s3://flex-updates/$ledgerKey"
+          if ($LASTEXITCODE -ne 0) {
+            throw "Ledger upload failed with exit code $LASTEXITCODE"
+          }
+          Write-Host "Uploaded to s3://flex-updates/$ledgerKey"
           aws s3 cp $patchPath "s3://flex-updates/$s3Key"
+          if ($LASTEXITCODE -ne 0) {
+            throw "Patch upload failed with exit code $LASTEXITCODE"
+          }
           Write-Host "Uploaded to s3://flex-updates/$s3Key"
 
       - name: Upload Build Logs

Build/Installer.legacy.targets

@@ -105,17 +105,16 @@
 		/>
 	</Target>
 	<!--
-		Removing files breaks the ability to patch and requires a new Base build to update.
-		To improve our user experience we may want to delay making a new Base build so this target adds zero byte files to
-		replace files that were removed (for instance during a library update).
-		A base build should warn if we have 'RemovedSinceLastBase' items to help us remember to clear these out.
+		Windows Installer patches must preserve component identities shipped by the base or an
+		earlier patch. Zero-byte stand-ins keep those components present until a new base.
 	-->
 	<Target Name="RescuePatching">
 		<ItemGroup>
 			<!-- <RemovedSinceLastBase Include="$(dir-outputBase)/Helps/WW-ConceptualIntro/ConceptualIntroduction.htm" /> -->
 			<!-- liblcm swapped its IoC container from StructureMap to Microsoft.Extensions.DependencyInjection
 				in SIL.LCModel 11.0.0-beta0176, so nothing copies StructureMap.dll to the output any more. -->
 			<RemovedSinceLastBase Include="$(dir-outputBase)/StructureMap.dll" />
+			<RemovedSinceLastBase Include="$(dir-outputBase)/Avalonia.Themes.Fluent.dll" />
 		</ItemGroup>
 		<WriteLinesToFile
 			File="%(RemovedSinceLastBase.FullPath)"
@@ -670,8 +669,12 @@
 	</PropertyGroup>
 	<Target Name="BuildProductBaseMsi" DependsOnTargets="InstallerVersionNumbers">
 		<Warning
-			Text="RemovedSinceLastBase should be cleared out before making a new base build."
-			Condition="'@(RemovedSinceLastBase)' != ''"
+			Text="RemovedSinceLastBase contains %(RemovedSinceLastBase.Identity). Remove its entry before creating a base release."
+			Condition="'@(RemovedSinceLastBase)' != '' and '$(FailOnRemovedSinceLastBase)' != 'true'"
+		/>
+		<Error
+			Text="Cannot create a base release while RemovedSinceLastBase contains %(RemovedSinceLastBase.Identity). Remove its entry before creating the base release."
+			Condition="'@(RemovedSinceLastBase)' != '' and '$(FailOnRemovedSinceLastBase)' == 'true'"
 		/>
 		<Message
 			Text="Building FieldWorks Base Msi for $(Arch)"

Docs/workflows/patch-component-removal.md

@@ -0,0 +1,60 @@
+# Removing a file from a patch line
+
+A patch must keep every file-backed component under MSI APPFOLDER (application output
+files) shipped by the base or by the immediately previous published patch. Windows
+Installer sees the base and newest patch together, so those file-backed components must
+remain in the new patch.
+
+## What CI checks
+
+`patch-installer-cd.yml` runs the file-backed component ledger check before signing or publishing.
+The check compares the new Update MSI with the union of:
+
+- the file-backed components under MSI APPFOLDER in the Master/base MSI; and
+- the complete file-backed update-minus-base ledger published beside the immediately previous MSP.
+
+The first published patch on a patch line creates the initial S3 ledger. After a
+ledger-bearing patch exists, the immediately previous MSP must have its matching
+`*_components.tsv` file. Version filtering keeps a release branch from consuming a ledger
+for a later patch. Ledgers are release artifacts in S3 and are not stored in this repository.
+
+Each successful patch writes its complete update-minus-base file-backed component set under MSI APPFOLDER to a ledger beside
+the MSP. A later patch uses the ledger beside its immediately previous MSP.
+
+## When the check fails
+
+The diagnostic names every missing file-backed component and file, regardless of whether it came from
+the base MSI or the previous-patch ledger:
+
+```
+Patch 9.3.12.2761 drops file-backed component {B3A225EB-3642-5FE4-8ED4-B2DAE6F8AC9B}
+  file:    Avalonia.Themes.Fluent.dll   (feature Complete)
+  base:    1452
+Remediation:
+Add each missing file's output path to RemovedSinceLastBase in Build/Installer.legacy.targets,
+preserving its relative output path:
+  <RemovedSinceLastBase Include="$(dir-outputBase)/Avalonia.Themes.Fluent.dll" />
+Create an issue to remove the placeholder before the next base build, unless one already exists for the current base.
+```
+
+The `RemovedSinceLastBase` entry makes the build write a zero-byte stand-in at that path.
+That file-backed component remains in the patch, so machines that already have the real file keep
+working while the removal issue is completed.
+
+## Before publishing a base
+
+A scheduled base verification build warns while any `RemovedSinceLastBase` entries remain.
+A base release build fails. Both messages list the stand-in paths. Remove each
+`RemovedSinceLastBase` entry from `Build/Installer.legacy.targets`. CI builds start clean;
+a dirty local build must also delete each corresponding zero-byte file from the build output.
+
+Complete the removal issue before creating the base. A new base establishes the file-backed component
+set under MSI APPFOLDER that future patches must preserve.
+
+## Running the check locally
+
+```powershell
+.\scripts\Installer\Check-PatchComponentLedger.ps1 -MasterMsi <base.msi> -UpdateMsi <update.msi> `
+    -BaseBuildNumber 1452 -PatchVersion 9.3.12.2761 `
+    -OutLedger out.tsv
+```

FLExInstaller/AGENTS.md

@@ -13,10 +13,11 @@ Minimal installer guidance for agents.
 - Heat exclusions: **`PatchableInstallerHeatExclude.xml`** is copied to **`PatchableInstaller/BaseInstallerBuild/heat-exclude.xml`** before Heat (see **`Build/Installer.legacy.targets`** `CopyFilesToInstall`).
 - **`buildMsi.bat`** passes **`-fv`** to **`light.exe`** so **`MsiAssemblyName`** includes **fileVersion** (same intent as MSBuild **`SetMsiAssemblyNameFileVersion=true`**), which helps GAC servicing when **`AssemblyVersion`** is unchanged but the binary’s **file version** increases.
 - Newtonsoft.Json and similar authored components live in **`CustomComponents.wxi`** (overlays **`PatchableInstaller/Common`**), with definitions guarded by **`<?ifdef MASTERBUILDDIR?>`** so patch/update authoring omits them when only **`UPDATEBUILDDIR`** is set. Add matching **`ComponentRef`** entries in **`FLExInstaller/CustomFeatures.wxi`** inside the **same** **`<?ifdef MASTERBUILDDIR?>...<?endif?>`** so patch builds do not emit dangling refs (**LGHT0094**). WiX 6 **`Framework.wxs`** uses the same pattern for **`Feature Complete`**. Do not use **`FeatureRef Id="Complete"`** from an include that appears before **`Framework.wxs`** defines `Complete` (Light **LGHT0095**).
-- **Patch error `PYRO0305: The File '<name>' was removed in the patch`:** a file present in the base/**Master** harvest is missing from the new **Update** harvest, and WiX 3 **`pyro.exe`** forbids removing files in a patch. Typical trigger: a code change stops emitting a file the base release still ships — e.g. reg-free COM manifests dropped by a "reduce COM usage" change (`ManagedLgIcuCollator.manifest`, `ManagedVwWindow.manifest`).
-	- **Stopgap fix (patch against the existing base):** add the dropped file to the **`RemovedSinceLastBase`** item list in the **`RescuePatching`** target of **`Build/Installer.legacy.targets`** (runs via `BuildProduct`). It writes a zero-byte placeholder into the build output (**`$(dir-outputBase)`**) so the file appears in both harvests and `pyro` treats it as *changed*, not *removed*. Mirror the existing entries (`ManagedVwWindow.manifest`, `SimpleRootSite.manifest`) and add **only** the basenames actually dropped — usually just the `*.manifest`, not a still-shipping `.dll`.
-	- **Permanent fix:** cut a new **Base** build so the file is absent from Master too, then delete the now-stale `RemovedSinceLastBase` entries (the target comment notes a base build should warn when these exist).
-	- **Do not** add it to **`PatchableInstallerHeatExclude.xml`** — that list is for artifacts that must never be harvested (build-output dedup / test-only files), not for reconciling files removed since the base.
+- **Patch file-backed component removal:** the patch ledger check compares file-backed components under MSI APPFOLDER in the new **Update** MSI and **Master** MSI with the complete file-backed ledger from the immediately previous published MSP. The first published patch creates the initial ledger in S3; after a ledger-bearing patch exists, its matching ledger must be beside the immediately previous MSP. Ledgers are not stored in this repository. Later patch versions are ignored.
+	- A missing file-backed component from either source uses the same fix: add its output path, preserving the file's relative output path, to the **`RemovedSinceLastBase`** item list in the **`RescuePatching`** target of **`Build/Installer.legacy.targets`**. The target writes a zero-byte stand-in into **`$(dir-outputBase)`** so the file-backed component remains in the patch.
+	- Create an issue to remove the stand-in before the next base, unless one already exists for the current base.
+	- While any **`RemovedSinceLastBase`** entries remain, a scheduled base verification build warns and a base release build fails. Remove each entry before creating the base; a dirty local build must also delete the zero-byte file.
+- **Do not** add a dropped file-backed component to **`PatchableInstallerHeatExclude.xml`**. That list is for artifacts that must never be harvested, not for preserving file-backed patch component identity.
 
 ## Constraints
 

scripts/Installer/Check-PatchComponentLedger.ps1

@@ -0,0 +1,77 @@
+<#
+.SYNOPSIS
+Fails a patch build when the new Update MSI drops a file-backed component under MSI APPFOLDER
+from the base or previous patch.
+
+.DESCRIPTION
+Compares the new Update MSI with file-backed components under MSI APPFOLDER from the base
+MSI and the complete file-backed ledger from the immediately previous published patch.
+Writes the complete update-minus-base file-backed ledger for this patch.
+
+.PARAMETER MasterMsi
+The base (Master) MSI rebuilt by the patch build.
+
+.PARAMETER UpdateMsi
+The upgraded (Update) MSI the patch was diffed from.
+
+.PARAMETER PatchVersion
+The new patch's product version, e.g. 9.3.12.2761.
+
+.PARAMETER OutLedger
+Where to write this patch's file-backed component ledger.
+
+#>
+[CmdletBinding()]
+param(
+	[Parameter(Mandatory = $true)][string]$MasterMsi,
+	[Parameter(Mandatory = $true)][string]$UpdateMsi,
+	[Parameter(Mandatory = $true)][string]$BaseBuildNumber,
+	[Parameter(Mandatory = $true)][string]$PatchVersion,
+	[Parameter(Mandatory = $true)][string]$OutLedger
+)
+
+Set-StrictMode -Version Latest
+$ErrorActionPreference = 'Stop'
+Import-Module (Join-Path $PSScriptRoot 'PatchComponentLedger.psm1') -Force
+
+$master = Get-MsiComponents -MsiPath $MasterMsi
+$update = Get-MsiComponents -MsiPath $UpdateMsi
+
+$publishedPatchKeys = @(Get-PublishedPatchKeys -BaseBuildNumber $BaseBuildNumber -Wildcard '*.msp')
+$publishedLedgerKeys = @(Get-PublishedPatchKeys -BaseBuildNumber $BaseBuildNumber -Wildcard '*_components.tsv')
+$previous = Select-PreviousPublishedPatch `
+	-PatchKeys $publishedPatchKeys `
+	-LedgerKeys $publishedLedgerKeys `
+	-BaseBuildNumber $BaseBuildNumber `
+	-PatchVersion $PatchVersion
+
+$ledgerFiles = New-Object System.Collections.Generic.List[string]
+if ($previous.LedgerKey) {
+	$downloads = Join-Path ([IO.Path]::GetTempPath()) "fw-patch-ledgers-b$BaseBuildNumber"
+	New-Item -ItemType Directory -Force -Path $downloads | Out-Null
+	$ledgerFiles.Add((Save-PublishedFile -Key $previous.LedgerKey -Directory $downloads))
+}
+
+$previousLedger = Read-ComponentLedger -LedgerFiles $ledgerFiles.ToArray()
+$required = @{}
+foreach ($entry in $master.Values) { $required[$entry.ComponentId] = $entry }
+foreach ($entry in $previousLedger.Values) {
+	if (-not $required.ContainsKey($entry.ComponentId)) { $required[$entry.ComponentId] = $entry }
+}
+$dropped = @(Get-MissingComponents -Required $required -Available $update)
+$newLedgerEntries = @(Get-UpdateMinusBaseLedgerEntries -Master $master -Update $update)
+Write-ComponentLedger -Path $OutLedger -Entries $newLedgerEntries -Heading "Complete update-minus-base file-backed ledger under MSI APPFOLDER for patch $PatchVersion on base $BaseBuildNumber"
+Write-Output "Patch $PatchVersion ledger contains $($newLedgerEntries.Count) update-minus-base file-backed components under MSI APPFOLDER; ledger written to $OutLedger"
+Write-Output "Checking against $($required.Count) required file-backed components under MSI APPFOLDER"
+
+if ($dropped.Count -eq 0) {
+	Write-Output '[OK] The patch keeps every required file-backed component under MSI APPFOLDER.'
+	exit 0
+}
+
+$message = Format-DroppedComponentMessage -PatchVersion $PatchVersion -BaseBuildNumber $BaseBuildNumber -Dropped $dropped
+Write-Output $message
+if ($env:GITHUB_ACTIONS -eq 'true') {
+	Write-Output ('::error title=Patch file-backed component removal::' + ($message -replace "`r?`n", ' '))
+}
+exit 1