diff --git a/.claude/rules/ci.md b/.claude/rules/ci.md index ef74be4c75..24d8319c79 100644 --- a/.claude/rules/ci.md +++ b/.claude/rules/ci.md @@ -27,6 +27,8 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ - Every Xcode job in the test and release workflows runs on the `xcode-27` runner label, a GitHub preview image. actionlint learns the label from `.github/actionlint.yaml`. `build-plugin.yml` picks its runner from the ref it builds (`RELEASING.md`). - The Xcode version is written once, in `.github/actions/setup-xcode/action.yml`, which runs `xcode-select --switch` and prints `xcodebuild -version`. A workflow never selects Xcode another way. - XcodeGen comes from `.github/actions/setup-xcodegen`, pinned by version and SHA-256. +- `scripts/release/feed.py` runs Sparkle's `generate_appcast` and `sign_update` from `SPARKLE_BIN`, which `.github/actions/setup-sparkle` sets. The action reads the version and revision from the app's `Package.resolved` (`project.yml` pins Sparkle with `exactVersion`) and checks the archive against the checksum in Sparkle's `Package.swift` at that revision. +- The tag release (`build.yml`) signs the feed with the copy `scripts/ci/sign-and-appcast.sh` downloads, pinned by its own `SPARKLE_VERSION` and `SPARKLE_SHA256`. No check compares them with `project.yml`: a Sparkle bump changes all three. - The compilation cache is off on CI (`COMPILATION_CACHE_ENABLE_CACHING=NO`); `Configs/Base.xcconfig` turns it on locally. ## macOS Tests (`macos-tests.yml`) @@ -88,7 +90,7 @@ The required check names are in `AGENTS.md` (CI). Reading a red check: `.claude/ - `check-lib-sourcing.py`, `check-plugin-manifest.py`, `check-deployment-target.sh`, `check-ios-shared-isolation.py`; - `localization.py plugins` and `plugins --target ios`; - `check-string-catalogs.py`, `check-license-inventory.py`, `check-entitlements.py`, `check-test-plans.py`; - - `check-pluginkit-bump-cadence.py`, `check-sparkle-version.py`; + - `check-pluginkit-bump-cadence.py`; - `scripts/audit-refactor-health.sh --check`, `check-hub-ratchet.py`; - the Go checks on the HANA helper, when the `hana` detector is true. - `audit-refactor-health.sh --check` fails a file defining a production `DatabaseType` outside `Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift`, unless `.github/duplicate-contract-baseline.txt` lists its path. Moving a listed file moves its baseline path in the same commit. Never add an entry to let a new copy pass. diff --git a/.github/actions/setup-sparkle/action.yml b/.github/actions/setup-sparkle/action.yml new file mode 100644 index 0000000000..41c936027e --- /dev/null +++ b/.github/actions/setup-sparkle/action.yml @@ -0,0 +1,51 @@ +name: Setup Sparkle +description: Installs Sparkle's generate_appcast and sign_update at the version the app resolves, and exports SPARKLE_BIN. + +runs: + using: composite + steps: + - name: Install the Sparkle tools + shell: bash + # These tools hold the EdDSA key that signs every update, so they come from the archive SwiftPM + # verifies for the framework: Sparkle's Package.swift at the pinned revision names its checksum. + env: + RESOLVED: TablePro.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved + run: | + set -euo pipefail + pin="$RUNNER_TEMP/sparkle-pin.txt" + python3 - "$RESOLVED" > "$pin" << 'PYTHON' + import json + import sys + + pins = [pin for pin in json.load(open(sys.argv[1]))["pins"] if pin["identity"] == "sparkle"] + if len(pins) != 1: + sys.exit(f"{sys.argv[1]} pins Sparkle {len(pins)} times") + print(pins[0]["state"]["version"], pins[0]["state"]["revision"]) + PYTHON + read -r version revision < "$pin" + case "$version" in *[!0-9.]* | '') echo "::error::Sparkle pin '$version' is not a version"; exit 1 ;; esac + case "$revision" in *[!0-9a-f]* | '') echo "::error::Sparkle pin '$revision' is not a commit"; exit 1 ;; esac + + manifest="$(curl -fsSL --retry 3 "https://raw.githubusercontent.com/sparkle-project/Sparkle/$revision/Package.swift")" + checksum="$(sed -n 's/^let checksum = "\([0-9a-f]\{64\}\)"$/\1/p' <<< "$manifest")" + tag="$(sed -n 's/^let tag = "\(.*\)"$/\1/p' <<< "$manifest")" + if [ -z "$checksum" ] || [ "$tag" != "$version" ]; then + echo "::error::Sparkle's Package.swift at $revision names tag '$tag' and checksum '$checksum', not $version" + exit 1 + fi + + archive="$RUNNER_TEMP/sparkle-$version.zip" + dir="$RUNNER_TEMP/sparkle-$version" + curl -fsSL --retry 3 -o "$archive" \ + "https://github.com/sparkle-project/Sparkle/releases/download/$version/Sparkle-for-Swift-Package-Manager.zip" + echo "$checksum $archive" | shasum -a 256 -c - + rm -rf "$dir" + unzip -q "$archive" -d "$dir" + for tool in generate_appcast sign_update; do + if [ ! -x "$dir/bin/$tool" ]; then + echo "::error::the Sparkle $version archive has no bin/$tool" + exit 1 + fi + done + echo "SPARKLE_BIN=$dir/bin" >> "$GITHUB_ENV" + echo "Sparkle $version tools in $dir/bin" diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index 940ff29759..9cab3d2a8f 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -115,9 +115,6 @@ jobs: - name: Check currentPluginKitVersion moved at most once since the last release run: python3 scripts/ci/check-pluginkit-bump-cadence.py --remote origin - - name: Check the Sparkle version agrees everywhere - run: python3 scripts/ci/check-sparkle-version.py - - name: Check the shared-contract drift gates run: scripts/audit-refactor-health.sh --check diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d80c4a568..037e03b4f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Theme Editor window with a live preview, undo and Revert Changes. Open windows show each edit as you make it. - iCloud sync of custom themes and the chosen light and dark themes, with Settings on in Settings > Sync. - Theme import by dropping files on the Themes list, and export by dragging a theme to Finder. +- Get beta updates in Settings > General, from a signed update feed at updates.tablepro.app. ### Changed diff --git a/TablePro/Core/Services/Infrastructure/SoftwareUpdater.swift b/TablePro/Core/Services/Infrastructure/SoftwareUpdater.swift index 201b6bd1c0..2c07be5776 100644 --- a/TablePro/Core/Services/Infrastructure/SoftwareUpdater.swift +++ b/TablePro/Core/Services/Infrastructure/SoftwareUpdater.swift @@ -3,16 +3,6 @@ import Foundation import os import Sparkle -/// Sparkle owns the update preferences, and this is the only place the app reads or writes them. -/// -/// `SPUUpdater.h:198-201` says it twice: do not keep an additional user default for -/// `automaticallyChecksForUpdates`, and do not set it on every launch unless the user's own -/// preference is meant to be ignored. The initial values belong in Info.plist instead, which is -/// what `SparkleUpdatePreferences` pins. -/// -/// The published values are a mirror, not a store. Each is fed by KVO from the property that owns -/// it, so a managed preference or Sparkle's own alert reaches the UI without anything having to -/// notice it happened. @MainActor final class SoftwareUpdater: ObservableObject { static let shared = SoftwareUpdater() @@ -27,8 +17,9 @@ final class SoftwareUpdater: ObservableObject { nonisolated private static let logger = Logger(subsystem: "com.TablePro", category: "SoftwareUpdater") private let controller: SPUStandardUpdaterController - /// Retained here because `SPUStandardUpdaterController` holds both delegates weakly. + // Retained here because `SPUStandardUpdaterController` holds both delegates weakly. private let delegate = SoftwareUpdaterDelegate() + private let defaults = AppStorageEnvironment.shared.defaults private var observations: [NSKeyValueObservation] = [] private var hasStarted = false @@ -38,10 +29,7 @@ final class SoftwareUpdater: ObservableObject { @Published private(set) var allowsAutomaticUpdates = true @Published private(set) var updateCheckInterval: TimeInterval = 0 @Published private(set) var lastUpdateCheckDate: Date? - - /// A scheduled update the app declined to put in front of the user. Both the app menu item and - /// the settings button read it, because a gentle reminder that shows in one place a person - /// never opens is the same as no reminder at all. + @Published private(set) var receivesBetaUpdates = false @Published private(set) var hasPendingUpdate = false private init() { @@ -67,8 +55,6 @@ final class SoftwareUpdater: ObservableObject { controller.updater } - /// The title both the app menu item and the settings button show, so a deferred update is - /// announced in the two places a person looks without either inventing its own wording. var checkForUpdatesTitle: String { Self.checkForUpdatesTitle(hasPendingUpdate: hasPendingUpdate) } @@ -93,21 +79,26 @@ final class SoftwareUpdater: ObservableObject { refreshFromUpdater() } - /// Hands the update preferences back to Info.plist. - /// - /// `AppSettingsManager.resetToDefaults()` reassigns the settings structs, and these values are - /// not in them, so without this "reset all settings across every section" would leave the - /// update section exactly as the user set it. - /// - /// The standard domain rather than `AppStorageEnvironment`: Sparkle reaches these keys through - /// `SUHost`, which reads the main bundle's own domain, so a redirected suite would clear a - /// copy nothing consults. + func setReceivesBetaUpdates(_ enabled: Bool) { + UpdateChannel.select(enabled ? .beta : .stable, in: defaults) + if hasStarted { + updater.resetUpdateCycleAfterShortDelay() + } + refreshFromUpdater() + } + func resetUpdatePreferences() { + // Sparkle reads these through `SUHost`, which uses the main bundle's own domain, never a redirected suite. SparkleUpdatePreferences.reset(in: UserDefaults.standard) + UpdateChannel.clearSelection(in: defaults) refreshFromUpdater() Self.logger.info("Update preferences reset to the values declared in Info.plist") } + fileprivate var allowedChannels: Set { + UpdateChannel.selected(in: defaults).sparkleChannels + } + fileprivate func recordDeferredUpdate() { hasPendingUpdate = true } @@ -130,17 +121,11 @@ final class SoftwareUpdater: ObservableObject { Self.logger.error("Update driver aborted: \(nsError.domain, privacy: .public) \(nsError.code, privacy: .public)") } - /// Every property here is documented KVO-compliant in `SPUUpdater.h`. - /// - /// The hop onto the main actor is not ceremony and must stay. `SPUUpdater` is - /// `NS_SWIFT_UI_ACTOR`, but these notifications do not all originate in the app: `SUHost.m:88` - /// puts a KVO observer on `NSUserDefaults`, and a change written from outside the process, which - /// is exactly what a managed preference is, arrives on whatever thread delivers it. Reading the - /// updater there would be off-actor, and `assumeIsolated` would trap rather than lag. private func observeUpdater() { let updater = controller.updater func mirror(_ keyPath: KeyPath) -> NSKeyValueObservation { updater.observe(keyPath, options: [.new]) { [weak self] _, _ in + // A managed preference reaches `SUHost`'s defaults observer off the main thread. Task { @MainActor [weak self] in self?.refreshFromUpdater() } } } @@ -153,9 +138,6 @@ final class SoftwareUpdater: ObservableObject { ] } - /// Called after every write and from every observation, so a missed KVO edge cannot leave a - /// control showing something the updater does not agree with. `lastUpdateCheckDate` is not - /// documented KVO-compliant, so it rides along here and on the update-cycle callback instead. private func refreshFromUpdater() { let updater = controller.updater canCheckForUpdates = updater.canCheckForUpdates @@ -163,18 +145,22 @@ final class SoftwareUpdater: ObservableObject { automaticallyDownloadsUpdates = updater.automaticallyDownloadsUpdates allowsAutomaticUpdates = updater.allowsAutomaticUpdates updateCheckInterval = updater.updateCheckInterval + // Not KVO-compliant, so it is read here and on the update-cycle callback. lastUpdateCheckDate = updater.lastUpdateCheckDate + receivesBetaUpdates = UpdateChannel.selected(in: defaults) == .beta } } -/// Sparkle's two delegate surfaces, kept off `SoftwareUpdater` so the app's model is not also an -/// `NSObject` bound to an Objective-C protocol. @MainActor private final class SoftwareUpdaterDelegate: NSObject { weak var owner: SoftwareUpdater? } extension SoftwareUpdaterDelegate: SPUUpdaterDelegate { + func allowedChannels(for updater: SPUUpdater) -> Set { + owner?.allowedChannels ?? [] + } + func updater(_ updater: SPUUpdater, didFinishUpdateCycleFor updateCheck: SPUUpdateCheck, error: Error?) { owner?.recordUpdateCycleFinished(error: error) } @@ -184,9 +170,6 @@ extension SoftwareUpdaterDelegate: SPUUpdaterDelegate { } } -/// Gentle scheduled reminders, the shape Sparkle documents for an app with a Dock icon: a check the -/// person did not ask for never takes the front, and the app carries the news in its own UI until -/// they come back to it. extension SoftwareUpdaterDelegate: @preconcurrency SPUStandardUserDriverDelegate { var supportsGentleScheduledUpdateReminders: Bool { true } diff --git a/TablePro/Core/Services/Infrastructure/SparkleUpdatePreferences.swift b/TablePro/Core/Services/Infrastructure/SparkleUpdatePreferences.swift index 6e9282065e..c2650645b5 100644 --- a/TablePro/Core/Services/Infrastructure/SparkleUpdatePreferences.swift +++ b/TablePro/Core/Services/Infrastructure/SparkleUpdatePreferences.swift @@ -1,49 +1,15 @@ -// -// SparkleUpdatePreferences.swift -// TablePro -// - import Foundation -/// The Sparkle preference keys the app depends on, and the rule that every one of them carries a -/// default the app ships rather than a default Sparkle infers. -/// -/// Sparkle resolves each key as user defaults first and the bundle's Info.plist second -/// (`SUHost.boolNumberForKey:`). A key declared in neither resolves to `NO`, and the three update -/// settings are chained: with `SUEnableAutomaticChecks` absent, `allowsAutomaticUpdates` is false, -/// and `SPUUpdaterSettings.setAutomaticallyDownloadsUpdates` then returns without writing anything. -/// So a `SUAutomaticallyUpdate` of `true` read as `false` at runtime, and the second toggle in the -/// settings pane was inert rather than merely dimmed. -/// -/// `SparkleUpdatePreferencesTests` reads the running bundle and fails when a key here has no -/// declared default, which is the check that absence defeated. enum SparkleUpdatePreferences { - /// Sparkle writes these into the main bundle's standard domain through `SUHost` when the user - /// changes a control. Removing them hands governance back to Info.plist, which is what - /// resetting settings has to mean. static let userSettableKeys = [ "SUEnableAutomaticChecks", "SUScheduledCheckInterval", "SUAutomaticallyUpdate", ] - /// `SUScheduledImpatientCheckInterval` is deliberately absent: Sparkle's own default is the - /// value we want, and a redeclared default is one more pair of numbers that can drift apart. - static let requiredInfoPlistKeys = [ - "SUFeedURL", - "SUPublicEDKey", - "SUEnableAutomaticChecks", - "SUScheduledCheckInterval", - "SUAutomaticallyUpdate", - ] - static func reset(in defaults: UserDefaults) { for key in userSettableKeys { defaults.removeObject(forKey: key) } } - - static func keysMissingDefaults(in bundle: Bundle) -> [String] { - requiredInfoPlistKeys.filter { bundle.object(forInfoDictionaryKey: $0) == nil } - } } diff --git a/TablePro/Core/Services/Infrastructure/UpdateChannel.swift b/TablePro/Core/Services/Infrastructure/UpdateChannel.swift new file mode 100644 index 0000000000..70f6898482 --- /dev/null +++ b/TablePro/Core/Services/Infrastructure/UpdateChannel.swift @@ -0,0 +1,35 @@ +import Foundation + +internal enum UpdateChannel: String, Sendable { + case stable + case beta +} + +internal extension UpdateChannel { + // Sparkle always adds its default channel, which carries every stable item. + var sparkleChannels: Set { + switch self { + case .stable: [] + case .beta: [Self.beta.rawValue] + } + } + + func accepts(_ releaseChannel: String) -> Bool { + switch self { + case .stable: releaseChannel == Self.stable.rawValue + case .beta: Self(rawValue: releaseChannel) != nil + } + } + + static func selected(in defaults: UserDefaults) -> UpdateChannel { + defaults.bool(forKey: PreferenceKeys.receivesBetaUpdates.name) ? .beta : .stable + } + + static func select(_ channel: UpdateChannel, in defaults: UserDefaults) { + defaults.set(channel == .beta, forKey: PreferenceKeys.receivesBetaUpdates.name) + } + + static func clearSelection(in defaults: UserDefaults) { + defaults.removeObject(forKey: PreferenceKeys.receivesBetaUpdates.name) + } +} diff --git a/TablePro/Core/Storage/Preferences/PreferenceKeys.swift b/TablePro/Core/Storage/Preferences/PreferenceKeys.swift index 40b61ce98d..1d38d8b54f 100644 --- a/TablePro/Core/Storage/Preferences/PreferenceKeys.swift +++ b/TablePro/Core/Storage/Preferences/PreferenceKeys.swift @@ -1,8 +1,3 @@ -// -// PreferenceKeys.swift -// TablePro -// - import Foundation enum PreferenceKeys { @@ -25,6 +20,8 @@ enum PreferenceKeys { static let connectionListFavoritesOrder = DefaultsKey<[String]>("com.TablePro.connectionList.favoritesOrder") static let connectionListShowsRecent = DefaultsKey("com.TablePro.connectionList.showsRecent") static let recentConnections = DefaultsKey("com.TablePro.connectionList.recentConnections") + /// Device-local, and in the standard domain so a configuration profile can pin it. + static let receivesBetaUpdates = DefaultsKey("com.TablePro.updates.receivesBetaUpdates") static let registeredKeyNames: [String] = [ connectionListSortMode.name, @@ -43,6 +40,7 @@ enum PreferenceKeys { queryPlanBarMetric.name, lastBackupDirectory.name, lastSeenAppVersion.name, + receivesBetaUpdates.name, ] static let columnDisplayFormatsPrefix = "com.TablePro.columns.displayFormat." diff --git a/TablePro/Info.plist b/TablePro/Info.plist index d34f4b71b4..9ac41c6a57 100644 --- a/TablePro/Info.plist +++ b/TablePro/Info.plist @@ -11,9 +11,13 @@ NSLocalNetworkUsageDescription TablePro connects to database servers and SSH tunnels running on your local network, including Bonjour (.local) hostnames. SUFeedURL - https://raw.githubusercontent.com/TableProApp/TablePro/main/appcast.xml + https://updates.tablepro.app/appcast.xml SUPublicEDKey - EongGFyuahKlYPZwgmnFx8nW3s1CqWlSSU5BDqY6n6Q= + PAQXJZpmlif12LfOoRkGQqf5lGQqPPf62E6dn7Jovhk= + SURequireSignedFeed + + SUVerifyUpdateBeforeExtraction + SUEnableAutomaticChecks SUScheduledCheckInterval diff --git a/TablePro/Resources/Localizable.xcstrings b/TablePro/Resources/Localizable.xcstrings index 6fc653e63e..b95187119f 100644 --- a/TablePro/Resources/Localizable.xcstrings +++ b/TablePro/Resources/Localizable.xcstrings @@ -30471,6 +30471,9 @@ "Bell" : { "extractionState" : "manual", "generatesSymbol" : false + }, + "Betas get new features first. Turning this off keeps the beta you have until a newer release ships." : { + }, "between" : { "localizations" : { @@ -84183,6 +84186,9 @@ }, "Geometry: %@" : { + }, + "Get beta updates" : { + }, "Get by key" : { "extractionState" : "manual", diff --git a/TablePro/Views/Settings/GeneralSettingsView.swift b/TablePro/Views/Settings/GeneralSettingsView.swift index a07909a525..1fe07d1c95 100644 --- a/TablePro/Views/Settings/GeneralSettingsView.swift +++ b/TablePro/Views/Settings/GeneralSettingsView.swift @@ -13,7 +13,7 @@ struct GeneralSettingsView: View { private static let standardTimeouts = [10, 20, 30, 40, 50, 60, 90, 120, 180, 300, 600] - /// Sparkle owns these, so `GeneralSettings` holds no copy for settings sync to overwrite. + // The updater owns these, so `GeneralSettings` holds no copy for settings sync to overwrite. private var automaticallyChecksForUpdates: Binding { Binding( get: { updater.automaticallyChecksForUpdates }, @@ -28,6 +28,13 @@ struct GeneralSettingsView: View { ) } + private var receivesBetaUpdates: Binding { + Binding( + get: { updater.receivesBetaUpdates }, + set: { updater.setReceivesBetaUpdates($0) } + ) + } + private var lastUpdateCheckDescription: String { guard let date = updater.lastUpdateCheckDate else { return String(localized: "Last checked: never") @@ -153,6 +160,9 @@ struct GeneralSettingsView: View { .accessibilityIdentifier("automatic-update-install-toggle") .help(String(localized: "A new version downloads in the background and installs the next time you quit TablePro.")) + Toggle("Get beta updates", isOn: receivesBetaUpdates) + .accessibilityIdentifier("beta-updates-toggle") + LabeledContent { Button(updater.checkForUpdatesTitle) { updater.checkForUpdates() @@ -175,6 +185,8 @@ struct GeneralSettingsView: View { .accessibilityIdentifier("whats-new-link") } header: { Text("Software Update") + } footer: { + Text("Betas get new features first. Turning this off keeps the beta you have until a newer release ships.") } Section { diff --git a/TableProTests/Services/SparkleUpdatePreferencesTests.swift b/TableProTests/Services/SparkleUpdatePreferencesTests.swift index 0f4ce0685e..816bb8e578 100644 --- a/TableProTests/Services/SparkleUpdatePreferencesTests.swift +++ b/TableProTests/Services/SparkleUpdatePreferencesTests.swift @@ -2,54 +2,48 @@ import Foundation @testable import TablePro import Testing -/// The regression these exist for: every Sparkle update control read `false` on a fresh install, -/// because `SUEnableAutomaticChecks` was declared nowhere. Sparkle resolves a key as user defaults -/// first and Info.plist second, so an undeclared key is `NO`, `allowsAutomaticUpdates` is then `NO` -/// too, and `setAutomaticallyDownloadsUpdates` returns without writing. `SUAutomaticallyUpdate` was -/// `true` in Info.plist and `false` everywhere it mattered. struct SparkleUpdatePreferencesTests { - /// The unit-test bundle is hosted by the app, so the process's main bundle is the app bundle. - /// A runner that ever stopped hosting would fail `everyRequiredKeyIsDeclared` rather than pass - /// it quietly, which is the failure direction to want. + // The unit tests are hosted by the app, so the main bundle is TablePro.app and its built Info.plist. private var appBundle: Bundle { Bundle.main } + // An undeclared key resolves to NO in Sparkle (#2865). `SUScheduledImpatientCheckInterval` keeps Sparkle's default. + private static let requiredInfoPlistKeys = [ + "SUFeedURL", + "SUPublicEDKey", + "SURequireSignedFeed", + "SUVerifyUpdateBeforeExtraction", + "SUEnableAutomaticChecks", + "SUScheduledCheckInterval", + "SUAutomaticallyUpdate", + ] + @Test("Every Sparkle key the app depends on carries a default in Info.plist") func everyRequiredKeyIsDeclared() { - let missing = SparkleUpdatePreferences.keysMissingDefaults(in: appBundle) + let missing = Self.requiredInfoPlistKeys.filter { appBundle.object(forInfoDictionaryKey: $0) == nil } #expect(missing.isEmpty, "Info.plist declares no default for \(missing.joined(separator: ", "))") } @Test("A key the user can change has a default to go back to") func everySettableKeyHasADeclaredDefault() { for key in SparkleUpdatePreferences.userSettableKeys { - #expect( - SparkleUpdatePreferences.requiredInfoPlistKeys.contains(key), - "\(key) is resettable but has no declared default, so resetting it re-arms Sparkle's permission prompt" - ) + #expect(appBundle.object(forInfoDictionaryKey: key) != nil, "\(key) is resettable but Info.plist declares no default") } } @Test("The shipped defaults are checks on, installs on, daily") - func shippedDefaults() throws { + func shippedDefaults() { #expect(appBundle.object(forInfoDictionaryKey: "SUEnableAutomaticChecks") as? Bool == true) #expect(appBundle.object(forInfoDictionaryKey: "SUAutomaticallyUpdate") as? Bool == true) - - let interval = try #require(appBundle.object(forInfoDictionaryKey: "SUScheduledCheckInterval") as? Int) - #expect(interval == 86_400) - #expect(interval >= 3_600, "Sparkle clamps the scheduled check to a one-hour floor") - #expect( - interval < 604_800, - "an interval at or above Sparkle's 604800 impatient default leaves a downloaded fix waiting longer than the reminder that chases it" - ) + #expect(appBundle.object(forInfoDictionaryKey: "SUScheduledCheckInterval") as? Int == 86_400) } - @Test("The feed is fetched over HTTPS and its signing key is declared") + @Test("The feed comes from updates.tablepro.app, signed, and every archive is verified before extraction") func feedIsSigned() throws { - let feed = try #require(appBundle.object(forInfoDictionaryKey: "SUFeedURL") as? String) - #expect(feed.hasPrefix("https://")) + #expect(appBundle.object(forInfoDictionaryKey: "SUFeedURL") as? String == "https://updates.tablepro.app/appcast.xml") + #expect(appBundle.object(forInfoDictionaryKey: "SURequireSignedFeed") as? Bool == true) + #expect(appBundle.object(forInfoDictionaryKey: "SUVerifyUpdateBeforeExtraction") as? Bool == true) let key = try #require(appBundle.object(forInfoDictionaryKey: "SUPublicEDKey") as? String) - #expect(!key.isEmpty) #expect(Data(base64Encoded: key)?.count == 32, "an EdDSA public key is 32 bytes") } @@ -71,9 +65,6 @@ struct SparkleUpdatePreferencesTests { #expect(defaults.string(forKey: "SUFeedURL") == "keep me", "reset cleared a key Sparkle does not let the user set") } - /// The only signal a deferred update leaves. Sparkle's standard driver shows nothing when - /// `standardUserDriverShouldHandleShowingScheduledUpdate` returns false, so a title that did not - /// change would mean the update was never announced at all. @Test("A deferred update renames the control, and nothing else does") @MainActor func deferredUpdateRenamesTheControl() { diff --git a/TableProTests/Services/UpdateChannelTests.swift b/TableProTests/Services/UpdateChannelTests.swift new file mode 100644 index 0000000000..3e6dbbb04f --- /dev/null +++ b/TableProTests/Services/UpdateChannelTests.swift @@ -0,0 +1,60 @@ +import Foundation +@testable import TablePro +import Testing + +struct UpdateChannelTests { + @Test("An unset preference asks Sparkle for the default channel only") + func unsetAllowsNoExtraChannel() throws { + let suite = try EphemeralDefaults() + + #expect(UpdateChannel.selected(in: suite.defaults) == .stable) + #expect(UpdateChannel.selected(in: suite.defaults).sparkleChannels == []) + } + + @Test("Turning beta updates on adds the beta channel") + func onAllowsBeta() throws { + let suite = try EphemeralDefaults() + + UpdateChannel.select(.beta, in: suite.defaults) + + #expect(suite.defaults.bool(forKey: PreferenceKeys.receivesBetaUpdates.name)) + #expect(UpdateChannel.selected(in: suite.defaults).sparkleChannels == ["beta"]) + } + + @Test("Turning beta updates off goes back to the default channel") + func offAllowsNoExtraChannel() throws { + let suite = try EphemeralDefaults() + + UpdateChannel.select(.beta, in: suite.defaults) + UpdateChannel.select(.stable, in: suite.defaults) + + #expect(UpdateChannel.selected(in: suite.defaults).sparkleChannels == []) + } + + @Test("Reset removes the key") + func resetRemovesTheKey() throws { + let suite = try EphemeralDefaults() + + UpdateChannel.select(.beta, in: suite.defaults) + UpdateChannel.clearSelection(in: suite.defaults) + + #expect(suite.defaults.object(forKey: PreferenceKeys.receivesBetaUpdates.name) == nil) + #expect(UpdateChannel.selected(in: suite.defaults) == .stable) + } + + @Test("Stable takes only stable releases") + func stableAcceptsStableOnly() { + #expect(UpdateChannel.stable.accepts("stable")) + #expect(!UpdateChannel.stable.accepts("beta")) + #expect(!UpdateChannel.stable.accepts("nightly")) + #expect(!UpdateChannel.stable.accepts("")) + } + + @Test("Beta takes stable and beta releases") + func betaAcceptsStableAndBeta() { + #expect(UpdateChannel.beta.accepts("stable")) + #expect(UpdateChannel.beta.accepts("beta")) + #expect(!UpdateChannel.beta.accepts("nightly")) + #expect(!UpdateChannel.beta.accepts("Beta")) + } +} diff --git a/project.yml b/project.yml index bc75aa5a70..4b1562726d 100644 --- a/project.yml +++ b/project.yml @@ -48,9 +48,10 @@ packages: path: Packages/TableProMacCore TableProOracle: path: Packages/TableProOracle + # Exact, because .github/actions/setup-sparkle installs the feed signing tools from this pin. Sparkle: url: https://github.com/sparkle-project/Sparkle - majorVersion: 2.10.0 + exactVersion: 2.10.0 Yams: url: https://github.com/jpsim/Yams majorVersion: 5.1.0 diff --git a/scripts/ci/check-sparkle-version.py b/scripts/ci/check-sparkle-version.py deleted file mode 100755 index bfada43686..0000000000 --- a/scripts/ci/check-sparkle-version.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""Fails on the Sparkle configuration invariants nothing else checks. - -The framework the app links and the generate_appcast that signs its feed are the same project, and -a release that mixes them is a release nobody can measure. Nothing checked this before: project.yml -declares `majorVersion`, which XcodeGen writes as upToNextMajorVersion, so a resolve can move the -framework on its own while the signing tool stays where it was. - -The second invariant is the deployment target. TablePro publishes two items per release and tells -them apart by `sparkle:hardwareRequirements`, which `generate_appcast` emits only while -`LSMinimumSystemVersion` is below 27.0 (ArchiveItem.swift:255-262, unchanged in 2.10.0): at or above -it, `mayNeedHardwareRequirement` is forced false and the element is never written. merge-appcast.py -then refuses the release, about forty minutes in, with both architectures already notarized and the -tag already pushed. - -Runs on Repo Hygiene, which has no Xcode. Pure text. -""" - -import pathlib -import re -import sys - -ROOT = pathlib.Path(__file__).resolve().parents[2] - -PROJECT_YML = ROOT / "project.yml" -PACKAGE_RESOLVED = ROOT / "TablePro.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved" -SIGN_SCRIPT = ROOT / "scripts/ci/sign-and-appcast.sh" -LICENSES = ROOT / "TablePro/Resources/ThirdPartyLicenses/licenses.yml" - - -class VersionError(Exception): - """A disagreement. Every one of these fails the job that raised it.""" - - -def read(path): - try: - return path.read_text(encoding="utf-8") - except OSError as error: - raise VersionError(f"{path.relative_to(ROOT)}: {error}") from error - - -def sole_match(pattern, text, path, what): - matches = re.findall(pattern, text, re.MULTILINE) - if len(matches) != 1: - raise VersionError( - f"{path.relative_to(ROOT)}: expected exactly one {what}, found {len(matches)}" - ) - return matches[0] - - -HARDWARE_REQUIREMENTS_CEILING = (27, 0) - - -def deployment_target(): - project = read(PROJECT_YML) - block = re.search(r"^ deploymentTarget:\n(?:^ {4}.*\n)+", project, re.MULTILINE) - if block is None: - raise VersionError("project.yml: no deploymentTarget block") - return sole_match(r'^ {4}macOS:\s*"?([0-9.]+)"?\s*$', block.group(0), PROJECT_YML, "macOS deployment target") - - -def check_deployment_target(): - raw = deployment_target() - parts = tuple(int(part) for part in raw.split(".")[:2]) - padded = parts + (0,) * (2 - len(parts)) - if padded >= HARDWARE_REQUIREMENTS_CEILING: - raise VersionError( - f"the macOS deployment target is {raw}, at or above " - f"{'.'.join(str(n) for n in HARDWARE_REQUIREMENTS_CEILING)}. generate_appcast stops " - "emitting sparkle:hardwareRequirements there, so the two per-architecture items become " - "indistinguishable and merge-appcast.py refuses the release after both notarizations. " - "Move the release to one Universal archive before raising the target" - ) - return raw - - -def declared_versions(): - """Every place a Sparkle version is written down, as {label: version}.""" - project = read(PROJECT_YML) - sparkle_block = re.search( - r"^ Sparkle:\n(?:^ {4}.*\n)+", project, re.MULTILINE - ) - if sparkle_block is None: - raise VersionError("project.yml: no Sparkle package block") - - versions = { - "project.yml majorVersion": sole_match( - r"^ {4}majorVersion:\s*(\S+)\s*$", sparkle_block.group(0), PROJECT_YML, "majorVersion" - ), - "Package.resolved": sole_match( - r'"identity"\s*:\s*"sparkle".*?"version"\s*:\s*"([^"]+)"', - read(PACKAGE_RESOLVED).replace("\n", " "), - PACKAGE_RESOLVED, - "sparkle pin", - ), - "sign-and-appcast.sh SPARKLE_VERSION": sole_match( - r'^SPARKLE_VERSION="([^"]+)"\s*$', read(SIGN_SCRIPT), SIGN_SCRIPT, "SPARKLE_VERSION" - ), - } - - licenses = read(LICENSES) - entry = re.search(r"^- id: sparkle\n(?:^(?![-]).*\n)+", licenses, re.MULTILINE) - if entry is None: - raise VersionError("licenses.yml: no sparkle entry") - versions["licenses.yml version"] = sole_match( - r"^ version:\s*(\S+)\s*$", entry.group(0), LICENSES, "version" - ) - versions["licenses.yml licenseTextURL"] = sole_match( - r"licenseTextURL:\s*https://github\.com/sparkle-project/Sparkle/blob/([^/]+)/LICENSE", - entry.group(0), - LICENSES, - "licenseTextURL", - ) - return versions - - -def main(): - try: - target = check_deployment_target() - versions = declared_versions() - except VersionError as error: - print(f"::error::check-sparkle-version: {error}", file=sys.stderr) - return 1 - - distinct = set(versions.values()) - if len(distinct) != 1: - print("::error::check-sparkle-version: the Sparkle version is not the same everywhere", file=sys.stderr) - for label, version in versions.items(): - print(f" {version:<10} {label}", file=sys.stderr) - return 1 - - version = distinct.pop() - print(f"Sparkle {version} agrees across {len(versions)} declarations") - print(f"macOS deployment target {target} still emits sparkle:hardwareRequirements") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/probes/check-sparkle-update-defaults.sh b/scripts/probes/check-sparkle-update-defaults.sh deleted file mode 100755 index c8ddc00f03..0000000000 --- a/scripts/probes/check-sparkle-update-defaults.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Asks the Sparkle we actually link what TablePro/Info.plist resolves to on a machine that has never -# run the app, and fails when the answer is not "checks on, installs on". -# -# Sparkle resolves every update preference as user defaults first and Info.plist second -# (SUHost.boolNumberForKey:), and the three settings are chained: an undeclared -# SUEnableAutomaticChecks makes allowsAutomaticUpdates false, which makes -# setAutomaticallyDownloadsUpdates return without writing. So SUAutomaticallyUpdate read true in -# Info.plist and false everywhere it mattered, every update control in Settings read off on a fresh -# install, and Sparkle asked for permission on the second launch instead. Nothing caught it, because -# every developer machine already had the keys written into its own defaults. -# -# SparkleUpdatePreferencesTests covers the half that is ours: the keys are declared. This covers the -# half that is Sparkle's: that declaring them still produces the values we expect. A framework bump -# is what would change that answer, which is why this is a script and not a transcription. -# -# Usage: scripts/probes/check-sparkle-update-defaults.sh [path/to/built/TablePro.app] -# With no argument it looks for the Debug build. Needs Xcode; macOS only. - -REPO_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -cd "$REPO_ROOT" - -export DEVELOPER_DIR="${DEVELOPER_DIR:-$(xcode-select -p)}" - -APP="${1:-}" -if [ -z "$APP" ]; then - DERIVED=$(xcodebuild -project TablePro.xcodeproj -scheme TablePro -configuration Debug \ - -showBuildSettings -skipPackagePluginValidation 2>/dev/null \ - | awk '/ BUILT_PRODUCTS_DIR = /{print $3; exit}') - APP="$DERIVED/TablePro.app" -fi - -FRAMEWORK_DIR="$(dirname "$APP")" -if [ ! -d "$FRAMEWORK_DIR/Sparkle.framework" ] && [ -d "$APP/Contents/Frameworks/Sparkle.framework" ]; then - FRAMEWORK_DIR="$APP/Contents/Frameworks" -fi - -if [ ! -d "$FRAMEWORK_DIR/Sparkle.framework" ]; then - echo "::error::no Sparkle.framework beside $APP; build the app first" >&2 - exit 1 -fi - -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT - -# The probe builds a throwaway host bundle from the repo's own Info.plist under an identifier no -# defaults domain has ever seen, which is the fresh-install condition the bug hid behind. -cat > "$WORK/probe.swift" <<'SWIFT' -import Foundation -import Sparkle - -let sourcePlist = CommandLine.arguments[1] -guard let declared = NSDictionary(contentsOfFile: sourcePlist) as? [String: Any] else { - FileHandle.standardError.write(Data("cannot read \(sourcePlist)\n".utf8)) - exit(2) -} - -let contents = URL(fileURLWithPath: NSTemporaryDirectory()) - .appendingPathComponent("sparkle-default-probe-\(ProcessInfo.processInfo.processIdentifier).app/Contents", isDirectory: true) -try? FileManager.default.createDirectory(at: contents, withIntermediateDirectories: true) - -var plist: [String: Any] = [ - "CFBundleIdentifier": "com.tablepro.sparkledefaultprobe.\(ProcessInfo.processInfo.processIdentifier)", - "CFBundleName": "Probe", - "CFBundleShortVersionString": "1.0", - "CFBundleVersion": "1", -] -for key in declared.keys where key.hasPrefix("SU") { - plist[key] = declared[key] -} -(plist as NSDictionary).write(to: contents.appendingPathComponent("Info.plist"), atomically: true) - -guard let host = Bundle(url: contents.deletingLastPathComponent()) else { - FileHandle.standardError.write(Data("cannot open the probe bundle\n".utf8)) - exit(2) -} - -let settings = SPUUpdaterSettings(hostBundle: host) -let sparkleVersion = Bundle(for: SPUUpdaterSettings.self) - .object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "unknown" - -print("Sparkle \(sparkleVersion), fresh defaults domain:") -print(" automaticallyChecksForUpdates \(settings.automaticallyChecksForUpdates)") -print(" allowsAutomaticUpdates \(settings.allowsAutomaticUpdates)") -print(" automaticallyDownloadsUpdates \(settings.automaticallyDownloadsUpdates)") -print(" updateCheckInterval \(Int(settings.updateCheckInterval))") -print(" impatientUpdateCheckInterval \(Int(settings.impatientUpdateCheckInterval))") - -var failures: [String] = [] -if !settings.automaticallyChecksForUpdates { - failures.append("automaticallyChecksForUpdates is false; declare SUEnableAutomaticChecks in Info.plist") -} -if !settings.allowsAutomaticUpdates { - failures.append("allowsAutomaticUpdates is false, so the install toggle is inert rather than merely dimmed") -} -if !settings.automaticallyDownloadsUpdates { - failures.append("automaticallyDownloadsUpdates is false despite SUAutomaticallyUpdate") -} -if settings.updateCheckInterval >= settings.impatientUpdateCheckInterval { - failures.append("updateCheckInterval is not below impatientUpdateCheckInterval") -} - -guard failures.isEmpty else { - for failure in failures { - FileHandle.standardError.write(Data("::error::\(failure)\n".utf8)) - } - exit(1) -} -print("\n✅ A fresh install checks for updates and installs them.") -SWIFT - -xcrun swiftc -O -o "$WORK/probe" "$WORK/probe.swift" \ - -F "$FRAMEWORK_DIR" -framework Sparkle \ - -Xlinker -rpath -Xlinker "$FRAMEWORK_DIR" - -"$WORK/probe" "$REPO_ROOT/TablePro/Info.plist" diff --git a/scripts/release/feed.py b/scripts/release/feed.py new file mode 100755 index 0000000000..701c3a1dc0 --- /dev/null +++ b/scripts/release/feed.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import lzma +import os +import plistlib +import re +import shutil +import subprocess +import sys +import tarfile +import tempfile +import xml.etree.ElementTree as ET +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Callable, Dict, Iterator, List, Optional, Tuple + +import registry +import version + +ROOT = Path(__file__).resolve().parent.parent.parent +SPARKLE = "{http://www.andymatuschak.org/xml-namespaces/sparkle}" +BRANCH = "feeds" +REMOTE_BRANCH = f"refs/remotes/origin/{BRANCH}" +APPCAST = "appcast.xml" +MANIFEST = "plugins/v3.json" +METADATA = ".github/plugin-registry.json" +PROJECT = "project.yml" +PLUGINKIT_CONFIG = "Configs/PluginKit.xcconfig" +PHASED_ROLLOUT_SECONDS = 21600 +PUSH_ATTEMPTS = 3 +BOT_NAME = "github-actions[bot]" +BOT_EMAIL = "41898282+github-actions[bot]@users.noreply.github.com" +ITEM_TEXT = re.compile(r"[ \t]*.*?[ \t]*\n?", re.DOTALL) +INTEGER = re.compile(r"[0-9]+") + + +class FeedError(Exception): + pass + + +@dataclass(frozen=True) +class Item: + build: Optional[str] + short_version: Optional[str] + channel: Optional[str] + is_critical: bool + critical_version: Optional[str] + url: Optional[str] + signature: Optional[str] + length: Optional[str] + + @property + def label(self) -> str: + return self.short_version or self.build or "an item" + + +@dataclass(frozen=True) +class Expected: + release: version.Version + url: str + + +def _sparkle_value(element: ET.Element, name: str) -> Optional[str]: + child = element.find(SPARKLE + name) + if child is not None: + return (child.text or "").strip() + enclosure = element.find("enclosure") + return enclosure.get(SPARKLE + name) if enclosure is not None else None + + +def parse_items(data: bytes) -> List[Item]: + try: + root = ET.fromstring(data) + except ET.ParseError as error: + raise FeedError(f"{APPCAST} is not well-formed XML: {error}") from error + channel = root.find("channel") + if root.tag != "rss" or channel is None: + raise FeedError(f"{APPCAST} has no rss channel") + items = [] + for element in channel.findall("item"): + enclosure = element.find("enclosure") + critical = element.find(f".//{SPARKLE}criticalUpdate") + name = element.find(SPARKLE + "channel") + items.append(Item( + build=_sparkle_value(element, "version"), + short_version=_sparkle_value(element, "shortVersionString"), + channel=(name.text or "").strip() if name is not None else None, + is_critical=critical is not None, + critical_version=critical.get(SPARKLE + "version") if critical is not None else None, + url=enclosure.get("url") if enclosure is not None else None, + signature=enclosure.get(SPARKLE + "edSignature") if enclosure is not None else None, + length=enclosure.get("length") if enclosure is not None else None, + )) + return items + + +def carried_critical(items: List[Item]) -> Optional[int]: + builds = [] + for item in items: + if item.channel is not None or not item.is_critical: + continue + # A critical update with no version is critical below its own build. + text = item.critical_version or item.build + if text is None or not INTEGER.fullmatch(text): + raise FeedError(f"{item.label}: the critical update version {text!r} is not an integer") + builds.append(int(text)) + return max(builds) if builds else None + + +def feed_problems(data: bytes, previous: Optional[bytes] = None, expected: Optional[Expected] = None) -> List[str]: + items = parse_items(data) + problems = [] + if previous is not None and any(item.channel is None for item in parse_items(previous)) \ + and not any(item.channel is None for item in items): + problems.append("the feed no longer has a default-channel item") + + versions: Dict[int, set] = {} + for item in items: + if item.url is None: + problems.append(f"{item.label} has no enclosure") + if not item.signature: + problems.append(f"{item.label}'s enclosure has no sparkle:edSignature") + if item.length is None or not INTEGER.fullmatch(item.length): + problems.append(f"{item.label}'s enclosure has no length") + if item.channel is not None and item.channel != "beta": + problems.append(f"{item.label} is on channel {item.channel!r}; the only channel is beta") + if item.channel == "beta" and item.is_critical: + problems.append(f"{item.label} is a beta marked critical") + if item.build is None or not INTEGER.fullmatch(item.build): + problems.append(f"{item.label}: sparkle:version {item.build!r} is not an integer") + continue + try: + parsed = version.parse_version(item.short_version or "") + except ValueError: + problems.append(f"build {item.build}: shortVersionString {item.short_version!r} is not a release version") + continue + versions.setdefault(int(item.build), set()).add(parsed) + + ordered = [] + for build, found in sorted(versions.items()): + if len(found) > 1: + problems.append(f"build {build} carries {', '.join(sorted(str(each) for each in found))}") + else: + ordered.append((build, found.pop())) + for (low_build, low), (high_build, high) in zip(ordered, ordered[1:]): + if not low.build < high.build: + problems.append(f"{high} has the higher build ({high_build} over {low_build}) but not the higher version") + + if expected is not None: + release = expected.release + channel = "beta" if release.is_beta else None + if not any(item.build == str(release.build) and item.short_version == str(release) + and item.channel == channel and item.url == expected.url for item in items): + problems.append(f"no {release.channel} item for {release} ({release.build}) at {expected.url}") + return problems + + +def remove_items(text: str, short_version: str) -> Tuple[str, int]: + items = parse_items(text.encode("utf-8")) + spans = list(ITEM_TEXT.finditer(text)) + if len(spans) != len(items): + raise FeedError(f"{APPCAST} has {len(items)} items but {len(spans)} blocks in its text; edit it by hand") + doomed = [span for span, item in zip(spans, items) if item.short_version == short_version] + for span in reversed(doomed): + text = text[:span.start()] + text[span.end():] + left = [item for item in items if item.short_version != short_version] + if parse_items(text.encode("utf-8")) != left: + raise FeedError(f"removing {short_version} from the text of {APPCAST} changed other items; edit it by hand") + return text, len(doomed) + + +def appcast_has(items: List[Item], release: version.Version) -> bool: + matches = [item for item in items if item.short_version == str(release) or item.build == str(release.build)] + for item in matches: + if item.short_version != str(release) or item.build != str(release.build): + raise FeedError(f"{APPCAST} has {item.short_version} at build {item.build}, " + f"which clashes with {release} at {release.build}") + return bool(matches) + + +def appcast_arguments(stage: Path, tag: str, release: version.Version, critical_build: Optional[int], + repository: str) -> List[str]: + command = [ + "--ed-key-file", "-", "--embed-release-notes", "--maximum-deltas", "0", + "--versions", str(release.build), + "--download-url-prefix", f"https://github.com/{repository}/releases/download/{tag}/", + "--full-release-notes-url", f"https://github.com/{repository}/releases/tag/{tag}", + ] + if release.is_beta: + command += ["--channel", "beta"] + else: + command += ["--phased-rollout-interval", str(PHASED_ROLLOUT_SECONDS)] + if critical_build is not None: + command += ["--critical-update-version", str(critical_build)] + return command + [str(stage)] + + +def pluginkit_version(root: Path) -> int: + path = root / PLUGINKIT_CONFIG + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + raise FeedError(f"{PLUGINKIT_CONFIG}: {error}") from error + found = re.search(r"^TABLEPRO_PLUGINKIT_VERSION\s*=\s*([0-9]+)\s*$", text, re.MULTILINE) + if found is None: + raise FeedError(f"{PLUGINKIT_CONFIG} sets no TABLEPRO_PLUGINKIT_VERSION") + return int(found.group(1)) + + +def check_archive(path: Path, release: version.Version) -> None: + name = "TablePro.app/Contents/Info.plist" + try: + with tarfile.open(path, "r:xz") as archive: + member = archive.extractfile(name) + info = plistlib.loads(member.read()) if member is not None else None + except KeyError: + info = None + except (tarfile.TarError, lzma.LZMAError, plistlib.InvalidFileException, ValueError, OSError) as error: + raise FeedError(f"{path.name}: {error}") from error + if info is None: + raise FeedError(f"{path.name} has no {name}") + found = (info.get("CFBundleShortVersionString"), info.get("CFBundleVersion")) + if found != (str(release), str(release.build)): + raise FeedError(f"{path.name} holds {found[0]} ({found[1]}), not {release} ({release.build})") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +class Releases: + def __init__(self, repository: str, run: Callable = subprocess.run): + self.repository = repository + self._run = run + + def _gh(self, *args: str) -> str: + result = self._run(["gh", *args, "--repo", self.repository], capture_output=True, text=True) + if result.returncode != 0: + raise FeedError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result.stdout + + def digests(self, tag: str) -> Dict[str, str]: + release = json.loads(self._gh("release", "view", tag, "--json", "isDraft,assets")) + if release.get("isDraft"): + raise FeedError(f"{tag} is a draft; the feed only points at a published release") + return {asset["name"]: asset.get("digest") or "" for asset in release.get("assets", [])} + + def download(self, tag: str, names: List[str], directory: Path) -> None: + patterns = [argument for name in names for argument in ("--pattern", name)] + self._gh("release", "download", tag, "--dir", str(directory), "--clobber", *patterns) + + +def fetch_assets(releases: Releases, tag: str, names: List[str], directory: Path) -> None: + wanted = [name for name in names if not (directory / name).is_file()] + if not wanted: + return + digests = releases.digests(tag) + missing = [name for name in wanted if name not in digests] + if missing: + raise FeedError(f"{tag} has no {', '.join(missing)}") + directory.mkdir(parents=True, exist_ok=True) + releases.download(tag, wanted, directory) + for name in wanted: + path = directory / name + if not path.is_file(): + raise FeedError(f"gh did not download {name} from {tag}") + found = f"sha256:{_sha256(path)}" + if digests[name] != found: + path.unlink() + raise FeedError(f"{name} downloaded as {found}, but {tag} lists {digests[name] or 'no digest'}") + + +def _git(root: Path, *args: str, run: Callable = subprocess.run, check: bool = True, + env: Optional[Dict[str, str]] = None) -> subprocess.CompletedProcess: + result = run(["git", "-C", str(root), *args], capture_output=True, text=True, + env={**os.environ, **env} if env else None) + if check and result.returncode != 0: + raise FeedError(f"git {' '.join(args)}: {result.stderr.strip()}") + return result + + +@contextmanager +def feeds_worktree(root: Path, run: Callable = subprocess.run) -> Iterator[Path]: + _git(root, "fetch", "--quiet", "--no-tags", "origin", f"+refs/heads/{BRANCH}:{REMOTE_BRANCH}", run=run) + parent = Path(tempfile.mkdtemp(prefix="feeds-")) + tree = parent / BRANCH + _git(root, "worktree", "add", "--quiet", "--detach", str(tree), REMOTE_BRANCH, run=run) + try: + yield tree + finally: + _git(root, "worktree", "remove", "--force", str(tree), run=run, check=False) + shutil.rmtree(parent, ignore_errors=True) + + +def commit(tree: Path, message: str, run: Callable = subprocess.run) -> bool: + paths = [path for path in (APPCAST, MANIFEST) if (tree / path).exists()] + _git(tree, "add", "--", *paths, run=run) + if _git(tree, "diff", "--cached", "--quiet", run=run, check=False).returncode == 0: + return False + identity = {f"GIT_{role}_{field}": value for role in ("AUTHOR", "COMMITTER") + for field, value in (("NAME", BOT_NAME), ("EMAIL", BOT_EMAIL))} + _git(tree, "commit", "--quiet", "-m", message, run=run, env=identity) + return True + + +def push(tree: Path, run: Callable = subprocess.run) -> bool: + result = _git(tree, "push", "--quiet", "origin", f"HEAD:refs/heads/{BRANCH}", run=run, check=False) + if result.returncode == 0: + return True + # A rejected fast-forward or a lost ref lock means another run pushed first. + if "[rejected]" in result.stderr or ("[remote rejected]" in result.stderr and "cannot lock ref" in result.stderr): + return False + raise FeedError(f"git push origin {BRANCH}: {result.stderr.strip()}") + + +def _tool(tools: Path, name: str, key: str, *args: str, run: Callable = subprocess.run) -> str: + result = run([str(tools / name), *args], input=key, capture_output=True, text=True) + output = (result.stdout + result.stderr).strip() + if result.returncode != 0: + raise FeedError(f"{name} failed: {output}") + return output + + +def sign(path: Path, key: str, tools: Path, run: Callable = subprocess.run) -> str: + return _tool(tools, "sign_update", key, "--ed-key-file", "-", str(path), run=run) + + +def verify_signature(path: Path, key: str, tools: Path, run: Callable = subprocess.run) -> List[str]: + try: + _tool(tools, "sign_update", key, "--verify", "--ed-key-file", "-", str(path), run=run) + except FeedError as error: + return [str(error)] + return [] + + +def _refuse(problems: List[str]) -> None: + if problems: + raise FeedError("the feed fails its checks, so nothing was pushed:\n " + "\n ".join(problems)) + + +def _publish(tree: Path, message: str, run: Callable, log: Callable[[str], None]) -> Optional[bool]: + if not commit(tree, message, run): + log(f"origin/{BRANCH} already has everything from '{message}'") + return False + if push(tree, run): + log(f"pushed '{message}' to origin/{BRANCH}") + return True + return None + + +def add(root: Path, tag: str, build: int, channel: str, notes: Path, critical: bool, tools: Path, key: str, + releases: Releases, repository: str, run: Callable = subprocess.run, + log: Callable[[str], None] = print) -> bool: + release = version.parse_tag(tag) + if build != release.build or channel != release.channel: + raise FeedError(f"{tag} is build {release.build} on {release.channel}, not build {build} on {channel}") + if critical and release.is_beta: + raise FeedError(f"{tag} is a beta, and a beta is never critical") + if not notes.is_file(): + raise FeedError(f"{notes} does not exist") + metadata = registry.load_metadata(root / METADATA) + bundled = registry.bundled_targets((root / PROJECT).read_text(encoding="utf-8")) + plugins = [f"{target}-{release}.zip" for target in registry.publish_set(metadata, bundled)] + kit = pluginkit_version(root) + archive = f"TablePro-{release}.tar.xz" + url_base = f"https://github.com/{repository}/releases/download/{tag}/" + expected = Expected(release, url_base + archive) + + with tempfile.TemporaryDirectory(prefix="feed-") as scratch: + downloads = Path(scratch) / "downloads" + entries = None + for attempt in range(1, PUSH_ATTEMPTS + 1): + wanted: Optional[Expected] = expected + with feeds_worktree(root, run) as tree: + appcast = tree / APPCAST + if not appcast.is_file(): + raise FeedError(f"origin/{BRANCH} has no {APPCAST}; seed the branch first (RELEASING.md)") + previous = appcast.read_bytes() + items = parse_items(previous) + if appcast_has(items, release): + log(f"{APPCAST} already has {release} ({release.build})") + else: + fetch_assets(releases, tag, [archive], downloads) + check_archive(downloads / archive, release) + stage = Path(scratch) / f"stage-{attempt}" + stage.mkdir() + shutil.copy2(downloads / archive, stage / archive) + shutil.copy2(notes, stage / f"TablePro-{release}.md") + shutil.copy2(appcast, stage / APPCAST) + critical_build = release.build if critical else None + if critical_build is None and not release.is_beta: + critical_build = carried_critical(items) + arguments = appcast_arguments(stage, tag, release, critical_build, repository) + log(_tool(tools, "generate_appcast", key, *arguments, run=run)) + if appcast_has(parse_items((stage / APPCAST).read_bytes()), release): + shutil.copy2(stage / APPCAST, appcast) + # generate_appcast signs the feed only once an archive sets SURequireSignedFeed. + log(sign(appcast, key, tools, run)) + else: + # The archive matched the tag, so only Sparkle's per-branch limit left it out. + log(f"generate_appcast keeps newer items on {release}'s branch, so {APPCAST} stays as it is") + wanted = None + if entries is None: + fetch_assets(releases, tag, plugins, downloads) + entries = registry.describe(downloads, str(release), release.build, kit, metadata, bundled) + registry.add(tree / MANIFEST, entries, release.channel, url_base, metadata["retainFromPluginKit"], log) + _refuse(feed_problems(appcast.read_bytes(), previous, wanted) + + verify_signature(appcast, key, tools, run)) + pushed = _publish(tree, f"release: {tag}", run, log) + if pushed is not None: + return pushed + log(f"origin/{BRANCH} moved (attempt {attempt} of {PUSH_ATTEMPTS}); starting again from a fresh fetch") + raise FeedError(f"origin/{BRANCH} moved during each of {PUSH_ATTEMPTS} attempts") + + +def withdraw(root: Path, text: str, tools: Path, key: str, run: Callable = subprocess.run, + log: Callable[[str], None] = print) -> bool: + release = version.parse_version(text) + for attempt in range(1, PUSH_ATTEMPTS + 1): + with feeds_worktree(root, run) as tree: + appcast = tree / APPCAST + previous = appcast.read_bytes() + edited, removed = remove_items(previous.decode("utf-8"), str(release)) + if removed: + if any(item.channel is None for item in parse_items(previous)) \ + and not any(item.channel is None for item in parse_items(edited.encode("utf-8"))): + raise FeedError(f"withdrawing {release} would leave the feed with no default-channel item") + appcast.write_text(edited, encoding="utf-8") + log(sign(appcast, key, tools, run)) + log(f"removed {removed} {release} item(s) from {APPCAST}") + else: + log(f"{APPCAST} has no {release} item") + if (tree / MANIFEST).exists(): + registry.withdraw(tree / MANIFEST, str(release), log) + _refuse(feed_problems(appcast.read_bytes(), previous) + verify_signature(appcast, key, tools, run)) + pushed = _publish(tree, f"withdraw: v{release}", run, log) + if pushed is not None: + return pushed + log(f"origin/{BRANCH} moved (attempt {attempt} of {PUSH_ATTEMPTS}); starting again from a fresh fetch") + raise FeedError(f"origin/{BRANCH} moved during each of {PUSH_ATTEMPTS} attempts") + + +def check(root: Path, tools: Path, key: str, run: Callable = subprocess.run) -> List[str]: + with feeds_worktree(root, run) as tree: + appcast = tree / APPCAST + if not appcast.is_file(): + return [f"origin/{BRANCH} has no {APPCAST}"] + problems = feed_problems(appcast.read_bytes()) + verify_signature(appcast, key, tools, run) + try: + if registry.load_manifest(tree / MANIFEST) is None: + problems.append(f"origin/{BRANCH} has no {MANIFEST}") + except registry.RegistryError as error: + problems.append(str(error)) + return problems + + +def published_items(root: Path, run: Callable = subprocess.run) -> List[Item]: + _git(root, "fetch", "--quiet", "--no-tags", "origin", f"+refs/heads/{BRANCH}:{REMOTE_BRANCH}", run=run) + result = run(["git", "-C", str(root), "show", f"{REMOTE_BRANCH}:{APPCAST}"], capture_output=True) + if result.returncode != 0: + raise FeedError(f"origin/{BRANCH} has no {APPCAST}: {result.stderr.decode(errors='replace').strip()}") + return parse_items(result.stdout) + + +def _environment() -> Tuple[Path, str]: + tools = os.environ.get("SPARKLE_BIN") + key = os.environ.get("SPARKLE_PRIVATE_KEY") + if not tools: + raise FeedError("SPARKLE_BIN is not set; run the setup-sparkle action first") + if not key: + raise FeedError("SPARKLE_PRIVATE_KEY is not set") + for name in ("generate_appcast", "sign_update"): + if not os.access(Path(tools) / name, os.X_OK): + raise FeedError(f"{tools} has no {name}") + return Path(tools), key + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Writes the appcast and the plugin manifest on the feeds branch.") + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("add", help="add a published release to the feed and the manifest") + command.add_argument("tag") + command.add_argument("--build", type=int, required=True) + command.add_argument("--channel", choices=registry.CHANNELS, required=True) + command.add_argument("--notes", type=Path, required=True) + command.add_argument("--critical", action="store_true") + commands.add_parser("withdraw", help="remove a version from the feed and the manifest").add_argument("version") + commands.add_parser("check", help="check the published feed") + args = parser.parse_args(argv) + + repository = os.environ.get("GITHUB_REPOSITORY", version.REPOSITORY) + try: + tools, key = _environment() + if args.command == "add": + add(ROOT, args.tag, args.build, args.channel, args.notes, args.critical, tools, key, + Releases(repository), repository) + elif args.command == "withdraw": + withdraw(ROOT, args.version, tools, key) + else: + problems = check(ROOT, tools, key) + for problem in problems: + print(f"error: {problem}", file=sys.stderr) + if problems: + return 1 + print(f"origin/{BRANCH} passes its checks") + except (FeedError, registry.RegistryError, ValueError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/publish.py b/scripts/release/publish.py new file mode 100755 index 0000000000..43902cc940 --- /dev/null +++ b/scripts/release/publish.py @@ -0,0 +1,224 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Callable, Dict, List, Optional + +import feed +import version + +ROOT = Path(__file__).resolve().parent.parent.parent +FIND_ATTEMPTS = 5 +FIND_DELAY_SECONDS = 2 + + +class PublishError(Exception): + pass + + +def draft_action(is_published: bool, has_draft: bool) -> str: + if is_published: + return "published" + return "reuse" if has_draft else "create" + + +def should_be_latest(release: version.Version, latest_tag: Optional[str]) -> bool: + if release.is_beta: + return False + latest = version.try_parse_tag(latest_tag) if latest_tag else None + return latest is None or release.build > latest.build + + +def release_flags(release: version.Version, latest_tag: Optional[str]) -> List[str]: + flags = ["--draft=false"] + if release.is_beta: + return flags + ["--prerelease", "--latest=false"] + return flags + [f"--latest={'true' if should_be_latest(release, latest_tag) else 'false'}"] + + +def highest_stable(items: List[feed.Item]) -> Optional[version.Version]: + stables = [] + for item in items: + if item.channel is not None: + continue + parsed = version.try_parse_tag(f"v{item.short_version}") if item.short_version else None + if parsed is not None and not parsed.is_beta: + stables.append(parsed) + return max(stables, key=lambda candidate: candidate.build) if stables else None + + +def asset_problems(files: Dict[str, str], assets: List[dict]) -> List[str]: + found = {asset.get("name"): asset for asset in assets} + problems = [] + for name, digest in sorted(files.items()): + asset = found.get(name) + if asset is None: + problems.append(f"the release has no {name}") + elif asset.get("state") != "uploaded": + problems.append(f"{name} is {asset.get('state')}, not uploaded") + elif asset.get("digest") != f"sha256:{digest}": + found_digest = asset.get("digest") or "missing its digest" + problems.append(f"{name} is {found_digest} on GitHub but sha256:{digest} here") + return problems + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +class GitHub: + def __init__(self, repository: str, run: Callable = subprocess.run): + self.repository = repository + self._run = run + + def _gh(self, *args: str, check: bool = True) -> subprocess.CompletedProcess: + result = self._run(["gh", *args], capture_output=True, text=True) + if check and result.returncode != 0: + raise PublishError(f"gh {' '.join(args[:3])}: {result.stderr.strip()}") + return result + + def _release(self, *args: str) -> List[str]: + return ["release", *args, "--repo", self.repository] + + def is_published(self, tag: str) -> bool: + result = self._gh("api", f"repos/{self.repository}/releases/tags/{tag}", "--jq", ".draft", check=False) + if result.returncode == 0: + return result.stdout.strip() == "false" + if "HTTP 404" in result.stderr: + return False + raise PublishError(f"gh api releases/tags/{tag}: {result.stderr.strip()}") + + # gh races its published and draft lookups, so this runs only once no published release holds the tag. + def draft(self, tag: str) -> Optional[dict]: + result = self._gh(*self._release("view", tag, "--json", "isDraft,assets"), check=False) + if result.returncode != 0: + if "release not found" in result.stderr: + return None + raise PublishError(f"gh release view {tag}: {result.stderr.strip()}") + release = json.loads(result.stdout) + if not release.get("isDraft"): + raise PublishError(f"{tag} was published while this ran") + return release + + def create_draft(self, tag: str, notes: Path, files: List[Path], prerelease: bool) -> None: + self._gh(*self._release("create", tag, "--draft", "--verify-tag", "--title", tag, "--notes-file", str(notes), + f"--prerelease={'true' if prerelease else 'false'}", *[str(path) for path in files])) + + def update_draft(self, tag: str, notes: Path, files: List[Path], prerelease: bool) -> None: + self._gh(*self._release("edit", tag, "--title", tag, "--notes-file", str(notes), + f"--prerelease={'true' if prerelease else 'false'}")) + self._gh(*self._release("upload", tag, "--clobber", *[str(path) for path in files])) + + def edit(self, tag: str, flags: List[str]) -> None: + self._gh(*self._release("edit", tag, *flags)) + + def latest_tag(self) -> Optional[str]: + result = self._gh("api", f"repos/{self.repository}/releases/latest", "--jq", ".tag_name", check=False) + if result.returncode == 0: + return result.stdout.strip() or None + if "HTTP 404" in result.stderr: + return None + raise PublishError(f"gh api releases/latest: {result.stderr.strip()}") + + +def draft(github: GitHub, tag: str, notes: Path, files: List[Path], sleep: Callable[[float], None] = time.sleep, + log: Callable[[str], None] = print) -> str: + parsed = version.parse_tag(tag) + missing = [str(path) for path in [notes, *files] if not path.is_file()] + if missing: + raise PublishError(f"{', '.join(missing)} does not exist") + names = [path.name for path in files] + if len(set(names)) != len(names): + raise PublishError("two assets share a file name") + + published = github.is_published(tag) + existing = None if published else github.draft(tag) + action = draft_action(published, existing is not None) + if action == "published": + log(f"{tag} is already published") + return "published" + if action == "create": + github.create_draft(tag, notes, files, parsed.is_beta) + log(f"created the draft for {tag}") + else: + github.update_draft(tag, notes, files, parsed.is_beta) + log(f"reused the draft an earlier attempt left for {tag}") + + current = None + for _ in range(FIND_ATTEMPTS): + current = github.draft(tag) + if current is not None: + break + sleep(FIND_DELAY_SECONDS) + if current is None: + raise PublishError(f"the draft for {tag} was created but cannot be found") + problems = asset_problems({path.name: _sha256(path) for path in files}, current.get("assets", [])) + if problems: + raise PublishError(f"the draft for {tag} does not hold this build, so it stays unpublished:\n " + + "\n ".join(problems)) + log(f"the draft for {tag} holds all {len(files)} assets") + return "draft" + + +def release(github: GitHub, tag: str, log: Callable[[str], None] = print) -> None: + parsed = version.parse_tag(tag) + if github.is_published(tag): + log(f"{tag} is already published") + return + if github.draft(tag) is None: + raise PublishError(f"{tag} has no draft; run publish.py draft first") + flags = release_flags(parsed, github.latest_tag()) + github.edit(tag, flags) + log(f"published {tag} ({' '.join(flags[1:])})") + + +def latest(github: GitHub, items: List[feed.Item], log: Callable[[str], None] = print) -> str: + highest = highest_stable(items) + if highest is None: + raise PublishError("the feed has no stable item") + github.edit(highest.tag, ["--latest=true"]) + log(f"{highest.tag} is Latest") + return highest.tag + + +def _stderr(message: str) -> None: + print(message, file=sys.stderr) + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description="Drafts, publishes and marks GitHub releases.") + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("draft", help="put every asset on a draft and check its digests") + command.add_argument("tag") + command.add_argument("--notes", type=Path, required=True) + command.add_argument("files", type=Path, nargs="+") + commands.add_parser("release", help="publish the draft").add_argument("tag") + commands.add_parser("latest", help="mark the highest stable left in the feed as Latest") + args = parser.parse_args(argv) + + github = GitHub(os.environ.get("GITHUB_REPOSITORY", version.REPOSITORY)) + try: + if args.command == "draft": + print(f"state={draft(github, args.tag, args.notes, args.files, log=_stderr)}") + elif args.command == "release": + release(github, args.tag, _stderr) + else: + latest(github, feed.published_items(ROOT), _stderr) + except (PublishError, feed.FeedError, ValueError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/registry.py b/scripts/release/registry.py new file mode 100755 index 0000000000..2bd77268ed --- /dev/null +++ b/scripts/release/registry.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 + +import argparse +import hashlib +import json +import plistlib +import re +import sys +import zipfile +from pathlib import Path +from typing import Callable, Dict, List, Optional, Set + +import version + +ROOT = Path(__file__).resolve().parent.parent.parent +METADATA = ROOT / ".github/plugin-registry.json" +PROJECT = ROOT / "project.yml" +SCHEMA_VERSION = 3 +CHANNELS = ("stable", "beta") +RELEASES_PER_KEY = 2 +PROVIDES = ("TableProProvidesDatabaseTypeIds", "TableProProvidesExportFormatIds", "TableProProvidesImportFormatIds") +SYSTEM_VERSION = re.compile(r"[0-9]+(\.[0-9]+){0,2}") +SHA256 = re.compile(r"[0-9a-f]{64}") + + +class RegistryError(Exception): + pass + + +def load_metadata(path: Path) -> dict: + try: + metadata = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise RegistryError(f"{path}: {error}") from error + shape = "{retainFromPluginKit, plugins: {: {name, iconName}}}" + plugins = metadata.get("plugins") if isinstance(metadata, dict) else None + if not isinstance(plugins, dict) or not _is_int(metadata.get("retainFromPluginKit")): + raise RegistryError(f"{path} is not in the {shape} shape") + for target, entry in plugins.items(): + if not isinstance(entry, dict) or not all(isinstance(entry.get(key), str) for key in ("name", "iconName")): + raise RegistryError(f"{path}: {target} needs a name and an iconName, as in {shape}") + return metadata + + +def bundled_targets(project: str) -> Set[str]: + block = re.search(r"^aggregateTargets:\n(?:^(?: .*)?\n)*?^ BundledPlugins:\n^ targets:\n((?:^ - .*\n)+)", + project, re.MULTILINE) + if block is None: + raise RegistryError("project.yml has no BundledPlugins aggregate with a targets list") + return {line.strip()[2:].strip() for line in block.group(1).splitlines()} + + +def publish_set(metadata: dict, bundled: Set[str]) -> List[str]: + targets = sorted(metadata["plugins"]) + inside = [target for target in targets if target in bundled] + if inside: + raise RegistryError(f"the app bundles {', '.join(inside)}, so the registry must not publish them") + if not targets: + raise RegistryError("the registry metadata publishes no plugin") + return targets + + +def _is_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_info(archive: Path, target: str) -> dict: + name = f"{target}.tableplugin/Contents/Info.plist" + try: + with zipfile.ZipFile(archive) as bundle: + return plistlib.loads(bundle.read(name)) + except KeyError as error: + raise RegistryError(f"{archive.name} has no {name}") from error + except (OSError, zipfile.BadZipFile, plistlib.InvalidFileException, ValueError) as error: + raise RegistryError(f"{archive.name}: {error}") from error + + +def describe_one(archive: Path, target: str, entry: dict, release: str, build: int, kit: int) -> dict: + info = read_info(archive, target) + problems = [] + if info.get("CFBundleShortVersionString") != release: + problems.append(f"CFBundleShortVersionString is {info.get('CFBundleShortVersionString')!r}, not {release!r}") + if info.get("CFBundleVersion") != str(build): + problems.append(f"CFBundleVersion is {info.get('CFBundleVersion')!r}, not '{build}'") + found_kit = info.get("TableProPluginKitVersion") + if not _is_int(found_kit) or found_kit != kit: + problems.append(f"TableProPluginKitVersion is {found_kit!r}, not the integer {kit}") + identifier = info.get("CFBundleIdentifier") + if not isinstance(identifier, str) or not identifier: + problems.append("CFBundleIdentifier is missing") + system = info.get("LSMinimumSystemVersion") + if not isinstance(system, str) or not SYSTEM_VERSION.fullmatch(system): + problems.append(f"LSMinimumSystemVersion is {system!r}") + type_ids = info.get(PROVIDES[0], []) + if not isinstance(type_ids, list) or not all(isinstance(type_id, str) for type_id in type_ids): + problems.append(f"{PROVIDES[0]} is not a list of strings") + # The app finds the plugin that serves a database type or a file format through these keys. + if not any(info.get(key) for key in PROVIDES): + problems.append(f"it declares none of {', '.join(PROVIDES)}") + if problems: + raise RegistryError(f"{archive.name}: " + "; ".join(problems)) + return { + "id": identifier, + "name": entry["name"], + "iconName": entry["iconName"], + "databaseTypeIds": type_ids, + "version": release, + "build": build, + "pluginKit": kit, + "minimumSystemVersion": system, + "file": archive.name, + "sha256": _sha256(archive), + } + + +def describe(zips: Path, release: str, build: int, kit: int, metadata: dict, bundled: Set[str]) -> List[dict]: + entries = [] + for target in publish_set(metadata, bundled): + archive = zips / f"{target}-{release}.zip" + if not archive.is_file(): + raise RegistryError(f"{archive} does not exist") + entries.append(describe_one(archive, target, metadata["plugins"][target], release, build, kit)) + seen: Dict[str, str] = {} + for entry in entries: + if entry["id"] in seen: + raise RegistryError(f"{entry['file']} and {seen[entry['id']]} share the bundle id {entry['id']}") + seen[entry["id"]] = entry["file"] + return entries + + +def empty_manifest(retain_from: int) -> dict: + return {"schemaVersion": SCHEMA_VERSION, "retainFromPluginKit": retain_from, "plugins": []} + + +def load_manifest(path: Path) -> Optional[dict]: + if not path.exists(): + return None + try: + manifest = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as error: + raise RegistryError(f"{path}: {error}") from error + if not isinstance(manifest, dict) or manifest.get("schemaVersion") != SCHEMA_VERSION: + found = manifest.get("schemaVersion") if isinstance(manifest, dict) else None + raise RegistryError(f"{path} has schemaVersion {found!r}; this tool writes only {SCHEMA_VERSION}") + if not _is_int(manifest.get("retainFromPluginKit")) or not isinstance(manifest.get("plugins"), list): + raise RegistryError(f"{path} needs an integer retainFromPluginKit and a plugins list") + for plugin in manifest["plugins"]: + if not isinstance(plugin, dict) or not isinstance(plugin.get("id"), str) \ + or not isinstance(plugin.get("releases"), list): + raise RegistryError(f"{path}: every plugin needs an id and a releases list") + for release in plugin["releases"]: + if not isinstance(release, dict) or not all(_is_int(release.get(key)) for key in ("build", "pluginKit")) \ + or release.get("channel") not in CHANNELS \ + or not all(isinstance(release.get(key), str) for key in ("version", "minimumSystemVersion")): + raise RegistryError(f"{path}: a release of {plugin['id']} lacks an integer build or pluginKit, " + "a channel, a version or a minimumSystemVersion") + return manifest + + +def write_manifest(path: Path, manifest: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + + +def retain(releases: List[dict], floor: int) -> List[dict]: + kept = [release for release in releases if release["pluginKit"] >= floor] + kept = [ + release for release in kept + if release["channel"] != "beta" or not any( + other["channel"] == "stable" and other["pluginKit"] == release["pluginKit"] + and other["minimumSystemVersion"] == release["minimumSystemVersion"] and other["build"] > release["build"] + for other in kept) + ] + groups: Dict[tuple, List[dict]] = {} + for release in sorted(kept, key=lambda candidate: candidate["build"], reverse=True): + key = (release["channel"], release["pluginKit"], release["minimumSystemVersion"]) + groups.setdefault(key, []).append(release) + survivors = [release for group in groups.values() for release in group[:RELEASES_PER_KEY]] + return sorted(survivors, key=lambda candidate: candidate["build"], reverse=True) + + +def _release(entry: dict, channel: str, url_base: str) -> dict: + return { + "version": entry["version"], + "build": entry["build"], + "channel": channel, + "pluginKit": entry["pluginKit"], + "minimumSystemVersion": entry["minimumSystemVersion"], + "url": url_base.rstrip("/") + "/" + entry["file"], + "sha256": entry["sha256"], + } + + +def _check_entry(entry: dict, channel: str) -> None: + missing = [key for key in ("id", "name", "iconName", "databaseTypeIds", "version", "build", "pluginKit", + "minimumSystemVersion", "file", "sha256") if key not in entry] + if missing: + raise RegistryError(f"an entry has no {', '.join(missing)}") + if not SHA256.fullmatch(str(entry["sha256"])): + raise RegistryError(f"{entry['id']}: sha256 {entry['sha256']!r} is not 64 lowercase hex digits") + parsed = version.parse_version(entry["version"]) + if parsed.build != entry["build"]: + raise RegistryError(f"{entry['id']}: build {entry['build']} is not {parsed.build}, the build of {parsed}") + if parsed.channel != channel: + raise RegistryError(f"{entry['id']}: {parsed} belongs to the {parsed.channel} channel, not {channel}") + + +def add(path: Path, entries: List[dict], channel: str, url_base: str, retain_from: int, + log: Callable[[str], None] = print) -> bool: + if channel not in CHANNELS: + raise RegistryError(f"channel {channel!r} is not one of {', '.join(CHANNELS)}") + if not url_base.startswith("https://"): + raise RegistryError(f"the URL base {url_base!r} is not https") + for entry in entries: + _check_entry(entry, channel) + loaded = load_manifest(path) + before = json.dumps(loaded, sort_keys=True) if loaded is not None else None + manifest = loaded if loaded is not None else empty_manifest(retain_from) + floor = max(manifest["retainFromPluginKit"], retain_from) + plugins = {plugin["id"]: plugin for plugin in manifest["plugins"]} + + arrived = [] + for entry in entries: + release = _release(entry, channel, url_base) + plugin = plugins.setdefault(entry["id"], {"id": entry["id"], "releases": []}) + same = [existing for existing in plugin["releases"] if existing["build"] == release["build"]] + if same and same[0].get("sha256") != release["sha256"]: + raise RegistryError(f"{entry['id']} already has build {release['build']} with sha256 " + f"{same[0].get('sha256')}, not {release['sha256']}") + if same: + log(f"{entry['id']} {release['version']}: already in the manifest") + continue + plugin["releases"].append(release) + arrived.append((entry, release)) + + for plugin in plugins.values(): + plugin["releases"] = retain(plugin["releases"], floor) + for entry, release in arrived: + plugin = plugins[entry["id"]] + if not any(kept is release for kept in plugin["releases"]): + log(f"{entry['id']} {release['version']}: pruned on arrival by retention, not added") + elif plugin["releases"][0] is release: + plugin.update({key: entry[key] for key in ("name", "iconName", "databaseTypeIds")}) + + manifest = { + "schemaVersion": SCHEMA_VERSION, + "retainFromPluginKit": floor, + "plugins": [_ordered(plugins[key]) for key in sorted(plugins) if plugins[key]["releases"]], + } + if json.dumps(manifest, sort_keys=True) == before: + return False + write_manifest(path, manifest) + return True + + +def _ordered(plugin: dict) -> dict: + keys = ("id", "name", "iconName", "databaseTypeIds") + return {**{key: plugin[key] for key in keys if key in plugin}, "releases": plugin["releases"]} + + +def withdraw(path: Path, release: str, log: Callable[[str], None] = print) -> bool: + version.parse_version(release) + manifest = load_manifest(path) + if manifest is None: + raise RegistryError(f"{path} does not exist") + removed = 0 + for plugin in manifest["plugins"]: + kept = [candidate for candidate in plugin["releases"] if candidate.get("version") != release] + removed += len(plugin["releases"]) - len(kept) + plugin["releases"] = kept + if not removed: + log(f"{path.name} has no {release} release") + return False + manifest["plugins"] = [plugin for plugin in manifest["plugins"] if plugin["releases"]] + write_manifest(path, manifest) + log(f"withdrew {removed} {release} release(s) from {path.name}") + return True + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description="The registry plugins and the v3 plugin manifest.") + commands = parser.add_subparsers(dest="command", required=True) + command = commands.add_parser("targets", help="print the targets the registry publishes") + command.add_argument("--metadata", type=Path, default=METADATA) + command.add_argument("--project", type=Path, default=PROJECT) + command = commands.add_parser("describe", help="read the built plugin zips into manifest entries") + command.add_argument("--zips", type=Path, required=True) + command.add_argument("--version", required=True) + command.add_argument("--build", type=int, required=True) + command.add_argument("--kit", type=int, required=True) + command.add_argument("--metadata", type=Path, required=True) + command.add_argument("--project", type=Path, default=PROJECT) + command.add_argument("--out", type=Path, required=True) + command = commands.add_parser("add", help="add described releases to the manifest") + command.add_argument("--manifest", type=Path, required=True) + command.add_argument("--entries", type=Path, required=True) + command.add_argument("--channel", choices=CHANNELS, required=True) + command.add_argument("--url-base", required=True) + command.add_argument("--retain-from", type=int, required=True) + command = commands.add_parser("withdraw", help="remove one version's releases from the manifest") + command.add_argument("--manifest", type=Path, required=True) + command.add_argument("--version", required=True) + args = parser.parse_args(argv) + + try: + if args.command == "targets": + bundled = bundled_targets(args.project.read_text(encoding="utf-8")) + print("\n".join(publish_set(load_metadata(args.metadata), bundled))) + elif args.command == "describe": + bundled = bundled_targets(args.project.read_text(encoding="utf-8")) + entries = describe(args.zips, args.version, args.build, args.kit, load_metadata(args.metadata), bundled) + args.out.write_text(json.dumps(entries, indent=2) + "\n", encoding="utf-8") + print(f"described {len(entries)} plugins in {args.out}") + elif args.command == "add": + entries = json.loads(args.entries.read_text(encoding="utf-8")) + if not isinstance(entries, list): + raise RegistryError(f"{args.entries} is not a list of entries") + changed = add(args.manifest, entries, args.channel, args.url_base, args.retain_from) + print(f"{args.manifest} {'updated' if changed else 'unchanged'}") + elif args.command == "withdraw": + withdraw(args.manifest, args.version) + except (RegistryError, ValueError, OSError) as error: + print(f"error: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/release/test_feed.py b/scripts/release/test_feed.py new file mode 100644 index 0000000000..d4eb227211 --- /dev/null +++ b/scripts/release/test_feed.py @@ -0,0 +1,550 @@ +#!/usr/bin/env python3 + +import io +import json +import os +import plistlib +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile +from contextlib import redirect_stderr +from pathlib import Path +from unittest import mock + +import feed +import publish +import registry +import version + +KEY = "cHJvYmUta2V5LW5vdC1hLXJlYWwtb25l" +REPOSITORY = "TableProApp/TablePro" +GIT_ENV = { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "Fixture", + "GIT_AUTHOR_EMAIL": "fixture@example.com", + "GIT_COMMITTER_NAME": "Fixture", + "GIT_COMMITTER_EMAIL": "fixture@example.com", +} + +SIGNATURE = r''' +import hashlib, re +TRAILER = re.compile(r"\s*$", re.S) +def content(text): + return TRAILER.sub("", text) +def signed(text, key): + body = content(text) + signature = hashlib.sha256((key.strip() + body).encode()).hexdigest() + return body + "\n" +''' + +FAKE_GH = SIGNATURE + r''' +import json, os, shutil, sys +from pathlib import Path +args = sys.argv[1:] +with open(os.environ["FAKE_LOG"], "a") as log: + log.write(json.dumps(["gh", *args]) + "\n") +releases = Path(os.environ["FAKE_RELEASES"]) +if args[:2] == ["release", "view"]: + folder = releases / args[2] + if not folder.is_dir(): + sys.exit("release not found") + assets = [] + for path in sorted(folder.iterdir()): + digest = "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() + if path.name == os.environ.get("FAKE_BAD_DIGEST"): + digest = "sha256:" + "0" * 64 + assets.append({"name": path.name, "digest": digest, "state": "uploaded"}) + print(json.dumps({"isDraft": os.environ.get("FAKE_DRAFT") == "1", "assets": assets})) +elif args[:2] == ["release", "download"]: + out = Path(args[args.index("--dir") + 1]) + for index, arg in enumerate(args): + if arg == "--pattern": + shutil.copy(releases / args[2] / args[index + 1], out / args[index + 1]) +elif args[:2] != ["release", "edit"]: + sys.exit("fake gh cannot answer " + " ".join(args)) +''' + +FAKE_GENERATE_APPCAST = SIGNATURE + r''' +import json, os, subprocess, sys +from pathlib import Path +args = sys.argv[1:] +key = sys.stdin.read() +with open(os.environ["FAKE_LOG"], "a") as log: + log.write(json.dumps(["generate_appcast", *args, "stdin=" + key.strip()]) + "\n") +race = os.environ.get("FAKE_RACE_REPO") +if race and not os.path.exists(race + ".done"): + open(race + ".done", "w").close() + subprocess.run(["git", "-C", race, "push", "--quiet", "origin", "HEAD:refs/heads/feeds"], check=True) +def option(name): + return args[args.index(name) + 1] if name in args else None +stage = Path(args[-1]) +appcast = stage / "appcast.xml" +if os.environ.get("FAKE_PRUNE"): + appcast.write_text(content(appcast.read_text())) + print("Wrote 0 new updates") + sys.exit(0) +archive = next(stage.glob("*.tar.xz")) +short = archive.name[len("TablePro-"):-len(".tar.xz")] +lines = ["", "" + short + ""] +if option("--channel"): + lines.append("" + option("--channel") + "") +if option("--phased-rollout-interval"): + interval = option("--phased-rollout-interval") + lines.append("" + interval + "") +if option("--critical-update-version") is not None: + lines.append('') +lines.append("" + option("--versions") + "") +lines.append("" + short + "") +notes = (stage / ("TablePro-" + short + ".md")).read_text() +lines.append('') +signature = "" if os.environ.get("FAKE_UNSIGNED_ENCLOSURE") else ' sparkle:edSignature="c2ln"' +lines.append('") +lines.append("") +text = content(appcast.read_text()) +index = text.index("") if "" in text else text.index("") +appcast.write_text(text[:index] + "\n".join(lines) + "\n" + text[index:]) +print("Wrote 1 new update") +''' + +FAKE_SIGN_UPDATE = SIGNATURE + r''' +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +key = sys.stdin.read() +with open(os.environ["FAKE_LOG"], "a") as log: + log.write(json.dumps(["sign_update", *args, "stdin=" + key.strip()]) + "\n") +path = Path(args[-1]) +text = path.read_text() +if "--verify" in args: + if os.environ.get("FAKE_VERIFY_FAIL") or signed(text, key) != text: + print("Error: failed to pass signing verification.") + sys.exit(1) + sys.exit(0) +path.write_text(signed(text, key)) +print("") +''' + +SEED = """ + + + TablePro + + 0.81.0 + + 140 + 0.81.0 + + + + + 0.80.0 + 21600 + 139 + 0.80.0 + + + + +""" +METADATA = { + "retainFromPluginKit": 39, + "plugins": { + "OracleDriver": {"name": "Oracle Driver", "iconName": "oracle-icon"}, + "ParquetExport": {"name": "Parquet Export", "iconName": "parquet-icon"}, + }, +} +PROJECT = """aggregateTargets: + BundledPlugins: + targets: + - MySQLDriver +""" + + +def item(build, short, channel=None, critical=None, signature="c2ln", length="10"): + lines = [""] + if channel is not None: + lines.append(f"{channel}") + if critical is not None: + lines.append(f'' if critical else + "") + lines += [f"{build}", + f"{short}"] + signed = f' sparkle:edSignature="{signature}"' if signature else "" + lines += [f'', ""] + return "\n".join(lines) + + +def appcast(*items): + return (f'' + f'TablePro{"".join(items)}').encode() + + +class FeedRulesTests(unittest.TestCase): + def test_a_clean_feed_passes(self): + self.assertEqual(feed.feed_problems(appcast(item(820001, "0.82.0-beta.1", "beta"), item(140, "0.81.0"))), []) + + def test_each_rule(self): + cases = [ + (appcast(item(140, "0.81.0", signature="")), "has no sparkle:edSignature"), + (appcast(item(140, "0.81.0", length="")), "has no length"), + (appcast(item("1.4", "0.81.0")), "sparkle:version '1.4' is not an integer"), + (appcast(item(140, "0.81")), "shortVersionString '0.81' is not a release version"), + (appcast(item(820001, "0.82.0-beta.1", "alpha")), "the only channel is beta"), + (appcast(item(820001, "0.82.0-beta.1", "beta", critical="")), "a beta marked critical"), + (appcast(item(141, "0.81.0"), item(140, "0.82.0")), "0.81.0 has the higher build (141 over 140)"), + (appcast(item(140, "0.81.0"), item(140, "0.80.0")), "build 140 carries 0.80.0, 0.81.0"), + ] + for data, message in cases: + with self.subTest(message=message): + self.assertTrue(any(message in problem for problem in feed.feed_problems(data)), + feed.feed_problems(data)) + + def test_the_last_default_channel_item_cannot_go(self): + before = appcast(item(140, "0.81.0")) + after = appcast(item(820001, "0.82.0-beta.1", "beta")) + self.assertIn("the feed no longer has a default-channel item", feed.feed_problems(after, before)) + + def test_the_new_item_must_be_present(self): + release = version.parse_tag("v0.82.0-beta.1") + url = "https://example.com/TablePro-0.82.0-beta.1.tar.xz" + self.assertEqual(feed.feed_problems(appcast(item(820001, "0.82.0-beta.1", "beta")), None, + feed.Expected(release, url)), []) + problems = feed.feed_problems(appcast(item(820001, "0.82.0-beta.1")), None, feed.Expected(release, url)) + self.assertEqual(problems, [f"no beta item for 0.82.0-beta.1 (820001) at {url}"]) + + def test_the_critical_build_carries_forward(self): + items = feed.parse_items(appcast(item(820099, "0.82.0", critical="140"), item(140, "0.81.0", critical=""), + item(830001, "0.83.0-beta.1", "beta"))) + self.assertEqual(feed.carried_critical(items), 140) + self.assertEqual(feed.carried_critical(feed.parse_items(SEED.encode())), 140) + self.assertIsNone(feed.carried_critical(feed.parse_items(appcast(item(139, "0.80.0"))))) + + def test_the_generate_appcast_arguments(self): + stage = Path("/stage") + prefix = ["--ed-key-file", "-", "--embed-release-notes", "--maximum-deltas", "0"] + beta = feed.appcast_arguments(stage, "v0.82.0-beta.1", version.parse_tag("v0.82.0-beta.1"), None, REPOSITORY) + self.assertEqual(beta, prefix + [ + "--versions", "820001", + "--download-url-prefix", "https://github.com/TableProApp/TablePro/releases/download/v0.82.0-beta.1/", + "--full-release-notes-url", "https://github.com/TableProApp/TablePro/releases/tag/v0.82.0-beta.1", + "--channel", "beta", "/stage"]) + stable = feed.appcast_arguments(stage, "v0.82.0", version.parse_tag("v0.82.0"), 140, REPOSITORY) + self.assertEqual(stable[stable.index("--phased-rollout-interval"):], + ["--phased-rollout-interval", "21600", "--critical-update-version", "140", "/stage"]) + plain = feed.appcast_arguments(stage, "v0.82.1", version.parse_tag("v0.82.1"), None, REPOSITORY) + self.assertNotIn("--critical-update-version", plain) + self.assertNotIn("--channel", plain) + + def test_removes_items_by_version_and_keeps_the_rest_of_the_text(self): + text = SEED + edited, removed = feed.remove_items(text, "0.80.0") + self.assertEqual(removed, 1) + self.assertEqual([each.short_version for each in feed.parse_items(edited.encode())], ["0.81.0"]) + self.assertEqual(edited, text[:text.index(" \n 0.80.0")] + + text[text.index(" </channel>"):]) + + def test_refuses_to_edit_text_it_cannot_match(self): + text = SEED.replace("# 0.81.0", "# 0.81.0 </item> in the notes") + with self.assertRaises(feed.FeedError): + feed.remove_items(text, "0.81.0") + + +def git(path, *args): + return subprocess.run(["git", "-C", str(path), *args], env={**os.environ, **GIT_ENV}, check=True, + capture_output=True, text=True).stdout.strip() + + +def fake_signed(text, key=KEY): + namespace = {} + exec(SIGNATURE, namespace) + return namespace["signed"](text, key) + + +def app_archive(path, text, build): + info = plistlib.dumps({"CFBundleShortVersionString": text, "CFBundleVersion": str(build)}) + with tarfile.open(path, "w:xz") as archive: + member = tarfile.TarInfo("TablePro.app/Contents/Info.plist") + member.size = len(info) + archive.addfile(member, io.BytesIO(info)) + + +def plugin_zip(path, target, text, build, kit=39, **extra): + info = {"CFBundleIdentifier": f"com.TablePro.{target}", "CFBundleShortVersionString": text, + "CFBundleVersion": str(build), "TableProPluginKitVersion": kit, "LSMinimumSystemVersion": "13.0", + **extra} + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(f"{target}.tableplugin/Contents/Info.plist", plistlib.dumps(info)) + + +class FeedBranchTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.base = Path(directory.name) + self.tools = self.base / "bin" + self.tools.mkdir() + for name, source in (("gh", FAKE_GH), ("generate_appcast", FAKE_GENERATE_APPCAST), + ("sign_update", FAKE_SIGN_UPDATE)): + path = self.tools / name + path.write_text(f"#!{sys.executable}\n{source}") + path.chmod(0o755) + self.log_path = self.base / "calls.log" + self.releases = self.base / "releases" + environment = mock.patch.dict(os.environ, { + **GIT_ENV, + "PATH": f"{self.tools}{os.pathsep}{os.environ['PATH']}", + "FAKE_LOG": str(self.log_path), + "FAKE_RELEASES": str(self.releases), + }) + environment.start() + self.addCleanup(environment.stop) + + self.origin = self.base / "origin.git" + git(self.base, "init", "--quiet", "--bare", "--initial-branch=main", str(self.origin)) + seed = self.base / "seed" + git(self.base, "init", "--quiet", "--initial-branch=feeds", str(seed)) + (seed / "appcast.xml").write_text(SEED) + (seed / "plugins").mkdir() + (seed / "plugins/v3.json").write_text(json.dumps(registry.empty_manifest(39))) + git(seed, "add", ".") + git(seed, "commit", "--quiet", "-m", "seed") + git(seed, "push", "--quiet", str(self.origin), "feeds") + + self.root = self.base / "tag" + git(self.base, "init", "--quiet", "--initial-branch=main", str(self.root)) + (self.root / ".github").mkdir() + (self.root / ".github/plugin-registry.json").write_text(json.dumps(METADATA)) + (self.root / "Configs").mkdir() + (self.root / "Configs/PluginKit.xcconfig").write_text( + "TABLEPRO_PLUGINKIT_VERSION = 39\nTABLEPRO_PLUGINKIT_MINIMUM_VERSION = 19\n") + (self.root / "project.yml").write_text(PROJECT) + git(self.root, "add", ".") + git(self.root, "commit", "--quiet", "-m", "tag") + git(self.root, "remote", "add", "origin", str(self.origin)) + self.notes = self.base / "feed.md" + self.notes.write_text("- A fix. (#1)\n") + + def publish(self, tag, kit=39, archive=None): + release = version.parse_tag(tag) + folder = self.releases / tag + folder.mkdir(parents=True) + app_archive(folder / f"TablePro-{release}.tar.xz", *(archive or (str(release), release.build))) + plugin_zip(folder / f"OracleDriver-{release}.zip", "OracleDriver", str(release), release.build, kit, + TableProProvidesDatabaseTypeIds=["Oracle"]) + plugin_zip(folder / f"ParquetExport-{release}.zip", "ParquetExport", str(release), release.build, kit, + TableProProvidesExportFormatIds=["parquet"]) + + def add(self, tag, critical=False): + release = version.parse_tag(tag) + messages = [] + changed = feed.add(self.root, tag, release.build, release.channel, self.notes, critical, self.tools, KEY, + feed.Releases(REPOSITORY), REPOSITORY, log=messages.append) + return changed, messages + + def replace_appcast(self, text): + clone = self.base / "replace" + git(self.base, "clone", "--quiet", "--branch", "feeds", str(self.origin), str(clone)) + (clone / "appcast.xml").write_text(text) + git(clone, "commit", "--quiet", "-am", "replace the appcast") + git(clone, "push", "--quiet", "origin", "feeds") + + def calls(self, tool): + if not self.log_path.exists(): + return [] + return [call[1:] for call in map(json.loads, self.log_path.read_text().splitlines()) if call[0] == tool] + + def published(self, path): + return subprocess.run(["git", "-C", str(self.origin), "show", f"feeds:{path}"], check=True, + capture_output=True, text=True).stdout + + def history(self): + return git(self.origin, "log", "--format=%an <%ae> %s", "feeds").splitlines() + + def test_adds_a_beta_and_its_plugins(self): + self.publish("v0.82.0-beta.1") + changed, _ = self.add("v0.82.0-beta.1") + self.assertTrue(changed) + self.assertEqual(self.history(), [ + "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> release: v0.82.0-beta.1", + "Fixture <fixture@example.com> seed", + ]) + (generate,) = self.calls("generate_appcast") + self.assertEqual(generate[-1], f"stdin={KEY}") + signing = [call for call in self.calls("sign_update") if "--verify" not in call] + self.assertEqual([call[:2] + call[-1:] for call in signing], [["--ed-key-file", "-", f"stdin={KEY}"]]) + self.assertIn("--channel", generate) + self.assertNotIn("--critical-update-version", generate) + self.assertNotIn("--phased-rollout-interval", generate) + items = feed.parse_items(self.published("appcast.xml").encode()) + self.assertEqual([(each.short_version, each.channel) for each in items], + [("0.82.0-beta.1", "beta"), ("0.81.0", None), ("0.80.0", None)]) + manifest = json.loads(self.published("plugins/v3.json")) + self.assertEqual([plugin["id"] for plugin in manifest["plugins"]], + ["com.TablePro.OracleDriver", "com.TablePro.ParquetExport"]) + oracle = manifest["plugins"][0] + self.assertEqual(oracle["databaseTypeIds"], ["Oracle"]) + self.assertEqual(oracle["releases"][0]["url"], + "https://github.com/TableProApp/TablePro/releases/download/v0.82.0-beta.1/" + "OracleDriver-0.82.0-beta.1.zip") + self.assertEqual(oracle["releases"][0]["channel"], "beta") + + def test_a_stable_carries_the_critical_build_and_phases(self): + self.publish("v0.82.0") + self.add("v0.82.0") + (generate,) = self.calls("generate_appcast") + self.assertEqual(generate[generate.index("--phased-rollout-interval") + 1], "21600") + self.assertEqual(generate[generate.index("--critical-update-version") + 1], "140") + + def test_a_critical_stable_is_critical_below_its_own_build(self): + self.publish("v0.82.0") + self.add("v0.82.0", critical=True) + (generate,) = self.calls("generate_appcast") + self.assertEqual(generate[generate.index("--critical-update-version") + 1], "820099") + + def test_a_critical_beta_is_refused(self): + with self.assertRaisesRegex(feed.FeedError, "a beta is never critical"): + self.add("v0.82.0-beta.1", critical=True) + + def test_a_build_or_channel_that_disagrees_with_the_tag_is_refused(self): + with self.assertRaisesRegex(feed.FeedError, "is build 820001 on beta"): + feed.add(self.root, "v0.82.0-beta.1", 820099, "stable", self.notes, False, self.tools, KEY, + feed.Releases(REPOSITORY), REPOSITORY, log=lambda message: None) + + def test_a_rerun_changes_nothing(self): + self.publish("v0.82.0-beta.1") + self.add("v0.82.0-beta.1") + head = git(self.origin, "rev-parse", "feeds") + changed, messages = self.add("v0.82.0-beta.1") + self.assertFalse(changed) + self.assertEqual(git(self.origin, "rev-parse", "feeds"), head) + self.assertEqual(len(self.calls("generate_appcast")), 1) + self.assertIn("appcast.xml already has 0.82.0-beta.1 (820001)", messages) + + def test_starts_again_when_another_run_pushed_first(self): + self.publish("v0.82.0-beta.1") + race = self.base / "race" + git(self.base, "clone", "--quiet", "--branch", "feeds", str(self.origin), str(race)) + (race / "other.txt").write_text("another run\n") + git(race, "add", "other.txt") + git(race, "commit", "--quiet", "-m", "release: v0.81.1") + with mock.patch.dict(os.environ, {"FAKE_RACE_REPO": str(race)}): + changed, messages = self.add("v0.82.0-beta.1") + self.assertTrue(changed) + self.assertEqual([line.split("> ")[1] for line in self.history()], + ["release: v0.82.0-beta.1", "release: v0.81.1", "seed"]) + self.assertEqual(len(self.calls("generate_appcast")), 2) + self.assertTrue(any("attempt 1 of 3" in message for message in messages)) + self.assertEqual(self.published("other.txt"), "another run\n") + + def test_an_archive_of_another_build_is_refused(self): + self.publish("v0.82.0-beta.1", archive=("0.82.0-beta.1", 140)) + with self.assertRaisesRegex(feed.FeedError, r"holds 0.82.0-beta.1 \(140\), not 0.82.0-beta.1 \(820001\)"): + self.add("v0.82.0-beta.1") + self.assertEqual(self.calls("generate_appcast"), []) + + def test_an_item_sparkle_prunes_on_arrival_leaves_the_appcast_alone(self): + self.replace_appcast(fake_signed(SEED)) + self.publish("v0.79.1") + with mock.patch.dict(os.environ, {"FAKE_PRUNE": "1"}): + changed, messages = self.add("v0.79.1") + self.assertTrue(changed) + self.assertEqual(self.published("appcast.xml"), fake_signed(SEED)) + self.assertIn("generate_appcast keeps newer items on 0.79.1's branch, so appcast.xml stays as it is", messages) + self.assertEqual(len(json.loads(self.published("plugins/v3.json"))["plugins"]), 2) + + def test_a_failed_check_pushes_nothing(self): + self.publish("v0.82.0-beta.1") + with mock.patch.dict(os.environ, {"FAKE_UNSIGNED_ENCLOSURE": "1"}): + with self.assertRaisesRegex(feed.FeedError, "0.82.0-beta.1's enclosure has no sparkle:edSignature"): + self.add("v0.82.0-beta.1") + self.assertEqual(len(self.history()), 1) + + def test_a_rejected_signature_pushes_nothing(self): + self.publish("v0.82.0-beta.1") + with mock.patch.dict(os.environ, {"FAKE_VERIFY_FAIL": "1"}): + with self.assertRaisesRegex(feed.FeedError, "sign_update failed"): + self.add("v0.82.0-beta.1") + self.assertEqual(len(self.history()), 1) + + def test_a_wrong_digest_is_refused(self): + self.publish("v0.82.0-beta.1") + with mock.patch.dict(os.environ, {"FAKE_BAD_DIGEST": "TablePro-0.82.0-beta.1.tar.xz"}): + with self.assertRaisesRegex(feed.FeedError, "but v0.82.0-beta.1 lists sha256:0000"): + self.add("v0.82.0-beta.1") + self.assertEqual(self.calls("generate_appcast"), []) + + def test_a_draft_is_refused(self): + self.publish("v0.82.0-beta.1") + with mock.patch.dict(os.environ, {"FAKE_DRAFT": "1"}): + with self.assertRaisesRegex(feed.FeedError, "is a draft"): + self.add("v0.82.0-beta.1") + + def test_a_plugin_built_against_another_kit_is_refused(self): + self.publish("v0.82.0-beta.1", kit=38) + with self.assertRaisesRegex(registry.RegistryError, "TableProPluginKitVersion is 38"): + self.add("v0.82.0-beta.1") + self.assertEqual(len(self.history()), 1) + + def test_withdraw_removes_the_version_and_re_signs(self): + self.publish("v0.82.0-beta.1") + self.publish("v0.82.0") + self.add("v0.82.0-beta.1") + self.add("v0.82.0") + signed_before = sum("--verify" not in call for call in self.calls("sign_update")) + self.assertTrue(feed.withdraw(self.root, "0.82.0", self.tools, KEY, log=lambda message: None)) + self.assertEqual(self.history()[0].split("> ")[1], "withdraw: v0.82.0") + published = self.published("appcast.xml") + self.assertEqual([each.short_version for each in feed.parse_items(published.encode())], + ["0.82.0-beta.1", "0.81.0", "0.80.0"]) + self.assertEqual(fake_signed(published), published) + self.assertEqual(sum("--verify" not in call for call in self.calls("sign_update")), signed_before + 1) + self.assertEqual(json.loads(self.published("plugins/v3.json"))["plugins"], []) + self.assertFalse(feed.withdraw(self.root, "0.82.0", self.tools, KEY, log=lambda message: None)) + + def test_withdraw_keeps_a_default_channel_item(self): + self.replace_appcast(fake_signed(feed.remove_items(SEED, "0.80.0")[0])) + with self.assertRaisesRegex(feed.FeedError, "no default-channel item"): + feed.withdraw(self.root, "0.81.0", self.tools, KEY, log=lambda message: None) + + def test_latest_after_a_withdraw_reads_the_pushed_branch(self): + self.publish("v0.82.0-beta.1") + self.publish("v0.82.0") + self.add("v0.82.0-beta.1") + self.add("v0.82.0") + before = git(self.origin, "rev-parse", "feeds") + feed.withdraw(self.root, "0.82.0", self.tools, KEY, log=lambda message: None) + git(self.root, "branch", "feeds", before) + git(self.root, "update-ref", feed.REMOTE_BRANCH, before) + self.assertEqual([each.short_version for each in feed.published_items(self.root)], + ["0.82.0-beta.1", "0.81.0", "0.80.0"]) + + git(self.root, "update-ref", feed.REMOTE_BRANCH, before) + with mock.patch.object(publish, "ROOT", self.root), \ + mock.patch.dict(os.environ, {"GITHUB_REPOSITORY": REPOSITORY}), redirect_stderr(io.StringIO()): + self.assertEqual(publish.main(["latest"]), 0) + self.assertEqual(self.calls("gh")[-1], ["release", "edit", "v0.81.0", "--latest=true", "--repo", REPOSITORY]) + + def test_check_reads_the_published_branch(self): + self.assertTrue(any("sign_update failed" in problem for problem in feed.check(self.root, self.tools, KEY))) + self.publish("v0.82.0-beta.1") + self.add("v0.82.0-beta.1") + self.assertEqual(feed.check(self.root, self.tools, KEY), []) + + def test_main_needs_the_sparkle_tools_and_key(self): + with mock.patch.dict(os.environ, {"SPARKLE_BIN": "", "SPARKLE_PRIVATE_KEY": KEY}), \ + redirect_stderr(io.StringIO()) as errors: + self.assertEqual(feed.main(["check"]), 1) + self.assertIn("SPARKLE_BIN is not set", errors.getvalue()) + with mock.patch.dict(os.environ, {"SPARKLE_BIN": str(self.tools), "SPARKLE_PRIVATE_KEY": ""}), \ + redirect_stderr(io.StringIO()) as errors: + self.assertEqual(feed.main(["check"]), 1) + self.assertIn("SPARKLE_PRIVATE_KEY is not set", errors.getvalue()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/release/test_publish.py b/scripts/release/test_publish.py new file mode 100644 index 0000000000..cdcb6cb161 --- /dev/null +++ b/scripts/release/test_publish.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 + +import hashlib +import io +import json +import os +import sys +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from unittest import mock + +import feed +import publish +import version + +REPOSITORY = "TableProApp/TablePro" + +FAKE_GH = r''' +import hashlib, json, os, sys +from pathlib import Path +args = sys.argv[1:] +state_path = Path(os.environ["FAKE_STATE"]) +state = json.loads(state_path.read_text()) +state["calls"].append(args) +releases = state["releases"] + +def save(): + state_path.write_text(json.dumps(state)) + +def fail(message): + save() + sys.exit(message) + +def option(name): + return args[args.index(name) + 1] if name in args else None + +def files(): + return [arg for arg in args[3:] if os.path.isfile(arg) and arg != option("--notes-file")] + +def upload(release, paths): + for path in paths: + digest = "sha256:" + hashlib.sha256(Path(path).read_bytes()).hexdigest() + if Path(path).name == os.environ.get("FAKE_CORRUPT"): + digest = "sha256:" + "0" * 64 + release["assets"][Path(path).name] = digest + +if args[0] == "api": + path = args[1] + if path.endswith("/releases/latest"): + if state["latest"] is None: + fail("gh: Not Found (HTTP 404)") + print(state["latest"]) + else: + tag = path.rsplit("/", 1)[1] + if tag not in releases or releases[tag]["draft"]: + fail("gh: Not Found (HTTP 404)") + print("false") +elif args[:2] == ["release", "view"]: + release = releases.get(args[2]) + if release is None: + fail("release not found") + if state.get("hide_drafts", 0) > 0: + state["hide_drafts"] -= 1 + fail("release not found") + assets = [{"name": name, "digest": digest, "state": "uploaded"} for name, digest in release["assets"].items()] + print(json.dumps({"isDraft": release["draft"], "assets": assets})) +elif args[:2] == ["release", "create"]: + if args[2] not in state["tags"]: + fail("tag " + args[2] + " doesn't exist in the repo") + release = {"draft": "--draft" in args, "prerelease": "--prerelease=true" in args, "latest": None, "assets": {}} + releases[args[2]] = release + upload(release, files()) +elif args[:2] == ["release", "upload"]: + upload(releases[args[2]], files()) +elif args[:2] == ["release", "edit"]: + release = releases[args[2]] + for arg in args[3:]: + if arg == "--draft=false": + release["draft"] = False + elif arg in ("--prerelease", "--prerelease=true"): + release["prerelease"] = True + elif arg == "--prerelease=false": + release["prerelease"] = False + elif arg == "--latest=true": + state["latest"] = args[2] + elif arg == "--latest=false": + release["latest"] = False +else: + fail("fake gh cannot answer " + " ".join(args)) +save() +''' + + +def items(*pairs): + return [feed.Item(build=str(build), short_version=short, channel=channel, is_critical=False, + critical_version=None, url="https://example.com", signature="c2ln", length="1") + for build, short, channel in pairs] + + +class DecisionTests(unittest.TestCase): + def test_the_draft_action(self): + self.assertEqual(publish.draft_action(True, False), "published") + self.assertEqual(publish.draft_action(True, True), "published") + self.assertEqual(publish.draft_action(False, True), "reuse") + self.assertEqual(publish.draft_action(False, False), "create") + + def test_latest_goes_only_to_the_highest_stable(self): + cases = [ + ("v0.82.0", "v0.81.0", True), + ("v0.82.0", None, True), + ("v0.82.0", "plugin-oracle-v1.0.0", True), + ("v0.81.1", "v0.82.0", False), + ("v0.82.0", "v0.82.0", False), + ("v0.83.0-beta.1", "v0.82.0", False), + ] + for tag, latest, expected in cases: + with self.subTest(tag=tag, latest=latest): + self.assertEqual(publish.should_be_latest(version.parse_tag(tag), latest), expected) + + def test_the_release_flags(self): + self.assertEqual(publish.release_flags(version.parse_tag("v0.82.0-beta.1"), "v0.81.0"), + ["--draft=false", "--prerelease", "--latest=false"]) + self.assertEqual(publish.release_flags(version.parse_tag("v0.82.0"), "v0.81.0"), + ["--draft=false", "--latest=true"]) + self.assertEqual(publish.release_flags(version.parse_tag("v0.81.1"), "v0.82.0"), + ["--draft=false", "--latest=false"]) + + def test_the_highest_stable_in_the_feed(self): + feed_items = items((830001, "0.83.0-beta.1", "beta"), (820199, "0.82.1", None), (820099, "0.82.0", None), + (140, "0.81.0", None)) + self.assertEqual(str(publish.highest_stable(feed_items)), "0.82.1") + self.assertEqual(str(publish.highest_stable(items((140, "0.81.0", None), (139, "0.80.0", None)))), "0.81.0") + self.assertIsNone(publish.highest_stable(items((830001, "0.83.0-beta.1", "beta")))) + + def test_asset_problems(self): + assets = [{"name": "a.zip", "digest": "sha256:" + "1" * 64, "state": "uploaded"}, + {"name": "b.zip", "digest": "sha256:" + "2" * 64, "state": "starter"}] + self.assertEqual(publish.asset_problems({"a.zip": "1" * 64}, assets), []) + self.assertEqual(publish.asset_problems({"a.zip": "3" * 64, "b.zip": "2" * 64, "c.zip": "4" * 64}, assets), [ + f"a.zip is sha256:{'1' * 64} on GitHub but sha256:{'3' * 64} here", + "b.zip is starter, not uploaded", + "the release has no c.zip", + ]) + + +class GitHubTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.base = Path(directory.name) + tools = self.base / "bin" + tools.mkdir() + (tools / "gh").write_text(f"#!{sys.executable}\n{FAKE_GH}") + (tools / "gh").chmod(0o755) + self.state_path = self.base / "state.json" + self.state({"releases": {"v0.81.0": {"draft": False, "prerelease": False, "assets": {}}}, + "latest": "v0.81.0", "tags": ["v0.81.0", "v0.82.0-beta.1", "v0.82.0", "v0.81.1"], "calls": []}) + environment = mock.patch.dict(os.environ, {"PATH": f"{tools}{os.pathsep}{os.environ['PATH']}", + "FAKE_STATE": str(self.state_path), + "GITHUB_REPOSITORY": REPOSITORY}) + environment.start() + self.addCleanup(environment.stop) + self.github = publish.GitHub(REPOSITORY) + self.notes = self.base / "body.md" + self.notes.write_text("Notes\n") + self.files = [] + for name in ("TablePro-0.82.0.dmg", "TablePro-0.82.0.tar.xz", "OracleDriver-0.82.0.zip"): + path = self.base / name + path.write_bytes(name.encode()) + self.files.append(path) + self.log = [] + + def state(self, value=None): + if value is not None: + self.state_path.write_text(json.dumps(value)) + return json.loads(self.state_path.read_text()) + + def verbs(self): + return [call[:2] if call[0] == "release" else call[:1] for call in self.state()["calls"]] + + def draft(self, tag="v0.82.0"): + return publish.draft(self.github, tag, self.notes, self.files, sleep=lambda seconds: None, + log=self.log.append) + + def digests(self): + return {path.name: "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() for path in self.files} + + def test_creates_a_draft_and_checks_its_digests(self): + self.assertEqual(self.draft(), "draft") + release = self.state()["releases"]["v0.82.0"] + self.assertEqual((release["draft"], release["prerelease"], release["assets"]), (True, False, self.digests())) + create = next(call for call in self.state()["calls"] if call[:2] == ["release", "create"]) + self.assertEqual(create[:9], ["release", "create", "v0.82.0", "--draft", "--verify-tag", "--title", "v0.82.0", + "--notes-file", str(self.notes)]) + + def test_a_beta_draft_is_a_prerelease(self): + self.files = [self.base / "TablePro-0.82.0.dmg"] + self.draft("v0.82.0-beta.1") + self.assertTrue(self.state()["releases"]["v0.82.0-beta.1"]["prerelease"]) + + def test_a_published_release_is_left_alone(self): + state = self.state() + state["releases"]["v0.82.0"] = {"draft": False, "prerelease": False, "assets": {}} + self.state(state) + self.assertEqual(self.draft(), "published") + self.assertEqual(self.verbs(), [["api"]]) + + def test_an_earlier_draft_is_reused(self): + self.draft() + state = self.state() + state["calls"] = [] + self.state(state) + self.files[0].write_bytes(b"rebuilt") + self.assertEqual(self.draft(), "draft") + self.assertNotIn(["release", "create"], self.verbs()) + self.assertIn(["release", "upload"], self.verbs()) + self.assertEqual(self.state()["releases"]["v0.82.0"]["assets"], self.digests()) + + def test_waits_for_a_new_draft_to_appear(self): + state = self.state() + state["hide_drafts"] = 2 + self.state(state) + self.assertEqual(self.draft(), "draft") + self.assertEqual(self.verbs().count(["release", "view"]), 4) + + def test_a_draft_that_never_appears_is_an_error(self): + state = self.state() + state["hide_drafts"] = publish.FIND_ATTEMPTS + self.state(state) + with self.assertRaisesRegex(publish.PublishError, "cannot be found"): + self.draft() + + def test_a_wrong_digest_keeps_the_draft_unpublished(self): + with mock.patch.dict(os.environ, {"FAKE_CORRUPT": "OracleDriver-0.82.0.zip"}): + with self.assertRaisesRegex(publish.PublishError, "OracleDriver-0.82.0.zip is sha256:0000"): + self.draft() + self.assertTrue(self.state()["releases"]["v0.82.0"]["draft"]) + + def test_a_missing_file_is_refused_before_any_call(self): + self.files.append(self.base / "missing.zip") + with self.assertRaisesRegex(publish.PublishError, "missing.zip does not exist"): + self.draft() + self.assertEqual(self.state()["calls"], []) + + def test_publishes_a_beta_as_a_prerelease_that_is_never_latest(self): + self.files = [self.base / "TablePro-0.82.0.dmg"] + self.draft("v0.82.0-beta.1") + publish.release(self.github, "v0.82.0-beta.1", self.log.append) + state = self.state() + self.assertEqual(state["releases"]["v0.82.0-beta.1"]["draft"], False) + self.assertEqual(state["latest"], "v0.81.0") + self.assertEqual(state["calls"][-1], ["release", "edit", "v0.82.0-beta.1", "--draft=false", "--prerelease", + "--latest=false", "--repo", REPOSITORY]) + + def test_a_stable_above_latest_becomes_latest(self): + self.draft() + publish.release(self.github, "v0.82.0", self.log.append) + self.assertEqual(self.state()["latest"], "v0.82.0") + + def test_a_patch_on_an_older_line_is_not_latest(self): + state = self.state() + state["latest"] = "v0.82.0" + self.state(state) + self.draft("v0.81.1") + publish.release(self.github, "v0.81.1", self.log.append) + self.assertEqual(self.state()["latest"], "v0.82.0") + self.assertIn("--latest=false", self.state()["calls"][-1]) + + def test_releasing_twice_edits_once(self): + self.draft() + publish.release(self.github, "v0.82.0", self.log.append) + publish.release(self.github, "v0.82.0", self.log.append) + self.assertEqual(sum(call[:2] == ["release", "edit"] for call in self.state()["calls"]), 1) + + def test_releasing_without_a_draft_is_refused(self): + with self.assertRaisesRegex(publish.PublishError, "has no draft"): + publish.release(self.github, "v0.82.0", self.log.append) + + def test_latest_marks_the_highest_stable_left_in_the_feed(self): + feed_items = items((830001, "0.83.0-beta.1", "beta"), (140, "0.81.0", None)) + self.assertEqual(publish.latest(self.github, feed_items, self.log.append), "v0.81.0") + self.assertEqual(self.state()["calls"][-1], ["release", "edit", "v0.81.0", "--latest=true", + "--repo", REPOSITORY]) + + def test_draft_prints_only_the_state_on_stdout(self): + arguments = ["draft", "v0.82.0", "--notes", str(self.notes), *[str(path) for path in self.files]] + with redirect_stdout(io.StringIO()) as output, redirect_stderr(io.StringIO()): + self.assertEqual(publish.main(arguments), 0) + self.assertEqual(output.getvalue(), "state=draft\n") + with redirect_stdout(io.StringIO()) as output, redirect_stderr(io.StringIO()): + self.assertEqual(publish.main(["release", "v0.82.0"]), 0) + self.assertEqual(publish.main(arguments), 0) + self.assertEqual(output.getvalue(), "state=published\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/release/test_registry.py b/scripts/release/test_registry.py new file mode 100644 index 0000000000..53eec9d538 --- /dev/null +++ b/scripts/release/test_registry.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 + +import io +import json +import plistlib +import tempfile +import unittest +import zipfile +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +import registry +import version + +URL_BASE = "https://github.com/TableProApp/TablePro/releases/download/v0.82.0/" +PROJECT = """\ +targets: + OracleDriver: + templates: [DriverPlugin] + +aggregateTargets: + BundledPlugins: + targets: + - MySQLDriver + - SQLiteDriver + + RegistryPlugins: + targets: + - OracleDriver + - ParquetExport + scheme: {} +""" +METADATA = { + "retainFromPluginKit": 39, + "plugins": { + "OracleDriver": {"name": "Oracle Driver", "iconName": "oracle-icon"}, + "ParquetExport": {"name": "Parquet Export", "iconName": "parquet-icon"}, + }, +} + + +def entry(text="0.82.0", kit=39, system="13.0", sha="a", plugin="com.TablePro.OracleDriver", name="Oracle Driver"): + return { + "id": plugin, "name": name, "iconName": "oracle-icon", "databaseTypeIds": ["Oracle"], + "version": text, "build": version.parse_version(text).build, "pluginKit": kit, + "minimumSystemVersion": system, "file": f"OracleDriver-{text}.zip", "sha256": sha * 64, + } + + +def select(manifest, plugin, channels, kit, system): + def system_key(text): + return tuple(int(part) for part in text.split(".")) + + releases = next(entry["releases"] for entry in manifest["plugins"] if entry["id"] == plugin) + eligible = [release for release in releases if release["channel"] in channels and release["pluginKit"] == kit + and system_key(release["minimumSystemVersion"]) <= system_key(system)] + return max(eligible, key=lambda release: release["build"])["version"] if eligible else None + + +class ManifestCase(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.path = Path(directory.name) / "plugins" / "v3.json" + self.log = [] + + def add(self, entries, channel="stable", retain_from=39): + return registry.add(self.path, entries, channel, URL_BASE, retain_from, self.log.append) + + def manifest(self): + return json.loads(self.path.read_text(encoding="utf-8")) + + def builds(self, plugin="com.TablePro.OracleDriver"): + return [release["build"] for entry in self.manifest()["plugins"] if entry["id"] == plugin + for release in entry["releases"]] + + +class AddTests(ManifestCase): + def test_creates_the_manifest_and_an_entry(self): + self.assertTrue(self.add([entry()])) + self.assertEqual(self.manifest(), { + "schemaVersion": 3, + "retainFromPluginKit": 39, + "plugins": [{ + "id": "com.TablePro.OracleDriver", "name": "Oracle Driver", "iconName": "oracle-icon", + "databaseTypeIds": ["Oracle"], + "releases": [{ + "version": "0.82.0", "build": 820099, "channel": "stable", "pluginKit": 39, + "minimumSystemVersion": "13.0", "url": URL_BASE + "OracleDriver-0.82.0.zip", "sha256": "a" * 64, + }], + }], + }) + + def test_a_rerun_is_a_no_op(self): + self.add([entry()]) + written = self.path.read_bytes() + self.assertFalse(self.add([entry()])) + self.assertEqual(self.path.read_bytes(), written) + + def test_a_changed_sha256_for_the_same_build_is_an_error(self): + self.add([entry()]) + with self.assertRaisesRegex(registry.RegistryError, "already has build 820099"): + self.add([entry(sha="b")]) + + def test_keeps_two_builds_per_channel_kit_and_system(self): + for text in ("0.82.0", "0.82.1", "0.82.2"): + self.add([entry(text)]) + self.assertEqual(self.builds(), [820299, 820199]) + + def test_a_beta_goes_once_a_higher_stable_shares_its_kit_and_system(self): + self.add([entry("0.82.0-beta.1")], channel="beta") + self.add([entry("0.82.0-beta.2")], channel="beta") + self.assertEqual(self.builds(), [820002, 820001]) + self.add([entry("0.82.0")]) + self.assertEqual(self.builds(), [820099]) + + def test_a_beta_on_another_kit_survives_a_stable(self): + self.add([entry("0.83.0-beta.1", kit=40)], channel="beta") + self.add([entry("0.82.1")]) + self.assertEqual(self.builds(), [830001, 820199]) + + def test_a_higher_minimum_system_keeps_the_older_build(self): + self.add([entry("0.82.0")]) + self.add([entry("0.82.1", system="14.0")]) + self.add([entry("0.82.2", system="14.0")]) + self.assertEqual(self.builds(), [820299, 820199, 820099]) + manifest = self.manifest() + self.assertEqual(select(manifest, "com.TablePro.OracleDriver", {"stable"}, 39, "13.5"), "0.82.0") + self.assertEqual(select(manifest, "com.TablePro.OracleDriver", {"stable"}, 39, "14.1"), "0.82.2") + + def test_releases_below_the_floor_are_pruned(self): + self.add([entry("0.82.0", kit=38)], retain_from=38) + self.add([entry("0.83.0", kit=39)], retain_from=39) + self.assertEqual(self.builds(), [830099]) + self.assertEqual(self.manifest()["retainFromPluginKit"], 39) + + def test_an_older_line_never_lowers_the_floor(self): + self.add([entry("0.83.0", kit=40)], retain_from=40) + self.add([entry("0.82.1", kit=40)], retain_from=39) + self.assertEqual(self.manifest()["retainFromPluginKit"], 40) + + def test_a_release_pruned_on_arrival_is_a_logged_no_op(self): + self.add([entry("0.83.0", kit=40)], retain_from=40) + written = self.path.read_bytes() + self.assertFalse(self.add([entry("0.82.1", kit=39)], retain_from=39)) + self.assertEqual(self.path.read_bytes(), written) + self.assertIn("com.TablePro.OracleDriver 0.82.1: pruned on arrival by retention, not added", self.log) + + def test_metadata_comes_only_from_the_highest_build(self): + newer = entry("0.83.0-beta.1", kit=40) + newer["name"] = "Oracle" + self.add([newer], channel="beta") + older = entry("0.82.1") + older["name"] = "Oracle Driver (old)" + self.add([older]) + self.assertEqual(self.manifest()["plugins"][0]["name"], "Oracle") + newest = entry("0.83.0", kit=40) + newest["name"] = "Oracle Database" + self.add([newest]) + self.assertEqual(self.manifest()["plugins"][0]["name"], "Oracle Database") + + def test_plugins_sort_by_id_and_releases_by_build(self): + self.add([entry("0.82.0", plugin="com.TablePro.ZDriver"), entry("0.82.0", plugin="com.TablePro.ADriver")]) + self.add([entry("0.82.1", plugin="com.TablePro.ADriver")]) + manifest = self.manifest() + self.assertEqual([plugin["id"] for plugin in manifest["plugins"]], + ["com.TablePro.ADriver", "com.TablePro.ZDriver"]) + self.assertEqual([release["build"] for release in manifest["plugins"][0]["releases"]], [820199, 820099]) + + def test_another_schema_is_refused(self): + self.path.parent.mkdir(parents=True) + self.path.write_text(json.dumps({"schemaVersion": 4, "retainFromPluginKit": 39, "plugins": []})) + with self.assertRaisesRegex(registry.RegistryError, "schemaVersion 4"): + self.add([entry()]) + + def test_a_build_that_is_not_the_versions_build_is_refused(self): + wrong = entry() + wrong["build"] = 140 + with self.assertRaisesRegex(registry.RegistryError, "build 140 is not 820099"): + self.add([wrong]) + + def test_a_version_on_the_wrong_channel_is_refused(self): + with self.assertRaisesRegex(registry.RegistryError, "belongs to the beta channel"): + self.add([entry("0.82.0-beta.1")], channel="stable") + + +class WithdrawTests(ManifestCase): + def test_a_withdraw_brings_the_previous_build_back_into_selection(self): + self.add([entry("0.82.0")]) + self.add([entry("0.82.1")]) + self.assertEqual(select(self.manifest(), "com.TablePro.OracleDriver", {"stable"}, 39, "26.0"), "0.82.1") + self.assertTrue(registry.withdraw(self.path, "0.82.1", self.log.append)) + self.assertEqual(select(self.manifest(), "com.TablePro.OracleDriver", {"stable"}, 39, "26.0"), "0.82.0") + + def test_withdrawing_a_missing_version_changes_nothing(self): + self.add([entry("0.82.0")]) + written = self.path.read_bytes() + self.assertFalse(registry.withdraw(self.path, "0.82.5", self.log.append)) + self.assertEqual(self.path.read_bytes(), written) + + def test_a_plugin_left_with_no_release_leaves_the_manifest(self): + self.add([entry("0.82.0")]) + registry.withdraw(self.path, "0.82.0", self.log.append) + self.assertEqual(self.manifest()["plugins"], []) + + +class TargetsTests(unittest.TestCase): + def test_reads_the_bundled_aggregate(self): + self.assertEqual(registry.bundled_targets(PROJECT), {"MySQLDriver", "SQLiteDriver"}) + + def test_reads_the_bundled_aggregate_of_this_repository(self): + bundled = registry.bundled_targets(registry.PROJECT.read_text(encoding="utf-8")) + self.assertIn("MySQLDriver", bundled) + self.assertNotIn("OracleDriver", bundled) + + def test_prints_the_publish_set(self): + with tempfile.TemporaryDirectory() as directory: + metadata, project = Path(directory) / "registry.json", Path(directory) / "project.yml" + metadata.write_text(json.dumps(METADATA)) + project.write_text(PROJECT) + with redirect_stdout(io.StringIO()) as output: + self.assertEqual(registry.main(["targets", "--metadata", str(metadata), "--project", str(project)]), 0) + self.assertEqual(output.getvalue(), "OracleDriver\nParquetExport\n") + + def test_refuses_a_bundled_target(self): + metadata = {"retainFromPluginKit": 39, "plugins": {**METADATA["plugins"], + "MySQLDriver": {"name": "MySQL", "iconName": "mysql"}}} + with self.assertRaisesRegex(registry.RegistryError, "the app bundles MySQLDriver"): + registry.publish_set(metadata, registry.bundled_targets(PROJECT)) + + def test_refuses_metadata_in_another_shape(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "registry.json" + path.write_text(json.dumps({"plugins": {"oracle": {"target": "OracleDriver"}}})) + with self.assertRaisesRegex(registry.RegistryError, "shape"): + registry.load_metadata(path) + + +class DescribeTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.zips = Path(directory.name) + + def plugin(self, target, **overrides): + info = { + "CFBundleIdentifier": f"com.TablePro.{target}", "CFBundleShortVersionString": "0.82.0", + "CFBundleVersion": "820099", "TableProPluginKitVersion": 39, "LSMinimumSystemVersion": "13.0", + "TableProProvidesDatabaseTypeIds": ["Oracle"], + } + info.update(overrides) + info = {key: value for key, value in info.items() if value is not None} + path = self.zips / f"{target}-0.82.0.zip" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr(f"{target}.tableplugin/Contents/Info.plist", plistlib.dumps(info)) + archive.writestr(f"{target}.tableplugin/Contents/MacOS/{target}", b"binary") + return path + + def describe(self, metadata=None): + return registry.describe(self.zips, "0.82.0", 820099, 39, metadata or METADATA, + registry.bundled_targets(PROJECT)) + + def test_reads_each_zip(self): + self.plugin("OracleDriver") + self.plugin("ParquetExport", CFBundleIdentifier="com.TablePro.ParquetExportPlugin", + TableProProvidesDatabaseTypeIds=None, TableProProvidesExportFormatIds=["parquet"]) + oracle, parquet = self.describe() + self.assertEqual(oracle["id"], "com.TablePro.OracleDriver") + self.assertEqual(oracle["databaseTypeIds"], ["Oracle"]) + self.assertEqual((oracle["version"], oracle["build"], oracle["pluginKit"], oracle["minimumSystemVersion"]), + ("0.82.0", 820099, 39, "13.0")) + self.assertEqual(oracle["file"], "OracleDriver-0.82.0.zip") + self.assertRegex(oracle["sha256"], "^[0-9a-f]{64}$") + self.assertEqual((parquet["id"], parquet["databaseTypeIds"], parquet["name"]), + ("com.TablePro.ParquetExportPlugin", [], "Parquet Export")) + + def test_rejects_a_wrong_version_build_or_kit(self): + cases = [ + ({"CFBundleShortVersionString": "0.81.0"}, "CFBundleShortVersionString is '0.81.0'"), + ({"CFBundleVersion": "140"}, "CFBundleVersion is '140'"), + ({"TableProPluginKitVersion": 38}, "TableProPluginKitVersion is 38"), + ({"TableProPluginKitVersion": "39"}, "TableProPluginKitVersion is '39'"), + ] + self.plugin("ParquetExport", TableProProvidesExportFormatIds=["parquet"]) + for overrides, message in cases: + with self.subTest(overrides=overrides): + self.plugin("OracleDriver", **overrides) + with self.assertRaisesRegex(registry.RegistryError, message): + self.describe() + + def test_rejects_a_plugin_that_provides_nothing(self): + self.plugin("OracleDriver", TableProProvidesDatabaseTypeIds=None) + self.plugin("ParquetExport", TableProProvidesExportFormatIds=["parquet"]) + with self.assertRaisesRegex(registry.RegistryError, "OracleDriver-0.82.0.zip: it declares none of"): + self.describe() + + def test_rejects_a_missing_zip(self): + self.plugin("OracleDriver") + with self.assertRaisesRegex(registry.RegistryError, "ParquetExport-0.82.0.zip does not exist"): + self.describe() + + def test_rejects_two_plugins_with_one_bundle_id(self): + self.plugin("OracleDriver") + self.plugin("ParquetExport", CFBundleIdentifier="com.TablePro.OracleDriver") + with self.assertRaisesRegex(registry.RegistryError, "share the bundle id"): + self.describe() + + def test_the_cli_writes_entries_that_add_reads(self): + self.plugin("OracleDriver") + self.plugin("ParquetExport", TableProProvidesExportFormatIds=["parquet"]) + metadata, project = self.zips / "registry.json", self.zips / "project.yml" + metadata.write_text(json.dumps(METADATA)) + project.write_text(PROJECT) + entries, manifest = self.zips / "entries.json", self.zips / "v3.json" + with redirect_stdout(io.StringIO()), redirect_stderr(io.StringIO()): + self.assertEqual(registry.main([ + "describe", "--zips", str(self.zips), "--version", "0.82.0", "--build", "820099", "--kit", "39", + "--metadata", str(metadata), "--project", str(project), "--out", str(entries)]), 0) + self.assertEqual(registry.main([ + "add", "--manifest", str(manifest), "--entries", str(entries), "--channel", "stable", + "--url-base", URL_BASE, "--retain-from", "39"]), 0) + self.assertEqual(registry.main(["withdraw", "--manifest", str(manifest), "--version", "0.82.0"]), 0) + self.assertEqual(json.loads(manifest.read_text())["plugins"], []) + + +if __name__ == "__main__": + unittest.main()