From 18af5846fd5b3b137720dc8057e6e1b5fd6273f4 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 4 Sep 2026 19:06:01 +0000 Subject: [PATCH 1/6] feat(repo): compare RPM repository inventories --- docs/user/explanation/repos.md | 41 +++ docs/user/reference/cli/azldev_repo.md | 1 + .../user/reference/cli/azldev_repo_compare.md | 48 +++ docs/user/reference/config/resources.md | 5 + internal/app/azldev/cmds/repo/compare.go | 188 ++++++++++ .../azldev/cmds/repo/compare_internal_test.go | 152 ++++++++ internal/app/azldev/cmds/repo/repo.go | 1 + internal/app/azldev/cmds/repo/repo_test.go | 22 ++ internal/projectconfig/resources.go | 3 + internal/repo/repocompare/diff.go | 236 ++++++++++++ internal/repo/repocompare/diff_test.go | 101 +++++ internal/repo/repocompare/metadata.go | 348 ++++++++++++++++++ internal/repo/repocompare/metadata_test.go | 102 +++++ ...ainer_config_generate-schema_stdout_1.snap | 10 + ...shots_config_generate-schema_stdout_1.snap | 10 + schemas/azldev.schema.json | 5 + 16 files changed, 1273 insertions(+) create mode 100644 docs/user/reference/cli/azldev_repo_compare.md create mode 100644 internal/app/azldev/cmds/repo/compare.go create mode 100644 internal/app/azldev/cmds/repo/compare_internal_test.go create mode 100644 internal/repo/repocompare/diff.go create mode 100644 internal/repo/repocompare/diff_test.go create mode 100644 internal/repo/repocompare/metadata.go create mode 100644 internal/repo/repocompare/metadata_test.go diff --git a/docs/user/explanation/repos.md b/docs/user/explanation/repos.md index 193b5be9a..02ed90a09 100644 --- a/docs/user/explanation/repos.md +++ b/docs/user/explanation/repos.md @@ -94,6 +94,47 @@ Why split RPM-build vs image-build? They have different security envelopes: - mock evaluates `gpg-key` URIs *inside* the chroot, so a local file path is invisible. azldev rejects local `gpg-key` values for `rpm-build` repos. - kiwi runs on the host, so any URI form works for `image-build`. +### Compare Koji and PMC inventories + +Define each repository as a named repo set: + +```toml +[resources.rpm-repo-sets.koji-build] +template = "koji-dist-repo" +base-uri = "https://koji.example.com/repos/azl-build/latest" +disable-gpg-check = true +# For a trusted internal endpoint with an invalid certificate only: +# disable-ssl-verify = true + +[resources.rpm-repo-sets.pmc-prod] +template = "azl-standard" +base-uri = "https://packages.example.com/azurelinux/4.0/prod" +disable-gpg-check = true +``` + +Then compare their inventories: + +```sh +azldev repo compare \ + --left koji-build \ + --right pmc-prod \ + --arch x86_64,aarch64 +``` + +Each repo set is expanded with its selected template and respects its `arches` +and `subrepos` restrictions. The command can also compare two PMC trees, two +Koji dist-repos, or custom layouts. The report contains one row per differing +package name, summary statuses, and the complete left and right NEVR +inventories. `missing-from-right` means at least one left identity is absent +from the right; `added-in-right` means at least one right identity is absent +from the left. Identity includes name, normalized epoch, version, release, RPM +architecture, and artifact kind (binary, debug, or source). Matching NEVRs with +different architecture sets also receive `architectures-differ`. Replicated +`noarch` packages are counted once. + +This comparison checks inventory only. It does not compare RPM bytes, +signatures, checksums, or publication routing. + ## Load-time vs use-time Expansion is deterministic and happens once during `ProjectConfig.Validate()`, after **all** config files (project, user, `--config-file` extras) have been merged. This means: diff --git a/docs/user/reference/cli/azldev_repo.md b/docs/user/reference/cli/azldev_repo.md index e9132d0b8..06bedc36d 100644 --- a/docs/user/reference/cli/azldev_repo.md +++ b/docs/user/reference/cli/azldev_repo.md @@ -37,5 +37,6 @@ under one or more URL prefixes. ### SEE ALSO * [azldev](azldev.md) - 🐧 Azure Linux Dev Tool +* [azldev repo compare](azldev_repo_compare.md) - Compare package inventories in two RPM repo sets * [azldev repo query](azldev_repo_query.md) - Run dnf against auto-discovered RPM repos diff --git a/docs/user/reference/cli/azldev_repo_compare.md b/docs/user/reference/cli/azldev_repo_compare.md new file mode 100644 index 000000000..3d5d35b33 --- /dev/null +++ b/docs/user/reference/cli/azldev_repo_compare.md @@ -0,0 +1,48 @@ + + +## azldev repo compare + +Compare package inventories in two RPM repo sets + +### Synopsis + +Compare package identities in two configured RPM repo sets. + +The command expands each named [resources.rpm-repo-sets] entry using its selected +template. The report groups differences by package name and shows summary +statuses plus the complete left and right NEVR inventories. Package content is +not compared. + +``` +azldev repo compare [flags] +``` + +### Options + +``` + --arch strings comma-separated target architectures (default [x86_64,aarch64]) + -h, --help help for compare + --left string left [resources.rpm-repo-sets] name + --right string right [resources.rpm-repo-sets] name +``` + +### Options inherited from parent commands + +``` + -y, --accept-all accept all prompts + --color mode output colorization mode {always, auto, never} (default auto) + --config-file stringArray additional TOML config file(s) to merge (may be repeated) + -n, --dry-run dry run only (do not take action) + --network-retries int maximum number of attempts for network operations (minimum 1) (default 3) + --no-default-config disable default configuration + -O, --output-format fmt output format {csv, json, markdown, table} (default table) + --permissive-config do not fail on unknown fields in TOML config files + -C, --project string path to Azure Linux project + -q, --quiet only enable minimal output + -v, --verbose enable verbose output +``` + +### SEE ALSO + +* [azldev repo](azldev_repo.md) - Inspect and manage RPM repositories + diff --git a/docs/user/reference/config/resources.md b/docs/user/reference/config/resources.md index b5f338199..25fa161fe 100644 --- a/docs/user/reference/config/resources.md +++ b/docs/user/reference/config/resources.md @@ -98,6 +98,7 @@ Defined under `[resources.rpm-repo-sets.]`. | Name prefix | `name-prefix` | string | Prepended to each sub-repo's `name` to form the synthesized repo ID. May be empty. | | GPG key | `gpg-key` | string | Shared GPG key for sub-repos in this set; same shape rules as the per-repo `gpg-key` | | Disable GPG check | `disable-gpg-check` | bool | Opt out of GPG verification for sub-repos in this set | +| Disable SSL verification | `disable-ssl-verify` | bool | Disable TLS certificate verification for every sub-repo in this set. Use only for an explicitly trusted endpoint. | | Arches | `arches` | list of string | Restrict every synthesized repo in this set to specific architectures | | Sub-repos | `subrepos` | list of string | Allowlist of sub-repo names to include from the template. Empty/unset = include all sub-repos. Names must match entries in the referenced template. | @@ -129,6 +130,10 @@ subrepos = ["base", "base-src", "sdk", "sdk-src"] # binary + sources, no de This expands into four `RpmRepoResource` entries: `azl4-base`, `azl4-base-src`, `azl4-sdk`, `azl4-sdk-src`. +Named repo sets can also be compared directly with `azldev repo compare +--left --right `; see the [repository comparison +example](../../explanation/repos.md#compare-koji-and-pmc-inventories). + ## Merging across files Like other top-level maps, `rpm-repos`, `rpm-repo-set-templates`, and `rpm-repo-sets` are merged across config files **by key with wholesale entry replacement**. A duplicate name in a later-loaded file fully replaces the earlier definition (including any zero-value fields). This makes `--config-file` overrides predictable: setting `disable-gpg-check = false` (the zero value) will override an earlier `true`. diff --git a/internal/app/azldev/cmds/repo/compare.go b/internal/app/azldev/cmds/repo/compare.go new file mode 100644 index 000000000..d1e3c8bf1 --- /dev/null +++ b/internal/app/azldev/cmds/repo/compare.go @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package repo + +import ( + "errors" + "fmt" + "log/slog" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/repo/repocompare" + "github.com/microsoft/azure-linux-dev-tools/internal/repo/repolayout" + "github.com/spf13/cobra" +) + +// CompareOptions are the CLI flags for `azldev repo compare`. +type CompareOptions struct { + Left string + Right string + Arches []string +} + +func compareOnAppInit(_ *azldev.App, parentCmd *cobra.Command) { + parentCmd.AddCommand(NewCompareCmd()) +} + +// NewCompareCmd constructs the `azldev repo compare` command. +func NewCompareCmd() *cobra.Command { + var options CompareOptions + + cmd := &cobra.Command{ + Use: "compare", + Short: "Compare package inventories in two RPM repo sets", + Long: `Compare package identities in two configured RPM repo sets. + +The command expands each named [resources.rpm-repo-sets] entry using its selected +template. The report groups differences by package name and shows summary +statuses plus the complete left and right NEVR inventories. Package content is +not compared.`, + } + + cmd.RunE = azldev.RunFunc(func(env *azldev.Env) (interface{}, error) { + return RunCompare(env, &options) + }) + + cmd.Flags().StringVar(&options.Left, "left", "", "left [resources.rpm-repo-sets] name") + cmd.Flags().StringVar(&options.Right, "right", "", "right [resources.rpm-repo-sets] name") + cmd.Flags().StringSliceVar(&options.Arches, "arch", repolayout.DefaultArches, + "comma-separated target architectures") + + for _, name := range []string{"left", "right"} { + _ = cmd.MarkFlagRequired(name) + } + + return cmd +} + +// RunCompare loads both repository inventories and returns their package identity differences. +func RunCompare(env *azldev.Env, options *CompareOptions) ([]repocompare.PackageReport, error) { + fetcher := &repocompare.HTTPFetcher{Attempts: env.NetworkRetries()} + + return runCompare(env, options, fetcher) +} + +func runCompare( + env *azldev.Env, + options *CompareOptions, + fetcher repocompare.Fetcher, +) ([]repocompare.PackageReport, error) { + if options.Left == options.Right { + return nil, errors.New("'--left' and '--right' must name different rpm-repo-sets") + } + + leftRepositories, err := comparisonRepositories( + &env.Config().Resources, + "left", + options.Left, + options.Arches, + ) + if err != nil { + return nil, fmt.Errorf("resolving left repositories:\n%w", err) + } + + rightRepositories, err := comparisonRepositories( + &env.Config().Resources, + "right", + options.Right, + options.Arches, + ) + if err != nil { + return nil, fmt.Errorf("resolving right repositories:\n%w", err) + } + + leftPackages, err := repocompare.LoadRepositories(env, fetcher, leftRepositories) + if err != nil { + return nil, fmt.Errorf("loading left repositories:\n%w", err) + } + + rightPackages, err := repocompare.LoadRepositories(env, fetcher, rightRepositories) + if err != nil { + return nil, fmt.Errorf("loading right repositories:\n%w", err) + } + + reports, err := repocompare.Compare(leftPackages, rightPackages) + if err != nil { + return nil, fmt.Errorf("comparing repository inventories:\n%w", err) + } + + return reports, nil +} + +func comparisonRepositories( + resources *projectconfig.ResourcesConfig, + side string, + setName string, + arches []string, +) ([]repocompare.Repository, error) { + set, ok := resources.RpmRepoSets[setName] + if !ok { + return nil, fmt.Errorf("rpm-repo-set %#q is not defined", setName) + } + + template, err := repolayout.ResolveTemplate(resources.RpmRepoSetTemplates, set.Template) + if err != nil { + return nil, fmt.Errorf("rpm-repo-set %#q:\n%w", setName, err) + } + + if set.DisableSSLVerify { + slog.Warn("TLS certificate verification is disabled", "rpmRepoSet", setName) + } + + allowlist := make(map[string]struct{}, len(set.Subrepos)) + for _, name := range set.Subrepos { + allowlist[name] = struct{}{} + } + + expanded := repolayout.ExpandTemplate(set.BaseURI, set.Template, template, arches) + selected := make([]repolayout.InputRepo, 0, len(expanded)) + + for _, repo := range expanded { + if len(allowlist) > 0 { + if _, ok := allowlist[repo.SubrepoName]; !ok { + continue + } + } + + if repo.Arch != "" && !archAllowed(set.Arches, repo.Arch) { + continue + } + + selected = append(selected, repo) + } + + selected = repolayout.DedupInputRepos(selected) + repositories := make([]repocompare.Repository, 0, len(selected)) + + for _, repo := range selected { + repoID := side + "-" + repo.SubrepoName + if repo.Arch != "" { + repoID += "-" + repo.Arch + } + + repositories = append(repositories, repocompare.Repository{ + ID: repoID, + Kind: repo.Kind, + URL: repo.URL, + DisableSSLVerify: set.DisableSSLVerify, + }) + } + + return repositories, nil +} + +func archAllowed(allowlist []string, arch string) bool { + if len(allowlist) == 0 { + return true + } + + for _, allowed := range allowlist { + if allowed == arch { + return true + } + } + + return false +} diff --git a/internal/app/azldev/cmds/repo/compare_internal_test.go b/internal/app/azldev/cmds/repo/compare_internal_test.go new file mode 100644 index 000000000..89cd8b34d --- /dev/null +++ b/internal/app/azldev/cmds/repo/compare_internal_test.go @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package repo + +import ( + "context" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/repo/repocompare" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type comparisonMapFetcher map[string][]byte + +func (f comparisonMapFetcher) Fetch(_ context.Context, rawURL string, _ bool) ([]byte, error) { + return f[rawURL], nil +} + +func TestComparisonRepositoriesExpandsTemplate(t *testing.T) { + t.Parallel() + + resources := &projectconfig.ResourcesConfig{ + RpmRepoSetTemplates: map[string]projectconfig.RpmRepoSetTemplate{ + "layout": { + Subrepos: []projectconfig.SubrepoSpec{ + {Name: "binary", Subpath: "$basearch", Kind: projectconfig.SubrepoKindBinary}, + {Name: "src", Subpath: "src", Kind: projectconfig.SubrepoKindSource}, + }, + }, + }, + RpmRepoSets: map[string]projectconfig.RpmRepoSet{ + "build": { + Template: "layout", + BaseURI: "https://example.com/repos/build/latest", + DisableSSLVerify: true, + }, + }, + } + + repositories, err := comparisonRepositories(resources, "left", "build", []string{"x86_64", "aarch64"}) + require.NoError(t, err) + require.Len(t, repositories, 3) + + assert.Equal(t, "left-binary-x86_64", repositories[0].ID) + assert.Equal(t, "https://example.com/repos/build/latest/x86_64", repositories[0].URL) + assert.Equal(t, projectconfig.SubrepoKindBinary, repositories[0].Kind) + assert.True(t, repositories[0].DisableSSLVerify) + assert.Equal(t, "left-binary-aarch64", repositories[1].ID) + assert.Equal(t, "left-src", repositories[2].ID) + assert.Equal(t, "https://example.com/repos/build/latest/src", repositories[2].URL) +} + +func TestComparisonRepositoriesAppliesSetFilters(t *testing.T) { + t.Parallel() + + resources := &projectconfig.ResourcesConfig{ + RpmRepoSetTemplates: map[string]projectconfig.RpmRepoSetTemplate{ + "layout": { + Subrepos: []projectconfig.SubrepoSpec{ + {Name: "binary", Subpath: "$basearch", Kind: projectconfig.SubrepoKindBinary}, + {Name: "src", Subpath: "src", Kind: projectconfig.SubrepoKindSource}, + }, + }, + }, + RpmRepoSets: map[string]projectconfig.RpmRepoSet{ + "build": { + Template: "layout", + BaseURI: "https://example.com/repos/build/latest", + Arches: []string{"aarch64"}, + Subrepos: []string{"binary"}, + }, + }, + } + + repositories, err := comparisonRepositories(resources, "left", "build", []string{"x86_64", "aarch64"}) + require.NoError(t, err) + require.Len(t, repositories, 1) + + assert.Equal(t, "left-binary-aarch64", repositories[0].ID) +} + +func TestComparisonRepositoriesRejectsUnknownSet(t *testing.T) { + t.Parallel() + + _, err := comparisonRepositories(&projectconfig.ResourcesConfig{}, "left", "missing", []string{"x86_64"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "rpm-repo-set") + assert.Contains(t, err.Error(), "missing") +} + +func TestRunCompareReportsInventoryDifferences(t *testing.T) { + t.Parallel() + + testEnv := testutils.NewTestEnv(t) + testEnv.Config.Resources.RpmRepoSetTemplates = map[string]projectconfig.RpmRepoSetTemplate{ + "left-layout": { + Subrepos: []projectconfig.SubrepoSpec{{ + Name: "binary", Subpath: "$basearch", Kind: projectconfig.SubrepoKindBinary, + }}, + }, + "right-layout": { + Subrepos: []projectconfig.SubrepoSpec{{ + Name: "base", Subpath: "base/$basearch", Kind: projectconfig.SubrepoKindBinary, + }}, + }, + } + testEnv.Config.Resources.RpmRepoSets = map[string]projectconfig.RpmRepoSet{ + "left-set": { + Template: "left-layout", + BaseURI: "https://left.example.com/latest", + }, + "right-set": { + Template: "right-layout", + BaseURI: "https://right.example.com/prod", + }, + } + + const ( + leftURL = "https://left.example.com/latest/x86_64" + rightURL = "https://right.example.com/prod/base/x86_64" + ) + + repomd := []byte(``) + primary := func(name string) []byte { + return []byte(`` + name + `x86_64` + + ``) + } + + findings, err := runCompare( + testEnv.Env, + &CompareOptions{ + Left: "left-set", + Right: "right-set", + Arches: []string{"x86_64"}, + }, + comparisonMapFetcher{ + leftURL + "/repodata/repomd.xml": repomd, + leftURL + "/repodata/primary.xml": primary("left-only"), + rightURL + "/repodata/repomd.xml": repomd, + rightURL + "/repodata/primary.xml": primary("right-only"), + }, + ) + require.NoError(t, err) + assert.Equal(t, []repocompare.PackageReport{ + {Name: "left-only", Summary: "missing-from-right", LeftNEVRs: "left-only-1-1.azl4"}, + {Name: "right-only", Summary: "added-in-right", RightNEVRs: "right-only-1-1.azl4"}, + }, findings) +} diff --git a/internal/app/azldev/cmds/repo/repo.go b/internal/app/azldev/cmds/repo/repo.go index 08378763a..0dcf72011 100644 --- a/internal/app/azldev/cmds/repo/repo.go +++ b/internal/app/azldev/cmds/repo/repo.go @@ -25,4 +25,5 @@ under one or more URL prefixes.`, app.AddTopLevelCommand(cmd) queryOnAppInit(app, cmd) + compareOnAppInit(app, cmd) } diff --git a/internal/app/azldev/cmds/repo/repo_test.go b/internal/app/azldev/cmds/repo/repo_test.go index 4a301767f..e7a5eb5a2 100644 --- a/internal/app/azldev/cmds/repo/repo_test.go +++ b/internal/app/azldev/cmds/repo/repo_test.go @@ -36,6 +36,28 @@ func TestNewQueryCmd_FlagsRegistered(t *testing.T) { } } +func TestNewCompareCmd_FlagsRegistered(t *testing.T) { + t.Parallel() + + cmd := repo.NewCompareCmd() + for _, name := range []string{"left", "right", "arch"} { + assert.NotNil(t, cmd.Flags().Lookup(name), "expected flag --%s", name) + } +} + +func TestNewCompareCmd_RequiresPrefixes(t *testing.T) { + t.Parallel() + + cmd := repo.NewCompareCmd() + cmd.SetArgs([]string{}) + cmd.SilenceErrors = true + cmd.SilenceUsage = true + + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "left") +} + func TestNewQueryCmd_OneOfRepoPrefixOrVersionRequired(t *testing.T) { t.Parallel() diff --git a/internal/projectconfig/resources.go b/internal/projectconfig/resources.go index 07349c038..9374994db 100644 --- a/internal/projectconfig/resources.go +++ b/internal/projectconfig/resources.go @@ -662,6 +662,9 @@ type RpmRepoSet struct { // this set; same default semantics as [RpmRepoResource.DisableGPGCheck]. DisableGPGCheck bool `toml:"disable-gpg-check,omitempty" json:"disableGpgCheck,omitempty" jsonschema:"title=Disable GPG check,description=Opt out of GPG signature verification for repos in this set"` + // DisableSSLVerify disables TLS certificate verification for every sub-repo in this set. + DisableSSLVerify bool `toml:"disable-ssl-verify,omitempty" json:"disableSslVerify,omitempty" jsonschema:"title=Disable SSL verification,description=Disable TLS certificate verification for repos in this set"` + // Arches optionally restricts every synthesized repo in this set to a // specific list of target architectures. Empty = all. Arches []string `toml:"arches,omitempty" json:"arches,omitempty" jsonschema:"title=Arches,description=Restrict to specific target architectures; empty = all"` diff --git a/internal/repo/repocompare/diff.go b/internal/repo/repocompare/diff.go new file mode 100644 index 000000000..ccd57f6d4 --- /dev/null +++ b/internal/repo/repocompare/diff.go @@ -0,0 +1,236 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package repocompare + +import ( + "fmt" + "sort" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/rpm" +) + +// PackageStatus summarizes why a package name appears in the comparison. +type PackageStatus string + +const ( + // PackageStatusMissingFromRight indicates that the left contains an identity absent from the right. + PackageStatusMissingFromRight PackageStatus = "missing-from-right" + // PackageStatusAddedInRight indicates that the right contains an identity absent from the left. + PackageStatusAddedInRight PackageStatus = "added-in-right" + // PackageStatusArchitecturesDiffer indicates that matching NEVRs have different architecture sets. + PackageStatusArchitecturesDiffer PackageStatus = "architectures-differ" +) + +// PackageReport summarizes both inventories for one differing package name. +type PackageReport struct { + Name string `json:"name" table:"Name"` + Summary string `json:"summary" table:"Summary"` + LeftNEVRs string `json:"leftNevrs" table:"Left NEVRs"` + RightNEVRs string `json:"rightNevrs" table:"Right NEVRs"` +} + +// Compare returns one package-centric report for each package name with inventory differences. +func Compare(left, right []Package) ([]PackageReport, error) { + leftByName := groupByName(left) + rightByName := groupByName(right) + names := unionNames(leftByName, rightByName) + reports := make([]PackageReport, 0, len(names)) + + for _, name := range names { + leftPackages := leftByName[name] + rightPackages := rightByName[name] + + statuses := packageStatuses(leftPackages, rightPackages) + if len(statuses) == 0 { + continue + } + + leftNEVRs, err := sortedNEVRs(leftPackages) + if err != nil { + return nil, fmt.Errorf("sorting left inventory for package %#q:\n%w", name, err) + } + + rightNEVRs, err := sortedNEVRs(rightPackages) + if err != nil { + return nil, fmt.Errorf("sorting right inventory for package %#q:\n%w", name, err) + } + + reports = append(reports, PackageReport{ + Name: name, + Summary: joinStatuses(statuses), + LeftNEVRs: strings.Join(leftNEVRs, ", "), + RightNEVRs: strings.Join(rightNEVRs, ", "), + }) + } + + return reports, nil +} + +func packageStatuses(left, right []Package) []PackageStatus { + leftIdentities := identitySet(left) + rightIdentities := identitySet(right) + + var statuses []PackageStatus + + if hasSetDifference(leftIdentities, rightIdentities) { + statuses = append(statuses, PackageStatusMissingFromRight) + } + + if hasSetDifference(rightIdentities, leftIdentities) { + statuses = append(statuses, PackageStatusAddedInRight) + } + + if architecturesDiffer(left, right) { + statuses = append(statuses, PackageStatusArchitecturesDiffer) + } + + return statuses +} + +func identitySet(packages []Package) map[string]struct{} { + result := make(map[string]struct{}, len(packages)) + for _, pkg := range packages { + result[pkg.Identity()] = struct{}{} + } + + return result +} + +func hasSetDifference(left, right map[string]struct{}) bool { + for identity := range left { + if _, ok := right[identity]; !ok { + return true + } + } + + return false +} + +func architecturesDiffer(left, right []Package) bool { + leftArches := archesByVersionAndKind(left) + rightArches := archesByVersionAndKind(right) + + for key, arches := range leftArches { + if other, ok := rightArches[key]; ok && !equalStrings(arches, other) { + return true + } + } + + return false +} + +func archesByVersionAndKind(packages []Package) map[string][]string { + sets := make(map[string]map[string]struct{}) + for _, pkg := range packages { + key := pkg.EVR() + "\x00" + string(pkg.Kind) + if sets[key] == nil { + sets[key] = make(map[string]struct{}) + } + + sets[key][pkg.Arch] = struct{}{} + } + + result := make(map[string][]string, len(sets)) + for key, set := range sets { + for arch := range set { + result[key] = append(result[key], arch) + } + + sort.Strings(result[key]) + } + + return result +} + +func sortedNEVRs(packages []Package) ([]string, error) { + type entry struct { + nevr string + version *rpm.Version + } + + byNEVR := make(map[string]entry) + + for _, pkg := range packages { + nevr := pkg.NEVR() + if _, ok := byNEVR[nevr]; ok { + continue + } + + version, err := rpm.NewVersionFromEVR(normalizedEpoch(pkg.Epoch), pkg.Version, pkg.Release) + if err != nil { + return nil, fmt.Errorf("invalid EVR for %s:\n%w", pkg.NEVRA(), err) + } + + byNEVR[nevr] = entry{nevr: nevr, version: version} + } + + entries := make([]entry, 0, len(byNEVR)) + for _, value := range byNEVR { + entries = append(entries, value) + } + + sort.Slice(entries, func(leftIndex, rightIndex int) bool { + return entries[leftIndex].version.GreaterThan(entries[rightIndex].version) + }) + + result := make([]string, 0, len(entries)) + for _, value := range entries { + result = append(result, value.nevr) + } + + return result, nil +} + +func groupByName(packages []Package) map[string][]Package { + result := make(map[string][]Package) + for _, pkg := range packages { + result[pkg.Name] = append(result[pkg.Name], pkg) + } + + return result +} + +func unionNames(left, right map[string][]Package) []string { + set := make(map[string]struct{}, len(left)+len(right)) + for name := range left { + set[name] = struct{}{} + } + + for name := range right { + set[name] = struct{}{} + } + + result := make([]string, 0, len(set)) + for name := range set { + result = append(result, name) + } + + sort.Strings(result) + + return result +} + +func joinStatuses(statuses []PackageStatus) string { + values := make([]string, 0, len(statuses)) + for _, status := range statuses { + values = append(values, string(status)) + } + + return strings.Join(values, ", ") +} + +func equalStrings(left, right []string) bool { + if len(left) != len(right) { + return false + } + + for index := range left { + if left[index] != right[index] { + return false + } + } + + return true +} diff --git a/internal/repo/repocompare/diff_test.go b/internal/repo/repocompare/diff_test.go new file mode 100644 index 000000000..f1470216a --- /dev/null +++ b/internal/repo/repocompare/diff_test.go @@ -0,0 +1,101 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package repocompare_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/repo/repocompare" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func testPackage(name, version, arch string) repocompare.Package { + return repocompare.Package{ + Name: name, Epoch: "0", Version: version, Release: "1.azl4", Arch: arch, + Kind: projectconfig.SubrepoKindBinary, + } +} + +func TestCompareReportsDirectionalInventoryDifferences(t *testing.T) { + t.Parallel() + + shared := testPackage("bash", "5.3", "x86_64") + leftOnly := testPackage("curl", "8.0", "x86_64") + rightOnly := testPackage("azurelinux-release", "4.0", "noarch") + + reports, err := repocompare.Compare( + []repocompare.Package{shared, leftOnly}, + []repocompare.Package{shared, rightOnly}, + ) + require.NoError(t, err) + + assert.Equal(t, []repocompare.PackageReport{ + { + Name: "azurelinux-release", Summary: "added-in-right", + RightNEVRs: "azurelinux-release-4.0-1.azl4", + }, + { + Name: "curl", Summary: "missing-from-right", + LeftNEVRs: "curl-8.0-1.azl4", + }, + }, reports) +} + +func TestCompareNormalizesEpochAndNoarchReplication(t *testing.T) { + t.Parallel() + + left := testPackage("docs", "1", "noarch") + left.Epoch = "" + right := left + right.Epoch = "0" + + reports, err := repocompare.Compare( + []repocompare.Package{left, left}, + []repocompare.Package{right}, + ) + require.NoError(t, err) + + assert.Empty(t, reports) +} + +func TestCompareKeepsArtifactKindsDistinct(t *testing.T) { + t.Parallel() + + binary := testPackage("pkg", "1", "x86_64") + source := binary + source.Arch = "src" + source.Kind = projectconfig.SubrepoKindSource + + reports, err := repocompare.Compare([]repocompare.Package{binary, source}, []repocompare.Package{binary}) + require.NoError(t, err) + + assert.Equal(t, []repocompare.PackageReport{{ + Name: "pkg", Summary: "missing-from-right", LeftNEVRs: "pkg-1-1.azl4", RightNEVRs: "pkg-1-1.azl4", + }}, reports) +} + +func TestCompareSummarizesVersionAndArchitectureDifferences(t *testing.T) { + t.Parallel() + + leftX64 := testPackage("pkg", "2", "x86_64") + leftArm := leftX64 + leftArm.Arch = "aarch64" + rightX64 := testPackage("pkg", "2", "x86_64") + rightOld := testPackage("pkg", "1", "x86_64") + + reports, err := repocompare.Compare( + []repocompare.Package{leftX64, leftArm}, + []repocompare.Package{rightX64, rightOld}, + ) + require.NoError(t, err) + + assert.Equal(t, []repocompare.PackageReport{{ + Name: "pkg", + Summary: "missing-from-right, added-in-right, architectures-differ", + LeftNEVRs: "pkg-2-1.azl4", + RightNEVRs: "pkg-2-1.azl4, pkg-1-1.azl4", + }}, reports) +} diff --git a/internal/repo/repocompare/metadata.go b/internal/repo/repocompare/metadata.go new file mode 100644 index 000000000..547432a20 --- /dev/null +++ b/internal/repo/repocompare/metadata.go @@ -0,0 +1,348 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package repocompare loads and compares RPM repository package inventories. +package repocompare + +import ( + "bytes" + "compress/bzip2" + "compress/gzip" + "context" + "crypto/tls" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + "github.com/klauspost/compress/zstd" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/retry" + "github.com/ulikunitz/xz" +) + +// Repository describes one physical RPM repository. +type Repository struct { + ID string + Kind projectconfig.SubrepoKind + URL string + DisableSSLVerify bool +} + +// Package identifies one package in RPM primary metadata. +type Package struct { + Name string + Epoch string + Version string + Release string + Arch string + Kind projectconfig.SubrepoKind +} + +// Identity returns the stable key used to compare package inventories. +func (p Package) Identity() string { + return strings.Join([]string{ + p.Name, normalizedEpoch(p.Epoch), p.Version, p.Release, p.Arch, string(p.Kind), + }, "\x00") +} + +// NEVRA returns the package's human-readable name, epoch, version, release, and architecture. +func (p Package) NEVRA() string { + epoch := "" + if normalizedEpoch(p.Epoch) != "0" { + epoch = normalizedEpoch(p.Epoch) + ":" + } + + return fmt.Sprintf("%s-%s%s-%s.%s", p.Name, epoch, p.Version, p.Release, p.Arch) +} + +// NEVR returns the package's human-readable name, epoch, version, and release. +func (p Package) NEVR() string { + epoch := "" + if normalizedEpoch(p.Epoch) != "0" { + epoch = normalizedEpoch(p.Epoch) + ":" + } + + return fmt.Sprintf("%s-%s%s-%s", p.Name, epoch, p.Version, p.Release) +} + +// EVR returns the package's epoch, version, and release without its architecture. +func (p Package) EVR() string { + epoch := "" + if normalizedEpoch(p.Epoch) != "0" { + epoch = normalizedEpoch(p.Epoch) + ":" + } + + return fmt.Sprintf("%s%s-%s", epoch, p.Version, p.Release) +} + +// Fetcher retrieves repository metadata. +type Fetcher interface { + // Fetch returns the bytes at rawURL. + Fetch(ctx context.Context, rawURL string, disableSSLVerify bool) ([]byte, error) +} + +// HTTPFetcher retrieves repository metadata over HTTP with bounded retries. +type HTTPFetcher struct { + Attempts int +} + +// Fetch implements [Fetcher]. +func (f *HTTPFetcher) Fetch(ctx context.Context, rawURL string, disableSSLVerify bool) ([]byte, error) { + const requestTimeout = 10 * time.Minute + + attempts := f.Attempts + if attempts < 1 { + attempts = 1 + } + + var result []byte + + retryConfig := retry.DefaultConfig() + retryConfig.MaxAttempts = attempts + + err := retry.Do(ctx, retryConfig, func() error { + defaultTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return errors.New("default HTTP transport is not an *http.Transport") + } + + transport := defaultTransport.Clone() + if disableSSLVerify { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit per-repo opt-out + } + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) + if err != nil { + return fmt.Errorf("creating request for %#q:\n%w", rawURL, err) + } + + request.Header.Set("Accept-Encoding", "identity") + + client := &http.Client{Transport: transport, Timeout: requestTimeout} + + response, err := client.Do(request) + if err != nil { + return fmt.Errorf("fetching %#q:\n%w", rawURL, err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return fmt.Errorf("fetching %#q returned status %#q", rawURL, response.Status) + } + + result, err = io.ReadAll(response.Body) + if err != nil { + return fmt.Errorf("reading %#q:\n%w", rawURL, err) + } + + return nil + }) + if err != nil { + return nil, fmt.Errorf("fetching repository metadata:\n%w", err) + } + + return result, nil +} + +type repoMD struct { + Data []repoMDData `xml:"data"` +} + +type repoMDData struct { + Type string `xml:"type,attr"` + Location xmlLocation `xml:"location"` +} + +type xmlLocation struct { + Href string `xml:"href,attr"` +} + +type primaryPackage struct { + Name string `xml:"name"` + Arch string `xml:"arch"` + Version struct { + Epoch string `xml:"epoch,attr"` + Version string `xml:"ver,attr"` + Release string `xml:"rel,attr"` + } `xml:"version"` +} + +type pendingRepository struct { + repository Repository + primary repoMDData +} + +// LoadRepositories snapshots each repository's repomd document before loading its primary metadata. +func LoadRepositories( + ctx context.Context, + fetcher Fetcher, + repositories []Repository, +) ([]Package, error) { + if fetcher == nil { + return nil, errors.New("metadata fetcher cannot be nil") + } + + pending := make([]pendingRepository, 0, len(repositories)) + for _, repository := range repositories { + repomdURL, err := joinURL(repository.URL, "repodata/repomd.xml") + if err != nil { + return nil, fmt.Errorf("building repomd URL for repository %#q:\n%w", repository.ID, err) + } + + data, err := fetcher.Fetch(ctx, repomdURL, repository.DisableSSLVerify) + if err != nil { + return nil, fmt.Errorf("loading repomd for repository %#q:\n%w", repository.ID, err) + } + + var metadata repoMD + if err := xml.Unmarshal(data, &metadata); err != nil { + return nil, fmt.Errorf("parsing repomd for repository %#q:\n%w", repository.ID, err) + } + + primary, err := findPrimary(metadata.Data) + if err != nil { + return nil, fmt.Errorf("repository %#q:\n%w", repository.ID, err) + } + + pending = append(pending, pendingRepository{repository: repository, primary: primary}) + } + + var packages []Package + + for _, entry := range pending { + primaryURL, err := joinURL(entry.repository.URL, entry.primary.Location.Href) + if err != nil { + return nil, fmt.Errorf("building primary metadata URL for repository %#q:\n%w", entry.repository.ID, err) + } + + data, err := fetcher.Fetch(ctx, primaryURL, entry.repository.DisableSSLVerify) + if err != nil { + return nil, fmt.Errorf("loading primary metadata for repository %#q:\n%w", entry.repository.ID, err) + } + + repositoryPackages, err := parsePrimary(data, entry.primary.Location.Href, entry.repository.Kind) + if err != nil { + return nil, fmt.Errorf("parsing primary metadata for repository %#q:\n%w", entry.repository.ID, err) + } + + packages = append(packages, repositoryPackages...) + } + + return packages, nil +} + +func findPrimary(data []repoMDData) (repoMDData, error) { + for _, entry := range data { + if entry.Type == "primary" { + if entry.Location.Href == "" { + return repoMDData{}, errors.New("primary metadata has no location") + } + + return entry, nil + } + } + + return repoMDData{}, errors.New("repomd contains no primary metadata") +} + +func parsePrimary(data []byte, name string, kind projectconfig.SubrepoKind) ([]Package, error) { + reader, closeReader, err := decompressedReader(bytes.NewReader(data), name) + if err != nil { + return nil, err + } + + if closeReader != nil { + defer closeReader() + } + + decoder := xml.NewDecoder(reader) + + var packages []Package + + for { + token, err := decoder.Token() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return nil, fmt.Errorf("reading primary XML:\n%w", err) + } + + start, ok := token.(xml.StartElement) + if !ok || start.Name.Local != "package" { + continue + } + + var raw primaryPackage + if err := decoder.DecodeElement(&raw, &start); err != nil { + return nil, fmt.Errorf("reading package element:\n%w", err) + } + + packages = append(packages, Package{ + Name: raw.Name, + Epoch: normalizedEpoch(raw.Version.Epoch), + Version: raw.Version.Version, + Release: raw.Version.Release, + Arch: raw.Arch, + Kind: kind, + }) + } + + return packages, nil +} + +func decompressedReader(reader io.Reader, name string) (io.Reader, func(), error) { + switch { + case strings.HasSuffix(name, ".gz"): + gzipReader, err := gzip.NewReader(reader) + if err != nil { + return nil, nil, fmt.Errorf("creating gzip reader:\n%w", err) + } + + return gzipReader, func() { _ = gzipReader.Close() }, nil + case strings.HasSuffix(name, ".zst"), strings.HasSuffix(name, ".zstd"): + zstdReader, err := zstd.NewReader(reader) + if err != nil { + return nil, nil, fmt.Errorf("creating zstd reader:\n%w", err) + } + + return zstdReader, zstdReader.Close, nil + case strings.HasSuffix(name, ".bz2"): + return bzip2.NewReader(reader), nil, nil + case strings.HasSuffix(name, ".xz"): + xzReader, err := xz.NewReader(reader) + if err != nil { + return nil, nil, fmt.Errorf("creating xz reader:\n%w", err) + } + + return xzReader, nil, nil + default: + return reader, nil, nil + } +} + +func joinURL(baseURL, relative string) (string, error) { + parsed, err := url.Parse(baseURL) + if err != nil { + return "", fmt.Errorf("parsing base URL %#q:\n%w", baseURL, err) + } + + parsed.Path = path.Join(parsed.Path, relative) + + return parsed.String(), nil +} + +func normalizedEpoch(epoch string) string { + if epoch == "" { + return "0" + } + + return epoch +} diff --git a/internal/repo/repocompare/metadata_test.go b/internal/repo/repocompare/metadata_test.go new file mode 100644 index 000000000..54069a560 --- /dev/null +++ b/internal/repo/repocompare/metadata_test.go @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package repocompare_test + +import ( + "bytes" + "compress/gzip" + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/repo/repocompare" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type mapFetcher map[string][]byte + +func (f mapFetcher) Fetch(_ context.Context, rawURL string, _ bool) ([]byte, error) { + return f[rawURL], nil +} + +func TestHTTPFetcherCanDisableSSLVerification(t *testing.T) { + t.Parallel() + + server := httptest.NewTLSServer(http.HandlerFunc(func(writer http.ResponseWriter, _ *http.Request) { + _, _ = writer.Write([]byte("metadata")) + })) + defer server.Close() + + fetcher := &repocompare.HTTPFetcher{Attempts: 1} + _, err := fetcher.Fetch(t.Context(), server.URL, false) + require.Error(t, err) + + data, err := fetcher.Fetch(t.Context(), server.URL, true) + require.NoError(t, err) + assert.NotEmpty(t, data) +} + +func TestLoadRepositories(t *testing.T) { + t.Parallel() + + const ( + baseURL = "https://example.com/repo" + href = "repodata/primary.xml.gz" + ) + + primary := []byte(` + + + bashx86_64 + + +`) + + var compressed bytes.Buffer + + writer := gzip.NewWriter(&compressed) + _, err := writer.Write(primary) + require.NoError(t, err) + require.NoError(t, writer.Close()) + + repomd := []byte(` +`) + + packages, err := repocompare.LoadRepositories( + t.Context(), + mapFetcher{ + baseURL + "/repodata/repomd.xml": repomd, + baseURL + "/" + href: compressed.Bytes(), + }, + []repocompare.Repository{{ + ID: "test-base-x86_64", + Kind: projectconfig.SubrepoKindBinary, + URL: baseURL, + }}, + ) + require.NoError(t, err) + require.Len(t, packages, 1) + + assert.Equal(t, "0", packages[0].Epoch) + assert.Equal(t, "bash-5.3-1.azl4.x86_64", packages[0].NEVRA()) + assert.Equal(t, projectconfig.SubrepoKindBinary, packages[0].Kind) +} + +func TestLoadRepositoriesRejectsMissingPrimaryMetadata(t *testing.T) { + t.Parallel() + + const baseURL = "https://example.com/repo" + + _, err := repocompare.LoadRepositories( + t.Context(), + mapFetcher{baseURL + "/repodata/repomd.xml": []byte(``)}, + []repocompare.Repository{{ID: "test", URL: baseURL}}, + ) + require.Error(t, err) + assert.Contains(t, err.Error(), "test") + assert.Contains(t, err.Error(), "no primary metadata") +} diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 974e5d428..4de939bb1 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -1249,6 +1249,11 @@ "type": "array", "title": "Arches", "description": "Restrict to specific target architectures; empty = all" + }, + "disable-ssl-verify": { + "type": "boolean", + "title": "Disable SSL verification", + "description": "Disable TLS certificate verification for this repository" } }, "additionalProperties": false, @@ -1289,6 +1294,11 @@ "title": "Disable GPG check", "description": "Opt out of GPG signature verification for repos in this set" }, + "disable-ssl-verify": { + "type": "boolean", + "title": "Disable SSL verification", + "description": "Disable TLS certificate verification for repos in this set" + }, "arches": { "items": { "type": "string" diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 974e5d428..4de939bb1 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -1249,6 +1249,11 @@ "type": "array", "title": "Arches", "description": "Restrict to specific target architectures; empty = all" + }, + "disable-ssl-verify": { + "type": "boolean", + "title": "Disable SSL verification", + "description": "Disable TLS certificate verification for this repository" } }, "additionalProperties": false, @@ -1289,6 +1294,11 @@ "title": "Disable GPG check", "description": "Opt out of GPG signature verification for repos in this set" }, + "disable-ssl-verify": { + "type": "boolean", + "title": "Disable SSL verification", + "description": "Disable TLS certificate verification for repos in this set" + }, "arches": { "items": { "type": "string" diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 974e5d428..df428cc8f 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -1289,6 +1289,11 @@ "title": "Disable GPG check", "description": "Opt out of GPG signature verification for repos in this set" }, + "disable-ssl-verify": { + "type": "boolean", + "title": "Disable SSL verification", + "description": "Disable TLS certificate verification for repos in this set" + }, "arches": { "items": { "type": "string" From 01e287e01a34dec9b486282bd855df02b9c922b2 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 4 Sep 2026 20:50:13 +0000 Subject: [PATCH 2/6] updated snapshots --- internal/repo/repocompare/metadata.go | 8 +++++++- ...napshotsContainer_config_generate-schema_stdout_1.snap | 5 ----- .../TestSnapshots_config_generate-schema_stdout_1.snap | 5 ----- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/internal/repo/repocompare/metadata.go b/internal/repo/repocompare/metadata.go index 547432a20..a1e82b067 100644 --- a/internal/repo/repocompare/metadata.go +++ b/internal/repo/repocompare/metadata.go @@ -114,7 +114,13 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, rawURL string, disableSSLVerify transport := defaultTransport.Clone() if disableSSLVerify { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit per-repo opt-out + if transport.TLSClientConfig != nil { + clone := transport.TLSClientConfig.Clone() + clone.InsecureSkipVerify = true + transport.TLSClientConfig = clone + } else { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit per-repo opt-out + } } request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 4de939bb1..df428cc8f 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -1249,11 +1249,6 @@ "type": "array", "title": "Arches", "description": "Restrict to specific target architectures; empty = all" - }, - "disable-ssl-verify": { - "type": "boolean", - "title": "Disable SSL verification", - "description": "Disable TLS certificate verification for this repository" } }, "additionalProperties": false, diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 4de939bb1..df428cc8f 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -1249,11 +1249,6 @@ "type": "array", "title": "Arches", "description": "Restrict to specific target architectures; empty = all" - }, - "disable-ssl-verify": { - "type": "boolean", - "title": "Disable SSL verification", - "description": "Disable TLS certificate verification for this repository" } }, "additionalProperties": false, From d898ec922d7651c0cb795f05b3c40b59c4f4b5fc Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 4 Sep 2026 21:22:34 +0000 Subject: [PATCH 3/6] Addr pr comments --- internal/repo/repocompare/diff.go | 2 +- internal/repo/repocompare/metadata.go | 32 +++++++++++++-------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/internal/repo/repocompare/diff.go b/internal/repo/repocompare/diff.go index ccd57f6d4..dc9a68069 100644 --- a/internal/repo/repocompare/diff.go +++ b/internal/repo/repocompare/diff.go @@ -160,7 +160,7 @@ func sortedNEVRs(packages []Package) ([]string, error) { version, err := rpm.NewVersionFromEVR(normalizedEpoch(pkg.Epoch), pkg.Version, pkg.Release) if err != nil { - return nil, fmt.Errorf("invalid EVR for %s:\n%w", pkg.NEVRA(), err) + return nil, fmt.Errorf("invalid EVR for %#q:\n%w", pkg.NEVRA(), err) } byNEVR[nevr] = entry{nevr: nevr, version: version} diff --git a/internal/repo/repocompare/metadata.go b/internal/repo/repocompare/metadata.go index a1e82b067..84c941d05 100644 --- a/internal/repo/repocompare/metadata.go +++ b/internal/repo/repocompare/metadata.go @@ -106,23 +106,25 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, rawURL string, disableSSLVerify retryConfig := retry.DefaultConfig() retryConfig.MaxAttempts = attempts - err := retry.Do(ctx, retryConfig, func() error { - defaultTransport, ok := http.DefaultTransport.(*http.Transport) - if !ok { - return errors.New("default HTTP transport is not an *http.Transport") - } + defaultTransport, ok := http.DefaultTransport.(*http.Transport) + if !ok { + return nil, errors.New("default HTTP transport is not an *http.Transport") + } - transport := defaultTransport.Clone() - if disableSSLVerify { - if transport.TLSClientConfig != nil { - clone := transport.TLSClientConfig.Clone() - clone.InsecureSkipVerify = true - transport.TLSClientConfig = clone - } else { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit per-repo opt-out - } + transport := defaultTransport.Clone() + if disableSSLVerify { + if transport.TLSClientConfig != nil { + clone := transport.TLSClientConfig.Clone() + clone.InsecureSkipVerify = true + transport.TLSClientConfig = clone + } else { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} //nolint:gosec // explicit per-repo opt-out } + } + + client := &http.Client{Transport: transport, Timeout: requestTimeout} + err := retry.Do(ctx, retryConfig, func() error { request, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) if err != nil { return fmt.Errorf("creating request for %#q:\n%w", rawURL, err) @@ -130,8 +132,6 @@ func (f *HTTPFetcher) Fetch(ctx context.Context, rawURL string, disableSSLVerify request.Header.Set("Accept-Encoding", "identity") - client := &http.Client{Transport: transport, Timeout: requestTimeout} - response, err := client.Do(request) if err != nil { return fmt.Errorf("fetching %#q:\n%w", rawURL, err) From 9a98431812f172b658bc7083b0a4ea7a403d475a Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 4 Sep 2026 22:23:41 +0000 Subject: [PATCH 4/6] fix(repo): compare shared artifact kinds --- docs/user/explanation/repos.md | 4 ++- internal/app/azldev/cmds/repo/compare.go | 36 +++++++++++++++++++ .../azldev/cmds/repo/compare_internal_test.go | 18 ++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/user/explanation/repos.md b/docs/user/explanation/repos.md index 02ed90a09..5db8b0a84 100644 --- a/docs/user/explanation/repos.md +++ b/docs/user/explanation/repos.md @@ -130,7 +130,9 @@ from the right; `added-in-right` means at least one right identity is absent from the left. Identity includes name, normalized epoch, version, release, RPM architecture, and artifact kind (binary, debug, or source). Matching NEVRs with different architecture sets also receive `architectures-differ`. Replicated -`noarch` packages are counted once. +`noarch` packages are counted once. Only artifact kinds available on both sides +are compared; for example, a binary-only Koji set does not compare against PMC +source or debug repositories. This comparison checks inventory only. It does not compare RPM bytes, signatures, checksums, or publication routing. diff --git a/internal/app/azldev/cmds/repo/compare.go b/internal/app/azldev/cmds/repo/compare.go index d1e3c8bf1..15d22c0f5 100644 --- a/internal/app/azldev/cmds/repo/compare.go +++ b/internal/app/azldev/cmds/repo/compare.go @@ -93,6 +93,11 @@ func runCompare( return nil, fmt.Errorf("resolving right repositories:\n%w", err) } + leftRepositories, rightRepositories = filterToSharedKinds(leftRepositories, rightRepositories) + if len(leftRepositories) == 0 { + return nil, errors.New("the selected rpm-repo-sets have no shared artifact kinds") + } + leftPackages, err := repocompare.LoadRepositories(env, fetcher, leftRepositories) if err != nil { return nil, fmt.Errorf("loading left repositories:\n%w", err) @@ -186,3 +191,34 @@ func archAllowed(allowlist []string, arch string) bool { return false } + +func filterToSharedKinds( + left []repocompare.Repository, + right []repocompare.Repository, +) ([]repocompare.Repository, []repocompare.Repository) { + leftKinds := make(map[projectconfig.SubrepoKind]struct{}) + for _, repository := range left { + leftKinds[repository.Kind] = struct{}{} + } + + rightKinds := make(map[projectconfig.SubrepoKind]struct{}) + for _, repository := range right { + rightKinds[repository.Kind] = struct{}{} + } + + filter := func( + repositories []repocompare.Repository, + otherKinds map[projectconfig.SubrepoKind]struct{}, + ) []repocompare.Repository { + result := make([]repocompare.Repository, 0, len(repositories)) + for _, repository := range repositories { + if _, ok := otherKinds[repository.Kind]; ok { + result = append(result, repository) + } + } + + return result + } + + return filter(left, rightKinds), filter(right, leftKinds) +} diff --git a/internal/app/azldev/cmds/repo/compare_internal_test.go b/internal/app/azldev/cmds/repo/compare_internal_test.go index 89cd8b34d..4e1d37e02 100644 --- a/internal/app/azldev/cmds/repo/compare_internal_test.go +++ b/internal/app/azldev/cmds/repo/compare_internal_test.go @@ -92,6 +92,24 @@ func TestComparisonRepositoriesRejectsUnknownSet(t *testing.T) { assert.Contains(t, err.Error(), "missing") } +func TestFilterToSharedKinds(t *testing.T) { + t.Parallel() + + left := []repocompare.Repository{{ID: "left-binary", Kind: projectconfig.SubrepoKindBinary}} + right := []repocompare.Repository{ + {ID: "right-binary", Kind: projectconfig.SubrepoKindBinary}, + {ID: "right-debug", Kind: projectconfig.SubrepoKindDebug}, + {ID: "right-source", Kind: projectconfig.SubrepoKindSource}, + } + + filteredLeft, filteredRight := filterToSharedKinds(left, right) + + assert.Equal(t, left, filteredLeft) + assert.Equal(t, []repocompare.Repository{{ + ID: "right-binary", Kind: projectconfig.SubrepoKindBinary, + }}, filteredRight) +} + func TestRunCompareReportsInventoryDifferences(t *testing.T) { t.Parallel() From 8190afbe4415040ace3027353aa1fc5b99b39409 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 4 Sep 2026 22:49:34 +0000 Subject: [PATCH 5/6] Fix diff again --- defaultconfigs/content/defaults.toml | 14 +++++++++----- docs/user/explanation/repos.md | 4 ++-- internal/repo/repocompare/diff.go | 4 ++-- internal/repo/repocompare/diff_test.go | 20 +++++++++++++++++++- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/defaultconfigs/content/defaults.toml b/defaultconfigs/content/defaults.toml index c97bab2f0..3745bb794 100644 --- a/defaultconfigs/content/defaults.toml +++ b/defaultconfigs/content/defaults.toml @@ -4,21 +4,25 @@ default-author-email = "azurelinux@microsoft.com" [tools.imageCustomizer] containerTag = "mcr.microsoft.com/azurelinux/imagecustomizer:1" -# A standard "channelized" layout with two channels (`base`, `sdk`) and three +# A standard layout with three repositories (`base`, `sdk`, `microsoft`) and three # kinds of sub-repo per channel (binary, debug, source). Pair this template # with one or more `[resources.rpm-repo-sets.*]` entries that set `base-uri` # to the URL prefix of a published tree. [resources.rpm-repo-set-templates.azl-standard] -description = "Standard base/sdk x binary/debug/source layout" +description = "Standard base/sdk/microsoft x binary/debug/source layout" subrepos = [ { name = "base", kind = "binary", subpath = "base/$basearch" }, { name = "base-debug", kind = "debug", subpath = "base/debuginfo/$basearch" }, { name = "base-src", kind = "source", subpath = "base/srpms" }, - { name = "sdk", kind = "binary", subpath = "sdk/$basearch" }, - { name = "sdk-debug", kind = "debug", subpath = "sdk/debuginfo/$basearch" }, - { name = "sdk-src", kind = "source", subpath = "sdk/srpms" }, + { name = "sdk", kind = "binary", subpath = "builddeps/$basearch" }, + { name = "sdk-debug", kind = "debug", subpath = "builddeps/debuginfo/$basearch" }, + { name = "sdk-src", kind = "source", subpath = "builddeps/srpms" }, + { name = "microsoft", kind = "binary", subpath = "microsoft/$basearch" }, + { name = "microsoft-debug", kind = "debug", subpath = "microsoft/debuginfo/$basearch" }, + { name = "microsoft-src", kind = "source", subpath = "microsoft/srpms" }, ] + # Standard Koji dist-repo layout. dist-repos publish one rpm-md tree per # arch under `/`, a parallel `/debug/` tree for debuginfo # packages, and a single `src/` tree for SRPMs. diff --git a/docs/user/explanation/repos.md b/docs/user/explanation/repos.md index 5db8b0a84..f2d5ffb61 100644 --- a/docs/user/explanation/repos.md +++ b/docs/user/explanation/repos.md @@ -126,8 +126,8 @@ and `subrepos` restrictions. The command can also compare two PMC trees, two Koji dist-repos, or custom layouts. The report contains one row per differing package name, summary statuses, and the complete left and right NEVR inventories. `missing-from-right` means at least one left identity is absent -from the right; `added-in-right` means at least one right identity is absent -from the left. Identity includes name, normalized epoch, version, release, RPM +from the right; `added-in-right` means the package name exists only on the +right. Identity includes name, normalized epoch, version, release, RPM architecture, and artifact kind (binary, debug, or source). Matching NEVRs with different architecture sets also receive `architectures-differ`. Replicated `noarch` packages are counted once. Only artifact kinds available on both sides diff --git a/internal/repo/repocompare/diff.go b/internal/repo/repocompare/diff.go index dc9a68069..fa9698b30 100644 --- a/internal/repo/repocompare/diff.go +++ b/internal/repo/repocompare/diff.go @@ -17,7 +17,7 @@ type PackageStatus string const ( // PackageStatusMissingFromRight indicates that the left contains an identity absent from the right. PackageStatusMissingFromRight PackageStatus = "missing-from-right" - // PackageStatusAddedInRight indicates that the right contains an identity absent from the left. + // PackageStatusAddedInRight indicates that the right contains a package name absent from the left. PackageStatusAddedInRight PackageStatus = "added-in-right" // PackageStatusArchitecturesDiffer indicates that matching NEVRs have different architecture sets. PackageStatusArchitecturesDiffer PackageStatus = "architectures-differ" @@ -78,7 +78,7 @@ func packageStatuses(left, right []Package) []PackageStatus { statuses = append(statuses, PackageStatusMissingFromRight) } - if hasSetDifference(rightIdentities, leftIdentities) { + if len(left) == 0 && len(right) > 0 { statuses = append(statuses, PackageStatusAddedInRight) } diff --git a/internal/repo/repocompare/diff_test.go b/internal/repo/repocompare/diff_test.go index f1470216a..a2e279ddb 100644 --- a/internal/repo/repocompare/diff_test.go +++ b/internal/repo/repocompare/diff_test.go @@ -94,8 +94,26 @@ func TestCompareSummarizesVersionAndArchitectureDifferences(t *testing.T) { assert.Equal(t, []repocompare.PackageReport{{ Name: "pkg", - Summary: "missing-from-right, added-in-right, architectures-differ", + Summary: "missing-from-right, architectures-differ", LeftNEVRs: "pkg-2-1.azl4", RightNEVRs: "pkg-2-1.azl4, pkg-1-1.azl4", }}, reports) } + +func TestCompareDoesNotTreatHistoricalRightVersionAsAddedPackage(t *testing.T) { + t.Parallel() + + left := testPackage("shared", "2", "x86_64") + right := testPackage("shared", "1", "x86_64") + + reports, err := repocompare.Compare( + []repocompare.Package{left}, + []repocompare.Package{right}, + ) + require.NoError(t, err) + + assert.Equal(t, []repocompare.PackageReport{{ + Name: "shared", Summary: "missing-from-right", + LeftNEVRs: "shared-2-1.azl4", RightNEVRs: "shared-1-1.azl4", + }}, reports) +} From 5b41332df9d2a0e3e7d6afa1840f485294dde731 Mon Sep 17 00:00:00 2001 From: Antonio Salinas Date: Fri, 4 Sep 2026 23:05:43 +0000 Subject: [PATCH 6/6] fix(repo): restore identity-based additions --- docs/user/explanation/repos.md | 4 ++-- internal/repo/repocompare/diff.go | 4 ++-- internal/repo/repocompare/diff_test.go | 20 +------------------- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/docs/user/explanation/repos.md b/docs/user/explanation/repos.md index f2d5ffb61..5db8b0a84 100644 --- a/docs/user/explanation/repos.md +++ b/docs/user/explanation/repos.md @@ -126,8 +126,8 @@ and `subrepos` restrictions. The command can also compare two PMC trees, two Koji dist-repos, or custom layouts. The report contains one row per differing package name, summary statuses, and the complete left and right NEVR inventories. `missing-from-right` means at least one left identity is absent -from the right; `added-in-right` means the package name exists only on the -right. Identity includes name, normalized epoch, version, release, RPM +from the right; `added-in-right` means at least one right identity is absent +from the left. Identity includes name, normalized epoch, version, release, RPM architecture, and artifact kind (binary, debug, or source). Matching NEVRs with different architecture sets also receive `architectures-differ`. Replicated `noarch` packages are counted once. Only artifact kinds available on both sides diff --git a/internal/repo/repocompare/diff.go b/internal/repo/repocompare/diff.go index fa9698b30..dc9a68069 100644 --- a/internal/repo/repocompare/diff.go +++ b/internal/repo/repocompare/diff.go @@ -17,7 +17,7 @@ type PackageStatus string const ( // PackageStatusMissingFromRight indicates that the left contains an identity absent from the right. PackageStatusMissingFromRight PackageStatus = "missing-from-right" - // PackageStatusAddedInRight indicates that the right contains a package name absent from the left. + // PackageStatusAddedInRight indicates that the right contains an identity absent from the left. PackageStatusAddedInRight PackageStatus = "added-in-right" // PackageStatusArchitecturesDiffer indicates that matching NEVRs have different architecture sets. PackageStatusArchitecturesDiffer PackageStatus = "architectures-differ" @@ -78,7 +78,7 @@ func packageStatuses(left, right []Package) []PackageStatus { statuses = append(statuses, PackageStatusMissingFromRight) } - if len(left) == 0 && len(right) > 0 { + if hasSetDifference(rightIdentities, leftIdentities) { statuses = append(statuses, PackageStatusAddedInRight) } diff --git a/internal/repo/repocompare/diff_test.go b/internal/repo/repocompare/diff_test.go index a2e279ddb..f1470216a 100644 --- a/internal/repo/repocompare/diff_test.go +++ b/internal/repo/repocompare/diff_test.go @@ -94,26 +94,8 @@ func TestCompareSummarizesVersionAndArchitectureDifferences(t *testing.T) { assert.Equal(t, []repocompare.PackageReport{{ Name: "pkg", - Summary: "missing-from-right, architectures-differ", + Summary: "missing-from-right, added-in-right, architectures-differ", LeftNEVRs: "pkg-2-1.azl4", RightNEVRs: "pkg-2-1.azl4, pkg-1-1.azl4", }}, reports) } - -func TestCompareDoesNotTreatHistoricalRightVersionAsAddedPackage(t *testing.T) { - t.Parallel() - - left := testPackage("shared", "2", "x86_64") - right := testPackage("shared", "1", "x86_64") - - reports, err := repocompare.Compare( - []repocompare.Package{left}, - []repocompare.Package{right}, - ) - require.NoError(t, err) - - assert.Equal(t, []repocompare.PackageReport{{ - Name: "shared", Summary: "missing-from-right", - LeftNEVRs: "shared-2-1.azl4", RightNEVRs: "shared-1-1.azl4", - }}, reports) -}