diff --git a/Dockerfile b/Dockerfile index d434e582..80e144f7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,11 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ FROM alpine:3.24.3 -RUN apk add --no-cache ca-certificates git +# Git 2.56+ comes from Alpine 3.25; drop the repository once the base image +# is 3.25. +RUN apk add --no-cache \ + --repository=https://dl-cdn.alpinelinux.org/alpine/v3.25/main \ + ca-certificates "git~=2.56" COPY --from=builder /proxy /usr/local/bin/proxy diff --git a/docs/configuration.md b/docs/configuration.md index 40e94343..0e040820 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -920,7 +920,8 @@ location. A `git-upload-pack` request checks the upstream only when the last check refused the repository or the copy was last checked more than a minute past `stale_max_age` ago, so a clone checks once, at its ref listing. -The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). +The proxy runs `git` 2.56 or newer, which must be installed and is checked +at startup (the Docker image includes it). The `git` processes never reach the network themselves: their fetches go through a loopback endpoint in the proxy that forwards them with the same upstream client as every other request, so the address checks @@ -951,6 +952,9 @@ A transfer that receives nothing for 10 minutes is abandoned. Git's automatic garbage collection runs as part of fetches, and files over 8MB are kept without delta compression, so one large file does not take a fetch's memory. +Repositories are cached with reftable reference storage. One cached in +another format is removed at startup and cloned again by its next request. + Packs for clients are built by at most two `upload-pack` processes per CPU the proxy may use, each with one delta-search thread and 64MB of delta window, and further clones wait for one to finish; listing refs does not diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 2bef68e7..7e71d4d0 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -245,6 +245,16 @@ func New(opts Options) (*Cache, error) { if opts.maxRequestBody > 0 { c.maxRequestBody = opts.maxRequestBody } + // Before load, which deletes repositories a git too old could not + // clone again. + out, err := c.command(context.Background(), nil, "version").Output() + if err == nil && tooOld(string(out)) { + err = fmt.Errorf("found %s", strings.TrimSpace(string(out))) + } + if err != nil { + _ = lock.Close() + return nil, fmt.Errorf("upstream.git needs git 2.56 or newer: %w", err) + } if err := c.load(); err != nil { _ = lock.Close() return nil, err @@ -305,7 +315,17 @@ func (c *Cache) load() error { return err } rel, _ := filepath.Rel(c.dir, path) - rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path) + key := c.repoKey(filepath.ToSlash(rel)) + if outdated(path) { + trash, err := c.moveAside(path, "outdated-") + if err == nil { + c.logger.Info("git cache: removed a repository with an old reference format; its next request clones it again", "repository", key) + _ = os.RemoveAll(trash) + return filepath.SkipDir + } + c.logger.Warn("git cache: failed to remove a repository with an old reference format; serving it as it is", "repository", key, "error", err) + } + rp := newRepo(key, path) rp.size.Store(dirSize(path)) rp.lastChecked.Store(stampTime(path, checkedStamp)) rp.lastUsed.Store(stampTime(path, usedStamp)) @@ -320,6 +340,37 @@ func (c *Cache) load() error { return nil } +// moveAside moves path into a new directory under the temporary directory +// and returns that directory for the caller to delete, so nothing finds +// path half deleted, and what a failed deletion leaves is cleared with the +// temporary directory at the next start. +func (c *Cache) moveAside(path, prefix string) (string, error) { + trash, err := os.MkdirTemp(c.tmpDir(), prefix) + if err != nil { + return "", err + } + return trash, os.Rename(path, filepath.Join(trash, filepath.Base(path))) +} + +// outdated reports whether the repository at dir has no reftable, the +// reference format the cache clones with. It looks for the table list +// rather than running git, so startup costs no process per repository, and +// only a list known to be missing counts: a file system error is not taken +// as a reason to delete a repository. +func outdated(dir string) bool { + _, err := os.Stat(filepath.Join(dir, "reftable", "tables.list")) + return errors.Is(err, fs.ErrNotExist) +} + +// tooOld reports whether out, the output of "git version", names a git +// older than 2.56, the first with the reftable fixes the cache relies on. +// Output that does not parse is a custom build, taken as new enough. +func tooOld(out string) bool { + var major, minor int + _, err := fmt.Sscanf(out, "git version %d.%d", &major, &minor) + return err == nil && (major < 2 || major == 2 && minor < 56) +} + // upstreamDirPattern matches a directory UpstreamDir names, capturing the // upstream's name. var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`) @@ -349,10 +400,7 @@ func (c *Cache) removeRepointed() { unconfigured = append(unconfigured, name) continue } - trash, err := os.MkdirTemp(c.tmpDir(), "repointed-") - if err == nil { - err = os.Rename(filepath.Join(c.dir, name), filepath.Join(trash, name)) - } + trash, err := c.moveAside(filepath.Join(c.dir, name), "repointed-") if err != nil { c.logger.Warn("git cache: failed to remove the cache of a repointed upstream", "directory", name, "error", err) continue @@ -478,10 +526,7 @@ func (c *Cache) evictLocked(extra int64) ([]string, int64) { if !tooLarge && (c.opts.MaxSize <= 0 || total+extra <= c.opts.MaxSize) { continue } - trash, err := os.MkdirTemp(c.tmpDir(), "evicted-") - if err == nil { - err = os.Rename(rp.dir, filepath.Join(trash, "repo")) - } + trash, err := c.moveAside(rp.dir, "evicted-") if err != nil && !errors.Is(err, fs.ErrNotExist) { c.logger.Warn("git cache: failed to evict repository", "repository", rp.key, "error", err) continue diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 2b2e388d..8506b0e2 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -669,7 +669,7 @@ func TestNewRemovesRepointedUpstreams(t *testing.T) { // Named after a configured upstream's name, but another upstream's. longer := UpstreamDir("up-api", "https://api.example") for _, name := range []string{current, repointed, unconfigured, longer} { - runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, name, "repo.git")) + runGit(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(dir, name, "repo.git")) } c, err := New(Options{ CacheDir: dir, @@ -692,6 +692,42 @@ func TestNewRemovesRepointedUpstreams(t *testing.T) { } } +// TestNewRemovesOldFormatRepositories starts a cache whose directory holds +// a repository in the files reference format: it is removed, and its next +// request clones it again with reftable. +func TestNewRemovesOldFormatRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + dir := t.TempDir() + bare := filepath.Join(dir, "up", "repo.git") + runGit(t, "", "clone", "--quiet", "--bare", "--ref-format=files", upstream.bare(), bare) + + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir }) + if got := tc.Repositories(); len(got) != 0 { + t.Fatalf("startup kept an old-format repository: %+v", got) + } + tc.clone(t, "repo") + waitIdle(t, tc.Cache) + if outdated(bare) { + t.Error("the repository was not cloned again with reftable") + } +} + +func TestTooOld(t *testing.T) { + for out, want := range map[string]bool{ + "git version 2.55.2\n": true, + "git version 2.39.1 (Apple Git-146)\n": true, + "git version 2.56.0.windows.1\n": false, + "git version 2.57.0-rc0\n": false, + "git version 3.0.0\n": false, + "git version 2.x\n": false, + } { + if got := tooOld(out); got != want { + t.Errorf("tooOld(%q) = %v, want %v", out, got, want) + } + } +} + // TestCacheDirSharedWithOtherData points the cache at a directory that also // holds an artifact store and a database, as git.cache_dir: ./cache would. func TestCacheDirSharedWithOtherData(t *testing.T) { diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 683be053..fbda5601 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -221,7 +221,7 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo } defer func() { _ = os.RemoveAll(tmp) }() - if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil { + if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare", "--ref-format=reftable"); err != nil { return err } if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil { diff --git a/internal/gitcache/stats_test.go b/internal/gitcache/stats_test.go index e159ee33..65e61afd 100644 --- a/internal/gitcache/stats_test.go +++ b/internal/gitcache/stats_test.go @@ -225,7 +225,9 @@ func TestRepositoriesKeepCaseWhereKeysAreFolded(t *testing.T) { // TestRepositoryStateRetriesAFailedRead checks a read that fails, here on // a branch whose commit is missing, is reported as an error and read again -// a while later even if the upstream isn't checked in between. +// a while later even if the upstream isn't checked in between. The broken +// branch is a hand-written loose ref, so the repository is swapped for one +// in the files format. func TestRepositoryStateRetriesAFailedRead(t *testing.T) { upstream := newFakeUpstream(t) upstream.newRepo("repo") @@ -233,6 +235,10 @@ func TestRepositoryStateRetriesAFailedRead(t *testing.T) { tc.clone(t, "repo") waitIdle(t, tc.Cache) rp := tc.repo("up/repo.git") + if err := os.RemoveAll(rp.dir); err != nil { + t.Fatal(err) + } + runGit(t, "", "clone", "--quiet", "--bare", "--ref-format=files", upstream.bare(), rp.dir) ref := filepath.Join(rp.dir, "refs", "heads", "main") head := runGit(t, rp.dir, "rev-parse", "main") if err := os.WriteFile(ref, []byte(strings.Repeat("1", len(head))+"\n"), 0o600); err != nil { diff --git a/internal/server/git_ui_test.go b/internal/server/git_ui_test.go index d5d2c558..89c9fbd6 100644 --- a/internal/server/git_ui_test.go +++ b/internal/server/git_ui_test.go @@ -55,7 +55,7 @@ func TestGitRepositoriesPageSaysWhatIsNotReadYet(t *testing.T) { upstreams := map[string]config.GitUpstream{"up": {URL: "https://example.com"}} upstreamDir := gitcache.UpstreamDir("up", "https://example.com") for _, name := range []string{"one", "two"} { - gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", filepath.Join(cacheDir, upstreamDir, "team", name+".git")) + gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(cacheDir, upstreamDir, "team", name+".git")) } logger := slog.New(slog.NewTextHandler(io.Discard, nil)) cache, err := gitcache.New(gitcache.Options{ @@ -97,11 +97,7 @@ func TestGitRepositoriesPageLogsAStateItCannotRead(t *testing.T) { cacheDir := t.TempDir() upstreamDir := gitcache.UpstreamDir("up", "https://example.com") repoDir := filepath.Join(cacheDir, upstreamDir, "team", "broken.git") - gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", repoDir) - // A branch naming a commit the repository doesn't have. - if err := os.WriteFile(filepath.Join(repoDir, "refs", "heads", "main"), []byte(strings.Repeat("1", 40)+"\n"), 0o600); err != nil { - t.Fatal(err) - } + gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", repoDir) var logs bytes.Buffer logger := slog.New(slog.NewTextHandler(&logs, nil)) cache, err := gitcache.New(gitcache.Options{ @@ -112,6 +108,15 @@ func TestGitRepositoriesPageLogsAStateItCannotRead(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = cache.Close() }) + // A branch naming a commit the repository doesn't have, as a loose ref + // in a files-format repository swapped in after the cache loaded. + if err := os.RemoveAll(repoDir); err != nil { + t.Fatal(err) + } + gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", repoDir) + if err := os.WriteFile(filepath.Join(repoDir, "refs", "heads", "main"), []byte(strings.Repeat("1", 40)+"\n"), 0o600); err != nil { + t.Fatal(err) + } s := &Server{cfg: &config.Config{BaseURL: "http://proxy.example"}, git: cache, logger: logger, templates: &Templates{}} s.cfg.Upstream.Git = map[string]config.GitUpstream{"up": {URL: "https://example.com"}} @@ -135,7 +140,7 @@ func TestGitRepositoryRowsKeepCase(t *testing.T) { cacheDir := t.TempDir() upstreams := map[string]config.GitUpstream{"Up": {URL: "https://example.com", Repos: []string{"Team/*"}}} upstreamDir := gitcache.UpstreamDir("Up", "https://example.com") - gitCommand(t, "", "init", "--quiet", "--bare", filepath.Join(cacheDir, upstreamDir, "Team", "Repo.git")) + gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(cacheDir, upstreamDir, "Team", "Repo.git")) cache, err := gitcache.New(gitcache.Options{ CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour, Client: &http.Client{}, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), Upstreams: map[string]string{"Up": upstreamDir}, @@ -235,7 +240,7 @@ func TestGitRepositoryRowsEscapeCloneURL(t *testing.T) { func TestGitCacheGauges(t *testing.T) { cacheDir := t.TempDir() upstreamDir := gitcache.UpstreamDir("up", "https://example.com") - gitCommand(t, "", "init", "--quiet", "--bare", filepath.Join(cacheDir, upstreamDir, "team", "repo.git")) + gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(cacheDir, upstreamDir, "team", "repo.git")) logger := slog.New(slog.NewTextHandler(io.Discard, nil)) cache, err := gitcache.New(gitcache.Options{ CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour,