Improve git cache performance by using reftable

Dockerfile

@@ -22,7 +22,11 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
 
 FROM alpine:3.24.3
 
-RUN apk add --no-cache ca-certificates git
+# Git 2.56+ comes from Alpine 3.25; drop the repository once the base image
+# is 3.25.
+RUN apk add --no-cache \
+    --repository=https://dl-cdn.alpinelinux.org/alpine/v3.25/main \
+    ca-certificates "git~=2.56"
 
 COPY --from=builder /proxy /usr/local/bin/proxy
 

docs/configuration.md

@@ -920,7 +920,8 @@ location. A `git-upload-pack` request checks the upstream only when the last
 check refused the repository or the copy was last checked more than a minute
 past `stale_max_age` ago, so a clone checks once, at its ref listing.
 
-The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it).
+The proxy runs `git` 2.56 or newer, which must be installed and is checked
+at startup (the Docker image includes it).
 The `git` processes never reach the network themselves: their fetches go
 through a loopback endpoint in the proxy that forwards them with the same
 upstream client as every other request, so the address checks
@@ -951,6 +952,9 @@ A transfer that receives nothing for 10 minutes is abandoned. Git's automatic
 garbage collection runs as part of fetches, and files over 8MB are kept
 without delta compression, so one large file does not take a fetch's memory.
 
+Repositories are cached with reftable reference storage. One cached in
+another format is removed at startup and cloned again by its next request.
+
 Packs for clients are built by at most two `upload-pack` processes per CPU
 the proxy may use, each with one delta-search thread and 64MB of delta
 window, and further clones wait for one to finish; listing refs does not

internal/gitcache/cache.go

@@ -245,6 +245,16 @@ func New(opts Options) (*Cache, error) {
 	if opts.maxRequestBody > 0 {
 		c.maxRequestBody = opts.maxRequestBody
 	}
+	// Before load, which deletes repositories a git too old could not
+	// clone again.
+	out, err := c.command(context.Background(), nil, "version").Output()
+	if err == nil && tooOld(string(out)) {
+		err = fmt.Errorf("found %s", strings.TrimSpace(string(out)))
+	}
+	if err != nil {
+		_ = lock.Close()
+		return nil, fmt.Errorf("upstream.git needs git 2.56 or newer: %w", err)
+	}
 	if err := c.load(); err != nil {
 		_ = lock.Close()
 		return nil, err
@@ -305,7 +315,17 @@ func (c *Cache) load() error {
 				return err
 			}
 			rel, _ := filepath.Rel(c.dir, path)
-			rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path)
+			key := c.repoKey(filepath.ToSlash(rel))
+			if outdated(path) {
+				trash, err := c.moveAside(path, "outdated-")
+				if err == nil {
+					c.logger.Info("git cache: removed a repository with an old reference format; its next request clones it again", "repository", key)
+					_ = os.RemoveAll(trash)
+					return filepath.SkipDir
+				}
+				c.logger.Warn("git cache: failed to remove a repository with an old reference format; serving it as it is", "repository", key, "error", err)
+			}
+			rp := newRepo(key, path)
 			rp.size.Store(dirSize(path))
 			rp.lastChecked.Store(stampTime(path, checkedStamp))
 			rp.lastUsed.Store(stampTime(path, usedStamp))
@@ -320,6 +340,37 @@ func (c *Cache) load() error {
 	return nil
 }
 
+// moveAside moves path into a new directory under the temporary directory
+// and returns that directory for the caller to delete, so nothing finds
+// path half deleted, and what a failed deletion leaves is cleared with the
+// temporary directory at the next start.
+func (c *Cache) moveAside(path, prefix string) (string, error) {
+	trash, err := os.MkdirTemp(c.tmpDir(), prefix)
+	if err != nil {
+		return "", err
+	}
+	return trash, os.Rename(path, filepath.Join(trash, filepath.Base(path)))
+}
+
+// outdated reports whether the repository at dir has no reftable, the
+// reference format the cache clones with. It looks for the table list
+// rather than running git, so startup costs no process per repository, and
+// only a list known to be missing counts: a file system error is not taken
+// as a reason to delete a repository.
+func outdated(dir string) bool {
+	_, err := os.Stat(filepath.Join(dir, "reftable", "tables.list"))
+	return errors.Is(err, fs.ErrNotExist)
+}
+
+// tooOld reports whether out, the output of "git version", names a git
+// older than 2.56, the first with the reftable fixes the cache relies on.
+// Output that does not parse is a custom build, taken as new enough.
+func tooOld(out string) bool {
+	var major, minor int
+	_, err := fmt.Sscanf(out, "git version %d.%d", &major, &minor)
+	return err == nil && (major < 2 || major == 2 && minor < 56)
+}
+
 // upstreamDirPattern matches a directory UpstreamDir names, capturing the
 // upstream's name.
 var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`)
@@ -349,10 +400,7 @@ func (c *Cache) removeRepointed() {
 			unconfigured = append(unconfigured, name)
 			continue
 		}
-		trash, err := os.MkdirTemp(c.tmpDir(), "repointed-")
-		if err == nil {
-			err = os.Rename(filepath.Join(c.dir, name), filepath.Join(trash, name))
-		}
+		trash, err := c.moveAside(filepath.Join(c.dir, name), "repointed-")
 		if err != nil {
 			c.logger.Warn("git cache: failed to remove the cache of a repointed upstream", "directory", name, "error", err)
 			continue
@@ -478,10 +526,7 @@ func (c *Cache) evictLocked(extra int64) ([]string, int64) {
 		if !tooLarge && (c.opts.MaxSize <= 0 || total+extra <= c.opts.MaxSize) {
 			continue
 		}
-		trash, err := os.MkdirTemp(c.tmpDir(), "evicted-")
-		if err == nil {
-			err = os.Rename(rp.dir, filepath.Join(trash, "repo"))
-		}
+		trash, err := c.moveAside(rp.dir, "evicted-")
 		if err != nil && !errors.Is(err, fs.ErrNotExist) {
 			c.logger.Warn("git cache: failed to evict repository", "repository", rp.key, "error", err)
 			continue

internal/gitcache/cache_test.go

@@ -669,7 +669,7 @@ func TestNewRemovesRepointedUpstreams(t *testing.T) {
 	// Named after a configured upstream's name, but another upstream's.
 	longer := UpstreamDir("up-api", "https://api.example")
 	for _, name := range []string{current, repointed, unconfigured, longer} {
-		runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, name, "repo.git"))
+		runGit(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(dir, name, "repo.git"))
 	}
 	c, err := New(Options{
 		CacheDir:    dir,
@@ -692,6 +692,42 @@ func TestNewRemovesRepointedUpstreams(t *testing.T) {
 	}
 }
 
+// TestNewRemovesOldFormatRepositories starts a cache whose directory holds
+// a repository in the files reference format: it is removed, and its next
+// request clones it again with reftable.
+func TestNewRemovesOldFormatRepositories(t *testing.T) {
+	upstream := newFakeUpstream(t)
+	upstream.newRepo("repo")
+	dir := t.TempDir()
+	bare := filepath.Join(dir, "up", "repo.git")
+	runGit(t, "", "clone", "--quiet", "--bare", "--ref-format=files", upstream.bare(), bare)
+
+	tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir })
+	if got := tc.Repositories(); len(got) != 0 {
+		t.Fatalf("startup kept an old-format repository: %+v", got)
+	}
+	tc.clone(t, "repo")
+	waitIdle(t, tc.Cache)
+	if outdated(bare) {
+		t.Error("the repository was not cloned again with reftable")
+	}
+}
+
+func TestTooOld(t *testing.T) {
+	for out, want := range map[string]bool{
+		"git version 2.55.2\n":                 true,
+		"git version 2.39.1 (Apple Git-146)\n": true,
+		"git version 2.56.0.windows.1\n":       false,
+		"git version 2.57.0-rc0\n":             false,
+		"git version 3.0.0\n":                  false,
+		"git version 2.x\n":                    false,
+	} {
+		if got := tooOld(out); got != want {
+			t.Errorf("tooOld(%q) = %v, want %v", out, got, want)
+		}
+	}
+}
+
 // TestCacheDirSharedWithOtherData points the cache at a directory that also
 // holds an artifact store and a database, as git.cache_dir: ./cache would.
 func TestCacheDirSharedWithOtherData(t *testing.T) {

internal/gitcache/refresh.go

@@ -221,7 +221,7 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo
 	}
 	defer func() { _ = os.RemoveAll(tmp) }()
 
-	if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil {
+	if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare", "--ref-format=reftable"); err != nil {
 		return err
 	}
 	if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil {

internal/gitcache/stats_test.go

@@ -225,14 +225,20 @@ func TestRepositoriesKeepCaseWhereKeysAreFolded(t *testing.T) {
 
 // TestRepositoryStateRetriesAFailedRead checks a read that fails, here on
 // a branch whose commit is missing, is reported as an error and read again
-// a while later even if the upstream isn't checked in between.
+// a while later even if the upstream isn't checked in between. The broken
+// branch is a hand-written loose ref, so the repository is swapped for one
+// in the files format.
 func TestRepositoryStateRetriesAFailedRead(t *testing.T) {
 	upstream := newFakeUpstream(t)
 	upstream.newRepo("repo")
 	tc := newTestCache(t, upstream.server.URL, nil)
 	tc.clone(t, "repo")
 	waitIdle(t, tc.Cache)
 	rp := tc.repo("up/repo.git")
+	if err := os.RemoveAll(rp.dir); err != nil {
+		t.Fatal(err)
+	}
+	runGit(t, "", "clone", "--quiet", "--bare", "--ref-format=files", upstream.bare(), rp.dir)
 	ref := filepath.Join(rp.dir, "refs", "heads", "main")
 	head := runGit(t, rp.dir, "rev-parse", "main")
 	if err := os.WriteFile(ref, []byte(strings.Repeat("1", len(head))+"\n"), 0o600); err != nil {

internal/server/git_ui_test.go

@@ -55,7 +55,7 @@ func TestGitRepositoriesPageSaysWhatIsNotReadYet(t *testing.T) {
 	upstreams := map[string]config.GitUpstream{"up": {URL: "https://example.com"}}
 	upstreamDir := gitcache.UpstreamDir("up", "https://example.com")
 	for _, name := range []string{"one", "two"} {
-		gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", filepath.Join(cacheDir, upstreamDir, "team", name+".git"))
+		gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(cacheDir, upstreamDir, "team", name+".git"))
 	}
 	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
 	cache, err := gitcache.New(gitcache.Options{
@@ -97,11 +97,7 @@ func TestGitRepositoriesPageLogsAStateItCannotRead(t *testing.T) {
 	cacheDir := t.TempDir()
 	upstreamDir := gitcache.UpstreamDir("up", "https://example.com")
 	repoDir := filepath.Join(cacheDir, upstreamDir, "team", "broken.git")
-	gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", repoDir)
-	// A branch naming a commit the repository doesn't have.
-	if err := os.WriteFile(filepath.Join(repoDir, "refs", "heads", "main"), []byte(strings.Repeat("1", 40)+"\n"), 0o600); err != nil {
-		t.Fatal(err)
-	}
+	gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", repoDir)
 	var logs bytes.Buffer
 	logger := slog.New(slog.NewTextHandler(&logs, nil))
 	cache, err := gitcache.New(gitcache.Options{
@@ -112,6 +108,15 @@ func TestGitRepositoriesPageLogsAStateItCannotRead(t *testing.T) {
 		t.Fatal(err)
 	}
 	t.Cleanup(func() { _ = cache.Close() })
+	// A branch naming a commit the repository doesn't have, as a loose ref
+	// in a files-format repository swapped in after the cache loaded.
+	if err := os.RemoveAll(repoDir); err != nil {
+		t.Fatal(err)
+	}
+	gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", repoDir)
+	if err := os.WriteFile(filepath.Join(repoDir, "refs", "heads", "main"), []byte(strings.Repeat("1", 40)+"\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
 	s := &Server{cfg: &config.Config{BaseURL: "http://proxy.example"}, git: cache, logger: logger, templates: &Templates{}}
 	s.cfg.Upstream.Git = map[string]config.GitUpstream{"up": {URL: "https://example.com"}}
 
@@ -135,7 +140,7 @@ func TestGitRepositoryRowsKeepCase(t *testing.T) {
 	cacheDir := t.TempDir()
 	upstreams := map[string]config.GitUpstream{"Up": {URL: "https://example.com", Repos: []string{"Team/*"}}}
 	upstreamDir := gitcache.UpstreamDir("Up", "https://example.com")
-	gitCommand(t, "", "init", "--quiet", "--bare", filepath.Join(cacheDir, upstreamDir, "Team", "Repo.git"))
+	gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(cacheDir, upstreamDir, "Team", "Repo.git"))
 	cache, err := gitcache.New(gitcache.Options{
 		CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour,
 		Client: &http.Client{}, Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), Upstreams: map[string]string{"Up": upstreamDir},
@@ -235,7 +240,7 @@ func TestGitRepositoryRowsEscapeCloneURL(t *testing.T) {
 func TestGitCacheGauges(t *testing.T) {
 	cacheDir := t.TempDir()
 	upstreamDir := gitcache.UpstreamDir("up", "https://example.com")
-	gitCommand(t, "", "init", "--quiet", "--bare", filepath.Join(cacheDir, upstreamDir, "team", "repo.git"))
+	gitCommand(t, "", "init", "--quiet", "--bare", "--ref-format=reftable", filepath.Join(cacheDir, upstreamDir, "team", "repo.git"))
 	logger := slog.New(slog.NewTextHandler(io.Discard, nil))
 	cache, err := gitcache.New(gitcache.Options{
 		CacheDir: cacheDir, RefreshWait: time.Second, StaleMaxAge: time.Hour,