From c6ebe25f547a8216165ae95c1b2cfa4c8dca52d1 Mon Sep 17 00:00:00 2001 From: Felipe Santos Date: Fri, 13 Nov 2026 00:20:31 -0300 Subject: [PATCH] Keep cached repositories in reftable when git is 2.56 or newer --- docs/configuration.md | 7 +++++ internal/gitcache/cache.go | 49 ++++++++++++++++++++++++++++++++- internal/gitcache/cache_test.go | 46 +++++++++++++++++++++++++++++++ internal/gitcache/refresh.go | 6 +++- internal/gitcache/stats_test.go | 10 +++---- 5 files changed, 110 insertions(+), 8 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 40e94343..ee6ee78c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -951,6 +951,13 @@ A transfer that receives nothing for 10 minutes is abandoned. Git's automatic garbage collection runs as part of fetches, and files over 8MB are kept without delta compression, so one large file does not take a fetch's memory. +With `git` 2.56 or newer, repositories are cached with reftable reference +storage, which writes a fetch's reference updates to one table rather than +a file per reference, so repositories with many branches clone and refresh +faster, most of all on network file systems. Repositories cached in the +files format are converted at startup; one that fails to convert is served +as it is. With an older `git`, repositories keep the files format. + Packs for clients are built by at most two `upload-pack` processes per CPU the proxy may use, each with one delta-search thread and 64MB of delta window, and further clones wait for one to finish; listing refs does not diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go index 2bef68e7..0a22e8c1 100644 --- a/internal/gitcache/cache.go +++ b/internal/gitcache/cache.go @@ -98,6 +98,9 @@ type Cache struct { // foldCase is set where the cache directory ignores case, so keys that // differ only in case, which name one directory, are one repository. foldCase bool + // reftable is set when git is new enough for the cache to keep + // references in the reftable format. + reftable bool // stateReads lets one RepositoryState read run git at a time. stateReads chan struct{} @@ -245,6 +248,12 @@ func New(opts Options) (*Cache, error) { if opts.maxRequestBody > 0 { c.maxRequestBody = opts.maxRequestBody } + out, err := c.command(context.Background(), nil, "version").Output() + c.reftable = err == nil && reftableReady(string(out)) + if !c.reftable { + logger.Info("git cache: repositories keep the files reference format, as reftable needs git 2.56 or newer", + "version", strings.TrimSpace(string(out))) + } if err := c.load(); err != nil { _ = lock.Close() return nil, err @@ -289,7 +298,8 @@ func (c *Cache) tmpDir() string { } // load clears what an earlier run left in the temporary directory and -// registers the repositories already cached. +// registers the repositories already cached, converting those in the files +// reference format to reftable when git supports it. func (c *Cache) load() error { if err := os.RemoveAll(c.tmpDir()); err != nil { return fmt.Errorf("clearing %s: %w", c.tmpDir(), err) @@ -306,6 +316,9 @@ func (c *Cache) load() error { } rel, _ := filepath.Rel(c.dir, path) rp := newRepo(c.repoKey(filepath.ToSlash(rel)), path) + if c.reftable && !hasReftable(path) { + c.migrate(rp) + } rp.size.Store(dirSize(path)) rp.lastChecked.Store(stampTime(path, checkedStamp)) rp.lastUsed.Store(stampTime(path, usedStamp)) @@ -320,6 +333,40 @@ func (c *Cache) load() error { return nil } +// reftableReady reports whether out, the output of "git version", names git +// 2.56 or newer, the first with the reftable fixes the cache relies on. +func reftableReady(out string) bool { + var major, minor int + _, err := fmt.Sscanf(out, "git version %d.%d", &major, &minor) + return err == nil && (major > 2 || major == 2 && minor >= 56) +} + +// hasReftable reports whether the repository at dir keeps its references in +// the reftable format. It looks for the table list rather than running git, +// so a startup costs no process per repository once all are converted. Only +// a list known to be missing counts as the files format. +func hasReftable(dir string) bool { + _, err := os.Stat(filepath.Join(dir, "reftable", "tables.list")) + return !errors.Is(err, fs.ErrNotExist) +} + +// migrate converts the references of the repository rp to the reftable +// format. load calls it before any request can write to the repository, +// which git refs migrate does not guard against. One that fails to convert +// is served as it is. +func (c *Cache) migrate(rp *repo) { + // GIT_DIR rather than -C, which would have git look for a repository in + // the directories above one that is not. + out, err := c.command(context.Background(), []string{"GIT_DIR=" + rp.dir}, + "refs", "migrate", "--ref-format=reftable").CombinedOutput() + if err != nil { + c.logger.Warn("git cache: failed to convert a repository to the reftable format; serving it as it is", + "repository", rp.key, "error", err, "output", strings.TrimSpace(string(out))) + return + } + c.logger.Info("git cache: converted a repository to the reftable format", "repository", rp.key) +} + // upstreamDirPattern matches a directory UpstreamDir names, capturing the // upstream's name. var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`) diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go index 2b2e388d..15e8ca6a 100644 --- a/internal/gitcache/cache_test.go +++ b/internal/gitcache/cache_test.go @@ -692,6 +692,52 @@ func TestNewRemovesRepointedUpstreams(t *testing.T) { } } +// TestNewMigratesFilesFormatRepositories starts a cache whose directory +// holds a repository in the files reference format: it is converted to +// reftable with its refs, and served without being cloned again. +func TestNewMigratesFilesFormatRepositories(t *testing.T) { + // Before the fixture, as git older than 2.45 has no --ref-format. + if !reftableReady(runGit(t, "", "version")) { + t.Skip("git is older than 2.56") + } + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + dir := t.TempDir() + bare := filepath.Join(dir, "up", "repo.git") + runGit(t, "", "clone", "--quiet", "--bare", "--ref-format=files", upstream.bare(), bare) + refs := runGit(t, bare, "for-each-ref", "--format=%(objectname) %(refname)") + + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.CacheDir = dir }) + if !hasReftable(bare) { + t.Fatal("the repository was not converted to reftable") + } + if got := tc.Repositories(); len(got) != 1 { + t.Fatalf("startup registered %+v, want the converted repository", got) + } + if got := tc.cachedRefs(t); got != refs { + t.Errorf("refs after conversion:\n%s\nwant:\n%s", got, refs) + } + tc.clone(t, "repo") +} + +// TestReftableReady checks which versions turn reftable on, across the ways +// git prints its version. +func TestReftableReady(t *testing.T) { + for out, want := range map[string]bool{ + "git version 2.56.1\n": true, + "git version 2.56.0.windows.1\n": true, + "git version 2.57.0-rc0\n": true, + "git version 3.0.0\n": true, + "git version 2.55.2\n": false, + "git version 2.39.1 (Apple Git-146)\n": false, + "git version 2.x\n": false, + } { + if got := reftableReady(out); got != want { + t.Errorf("reftableReady(%q) = %v, want %v", out, got, want) + } + } +} + // TestCacheDirSharedWithOtherData points the cache at a directory that also // holds an artifact store and a database, as git.cache_dir: ./cache would. func TestCacheDirSharedWithOtherData(t *testing.T) { diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go index 683be053..d07d110c 100644 --- a/internal/gitcache/refresh.go +++ b/internal/gitcache/refresh.go @@ -221,7 +221,11 @@ func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remo } defer func() { _ = os.RemoveAll(tmp) }() - if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil { + args := []string{"init", "--quiet", "--bare"} + if c.reftable { + args = append(args, "--ref-format=reftable") + } + if _, err := c.run(ctx, tmp, args...); err != nil { return err } if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, res, refreshFlags, branchesAndTags...); err != nil { diff --git a/internal/gitcache/stats_test.go b/internal/gitcache/stats_test.go index e159ee33..f099cce7 100644 --- a/internal/gitcache/stats_test.go +++ b/internal/gitcache/stats_test.go @@ -6,7 +6,6 @@ import ( "net/http" "os" "path/filepath" - "strings" "testing" "time" ) @@ -233,18 +232,17 @@ func TestRepositoryStateRetriesAFailedRead(t *testing.T) { tc.clone(t, "repo") waitIdle(t, tc.Cache) rp := tc.repo("up/repo.git") - ref := filepath.Join(rp.dir, "refs", "heads", "main") head := runGit(t, rp.dir, "rev-parse", "main") - if err := os.WriteFile(ref, []byte(strings.Repeat("1", len(head))+"\n"), 0o600); err != nil { + missing := runGit(t, rp.dir, "commit-tree", "-p", head, "-m", "missing", head+"^{tree}") + runGit(t, rp.dir, "update-ref", "refs/heads/main", missing) + if err := os.Remove(filepath.Join(rp.dir, "objects", missing[:2], missing[2:])); err != nil { t.Fatal(err) } if st, err := tc.RepositoryState(context.Background(), "up/repo.git"); err == nil { t.Fatalf("state %+v of a branch with no commit, want an error", st) } - if err := os.WriteFile(ref, []byte(head+"\n"), 0o600); err != nil { - t.Fatal(err) - } + runGit(t, rp.dir, "update-ref", "refs/heads/main", head) if _, err := tc.RepositoryState(context.Background(), "up/repo.git"); err == nil { t.Error("failed read not kept for a while") }