diff --git a/README.md b/README.md index 652b490..d5149c8 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ brief crate:serde brief pypi:requests ``` -Remote sources are shallow-cloned by default. Use `--depth 0` for a full clone, `--keep` to preserve the clone, or `--dir ./somewhere` to clone into a specific directory. +Remote sources are shallow-cloned by default. Use `--depth 0` for a full clone, `--keep` to preserve the clone, or `--dir ./somewhere` to clone into a specific directory. Use `--cache ./cache` to keep one shallow checkout per HTTPS URL and reuse it across runs. Cache mode cannot be combined with `--depth 0` or `--dir`. JSON when piped, human-readable on a TTY. Force either with `--json` or `--human`. Use `--category test` to filter to a single category. diff --git a/cmd/brief/enrich.go b/cmd/brief/enrich.go index 0d6c000..cfa5dc9 100644 --- a/cmd/brief/enrich.go +++ b/cmd/brief/enrich.go @@ -30,6 +30,7 @@ func cmdEnrich(args []string) { keep := fs.Bool("keep", false, "Keep downloaded remote source") depth := fs.Int("depth", -1, "Git clone depth (0 = full clone, default shallow)") dir := fs.String("dir", "", "Directory to clone remote source into") + cache := fs.String("cache", "", "Persistent shallow cache for HTTPS remotes (incompatible with -depth 0 and -dir)") scanDepth := fs.Int("scan-depth", 0, "Max directory depth for language detection (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") _ = fs.Parse(args) @@ -43,6 +44,7 @@ func cmdEnrich(args []string) { Keep: *keep, Depth: *depth, Dir: *dir, + Cache: *cache, }) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) diff --git a/cmd/brief/main.go b/cmd/brief/main.go index 520185b..ee6d73e 100644 --- a/cmd/brief/main.go +++ b/cmd/brief/main.go @@ -73,6 +73,7 @@ func cmdScan(args []string) { keep := fs.Bool("keep", false, "Keep downloaded remote source") depth := fs.Int("depth", -1, "Git clone depth (0 = full clone, default shallow)") dir := fs.String("dir", "", "Directory to clone remote source into") + cache := fs.String("cache", "", "Persistent shallow cache for HTTPS remotes (incompatible with -depth 0 and -dir)") scanDepth := fs.Int("scan-depth", 0, "Max directory depth for language detection (0 = unlimited)") skip := fs.String("skip", "", "Additional directories to skip, comma-separated") tracked := fs.Bool("tracked", false, "Only consider files tracked by git") @@ -104,6 +105,7 @@ func cmdScan(args []string) { Keep: *keep, Depth: *depth, Dir: *dir, + Cache: *cache, }) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) diff --git a/cmd/brief/outline.go b/cmd/brief/outline.go index c876534..07065b2 100644 --- a/cmd/brief/outline.go +++ b/cmd/brief/outline.go @@ -29,6 +29,7 @@ func cmdOutline(args []string) { keep := fs.Bool("keep", false, "Keep downloaded remote source") depth := fs.Int("depth", -1, "Git clone depth (0 = full clone, default shallow)") dir := fs.String("dir", "", "Directory to clone remote source into") + cache := fs.String("cache", "", "Persistent shallow cache for HTTPS remotes (incompatible with -depth 0 and -dir)") _ = fs.Parse(args) path := "." @@ -40,6 +41,7 @@ func cmdOutline(args []string) { Keep: *keep, Depth: *depth, Dir: *dir, + Cache: *cache, }) if err != nil { _, _ = fmt.Fprintf(os.Stderr, "error: %v\n", err) diff --git a/go.mod b/go.mod index 6f3c9a8..3abdbda 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ toolchain go1.26.6 require ( github.com/BurntSushi/toml v1.6.0 github.com/git-pkgs/archives v0.5.0 + github.com/git-pkgs/clone v0.5.0 github.com/git-pkgs/enrichment v0.6.5 github.com/git-pkgs/forge v0.8.0 github.com/git-pkgs/licensecheck v0.4.1 diff --git a/go.sum b/go.sum index 12af250..974e2a2 100644 --- a/go.sum +++ b/go.sum @@ -16,6 +16,8 @@ github.com/ecosyste-ms/ecosystems-go v0.4.0 h1:5A+zF+XWT8sYYYjlc2/tI1SmiDGzbHLyT github.com/ecosyste-ms/ecosystems-go v0.4.0/go.mod h1:FVswCrp3DQkur1HjVqfDF/gYrDSEmiFflntcB1G0DbA= github.com/git-pkgs/archives v0.5.0 h1:QdowC1jTOSbEOKTYGqVt8ZjIr+yJzy7cmLNWcLPj9Y8= github.com/git-pkgs/archives v0.5.0/go.mod h1:tfio0OIuPKEBKHs/UCL5XBUvYmKpnvtnba2iDlfSd6g= +github.com/git-pkgs/clone v0.5.0 h1:0cYpr8PQpylqtvA9v1NBE9JXJeLLlrspJbvl2JYVLxU= +github.com/git-pkgs/clone v0.5.0/go.mod h1:nixjE44maKRXbu/dEiunJmHApWgLYp0IR7J2yWaYHv8= github.com/git-pkgs/enrichment v0.6.5 h1:U0SPzWVGoK4R8TwojCTASBRTEV+QSs0IitdLmzI/g/k= github.com/git-pkgs/enrichment v0.6.5/go.mod h1:Vt2PLMvWPOio9DLyC8Gdhh1yxsHwcRcG+L2Kkc9+kak= github.com/git-pkgs/forge v0.8.0 h1:rfG6YujMGEN27Sj5Bd2cYIA1XR8tYd0fU5RdsvSunC4= diff --git a/remote/remote.go b/remote/remote.go index d5cc939..b551d27 100644 --- a/remote/remote.go +++ b/remote/remote.go @@ -4,11 +4,13 @@ package remote import ( "context" + "errors" "fmt" "os" "os/exec" "strings" + gitclone "github.com/git-pkgs/clone" forges "github.com/git-pkgs/forge" "github.com/git-pkgs/purl" "github.com/git-pkgs/registries" @@ -27,6 +29,7 @@ type Options struct { Keep bool // don't delete temp dir after scanning Depth int // git clone depth (0 = full clone, -1 or unset = default shallow) Dir string // directory to clone into (empty = temp dir) + Cache string // persistent shallow clone cache directory (empty = no cache) } // Resolve takes a source string and returns a local directory to scan. @@ -134,22 +137,11 @@ func cloneURL(ctx context.Context, url, name string, opts Options) (*Source, err managed = true } - args := []string{"clone"} - if opts.Depth > 0 { - args = append(args, "--depth", fmt.Sprintf("%d", opts.Depth)) - } - args = append(args, url, dir) - - cmd := exec.CommandContext(ctx, "git", args...) - cmd.Stdout = os.Stderr - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { + if err := cloneInto(ctx, url, dir, opts); err != nil { if managed { _ = os.RemoveAll(dir) } - // git already wrote a (credential-redacted) failure message to - // stderr, so don't repeat the raw URL here. - return nil, fmt.Errorf("git clone failed: %w", err) + return nil, err } cleanup := func() { @@ -160,3 +152,53 @@ func cloneURL(ctx context.Context, url, name string, opts Options) (*Source, err return &Source{Dir: dir, Cleanup: cleanup, Origin: url}, nil } + +var ensureClone = func(ctx context.Context, url, dst string, full bool) error { + return gitclone.Ensure(ctx, gitclone.Retry{}, url, dst, "", full) +} + +var prepareCloneCache = func(ctx context.Context, root, url, dst string) error { + cache := gitclone.Cache{Root: root} + _, err := cache.Prepare(ctx, url, "", dst) + return err +} + +var execClone = func(ctx context.Context, url, dir string, depth int) error { + args := []string{"clone"} + if depth > 0 { + args = append(args, "--depth", fmt.Sprintf("%d", depth)) + } + args = append(args, url, dir) + + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + // Git already wrote a credential-redacted failure to stderr, so do + // not repeat the raw URL here. + return fmt.Errorf("git clone failed: %w", err) + } + return nil +} + +func cloneInto(ctx context.Context, url, dir string, opts Options) error { + if !strings.HasPrefix(url, "https://") { + return execClone(ctx, url, dir, opts.Depth) + } + if opts.Cache != "" { + if opts.Dir != "" { + return errors.New("clone cache cannot be combined with an explicit clone directory") + } + if opts.Depth == 0 { + return errors.New("clone cache does not support full clones (depth 0)") + } + if err := prepareCloneCache(ctx, opts.Cache, url, dir); err != nil { + return fmt.Errorf("preparing clone cache: %w", err) + } + return nil + } + if err := ensureClone(ctx, url, dir, opts.Depth == 0); err != nil { + return fmt.Errorf("cloning remote: %w", err) + } + return nil +} diff --git a/remote/remote_test.go b/remote/remote_test.go new file mode 100644 index 0000000..56934fe --- /dev/null +++ b/remote/remote_test.go @@ -0,0 +1,281 @@ +package remote + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestResolveLocalPath(t *testing.T) { + source, err := Resolve(context.Background(), "./project", Options{}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if source.Dir != "./project" { + t.Errorf("Dir = %q", source.Dir) + } + if source.Origin != "" { + t.Errorf("Origin = %q", source.Origin) + } +} + +func TestResolveHTTPSDefaultShallowAndCleanup(t *testing.T) { + originalEnsure := ensureClone + t.Cleanup(func() { ensureClone = originalEnsure }) + + var clonedDir string + ensureClone = func(_ context.Context, url, dst string, full bool) error { + if url != "https://github.com/example/project" { + t.Errorf("url = %q", url) + } + if full { + t.Error("default clone should be shallow") + } + clonedDir = dst + return nil + } + + source, err := Resolve(context.Background(), "https://github.com/example/project", Options{Depth: -1}) + if err != nil { + t.Fatalf("Resolve: %v", err) + } + if source.Dir != clonedDir { + t.Errorf("Dir = %q, cloned into %q", source.Dir, clonedDir) + } + if source.Origin != "https://github.com/example/project" { + t.Errorf("Origin = %q", source.Origin) + } + if _, err := os.Stat(source.Dir); err != nil { + t.Fatalf("temporary clone directory: %v", err) + } + source.Cleanup() + if _, err := os.Stat(source.Dir); !os.IsNotExist(err) { + t.Errorf("temporary clone directory remains after cleanup: %v", err) + } +} + +func TestResolveHTTPSRemovesTemporaryDirectoryOnFailure(t *testing.T) { + originalEnsure := ensureClone + t.Cleanup(func() { ensureClone = originalEnsure }) + + var clonedDir string + ensureClone = func(_ context.Context, _, dst string, _ bool) error { + clonedDir = dst + return errors.New("clone failed") + } + + _, err := Resolve(context.Background(), "https://github.com/example/project", Options{}) + if err == nil || !strings.Contains(err.Error(), "clone failed") { + t.Fatalf("Resolve error = %v", err) + } + if _, err := os.Stat(clonedDir); !os.IsNotExist(err) { + t.Errorf("temporary clone directory remains after failure: %v", err) + } +} + +func TestHTTPSCloneDepthMapping(t *testing.T) { + originalEnsure := ensureClone + originalCache := prepareCloneCache + originalExec := execClone + t.Cleanup(func() { + ensureClone = originalEnsure + prepareCloneCache = originalCache + execClone = originalExec + }) + + execClone = func(context.Context, string, string, int) error { + return errors.New("unexpected exec clone") + } + prepareCloneCache = func(context.Context, string, string, string) error { + return errors.New("unexpected cache clone") + } + + for _, tt := range []struct { + name string + depth int + full bool + }{ + {name: "full", depth: 0, full: true}, + {name: "shallow", depth: 1, full: false}, + {name: "positive depth is shallow", depth: 20, full: false}, + } { + t.Run(tt.name, func(t *testing.T) { + called := false + ensureClone = func(_ context.Context, url, dst string, full bool) error { + called = true + if url != "https://example.com/owner/repo" { + t.Errorf("url = %q", url) + } + if dst != "/tmp/brief-checkout" { + t.Errorf("dst = %q", dst) + } + if full != tt.full { + t.Errorf("full = %v, want %v", full, tt.full) + } + return nil + } + + err := cloneInto( + context.Background(), + "https://example.com/owner/repo", + "/tmp/brief-checkout", + Options{Depth: tt.depth}, + ) + if err != nil { + t.Fatalf("cloneInto: %v", err) + } + if !called { + t.Error("clone Ensure was not called") + } + }) + } +} + +func TestHTTPSCloneCache(t *testing.T) { + originalEnsure := ensureClone + originalCache := prepareCloneCache + t.Cleanup(func() { + ensureClone = originalEnsure + prepareCloneCache = originalCache + }) + + ensureClone = func(context.Context, string, string, bool) error { + return errors.New("unexpected clone Ensure") + } + called := false + prepareCloneCache = func(_ context.Context, root, url, dst string) error { + called = true + if root != "/tmp/brief-cache" { + t.Errorf("cache root = %q", root) + } + if url != "https://example.com/owner/repo" { + t.Errorf("url = %q", url) + } + if dst != "/tmp/brief-checkout" { + t.Errorf("dst = %q", dst) + } + return nil + } + + err := cloneInto( + context.Background(), + "https://example.com/owner/repo", + "/tmp/brief-checkout", + Options{Depth: 1, Cache: "/tmp/brief-cache"}, + ) + if err != nil { + t.Fatalf("cloneInto: %v", err) + } + if !called { + t.Error("clone cache was not prepared") + } +} + +func TestHTTPSCloneCacheError(t *testing.T) { + originalCache := prepareCloneCache + t.Cleanup(func() { prepareCloneCache = originalCache }) + + prepareCloneCache = func(context.Context, string, string, string) error { + return errors.New("cache failed") + } + err := cloneInto( + context.Background(), + "https://example.com/owner/repo", + "/tmp/brief-checkout", + Options{Depth: 1, Cache: "/tmp/brief-cache"}, + ) + if err == nil || !strings.Contains(err.Error(), "preparing clone cache: cache failed") { + t.Fatalf("cloneInto error = %v", err) + } +} + +func TestHTTPSCloneCacheRejectsFullClone(t *testing.T) { + originalCache := prepareCloneCache + t.Cleanup(func() { prepareCloneCache = originalCache }) + + prepareCloneCache = func(context.Context, string, string, string) error { + return errors.New("unexpected cache preparation") + } + err := cloneInto( + context.Background(), + "https://example.com/owner/repo", + "/tmp/brief-checkout", + Options{Depth: 0, Cache: "/tmp/brief-cache"}, + ) + if err == nil || !strings.Contains(err.Error(), "clone cache does not support full clones (depth 0)") { + t.Fatalf("cloneInto error = %v", err) + } +} + +func TestHTTPSCloneCacheRejectsExplicitDirectoryWithoutRemovingIt(t *testing.T) { + originalCache := prepareCloneCache + t.Cleanup(func() { prepareCloneCache = originalCache }) + + prepareCloneCache = func(context.Context, string, string, string) error { + return errors.New("unexpected cache preparation") + } + dir := t.TempDir() + sentinel := filepath.Join(dir, "keep.txt") + if err := os.WriteFile(sentinel, []byte("keep"), 0o644); err != nil { + t.Fatalf("write sentinel: %v", err) + } + + _, err := Resolve( + context.Background(), + "https://example.com/owner/repo", + Options{Depth: 1, Dir: dir, Cache: t.TempDir()}, + ) + if err == nil || !strings.Contains(err.Error(), "clone cache cannot be combined with an explicit clone directory") { + t.Fatalf("Resolve error = %v", err) + } + content, err := os.ReadFile(sentinel) + if err != nil { + t.Fatalf("read sentinel: %v", err) + } + if string(content) != "keep" { + t.Fatalf("sentinel content = %q", content) + } +} + +func TestNonHTTPSCloneUsesGitCommand(t *testing.T) { + originalEnsure := ensureClone + originalExec := execClone + t.Cleanup(func() { + ensureClone = originalEnsure + execClone = originalExec + }) + + ensureClone = func(context.Context, string, string, bool) error { + return errors.New("unexpected clone Ensure") + } + called := false + execClone = func(_ context.Context, url, dst string, depth int) error { + called = true + if url != "git@example.com:owner/repo.git" { + t.Errorf("url = %q", url) + } + if dst != "/tmp/brief-checkout" { + t.Errorf("dst = %q", dst) + } + if depth != 7 { + t.Errorf("depth = %d", depth) + } + return nil + } + + err := cloneInto( + context.Background(), + "git@example.com:owner/repo.git", + "/tmp/brief-checkout", + Options{Depth: 7}, + ) + if err != nil { + t.Fatalf("cloneInto: %v", err) + } + if !called { + t.Error("Git clone command was not called") + } +}