Use shared integrity verification

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/git-pkgs/archives v0.5.1
 	github.com/git-pkgs/cooldown v0.1.1
 	github.com/git-pkgs/enrichment v0.6.5
+	github.com/git-pkgs/integrity v0.1.1
 	github.com/git-pkgs/magic v0.2.0
 	github.com/git-pkgs/purl v0.1.17
 	github.com/git-pkgs/registries v0.7.0

go.sum

@@ -250,6 +250,8 @@ github.com/git-pkgs/cooldown v0.1.1 h1:9OqqzCB8gANz/y44SmqGD0Jp8Qtu81D1sCbKl6Ehg
 github.com/git-pkgs/cooldown v0.1.1/go.mod h1:v7APuK/UouTiu8mWQZbdDmj7DfxxkGUeuhjaRB5gv9E=
 github.com/git-pkgs/enrichment v0.6.5 h1:U0SPzWVGoK4R8TwojCTASBRTEV+QSs0IitdLmzI/g/k=
 github.com/git-pkgs/enrichment v0.6.5/go.mod h1:Vt2PLMvWPOio9DLyC8Gdhh1yxsHwcRcG+L2Kkc9+kak=
+github.com/git-pkgs/integrity v0.1.1 h1:nHQ7SktOiGM1dOb5BFnkdtttG/6FCgE6r5ru6QnsGts=
+github.com/git-pkgs/integrity v0.1.1/go.mod h1:hxu24lcd230377hCF28JQW7sGcCbuNLqo/0ULeb+F1Q=
 github.com/git-pkgs/magic v0.2.0 h1:c7HqVxnP8c88EaVMH0/KraDFVTcmiXckRiSvNZEnvMQ=
 github.com/git-pkgs/magic v0.2.0/go.mod h1:3ndidt+yvFaI1M0aEkkzkOlFnLPkeVQASIUojazcxCI=
 github.com/git-pkgs/packageurl-go v0.3.1 h1:WM3RBABQZLaRBxgKyYughc3cVBE8KyQxbSC6Jt5ak7M=

internal/database/database_test.go

@@ -8,6 +8,11 @@ import (
 	"time"
 )
 
+const (
+	testContentHash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	testIntegrity   = "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg=="
+)
+
 func TestCreateAndOpen(t *testing.T) {
 	dir := t.TempDir()
 	dbPath := filepath.Join(dir, "test.db")
@@ -132,7 +137,7 @@ func TestVersionCRUD(t *testing.T) {
 		v := &Version{
 			PURL:        "pkg:npm/lodash@4.17.21",
 			PackagePURL: "pkg:npm/lodash",
-			Integrity:   sql.NullString{String: "sha512-abc123", Valid: true},
+			Integrity:   sql.NullString{String: testIntegrity, Valid: true},
 		}
 
 		err = db.UpsertVersion(v)
@@ -200,7 +205,7 @@ func TestArtifactCRUD(t *testing.T) {
 			t.Error("expected artifact to not be cached yet")
 		}
 
-		err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", "sha256-abc", 12345, "application/gzip")
+		err = db.MarkArtifactCached(versionPURL, "lodash-4.17.21.tgz", "/cache/npm/lodash-4.17.21.tgz", testContentHash, 12345, "application/gzip")
 		if err != nil {
 			t.Fatalf("MarkArtifactCached failed: %v", err)
 		}
@@ -257,7 +262,7 @@ func TestGetCachedArtifact(t *testing.T) {
 		}
 
 		if err := db.MarkArtifactCached(versionPURL, filename, "/cache/npm/"+filename,
-			"sha256-abc", 12345, "application/gzip"); err != nil {
+			testContentHash, 12345, "application/gzip"); err != nil {
 			t.Fatalf("MarkArtifactCached failed: %v", err)
 		}
 
@@ -274,7 +279,7 @@ func TestGetCachedArtifact(t *testing.T) {
 		if cached.StoragePath != "/cache/npm/"+filename {
 			t.Errorf("expected cached storage path, got %q", cached.StoragePath)
 		}
-		if cached.ContentHash.String != "sha256-abc" {
+		if cached.ContentHash.String != testContentHash {
 			t.Errorf("expected cached content hash, got %q", cached.ContentHash.String)
 		}
 		if cached.Size.Int64 != 12345 {
@@ -283,7 +288,7 @@ func TestGetCachedArtifact(t *testing.T) {
 		if cached.ContentType.String != "application/gzip" {
 			t.Errorf("expected cached content type, got %q", cached.ContentType.String)
 		}
-		if cached.Integrity.String != "sha512-abc123" {
+		if cached.Integrity.String != testIntegrity {
 			t.Errorf("expected cached integrity, got %q", cached.Integrity.String)
 		}
 
@@ -306,7 +311,7 @@ func seedCachedArtifactTestData(t *testing.T, db *DB, packagePURL, versionPURL,
 	if err := db.UpsertVersion(&Version{
 		PURL:        versionPURL,
 		PackagePURL: packagePURL,
-		Integrity:   sql.NullString{String: "sha512-abc123", Valid: true},
+		Integrity:   sql.NullString{String: testIntegrity, Valid: true},
 	}); err != nil {
 		t.Fatalf("UpsertVersion failed: %v", err)
 	}

internal/handler/container_test.go

@@ -482,8 +482,9 @@ func TestContainerHandler_BlobHead_DirectServeRedirects(t *testing.T) {
 	if got := w.Header().Get("Location"); got != store.signedURL {
 		t.Errorf("Location = %q, want %q", got, store.signedURL)
 	}
-	if got := w.Header().Get("ETag"); got != `"abc123"` {
-		t.Errorf("ETag = %q, want %q", got, `"abc123"`)
+	wantETag := `"` + sha256Hex("cached blob") + `"`
+	if got := w.Header().Get("ETag"); got != wantETag {
+		t.Errorf("ETag = %q, want %q", got, wantETag)
 	}
 	if w.Body.Len() != 0 {
 		t.Errorf("HEAD response body length = %d, want 0", w.Body.Len())

internal/handler/download_test.go

@@ -49,7 +49,7 @@ func seedPackageWithPURL(t *testing.T, db *database.DB, store *mockStorage, ecos
 		Filename:    filename,
 		UpstreamURL: "https://example.com/" + filename,
 		StoragePath: sql.NullString{String: storagePath, Valid: true},
-		ContentHash: sql.NullString{String: "abc123", Valid: true},
+		ContentHash: sql.NullString{String: sha256Hex(content), Valid: true},
 		Size:        sql.NullInt64{Int64: int64(len(content)), Valid: true},
 		ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
 		FetchedAt:   sql.NullTime{Time: time.Now(), Valid: true},

internal/handler/handler.go

@@ -194,6 +194,11 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
 	if artifact == nil {
 		return nil, nil
 	}
+	checks, err := newIntegrityChecks(artifact.ContentHash.String, artifact.Integrity.String)
+	if err != nil {
+		p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err)
+		return nil, nil
+	}
 
 	result := &CacheResult{
 		Size:        artifact.Size.Int64,
@@ -225,16 +230,21 @@ func (p *Proxy) checkCache(ctx context.Context, pkgPURL, versionPURL, filename s
 		return nil, nil
 	}
 
-	result.Reader = newVerifyingReader(reader, artifact.ContentHash.String, artifact.Integrity.String,
+	result.Reader, err = checks.wrap(reader,
 		func(reason string) {
 			p.Logger.Error("cached artifact failed integrity check",
 				"purl", versionPURL, "filename", filename,
 				"path", artifact.StoragePath, "reason", reason)
-			metrics.RecordIntegrityFailure(artifact.Ecosystem)
+			metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem))
 			if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil {
 				p.Logger.Warn("failed to clear corrupt artifact from cache", "error", err)
 			}
 		})
+	if err != nil {
+		_ = reader.Close()
+		p.rejectUnusableCacheRecord(artifact, versionPURL, filename, err)
+		return nil, nil
+	}
 	p.recordCacheHit(artifact.Ecosystem, versionPURL, filename)
 	return result, nil
 }
@@ -264,6 +274,16 @@ func (p *Proxy) recordCacheHit(ecosystem, versionPURL, filename string) {
 	metrics.RecordCacheHit(ecosystem)
 }
 
+func (p *Proxy) rejectUnusableCacheRecord(artifact *database.CachedArtifact, versionPURL, filename string, cause error) {
+	p.Logger.Warn("cached artifact has unusable integrity metadata",
+		"purl", versionPURL, "filename", filename,
+		"path", artifact.StoragePath, "error", cause)
+	metrics.RecordIntegrityFailure(purl.NormalizeEcosystem(artifact.Ecosystem))
+	if err := p.DB.ClearArtifactCache(versionPURL, filename); err != nil {
+		p.Logger.Warn("failed to clear unusable artifact from cache", "error", err)
+	}
+}
+
 func (p *Proxy) fetchAndCache(ctx context.Context, ecosystem, name, version, filename, pkgPURL, versionPURL string) (*CacheResult, error) {
 	// Resolve download URL
 	info, err := p.Resolver.Resolve(ctx, ecosystem, name, version)

internal/handler/handler_test.go

@@ -3,9 +3,7 @@ package handler
 import (
 	"bytes"
 	"context"
-	"crypto/sha256"
 	"database/sql"
-	"encoding/hex"
 	"errors"
 	"io"
 	"log/slog"
@@ -46,8 +44,7 @@ func (s *mockStorage) Store(_ context.Context, path string, r io.Reader) (int64,
 		return 0, "", err
 	}
 	s.files[path] = data
-	digest := sha256.Sum256(data)
-	return int64(len(data)), hex.EncodeToString(digest[:]), nil
+	return int64(len(data)), sha256Hex(string(data)), nil
 }
 
 func (s *mockStorage) Open(_ context.Context, path string) (io.ReadCloser, error) {
@@ -182,7 +179,7 @@ func seedPackage(t testing.TB, db *database.DB, store *mockStorage, ecosystem, n
 		Filename:    filename,
 		UpstreamURL: "https://example.com/" + filename,
 		StoragePath: sql.NullString{String: storagePath, Valid: true},
-		ContentHash: sql.NullString{String: "abc123", Valid: true},
+		ContentHash: sql.NullString{String: sha256Hex(content), Valid: true},
 		Size:        sql.NullInt64{Int64: int64(len(content)), Valid: true},
 		ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
 		FetchedAt:   sql.NullTime{Time: time.Now(), Valid: true},
@@ -271,8 +268,74 @@ func TestGetOrFetchArtifact_CacheHit(t *testing.T) {
 	if result.ContentType != "application/octet-stream" {
 		t.Errorf("got content type %q, want %q", result.ContentType, "application/octet-stream")
 	}
-	if result.Hash != "abc123" {
-		t.Errorf("got hash %q, want %q", result.Hash, "abc123")
+	if result.Hash != sha256Hex("cached content") {
+		t.Errorf("got hash %q, want %q", result.Hash, sha256Hex("cached content"))
+	}
+}
+
+func TestGetCachedArtifactRejectsMalformedIntegrityMetadata(t *testing.T) {
+	tests := []struct {
+		name               string
+		malformedHash      string
+		malformedIntegrity string
+	}{
+		{name: "content hash", malformedHash: "abc123"},
+		{name: "native integrity", malformedIntegrity: "sha512-abc123"},
+	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			assertMalformedCacheRejected(t, test.malformedHash, test.malformedIntegrity)
+		})
+	}
+}
+
+func assertMalformedCacheRejected(t *testing.T, malformedHash, malformedIntegrity string) {
+	t.Helper()
+	proxy, db, store, _ := setupTestProxy(t)
+	const (
+		packageName = "broken"
+		version     = "1.0.0"
+		filename    = "broken-1.0.0.tgz"
+	)
+	seedPackage(t, db, store, "npm", packageName, version, filename, "cached content")
+	versionPURL := purl.MakePURLString("npm", packageName, version)
+
+	if malformedHash != "" {
+		artifact, err := db.GetArtifact(versionPURL, filename)
+		if err != nil {
+			t.Fatal(err)
+		}
+		artifact.ContentHash = sql.NullString{String: malformedHash, Valid: true}
+		if err := db.UpsertArtifact(artifact); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if malformedIntegrity != "" {
+		versionRecord := &database.Version{
+			PURL:        versionPURL,
+			PackagePURL: purl.MakePURLString("npm", packageName, ""),
+			Integrity:   sql.NullString{String: malformedIntegrity, Valid: true},
+		}
+		if err := db.UpsertVersion(versionRecord); err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	proxy.DirectServe = true
+	store.signedURL = "https://cache.example/broken"
+	result, err := proxy.GetCachedArtifact(context.Background(), "npm", packageName, version, filename)
+	if err != nil {
+		t.Fatalf("GetCachedArtifact: %v", err)
+	}
+	if result != nil {
+		t.Errorf("GetCachedArtifact = %+v, want nil", result)
+	}
+	artifact, err := db.GetArtifact(versionPURL, filename)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if artifact.StoragePath.Valid {
+		t.Error("unusable cache record retained its storage path")
 	}
 }
 
@@ -307,7 +370,7 @@ func TestGetOrFetchArtifactFromURL_CacheMiss_StorageMissing(t *testing.T) {
 		Filename:    "missing-1.0.0.tgz",
 		UpstreamURL: "https://example.com/missing.tgz",
 		StoragePath: sql.NullString{String: "nonexistent/path.tgz", Valid: true},
-		ContentHash: sql.NullString{String: "hash", Valid: true},
+		ContentHash: sql.NullString{String: sha256Hex("missing content"), Valid: true},
 		Size:        sql.NullInt64{Int64: 100, Valid: true},
 		ContentType: sql.NullString{String: "application/octet-stream", Valid: true},
 		FetchedAt:   sql.NullTime{Time: time.Now(), Valid: true},