fix(oci): cache tag lists and normalize manifest variants

docs/architecture.md

@@ -353,7 +353,7 @@ Eviction can be implemented as:
 - Fresh data - new versions visible immediately
 - Metadata is small, upstream fetch is fast
 - Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table
-- OCI manifests are the exception: they are cached automatically so previously fetched images remain pullable when the registry or token service is unavailable
+- OCI manifests and tag lists are exceptions: they are cached automatically so previously fetched images remain pullable and tag resolution works when the registry or token service is unavailable
 
 **Why stream artifacts?**
 - Memory efficient - don't load large files into RAM

docs/configuration.md

@@ -292,7 +292,7 @@ Note: Hex cooldown requires disabling registry signature verification since the
 
 By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata.
 
-OCI manifests are always cached because cached image blobs cannot be pulled without their manifests. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable.
+OCI manifests and tag lists are always cached because cached image blobs cannot be pulled without their manifests and offline clients may need tag resolution. Digest-addressed manifests are immutable and served directly from cache. Tag-addressed manifests and tag lists follow `metadata_ttl`, revalidate when stale, and fall back to the last cached response when the registry is unavailable.
 
 ```yaml
 cache_metadata: true

internal/handler/container.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net/http"
 	"regexp"
 	"strings"
@@ -182,7 +181,7 @@ func (h *ContainerHandler) handleManifest(w http.ResponseWriter, r *http.Request
 	h.serveManifest(w, r, registryURL, upstreamName, reference)
 }
 
-// handleTagsList proxies tag list requests to upstream.
+// handleTagsList caches tag list responses for offline OCI pulls.
 func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request, path string) {
 	if r.Method != http.MethodGet {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
@@ -201,27 +200,7 @@ func (h *ContainerHandler) handleTagsList(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", registryURL, upstreamName)
-	if r.URL.RawQuery != "" {
-		upstreamURL += "?" + r.URL.RawQuery
-	}
-
-	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil)
-	if err != nil {
-		h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request")
-		return
-	}
-
-	resp, err := h.proxy.HTTPClient.Do(req)
-	if err != nil {
-		h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
-		return
-	}
-	defer func() { _ = resp.Body.Close() }()
-
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(resp.StatusCode)
-	_, _ = io.Copy(w, resp.Body)
+	h.serveTagsList(w, r, registryURL, upstreamName)
 }
 
 // proxyBlobHead handles HEAD requests for blobs.

internal/handler/container_manifest.go

@@ -8,8 +8,10 @@ import (
 	"encoding/hex"
 	"fmt"
 	"io"
+	"mime"
 	"net/http"
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -35,12 +37,16 @@ type cachedContainerManifest struct {
 
 func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request, registryURL, name, reference string) {
 	accept := containerManifestAccept(r)
-	cacheKey := h.containerManifestCacheKey(registryURL, name, reference, accept)
+	cacheAccept := normalizeContainerManifestAccept(accept)
+	cacheKey := h.containerManifestCacheKey(registryURL, name, reference, cacheAccept)
 	cached, err := h.loadContainerManifest(r.Context(), cacheKey)
 	if err != nil {
 		h.proxy.Logger.Warn("failed to read cached container manifest", "error", err)
 		cached = nil
 	}
+	if cached != nil && cached.contentType != "" && !containerManifestAccepts(accept, cached.contentType) {
+		cached = nil
+	}
 
 	immutable := manifestDigestReferencePattern.MatchString(reference)
 	if cached != nil && (immutable || h.containerManifestFresh(cached)) {
@@ -111,7 +117,7 @@ func (h *ContainerHandler) serveManifest(w http.ResponseWriter, r *http.Request,
 		h.proxy.Logger.Warn("failed to cache container manifest", "error", err)
 	}
 	if manifest.contentDigest != reference && manifestDigestReferencePattern.MatchString(manifest.contentDigest) {
-		digestKey := h.containerManifestCacheKey(registryURL, name, manifest.contentDigest, accept)
+		digestKey := h.containerManifestCacheKey(registryURL, name, manifest.contentDigest, cacheAccept)
 		if err := h.storeContainerManifest(r.Context(), digestKey, manifest); err != nil {
 			h.proxy.Logger.Warn("failed to cache container manifest by digest", "error", err)
 		}
@@ -233,6 +239,83 @@ func containerManifestAccept(r *http.Request) string {
 	}, ", ")
 }
 
+func normalizeContainerManifestAccept(accept string) string {
+	mediaTypes := make(map[string]struct{})
+	for _, value := range strings.Split(accept, ",") {
+		value = strings.TrimSpace(value)
+		if value == "" {
+			continue
+		}
+		mediaType, params, err := mime.ParseMediaType(value)
+		if err != nil {
+			mediaTypes[strings.ToLower(value)] = struct{}{}
+			continue
+		}
+		paramKeys := make([]string, 0, len(params))
+		for key := range params {
+			paramKeys = append(paramKeys, key)
+		}
+		sort.Strings(paramKeys)
+		canonical := strings.ToLower(mediaType)
+		for _, key := range paramKeys {
+			value := params[key]
+			if strings.EqualFold(key, "q") {
+				if quality, err := strconv.ParseFloat(value, 64); err == nil {
+					if quality == 1 {
+						continue
+					}
+					value = strconv.FormatFloat(quality, 'g', -1, 64)
+				}
+			}
+			canonical += ";" + strings.ToLower(key) + "=" + value
+		}
+		mediaTypes[canonical] = struct{}{}
+	}
+	canonicalMediaTypes := make([]string, 0, len(mediaTypes))
+	for mediaType := range mediaTypes {
+		canonicalMediaTypes = append(canonicalMediaTypes, mediaType)
+	}
+	sort.Strings(canonicalMediaTypes)
+	return strings.Join(canonicalMediaTypes, ",")
+}
+
+func containerManifestAccepts(accept, contentType string) bool {
+	contentType, _, err := mime.ParseMediaType(contentType)
+	if err != nil {
+		return false
+	}
+	contentType = strings.ToLower(contentType)
+	contentMajor, contentMinor, found := strings.Cut(contentType, "/")
+	if !found {
+		return false
+	}
+
+	for _, value := range strings.Split(accept, ",") {
+		mediaType, params, err := mime.ParseMediaType(strings.TrimSpace(value))
+		if err != nil || containerAcceptQuality(params) == 0 {
+			continue
+		}
+		mediaType = strings.ToLower(mediaType)
+		major, minor, found := strings.Cut(mediaType, "/")
+		if found && (major == "*" || major == contentMajor) && (minor == "*" || minor == contentMinor) {
+			return true
+		}
+	}
+	return false
+}
+
+func containerAcceptQuality(params map[string]string) float64 {
+	value, ok := params["q"]
+	if !ok {
+		return 1
+	}
+	quality, err := strconv.ParseFloat(value, 64)
+	if err != nil || quality < 0 || quality > 1 {
+		return 0
+	}
+	return quality
+}
+
 func copyContainerManifestHeaders(destination, source http.Header) {
 	for _, header := range []string{"Content-Type", "Content-Length", "Docker-Content-Digest", "ETag", "WWW-Authenticate"} {
 		if value := source.Get(header); value != "" {

internal/handler/container_tags.go

@@ -0,0 +1,196 @@
+package handler
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/git-pkgs/proxy/internal/database"
+)
+
+const containerTagsCacheEcosystem = "oci-tags"
+
+type cachedContainerTags struct {
+	body        []byte
+	contentType string
+	etag        string
+	size        int64
+	fetchedAt   time.Time
+}
+
+func (h *ContainerHandler) serveTagsList(w http.ResponseWriter, r *http.Request, registryURL, name string) {
+	cacheKey := h.containerTagsCacheKey(registryURL, name, r.URL.Query())
+	cached, err := h.loadContainerTags(r.Context(), cacheKey)
+	if err != nil {
+		h.proxy.Logger.Warn("failed to read cached container tag list", "error", err)
+		cached = nil
+	}
+	if cached != nil && h.containerTagsFresh(cached) {
+		writeContainerTags(w, cached, false)
+		return
+	}
+
+	upstreamURL := fmt.Sprintf("%s/v2/%s/tags/list", registryURL, name)
+	if query := r.URL.Query().Encode(); query != "" {
+		upstreamURL += "?" + query
+	}
+	req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, upstreamURL, nil)
+	if err != nil {
+		h.containerError(w, http.StatusInternalServerError, "INTERNAL_ERROR", "failed to create request")
+		return
+	}
+	req.Header.Set("Accept", "application/json")
+	if cached != nil && cached.etag != "" {
+		req.Header.Set("If-None-Match", cached.etag)
+	}
+
+	resp, err := h.proxy.HTTPClient.Do(req)
+	if err != nil {
+		h.serveStaleTagsOrError(w, cached, err)
+		return
+	}
+	defer func() { _ = resp.Body.Close() }()
+
+	if resp.StatusCode == http.StatusNotModified && cached != nil {
+		cached.fetchedAt = time.Now()
+		if err := h.storeContainerTags(r.Context(), cacheKey, cached); err != nil {
+			h.proxy.Logger.Warn("failed to refresh cached container tag list", "error", err)
+		}
+		writeContainerTags(w, cached, false)
+		return
+	}
+	if resp.StatusCode != http.StatusOK {
+		if cached != nil && shouldServeStaleManifest(resp.StatusCode) {
+			writeContainerTags(w, cached, true)
+			return
+		}
+		copyContainerTagsHeaders(w.Header(), resp.Header)
+		w.WriteHeader(resp.StatusCode)
+		_, _ = io.Copy(w, resp.Body)
+		return
+	}
+
+	body, err := h.proxy.ReadMetadata(resp.Body)
+	if err != nil {
+		h.serveStaleTagsOrError(w, cached, fmt.Errorf("reading tag list: %w", err))
+		return
+	}
+	tags := &cachedContainerTags{
+		body:        body,
+		contentType: resp.Header.Get("Content-Type"),
+		etag:        resp.Header.Get("ETag"),
+		size:        int64(len(body)),
+		fetchedAt:   time.Now(),
+	}
+	if tags.contentType == "" {
+		tags.contentType = contentTypeJSON
+	}
+	if err := h.storeContainerTags(r.Context(), cacheKey, tags); err != nil {
+		h.proxy.Logger.Warn("failed to cache container tag list", "error", err)
+	}
+	writeContainerTags(w, tags, false)
+}
+
+func (h *ContainerHandler) serveStaleTagsOrError(w http.ResponseWriter, cached *cachedContainerTags, err error) {
+	if cached != nil {
+		h.proxy.Logger.Warn("upstream tag list fetch failed, serving stale cache", "error", err)
+		writeContainerTags(w, cached, true)
+		return
+	}
+	h.proxy.Logger.Error("failed to fetch container tag list", "error", err)
+	h.containerError(w, http.StatusBadGateway, "INTERNAL_ERROR", "failed to fetch from upstream")
+}
+
+func (h *ContainerHandler) containerTagsCacheKey(registryURL, name string, query url.Values) string {
+	identity := registryURL + "\x00" + name + "\x00" + query.Encode()
+	sum := sha256.Sum256([]byte(identity))
+	return hex.EncodeToString(sum[:])
+}
+
+func (h *ContainerHandler) containerTagsFresh(tags *cachedContainerTags) bool {
+	return h.proxy.MetadataTTL > 0 && !tags.fetchedAt.IsZero() && time.Since(tags.fetchedAt) < h.proxy.MetadataTTL
+}
+
+func (h *ContainerHandler) loadContainerTags(ctx context.Context, cacheKey string) (*cachedContainerTags, error) {
+	if h.proxy.DB == nil || h.proxy.Storage == nil {
+		return nil, nil
+	}
+	entry, err := h.proxy.DB.GetMetadataCache(containerTagsCacheEcosystem, cacheKey)
+	if err != nil || entry == nil {
+		return nil, err
+	}
+	reader, err := h.proxy.Storage.Open(ctx, entry.StoragePath)
+	if err != nil {
+		return nil, nil
+	}
+	defer func() { _ = reader.Close() }()
+	body, err := h.proxy.ReadMetadata(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	tags := &cachedContainerTags{body: body, contentType: contentTypeJSON, size: int64(len(body))}
+	if entry.ContentType.Valid {
+		tags.contentType = entry.ContentType.String
+	}
+	if entry.ETag.Valid {
+		tags.etag = entry.ETag.String
+	}
+	if entry.Size.Valid {
+		tags.size = entry.Size.Int64
+	}
+	if entry.FetchedAt.Valid {
+		tags.fetchedAt = entry.FetchedAt.Time
+	}
+	return tags, nil
+}
+
+func (h *ContainerHandler) storeContainerTags(ctx context.Context, cacheKey string, tags *cachedContainerTags) error {
+	if h.proxy.DB == nil || h.proxy.Storage == nil {
+		return nil
+	}
+	storagePath := metadataStoragePath(containerTagsCacheEcosystem, cacheKey)
+	size, _, err := h.proxy.Storage.Store(ctx, storagePath, bytes.NewReader(tags.body))
+	if err != nil {
+		return fmt.Errorf("storing tag list: %w", err)
+	}
+	tags.size = size
+	return h.proxy.DB.UpsertMetadataCache(&database.MetadataCacheEntry{
+		Ecosystem:   containerTagsCacheEcosystem,
+		Name:        cacheKey,
+		StoragePath: storagePath,
+		ETag:        sql.NullString{String: tags.etag, Valid: tags.etag != ""},
+		ContentType: sql.NullString{String: tags.contentType, Valid: tags.contentType != ""},
+		Size:        sql.NullInt64{Int64: size, Valid: true},
+		FetchedAt:   sql.NullTime{Time: tags.fetchedAt, Valid: !tags.fetchedAt.IsZero()},
+	})
+}
+
+func writeContainerTags(w http.ResponseWriter, tags *cachedContainerTags, stale bool) {
+	w.Header().Set("Content-Type", tags.contentType)
+	w.Header().Set("Content-Length", strconv.FormatInt(tags.size, 10))
+	if tags.etag != "" {
+		w.Header().Set("ETag", tags.etag)
+	}
+	if stale {
+		w.Header().Set("Warning", containerStaleWarning)
+	}
+	w.WriteHeader(http.StatusOK)
+	_, _ = w.Write(tags.body)
+}
+
+func copyContainerTagsHeaders(destination, source http.Header) {
+	for _, header := range []string{"Content-Type", "Content-Length", "ETag", "WWW-Authenticate"} {
+		if value := source.Get(header); value != "" {
+			destination.Set(header, value)
+		}
+	}
+}