Use Docker Hub's server-side manifest copy when possible

registry/client.go

@@ -51,13 +51,12 @@ func Client(host string, opts *ociclient.Options) (ociregistry.Interface, error)
 			Transport: clientOptions.Transport,
 		})
 
-		connectHost := host
-		if host == dockerHubCanonical {
-			connectHost = dockerHubConnect
-		} else if host == "localhost" || strings.HasPrefix(host, "localhost:") {
-			// assume localhost means HTTP
+		// keep the finished transport (auth, rate limiting, User-Agent) around for any requests we have to make ourselves because "ociclient" does not know about them (see "serverSideCopyManifest")
+		transportCache.Store(host, clientOptions.Transport)
+
+		connectHost, insecure := registryConnectHost(host)
+		if insecure {
 			clientOptions.Insecure = true
-			// TODO some way for callers to specify that their "localhost" *does* require TLS (maybe only do this if `opts == nil`, but then users cannot supply *any* options and still get help setting Insecure for localhost 🤔 -- at least this is a more narrow use case than the opposite of not having a way to have non-localhost insecure registries)
 		}
 
 		hostOptions := clientOptions // make a copy, since "ociclient.New" mutates it (such that sharing the object afterwards probably isn't the best idea -- they'll have the same DebugID if so, which isn't ideal)
@@ -133,6 +132,16 @@ func Client(host string, opts *ociclient.Options) (ociregistry.Interface, error)
 	return f.(func() (ociregistry.Interface, error))()
 }
 
+// returns the actual "host[:port]" requests for the given registry host should be sent to (see [dockerHubConnect]) and whether it should be assumed to be plain HTTP
+func registryConnectHost(host string) (connectHost string, insecure bool) {
+	if host == dockerHubCanonical {
+		return dockerHubConnect, false
+	}
+	// assume localhost means HTTP
+	// TODO some way for callers to specify that their "localhost" *does* require TLS (maybe only do this if `opts == nil`, but then users cannot supply *any* options and still get help setting Insecure for localhost 🤔 -- at least this is a more narrow use case than the opposite of not having a way to have non-localhost insecure registries)
+	return host, host == "localhost" || strings.HasPrefix(host, "localhost:")
+}
+
 type dockerAuthConfigWrapper struct {
 	ociauth.Config
 }
@@ -172,4 +181,6 @@ var (
 		return dockerAuthConfigWrapper{config}, nil
 	})
 	clientCache = sync.Map{} // "(normalized) host" => OnceValues() => ociregistry.Interface, error
+
+	transportCache = sync.Map{} // "host exactly as passed to Client" => http.RoundTripper (populated as a side effect of the corresponding clientCache entry, so look up the same string you gave Client; neither normalizes, that is up to the caller, see Reference.Normalize)
 )

registry/push.go

@@ -96,6 +96,10 @@ func EnsureManifest(ctx context.Context, ref Reference, manifest json.RawMessage
 
 			for _, child := range manifestChildren.Manifests {
 				childRef, childTargetRef := childToRefs(child)
+				if _, ok := serverSideCopyManifest(ctx, childRef, childTargetRef); ok {
+					// the registry copied the child (and all *its* children) for us
+					continue
+				}
 				r, err := Lookup(ctx, childRef, nil)
 				if err != nil {
 					return desc, fmt.Errorf("%s: manifest lookup failed: %w", childRef, err)
@@ -155,7 +159,11 @@ func EnsureManifest(ctx context.Context, ref Reference, manifest json.RawMessage
 func CopyManifest(ctx context.Context, srcRef, dstRef Reference, childRefs map[ociregistry.Digest]Reference) (ociregistry.Descriptor, error) {
 	var desc ociregistry.Descriptor
 
-	// wouldn't it be nice if MountBlob for manifests was a thing? 🥺
+	// wouldn't it be nice if MountBlob for manifests was a thing? 🥺 (on Docker Hub it kind of is, see "serverSideCopyManifest", and if that does not work out, we fall back to copying client side)
+	if copied, ok := serverSideCopyManifest(ctx, srcRef, dstRef); ok {
+		return copied, nil
+	}
+
 	r, err := Lookup(ctx, srcRef, nil)
 	if err != nil {
 		return desc, fmt.Errorf("%s: lookup failed: %w", srcRef, err)

registry/server-side-copy.go

@@ -0,0 +1,174 @@
+package registry
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+
+	"cuelabs.dev/go/oci/ociregistry"
+	"cuelabs.dev/go/oci/ociregistry/ociauth"
+)
+
+var (
+	// "host" or "host|srcNamespace|dstNamespace" => true once we have good reason to believe a server-side manifest copy cannot work there (so we stop wasting a request on every manifest)
+	serverSideCopyDisabled = sync.Map{}
+)
+
+func repoNamespace(repo string) string {
+	ns, _, _ := strings.Cut(repo, "/")
+	return ns
+}
+
+// asks the registry to copy the manifest srcRef (by tag or by digest) into dstRef's repository *server side*, including every child manifest, config, and layer (so we don't have to do any per-blob or per-child requests ourselves)
+//
+// this is Docker Hub's "POST /v2/<dst>/_docker/copy/image?from=<src>&reference=<digest>[&tag=<tag>]" endpoint (basically a blob mount, but for a manifest), which Hub only enables for specific namespaces, so this is explicitly *best effort*: it returns true only if the registry confirmed it copied the right digest (and every child), and false for every other outcome (including "this registry has never heard of that endpoint") -- callers are expected to fall back to copying client side, which will also produce the authoritative error if something is genuinely wrong
+//
+// a failed attempt has no side effects (the registry stores nothing unless it was able to mount every child)
+//
+// the returned descriptor always has the right Digest on success, but MediaType and Size are only filled in if we happened to have to look them up anyhow (a source with a digest does not need a lookup, so we do not do one)
+func serverSideCopyManifest(ctx context.Context, srcRef, dstRef Reference) (ociregistry.Descriptor, bool) {
+	if srcRef.Host != dstRef.Host || srcRef.Repository == dstRef.Repository {
+		return ociregistry.Descriptor{}, false
+	}
+
+	// this is checked first so that a registry (or pair of namespaces) we already know does not work never costs us even a lookup
+	host := srcRef.Host
+	pairKey := host + "|" + repoNamespace(srcRef.Repository) + "|" + repoNamespace(dstRef.Repository)
+	if _, disabled := serverSideCopyDisabled.Load(host); disabled {
+		return ociregistry.Descriptor{}, false
+	}
+	if _, disabled := serverSideCopyDisabled.Load(pairKey); disabled {
+		return ociregistry.Descriptor{}, false
+	}
+
+	// we always ask by digest, so if we were not given one, we need to find it (and a source that does not exist is a good reason to let the client side code produce the error)
+	desc := ociregistry.Descriptor{Digest: srcRef.Digest}
+	if srcRef.Digest == "" {
+		r, err := Lookup(ctx, srcRef, &LookupOptions{Head: true})
+		if err != nil || r == nil {
+			return ociregistry.Descriptor{}, false
+		}
+		desc = r.Descriptor()
+		r.Close()
+		srcRef.Tag, srcRef.Digest = "", desc.Digest
+	}
+	if dstRef.Digest != "" && dstRef.Digest != srcRef.Digest {
+		// a digest mismatch is an error, but the client side code is better equipped to report it
+		return ociregistry.Descriptor{}, false
+	}
+
+	// if the destination already has this exact manifest (at this tag, if we were given one), there's nothing to do (the endpoint is not a cheap no-op: it redoes the whole push, including checking every child), see also the same assumption in "EnsureManifest" that this means child objects exist as well
+	headRef := dstRef
+	if headRef.Tag != "" {
+		// a lookup with both prefers the digest, which would not tell us anything about the tag
+		headRef.Digest = ""
+	} else {
+		headRef.Digest = srcRef.Digest
+	}
+	if r, err := Lookup(ctx, headRef, &LookupOptions{Head: true}); err == nil && r != nil {
+		head := r.Descriptor()
+		r.Close()
+		if head.Digest == srcRef.Digest {
+			return head, true
+		}
+	}
+
+	if !requestServerSideCopy(ctx, srcRef, dstRef) {
+		return ociregistry.Descriptor{}, false
+	}
+	return desc, true
+}
+
+// the actual request for [serverSideCopyManifest] (srcRef must have a Digest, and every precondition has been checked), which also records anything we learn about what the registry does not support
+func requestServerSideCopy(ctx context.Context, srcRef, dstRef Reference) bool {
+	host := srcRef.Host
+	pairKey := host + "|" + repoNamespace(srcRef.Repository) + "|" + repoNamespace(dstRef.Repository)
+
+	// making sure "Client" has run for this host is what populates "transportCache"
+	if _, err := Client(host, nil); err != nil {
+		return false
+	}
+	rt, ok := transportCache.Load(host)
+	if !ok {
+		return false
+	}
+
+	connectHost, insecure := registryConnectHost(host)
+	scheme := "https"
+	if insecure {
+		scheme = "http"
+	}
+
+	query := url.Values{
+		"from":      {srcRef.Repository},
+		"reference": {string(srcRef.Digest)}, // by digest so there's no implicit tag (and so we know exactly what we're asking for)
+	}
+	if dstRef.Tag != "" {
+		query.Set("tag", dstRef.Tag)
+	}
+	u := url.URL{
+		Scheme:   scheme,
+		Host:     connectHost,
+		Path:     "/v2/" + dstRef.Repository + "/_docker/copy/image",
+		RawQuery: query.Encode(),
+	}
+
+	// the server requires push on the destination and pull on the source (the same as it does for a blob mount)
+	ctx = ociauth.ContextWithRequestInfo(ctx, ociauth.RequestInfo{
+		RequiredScope: ociauth.NewScope(
+			ociauth.ResourceScope{ResourceType: ociauth.TypeRepository, Resource: dstRef.Repository, Action: ociauth.ActionPush},
+			ociauth.ResourceScope{ResourceType: ociauth.TypeRepository, Resource: dstRef.Repository, Action: ociauth.ActionPull},
+			ociauth.ResourceScope{ResourceType: ociauth.TypeRepository, Resource: srcRef.Repository, Action: ociauth.ActionPull},
+		),
+	})
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), http.NoBody)
+	if err != nil {
+		return false
+	}
+	resp, err := (&http.Client{Transport: rt.(http.RoundTripper)}).Do(req)
+	if err != nil {
+		return false
+	}
+	defer resp.Body.Close()
+	body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
+
+	switch {
+	case resp.StatusCode == http.StatusCreated:
+		// the digest we get back is the one the server actually copied, so it had better be the one we asked for
+		return resp.Header.Get("Docker-Content-Digest") == string(srcRef.Digest)
+
+	case resp.StatusCode == http.StatusMethodNotAllowed,
+		resp.StatusCode == http.StatusNotFound && !strings.HasPrefix(resp.Header.Get("Content-Type"), "application/json"):
+		// a non-JSON 404 (like the plain text "404 page not found") means the endpoint does not exist at all, so this registry will never support this and we should stop asking it
+		serverSideCopyDisabled.Store(host, true)
+
+	case resp.StatusCode == http.StatusBadRequest && responseHasErrorCode(body, "MANIFEST_BLOB_UNKNOWN"):
+		// the endpoint exists, but it did not mount anything (it falls back to a normal manifest push with no mounting if the feature is not enabled for either namespace), so stop asking about this specific pair of namespaces
+		serverSideCopyDisabled.Store(pairKey, true)
+	}
+
+	// anything else (401, 403, 5xx, a JSON 404 for an unknown manifest, etc) might be specific to this one manifest and/or temporary, so we do not remember it, we just fall back
+	return false
+}
+
+// whether body is an OCI distribution error response containing the given error code
+func responseHasErrorCode(body []byte, code string) bool {
+	var errs struct {
+		Errors []struct {
+			Code string `json:"code"`
+		} `json:"errors"`
+	}
+	if err := json.Unmarshal(body, &errs); err != nil {
+		return false
+	}
+	for _, e := range errs.Errors {
+		if e.Code == code {
+			return true
+		}
+	}
+	return false
+}