verify-ml-dsa-provider-documents

README.md

@@ -24,6 +24,21 @@ Based upon the fact that [golang text templates are Turing complete](https://lin
 
 As a side note, [this review piece](https://solutionspace.blog/2021/12/04/every-simple-language-will-eventually-end-up-turing-complete/) gives some insight into whay all DSLs wind up Turing complete and also therefore rather messy for narrowband purposes.
 
+## Provider document verification
+
+Registry verification uses FIPS 204 ML-DSA-65 from Go's standard library.
+`verifyConfig.nopVerify: true` preserves unverified reads; `false` requires
+`x-stackql-signature` on every provider, service and referenced document read
+from a local registry, cache, or remote source. There is no `.sig` or certificate
+fallback. Verified readers parse the exact authenticated YAML payload returned by
+the shared `github.com/stackql/stackql-provider-registry/signing/pqsig` package.
+
+`verifyConfig.publicKeys` may supply a `kid` to hex public-key map for a private
+registry; it replaces the embedded production roots. An empty map is invalid.
+The SDK's existing nil-config behavior is unchanged; StackQL supplies
+`nopVerify: true` by default. Archive extraction is outside this verification
+boundary.
+
 ## Acknowledgements
 
 Extensions, adaptations and shims of the following support our work:

cmd/argparse/aot.go

@@ -12,7 +12,7 @@ import (
 	"github.com/stackql/any-sdk/internal/anysdk"
 	"github.com/stackql/any-sdk/pkg/dto"
 	"github.com/stackql/any-sdk/public/discovery"
-	"github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto"
+	"github.com/stackql/stackql-provider-registry/signing/pqsig"
 )
 
 var aotCmd = &cobra.Command{
@@ -55,7 +55,7 @@ func getNewLocalRegistry(relativePath string) (anysdk.RegistryAPI, error) {
 			RegistryURL:      fmt.Sprintf("file://%s", cleanPath),
 			LocalDocRoot:     relativePath,
 			AllowSrcDownload: false,
-			VerifyConfig: &edcrypto.VerifierConfig{
+			VerifyConfig: &pqsig.VerifierConfig{
 				NopVerify: true,
 			},
 		},

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/stackql/go-suffix-map v0.0.1-alpha01
 	github.com/stackql/stackql-parser v0.0.15-alpha06
 	github.com/stackql/stackql-provider-registry v0.0.2-alpha01
+	github.com/stackql/stackql-provider-registry/signing v0.1.0-alpha01
 	github.com/stretchr/testify v1.11.1
 	github.com/xo/dburl v0.23.2
 	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0

go.sum

@@ -396,6 +396,8 @@ github.com/stackql/stackql-parser v0.0.15-alpha06 h1:bdaudybbEmrR9m88CO1HpX/eKda
 github.com/stackql/stackql-parser v0.0.15-alpha06/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ=
 github.com/stackql/stackql-provider-registry v0.0.2-alpha01 h1:fPz6MhfXaMC/VPNJnjUo0sZyjTYhlifzaUWZ9Yj8NVE=
 github.com/stackql/stackql-provider-registry v0.0.2-alpha01/go.mod h1:87rVxnS2aRASK20lBQgoYA0o7FSJTZBGGRaWFR7IDm4=
+github.com/stackql/stackql-provider-registry/signing v0.1.0-alpha01 h1:KtrwwTJLMWn5+yEzguADFx96pA59k7P2JxmpAKXae6g=
+github.com/stackql/stackql-provider-registry/signing v0.1.0-alpha01/go.mod h1:2rrVmn6zLIonwq72QX50qtixWRZqx61tSDxPeOt/E1s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=

internal/anysdk/mock_http.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/stackql/any-sdk/pkg/fileutil"
-	"github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto"
+	"github.com/stackql/stackql-provider-registry/signing/pqsig"
 )
 
 var ()
@@ -86,7 +86,7 @@ func getNewTestDataMockRegistry(relativePath string) (RegistryAPI, error) {
 			RegistryURL:      fmt.Sprintf("file://%s", relativePath),
 			LocalDocRoot:     relativePath,
 			AllowSrcDownload: false,
-			VerifyConfig: &edcrypto.VerifierConfig{
+			VerifyConfig: &pqsig.VerifierConfig{
 				NopVerify: true,
 			},
 		},

internal/anysdk/registry.go

@@ -2,18 +2,18 @@ package anysdk
 
 import (
 	"bytes"
-	"crypto/x509"
 	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
 	"strings"
 
 	"github.com/stackql/any-sdk/pkg/compression"
-	"github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto"
+	"github.com/stackql/stackql-provider-registry/signing/pqsig"
 	"gopkg.in/yaml.v3"
 
 	"github.com/stackql/any-sdk/pkg/semver"
@@ -59,12 +59,12 @@ type RegistryAPI interface {
 }
 
 type RegistryConfig struct {
-	RegistryURL      string                   `json:"url" yaml:"url"`
-	SrcPrefix        *string                  `json:"srcPrefix" yaml:"srcPrefix"`
-	DistPrefix       *string                  `json:"distPrefix" yaml:"distPrefix"`
-	AllowSrcDownload bool                     `json:"allowSrcDownload" yaml:"allowSrcDownload"`
-	LocalDocRoot     string                   `json:"localDocRoot" yaml:"localDocRoot"`
-	VerifyConfig     *edcrypto.VerifierConfig `json:"verifyConfig" yaml:"verifyConfig"`
+	RegistryURL      string                `json:"url" yaml:"url"`
+	SrcPrefix        *string               `json:"srcPrefix" yaml:"srcPrefix"`
+	DistPrefix       *string               `json:"distPrefix" yaml:"distPrefix"`
+	AllowSrcDownload bool                  `json:"allowSrcDownload" yaml:"allowSrcDownload"`
+	LocalDocRoot     string                `json:"localDocRoot" yaml:"localDocRoot"`
+	VerifyConfig     *pqsig.VerifierConfig `json:"verifyConfig" yaml:"verifyConfig"`
 }
 
 type Registry struct {
@@ -76,7 +76,7 @@ type Registry struct {
 	localSrcPrefix   string
 	localDistPrefix  string
 	transport        http.RoundTripper
-	verifier         *edcrypto.Verifier
+	verifier         pqsig.Verifier
 	nopVerifier      bool
 }
 
@@ -121,16 +121,18 @@ func newRegistry(registryCfg RegistryConfig, transport http.RoundTripper) (Regis
 	if err != nil {
 		return nil, err
 	}
-	var ver *edcrypto.Verifier
+	var ver pqsig.Verifier
 	nopVerify := false
-	if registryCfg.VerifyConfig == nil {
-		ver, err = edcrypto.NewVerifier(edcrypto.NewVerifierConfig("", "", ""))
-	} else {
-		ver, err = edcrypto.NewVerifier(*registryCfg.VerifyConfig)
+	var keys map[string]string
+	if registryCfg.VerifyConfig != nil {
 		nopVerify = registryCfg.VerifyConfig.NopVerify
+		keys = registryCfg.VerifyConfig.PublicKeys
 	}
-	if err != nil {
-		return nil, err
+	if !nopVerify {
+		ver, err = pqsig.NewVerifier(keys)
+		if err != nil {
+			return nil, err
+		}
 	}
 	rv := &Registry{
 		allowSrcDownload: registryCfg.AllowSrcDownload,
@@ -380,15 +382,6 @@ func (r *Registry) GetServiceFragment(ps ProviderService, resourceKey string) (S
 	return svc, nil
 }
 
-func (r *Registry) checkSignature(docUrl string, verFile, sigFile io.ReadCloser) (*edcrypto.VerifierResponse, error) {
-	if sigFile == nil {
-		return nil, fmt.Errorf("nil signature")
-	}
-	vc := edcrypto.NewVerifyContext(docUrl, sigFile, verFile, "base64", true, x509.VerifyOptions{})
-	vr, err := r.verifier.VerifyFileFromCertificateBytes(vc)
-	return &vr, err
-}
-
 func (r *Registry) pullArchive(archivePath string) (io.ReadCloser, error) {
 	return r.getUnVerifiedArchive(archivePath)
 }
@@ -576,127 +569,72 @@ func (r *Registry) getUnVerifiedArchive(docPath string) (io.ReadCloser, error) {
 	return nil, fmt.Errorf("registry scheme '%s' currently not supported", r.regUrl.Scheme)
 }
 
-func (r *Registry) getEmbeddedVerifiedDocResponse(docPath string) (*edcrypto.VerifierResponse, error) {
-	lf, err := getServiceDoc(docPath, "")
-	if err != nil {
-		return nil, err
-	}
-	sf, err := r.getLocalDoc(fmt.Sprintf("%s.sig", docPath))
-	if err != nil {
-		lf.Close()
-		return nil, fmt.Errorf("embedded document present but signature file not present")
-	}
-	return r.checkSignature(docPath, lf, sf)
-}
-
-func (r *Registry) getVerifiedDocResponse(docPath string) (*edcrypto.VerifierResponse, error) {
-	var embeddedErr error
+func (r *Registry) getDocReader(docPath string) (io.ReadCloser, error) {
 	if r.isLocalFile() {
-		localPath := r.srcUrl.Path
-		if r.srcUrl.Host == "." {
-			localPath = "." + localPath
-		}
-		rb, err := os.Open(path.Join(localPath, docPath))
-		if err != nil {
-			return nil, fmt.Errorf("cannot read local registry file: '%s'", err.Error())
-		}
-		defer rb.Close()
-		rBytes, rErr := io.ReadAll(rb)
-		if rErr != nil {
-			return nil, fmt.Errorf("cannot read local registry file: '%s'", rErr.Error())
-		}
-		fileLockSafeReadCloser := io.NopCloser(bytes.NewReader(rBytes))
-		if r.nopVerifier {
-			rv := edcrypto.NewVerifierResponse(true, nil, fileLockSafeReadCloser, nil)
-			return &rv, nil
-		}
-		sb, err := os.Open(path.Join(localPath, fmt.Sprintf("%s.sig", docPath)))
-		if err != nil {
-			return nil, fmt.Errorf("cannot read local signature file: '%s'", err.Error())
-		}
-		defer sb.Close()
-		sBytes, sErr := io.ReadAll(sb)
-		if sErr != nil {
-			return nil, fmt.Errorf("cannot read signature file: '%s'", sErr.Error())
-		}
-		return r.checkSignature(
-			docPath,
-			fileLockSafeReadCloser,
-			io.NopCloser(bytes.NewReader(sBytes)), // file lock safe signature file
-		)
+		return os.Open(path.Join(r.localSourcePath(), docPath))
 	}
 	if r.localDocRoot != "" {
 		localPath := r.getLocalDocPath(docPath)
 		lf, err := r.getLocalDoc(localPath)
 		if err == nil {
-			if r.nopVerifier {
-				rv := edcrypto.NewVerifierResponse(true, nil, lf, nil)
-				return &rv, nil
-			}
-			sf, err := r.getLocalDoc(fmt.Sprintf("%s.sig", localPath))
-			if err != nil {
-				if lf != nil {
-					lf.Close()
-				}
-				return nil, fmt.Errorf("local document present but signature file not present")
-			}
-			return r.checkSignature(localPath, lf, sf)
+			return lf, nil
 		}
 	}
-	fullUrl, err := url.Parse(r.regUrl.String())
-	if err != nil {
-		return nil, err
-	}
-	fullUrl.Path = path.Join(fullUrl.Path, docPath)
-	verifyUrl := fullUrl.String()
 	if r.isHttp() {
 		if !r.allowSrcDownload {
 			return nil, fmt.Errorf("download of individual docs disallowed; please attempt to pull provider docs")
 		}
-		cl := &http.Client{}
-		if r.transport != nil {
-			cl.Transport = r.transport
-		}
 		response, err := r.getRemoteDoc(docPath)
 		if err != nil {
 			return nil, err
 		}
 		if response == nil {
 			return nil, fmt.Errorf("no response body from remote")
 		}
-		if r.nopVerifier {
-			rv := edcrypto.NewVerifierResponse(true, nil, response, nil)
-			return &rv, nil
-		}
-		sigResponse, err := r.getRemoteDoc(fmt.Sprintf("%s.sig", docPath))
-		if err != nil {
-			response.Close()
-			return nil, fmt.Errorf("remote document '%s' present but signature file not present", verifyUrl)
-		}
-		return r.checkSignature(verifyUrl, response, sigResponse)
-	}
-	if embeddedErr != nil {
-		return nil, fmt.Errorf("error retrieving from embedded: %s", embeddedErr.Error())
+		return response, nil
 	}
 	return nil, fmt.Errorf("registry scheme '%s' currently not supported", r.regUrl.Scheme)
 }
 
 func (r *Registry) getVerifiedDocBytes(docPath string) ([]byte, error) {
-	vr, err := r.getVerifiedDocResponse(docPath)
+	reader, err := r.getDocReader(docPath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("registry document %q: %w", docPath, err)
+	}
+	defer reader.Close()
+	doc, err := io.ReadAll(reader)
+	if err != nil {
+		return nil, fmt.Errorf("read registry document %q: %w", docPath, err)
+	}
+	if r.nopVerifier {
+		return doc, nil
 	}
-	return io.ReadAll(vr.VerifyFile)
+	doc, err = r.verifier.Verify(doc)
+	if err != nil {
+		return nil, fmt.Errorf("verify registry document %q: %w", docPath, err)
+	}
+	return doc, nil
 }
 
 func (r *Registry) GetLatestAvailableVersion(providerName string) (string, error) {
 	return r.getLatestAvailableVersion(providerName)
 }
 
+func (r *Registry) localSourcePath() string {
+	localPath := r.srcUrl.Path
+	if r.srcUrl.Host == "." {
+		localPath = "." + localPath
+	}
+	if strings.HasPrefix(localPath, "/") && filepath.VolumeName(localPath[1:]) != "" {
+		localPath = localPath[1:]
+	}
+	return localPath
+}
+
 func (r *Registry) getLatestAvailableVersion(providerName string) (string, error) {
 	providerName = r.renameProvider(providerName)
 	if r.isLocalFile() {
-		deSlice, err := os.ReadDir(path.Join(r.srcUrl.Path, providerName))
+		deSlice, err := os.ReadDir(path.Join(r.localSourcePath(), providerName))
 		if err != nil {
 			return "", err
 		}