diff --git a/README.md b/README.md index 93c70aad..d5737bb7 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,21 @@ Based upon the fact that [golang text templates are Turing complete](https://lin As a side note, [this review piece](https://solutionspace.blog/2021/12/04/every-simple-language-will-eventually-end-up-turing-complete/) gives some insight into whay all DSLs wind up Turing complete and also therefore rather messy for narrowband purposes. +## Provider document verification + +Registry verification uses FIPS 204 ML-DSA-65 from Go's standard library. +`verifyConfig.nopVerify: true` preserves unverified reads; `false` requires +`x-stackql-signature` on every provider, service and referenced document read +from a local registry, cache, or remote source. There is no `.sig` or certificate +fallback. Verified readers parse the exact authenticated YAML payload returned by +the shared `github.com/stackql/stackql-provider-registry/signing/pqsig` package. + +`verifyConfig.publicKeys` may supply a `kid` to hex public-key map for a private +registry; it replaces the embedded production roots. An empty map is invalid. +The SDK's existing nil-config behavior is unchanged; StackQL supplies +`nopVerify: true` by default. Archive extraction is outside this verification +boundary. + ## Acknowledgements Extensions, adaptations and shims of the following support our work: diff --git a/cmd/argparse/aot.go b/cmd/argparse/aot.go index 5d31dbf9..ab459dd6 100644 --- a/cmd/argparse/aot.go +++ b/cmd/argparse/aot.go @@ -12,7 +12,7 @@ import ( "github.com/stackql/any-sdk/internal/anysdk" "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/any-sdk/public/discovery" - "github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto" + "github.com/stackql/stackql-provider-registry/signing/pqsig" ) var aotCmd = &cobra.Command{ @@ -55,7 +55,7 @@ func getNewLocalRegistry(relativePath string) (anysdk.RegistryAPI, error) { RegistryURL: fmt.Sprintf("file://%s", cleanPath), LocalDocRoot: relativePath, AllowSrcDownload: false, - VerifyConfig: &edcrypto.VerifierConfig{ + VerifyConfig: &pqsig.VerifierConfig{ NopVerify: true, }, }, diff --git a/go.mod b/go.mod index 96ebed8e..4ceabff6 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/stackql/go-suffix-map v0.0.1-alpha01 github.com/stackql/stackql-parser v0.0.15-alpha06 github.com/stackql/stackql-provider-registry v0.0.2-alpha01 + github.com/stackql/stackql-provider-registry/signing v0.1.0-alpha01 github.com/stretchr/testify v1.11.1 github.com/xo/dburl v0.23.2 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 diff --git a/go.sum b/go.sum index bbe2295a..0e6fd0aa 100644 --- a/go.sum +++ b/go.sum @@ -396,6 +396,8 @@ github.com/stackql/stackql-parser v0.0.15-alpha06 h1:bdaudybbEmrR9m88CO1HpX/eKda github.com/stackql/stackql-parser v0.0.15-alpha06/go.mod h1:iyB47SvRS+Fvpn7joF7mHAkeiWSq83TbUhglRmLzPLQ= github.com/stackql/stackql-provider-registry v0.0.2-alpha01 h1:fPz6MhfXaMC/VPNJnjUo0sZyjTYhlifzaUWZ9Yj8NVE= github.com/stackql/stackql-provider-registry v0.0.2-alpha01/go.mod h1:87rVxnS2aRASK20lBQgoYA0o7FSJTZBGGRaWFR7IDm4= +github.com/stackql/stackql-provider-registry/signing v0.1.0-alpha01 h1:KtrwwTJLMWn5+yEzguADFx96pA59k7P2JxmpAKXae6g= +github.com/stackql/stackql-provider-registry/signing v0.1.0-alpha01/go.mod h1:2rrVmn6zLIonwq72QX50qtixWRZqx61tSDxPeOt/E1s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= diff --git a/internal/anysdk/mock_http.go b/internal/anysdk/mock_http.go index c2e062a7..a4e9c13a 100644 --- a/internal/anysdk/mock_http.go +++ b/internal/anysdk/mock_http.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/stackql/any-sdk/pkg/fileutil" - "github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto" + "github.com/stackql/stackql-provider-registry/signing/pqsig" ) var () @@ -86,7 +86,7 @@ func getNewTestDataMockRegistry(relativePath string) (RegistryAPI, error) { RegistryURL: fmt.Sprintf("file://%s", relativePath), LocalDocRoot: relativePath, AllowSrcDownload: false, - VerifyConfig: &edcrypto.VerifierConfig{ + VerifyConfig: &pqsig.VerifierConfig{ NopVerify: true, }, }, diff --git a/internal/anysdk/registry.go b/internal/anysdk/registry.go index a660f8e4..0cec7317 100644 --- a/internal/anysdk/registry.go +++ b/internal/anysdk/registry.go @@ -2,18 +2,18 @@ package anysdk import ( "bytes" - "crypto/x509" "fmt" "io" "net/http" "net/url" "os" "path" + "path/filepath" "regexp" "strings" "github.com/stackql/any-sdk/pkg/compression" - "github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto" + "github.com/stackql/stackql-provider-registry/signing/pqsig" "gopkg.in/yaml.v3" "github.com/stackql/any-sdk/pkg/semver" @@ -59,12 +59,12 @@ type RegistryAPI interface { } type RegistryConfig struct { - RegistryURL string `json:"url" yaml:"url"` - SrcPrefix *string `json:"srcPrefix" yaml:"srcPrefix"` - DistPrefix *string `json:"distPrefix" yaml:"distPrefix"` - AllowSrcDownload bool `json:"allowSrcDownload" yaml:"allowSrcDownload"` - LocalDocRoot string `json:"localDocRoot" yaml:"localDocRoot"` - VerifyConfig *edcrypto.VerifierConfig `json:"verifyConfig" yaml:"verifyConfig"` + RegistryURL string `json:"url" yaml:"url"` + SrcPrefix *string `json:"srcPrefix" yaml:"srcPrefix"` + DistPrefix *string `json:"distPrefix" yaml:"distPrefix"` + AllowSrcDownload bool `json:"allowSrcDownload" yaml:"allowSrcDownload"` + LocalDocRoot string `json:"localDocRoot" yaml:"localDocRoot"` + VerifyConfig *pqsig.VerifierConfig `json:"verifyConfig" yaml:"verifyConfig"` } type Registry struct { @@ -76,7 +76,7 @@ type Registry struct { localSrcPrefix string localDistPrefix string transport http.RoundTripper - verifier *edcrypto.Verifier + verifier pqsig.Verifier nopVerifier bool } @@ -121,16 +121,18 @@ func newRegistry(registryCfg RegistryConfig, transport http.RoundTripper) (Regis if err != nil { return nil, err } - var ver *edcrypto.Verifier + var ver pqsig.Verifier nopVerify := false - if registryCfg.VerifyConfig == nil { - ver, err = edcrypto.NewVerifier(edcrypto.NewVerifierConfig("", "", "")) - } else { - ver, err = edcrypto.NewVerifier(*registryCfg.VerifyConfig) + var keys map[string]string + if registryCfg.VerifyConfig != nil { nopVerify = registryCfg.VerifyConfig.NopVerify + keys = registryCfg.VerifyConfig.PublicKeys } - if err != nil { - return nil, err + if !nopVerify { + ver, err = pqsig.NewVerifier(keys) + if err != nil { + return nil, err + } } rv := &Registry{ allowSrcDownload: registryCfg.AllowSrcDownload, @@ -380,15 +382,6 @@ func (r *Registry) GetServiceFragment(ps ProviderService, resourceKey string) (S return svc, nil } -func (r *Registry) checkSignature(docUrl string, verFile, sigFile io.ReadCloser) (*edcrypto.VerifierResponse, error) { - if sigFile == nil { - return nil, fmt.Errorf("nil signature") - } - vc := edcrypto.NewVerifyContext(docUrl, sigFile, verFile, "base64", true, x509.VerifyOptions{}) - vr, err := r.verifier.VerifyFileFromCertificateBytes(vc) - return &vr, err -} - func (r *Registry) pullArchive(archivePath string) (io.ReadCloser, error) { return r.getUnVerifiedArchive(archivePath) } @@ -576,87 +569,21 @@ func (r *Registry) getUnVerifiedArchive(docPath string) (io.ReadCloser, error) { return nil, fmt.Errorf("registry scheme '%s' currently not supported", r.regUrl.Scheme) } -func (r *Registry) getEmbeddedVerifiedDocResponse(docPath string) (*edcrypto.VerifierResponse, error) { - lf, err := getServiceDoc(docPath, "") - if err != nil { - return nil, err - } - sf, err := r.getLocalDoc(fmt.Sprintf("%s.sig", docPath)) - if err != nil { - lf.Close() - return nil, fmt.Errorf("embedded document present but signature file not present") - } - return r.checkSignature(docPath, lf, sf) -} - -func (r *Registry) getVerifiedDocResponse(docPath string) (*edcrypto.VerifierResponse, error) { - var embeddedErr error +func (r *Registry) getDocReader(docPath string) (io.ReadCloser, error) { if r.isLocalFile() { - localPath := r.srcUrl.Path - if r.srcUrl.Host == "." { - localPath = "." + localPath - } - rb, err := os.Open(path.Join(localPath, docPath)) - if err != nil { - return nil, fmt.Errorf("cannot read local registry file: '%s'", err.Error()) - } - defer rb.Close() - rBytes, rErr := io.ReadAll(rb) - if rErr != nil { - return nil, fmt.Errorf("cannot read local registry file: '%s'", rErr.Error()) - } - fileLockSafeReadCloser := io.NopCloser(bytes.NewReader(rBytes)) - if r.nopVerifier { - rv := edcrypto.NewVerifierResponse(true, nil, fileLockSafeReadCloser, nil) - return &rv, nil - } - sb, err := os.Open(path.Join(localPath, fmt.Sprintf("%s.sig", docPath))) - if err != nil { - return nil, fmt.Errorf("cannot read local signature file: '%s'", err.Error()) - } - defer sb.Close() - sBytes, sErr := io.ReadAll(sb) - if sErr != nil { - return nil, fmt.Errorf("cannot read signature file: '%s'", sErr.Error()) - } - return r.checkSignature( - docPath, - fileLockSafeReadCloser, - io.NopCloser(bytes.NewReader(sBytes)), // file lock safe signature file - ) + return os.Open(path.Join(r.localSourcePath(), docPath)) } if r.localDocRoot != "" { localPath := r.getLocalDocPath(docPath) lf, err := r.getLocalDoc(localPath) if err == nil { - if r.nopVerifier { - rv := edcrypto.NewVerifierResponse(true, nil, lf, nil) - return &rv, nil - } - sf, err := r.getLocalDoc(fmt.Sprintf("%s.sig", localPath)) - if err != nil { - if lf != nil { - lf.Close() - } - return nil, fmt.Errorf("local document present but signature file not present") - } - return r.checkSignature(localPath, lf, sf) + return lf, nil } } - fullUrl, err := url.Parse(r.regUrl.String()) - if err != nil { - return nil, err - } - fullUrl.Path = path.Join(fullUrl.Path, docPath) - verifyUrl := fullUrl.String() if r.isHttp() { if !r.allowSrcDownload { return nil, fmt.Errorf("download of individual docs disallowed; please attempt to pull provider docs") } - cl := &http.Client{} - if r.transport != nil { - cl.Transport = r.transport - } response, err := r.getRemoteDoc(docPath) if err != nil { return nil, err @@ -664,39 +591,50 @@ func (r *Registry) getVerifiedDocResponse(docPath string) (*edcrypto.VerifierRes if response == nil { return nil, fmt.Errorf("no response body from remote") } - if r.nopVerifier { - rv := edcrypto.NewVerifierResponse(true, nil, response, nil) - return &rv, nil - } - sigResponse, err := r.getRemoteDoc(fmt.Sprintf("%s.sig", docPath)) - if err != nil { - response.Close() - return nil, fmt.Errorf("remote document '%s' present but signature file not present", verifyUrl) - } - return r.checkSignature(verifyUrl, response, sigResponse) - } - if embeddedErr != nil { - return nil, fmt.Errorf("error retrieving from embedded: %s", embeddedErr.Error()) + return response, nil } return nil, fmt.Errorf("registry scheme '%s' currently not supported", r.regUrl.Scheme) } func (r *Registry) getVerifiedDocBytes(docPath string) ([]byte, error) { - vr, err := r.getVerifiedDocResponse(docPath) + reader, err := r.getDocReader(docPath) if err != nil { - return nil, err + return nil, fmt.Errorf("registry document %q: %w", docPath, err) + } + defer reader.Close() + doc, err := io.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("read registry document %q: %w", docPath, err) + } + if r.nopVerifier { + return doc, nil } - return io.ReadAll(vr.VerifyFile) + doc, err = r.verifier.Verify(doc) + if err != nil { + return nil, fmt.Errorf("verify registry document %q: %w", docPath, err) + } + return doc, nil } func (r *Registry) GetLatestAvailableVersion(providerName string) (string, error) { return r.getLatestAvailableVersion(providerName) } +func (r *Registry) localSourcePath() string { + localPath := r.srcUrl.Path + if r.srcUrl.Host == "." { + localPath = "." + localPath + } + if strings.HasPrefix(localPath, "/") && filepath.VolumeName(localPath[1:]) != "" { + localPath = localPath[1:] + } + return localPath +} + func (r *Registry) getLatestAvailableVersion(providerName string) (string, error) { providerName = r.renameProvider(providerName) if r.isLocalFile() { - deSlice, err := os.ReadDir(path.Join(r.srcUrl.Path, providerName)) + deSlice, err := os.ReadDir(path.Join(r.localSourcePath(), providerName)) if err != nil { return "", err } diff --git a/internal/anysdk/registry_signature_test.go b/internal/anysdk/registry_signature_test.go new file mode 100644 index 00000000..85c814a0 --- /dev/null +++ b/internal/anysdk/registry_signature_test.go @@ -0,0 +1,108 @@ +package anysdk_test + +import ( + "bytes" + "crypto/mldsa" + "encoding/hex" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stackql/any-sdk/internal/anysdk" + "github.com/stackql/stackql-provider-registry/signing/pqsig" +) + +func TestRegistryMLDSASignatures(t *testing.T) { + key, err := mldsa.GenerateKey(mldsa.MLDSA65()) + if err != nil { + t.Fatal(err) + } + keys := map[string]string{"test": hex.EncodeToString(key.PublicKey().Bytes())} + source := []byte("name: example\n") + signed, err := pqsig.Sign(source, "test", key) + if err != nil { + t.Fatal(err) + } + for _, mode := range []string{"file", "cache", "remote"} { + for _, tc := range []struct { + name string + doc []byte + nop bool + wantError bool + }{ + {"valid", signed, false, false}, + {"tampered", bytes.Replace(signed, []byte("name: example"), []byte("name: changed"), 1), false, true}, + {"unknown", bytes.Replace(signed, []byte("kid: test"), []byte("kid: unknown"), 1), false, true}, + {"unsigned", source, false, true}, + {"opt-out", source, true, false}, + {"opt-out-tampered", []byte("x-stackql-signature: invalid\n"), true, false}, + } { + t.Run(mode+"/"+tc.name, func(t *testing.T) { + const docPath = "example/v1/provider.yaml" + root := t.TempDir() + localPath := filepath.Join(root, "src", filepath.FromSlash(docPath)) + if err := os.MkdirAll(filepath.Dir(localPath), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(localPath, tc.doc, 0600); err != nil { + t.Fatal(err) + } + // A sidecar must neither rescue an invalid embedded signature nor be fetched. + if err := os.WriteFile(localPath+".sig", []byte("legacy"), 0600); err != nil { + t.Fatal(err) + } + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if r.URL.Path != "/src/"+docPath { + t.Errorf("unexpected request: %s", r.URL.Path) + } + _, _ = w.Write(tc.doc) + })) + defer server.Close() + cfg := anysdk.RegistryConfig{ + RegistryURL: server.URL, AllowSrcDownload: true, + VerifyConfig: &pqsig.VerifierConfig{NopVerify: tc.nop, PublicKeys: keys}, + } + switch mode { + case "file": + p := filepath.ToSlash(root) + if !strings.HasPrefix(p, "/") { + p = "/" + p + } + cfg.RegistryURL = (&url.URL{Scheme: "file", Path: p}).String() + case "cache": + cfg.LocalDocRoot = root + } + reg, err := anysdk.NewRegistry(cfg, nil) + if err != nil { + t.Fatal(err) + } + got, err := reg.GetDocBytes(docPath) + if (err != nil) != tc.wantError { + t.Fatalf("GetDocBytes: %s, %v", got, err) + } + if err != nil && !strings.Contains(err.Error(), docPath) { + t.Fatalf("error omitted document path: %v", err) + } + if !tc.wantError && !tc.nop && !bytes.Equal(got, source) { + t.Fatalf("expected exact authenticated payload, got %s", got) + } + if tc.nop && !bytes.Equal(got, tc.doc) { + t.Fatal("opt-out changed the document bytes") + } + wantRequests := 0 + if mode == "remote" { + wantRequests = 1 + } + if requests != wantRequests { + t.Fatalf("made %d HTTP requests; want %d", requests, wantRequests) + } + }) + } + } +} diff --git a/public/discovery/shallow_discovery_test.go b/public/discovery/shallow_discovery_test.go index 6cc449cd..e4fb7409 100644 --- a/public/discovery/shallow_discovery_test.go +++ b/public/discovery/shallow_discovery_test.go @@ -10,7 +10,7 @@ import ( "github.com/stackql/any-sdk/public/discovery" "github.com/stackql/any-sdk/public/persistence" "github.com/stackql/any-sdk/public/sqlengine" - "github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto" + "github.com/stackql/stackql-provider-registry/signing/pqsig" ) const ( @@ -23,7 +23,7 @@ func getNewTestDataMockRegistry(relativePath string) (anysdk.RegistryAPI, error) RegistryURL: fmt.Sprintf("file://%s", relativePath), LocalDocRoot: relativePath, AllowSrcDownload: false, - VerifyConfig: &edcrypto.VerifierConfig{ + VerifyConfig: &pqsig.VerifierConfig{ NopVerify: true, }, }, diff --git a/public/discovery/static_analyzer.go b/public/discovery/static_analyzer.go index 1391e6ab..1fb16d56 100644 --- a/public/discovery/static_analyzer.go +++ b/public/discovery/static_analyzer.go @@ -14,7 +14,7 @@ import ( "github.com/stackql/any-sdk/public/persistence" "github.com/stackql/any-sdk/public/radix_tree_address_space" "github.com/stackql/any-sdk/public/sqlengine" - "github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto" + "github.com/stackql/stackql-provider-registry/signing/pqsig" ) var ( @@ -139,7 +139,7 @@ func getNewLocalRegistry(relativePath string) (anysdk.RegistryAPI, error) { RegistryURL: fmt.Sprintf("file://%s", cleanPath), LocalDocRoot: relativePath, AllowSrcDownload: false, - VerifyConfig: &edcrypto.VerifierConfig{ + VerifyConfig: &pqsig.VerifierConfig{ NopVerify: true, }, }, @@ -1551,12 +1551,12 @@ func analyzeMethod( varName := MockResponseVarName(actx.Provider, actx.Service, actx.Resource, actx.Method) sampleResponse.VarName = varName f := AnalysisFinding{ - Level: "info", - Provider: actx.Provider, - Service: actx.Service, - Resource: actx.Resource, - Method: actx.Method, - Message: "mock generated from response schema", + Level: "info", + Provider: actx.Provider, + Service: actx.Service, + Resource: actx.Resource, + Method: actx.Method, + Message: "mock generated from response schema", SampleResponse: sampleResponse, MockRoute: GenerateMockRoute( actx.Provider, diff --git a/public/formulation/interfaces.go b/public/formulation/interfaces.go index 33e8fa81..c3f05563 100644 --- a/public/formulation/interfaces.go +++ b/public/formulation/interfaces.go @@ -24,7 +24,7 @@ import ( "github.com/stackql/any-sdk/public/discovery" "github.com/stackql/any-sdk/public/radix_tree_address_space" "github.com/stackql/stackql-parser/go/sqltypes" - "github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto" + "github.com/stackql/stackql-provider-registry/signing/pqsig" ) // RuntimeCtx mirrors methods on RuntimeCtx @@ -351,12 +351,12 @@ type ExecContext interface { } type RegistryConfig struct { - RegistryURL string `json:"url" yaml:"url"` - SrcPrefix *string `json:"srcPrefix" yaml:"srcPrefix"` - DistPrefix *string `json:"distPrefix" yaml:"distPrefix"` - AllowSrcDownload bool `json:"allowSrcDownload" yaml:"allowSrcDownload"` - LocalDocRoot string `json:"localDocRoot" yaml:"localDocRoot"` - VerifyConfig *edcrypto.VerifierConfig `json:"verifyConfig" yaml:"verifyConfig"` + RegistryURL string `json:"url" yaml:"url"` + SrcPrefix *string `json:"srcPrefix" yaml:"srcPrefix"` + DistPrefix *string `json:"distPrefix" yaml:"distPrefix"` + AllowSrcDownload bool `json:"allowSrcDownload" yaml:"allowSrcDownload"` + LocalDocRoot string `json:"localDocRoot" yaml:"localDocRoot"` + VerifyConfig *pqsig.VerifierConfig `json:"verifyConfig" yaml:"verifyConfig"` } func (rc RegistryConfig) toAnySdkRegistryConfig() anysdk.RegistryConfig { diff --git a/public/radix_tree_address_space/address_space_test.go b/public/radix_tree_address_space/address_space_test.go index 075af49c..a6e7ccdc 100644 --- a/public/radix_tree_address_space/address_space_test.go +++ b/public/radix_tree_address_space/address_space_test.go @@ -12,7 +12,7 @@ import ( "github.com/stackql/any-sdk/pkg/dto" "github.com/stackql/any-sdk/public/discovery" "github.com/stackql/any-sdk/public/radix_tree_address_space" - "github.com/stackql/stackql-provider-registry/signing/Ed25519/app/edcrypto" + "github.com/stackql/stackql-provider-registry/signing/pqsig" ) func getNewTestDataMockRegistry(relativePath string) (anysdk.RegistryAPI, error) { @@ -21,7 +21,7 @@ func getNewTestDataMockRegistry(relativePath string) (anysdk.RegistryAPI, error) RegistryURL: fmt.Sprintf("file://%s", relativePath), LocalDocRoot: relativePath, AllowSrcDownload: false, - VerifyConfig: &edcrypto.VerifierConfig{ + VerifyConfig: &pqsig.VerifierConfig{ NopVerify: true, }, },