From 8ac5a363d956caa4f3a71d665951283e74c418da Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Mon, 7 Sep 2026 11:03:43 -0300 Subject: [PATCH 1/9] feat: support new config and add manually vendored files for testing --- README.md | 1 + config_schema.json | 12 +- docs/connector.mdx | 23 ++ go.mod | 2 + go.sum | 2 - pkg/config/conf.gen.go | 1 + pkg/config/config.go | 11 +- pkg/connector/connector.go | 1 + .../baton-github/pkg/config/conf.gen.go | 2 + .../baton-github/pkg/config/config.go | 32 +- .../baton-github/pkg/connector/api_token.go | 7 +- .../baton-github/pkg/connector/app.go | 4 +- .../baton-github/pkg/connector/connector.go | 38 ++- .../baton-github/pkg/connector/invitation.go | 11 +- .../baton-github/pkg/connector/org_role.go | 6 +- .../baton-github/pkg/connector/team.go | 23 +- .../pkg/connector/usage_event_feed.go | 296 ++++++++++++++++++ .../baton-github/pkg/connector/user.go | 22 +- vendor/modules.txt | 3 +- 19 files changed, 458 insertions(+), 39 deletions(-) create mode 100644 vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go diff --git a/README.md b/README.md index 005c30b0..f2aa6c4d 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Flags: --skip-full-sync This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC) --ticketing This must be set to enable ticketing support ($BATON_TICKETING) --token string required: The GitHub access token used to connect to the GitHub API. ($BATON_TOKEN) + --sync-last-activity See when members were last active in your organizations. ($BATON_SYNC_LAST_ACTIVITY) --sync-secrets Whether to sync secrets or not ($BATON_SYNC_SECRETS) -v, --version version for baton-github-enterprise diff --git a/config_schema.json b/config_schema.json index 51cf1358..dae6b7ee 100644 --- a/config_schema.json +++ b/config_schema.json @@ -183,6 +183,12 @@ "displayName": "Optimize sync for large organizations", "description": "Reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. Recommended for large orgs.", "boolField": {} + }, + { + "name": "sync-last-activity", + "displayName": "Sync user last activity", + "description": "See when members were last active in your organizations.", + "boolField": {} } ], "displayName": "GitHub Enterprise", @@ -199,7 +205,8 @@ "orgs", "enterprises", "omit-archived-repositories", - "direct-collaborators-only" + "direct-collaborators-only", + "sync-last-activity" ], "default": true }, @@ -215,7 +222,8 @@ "enterprises", "sync-secrets", "omit-archived-repositories", - "direct-collaborators-only" + "direct-collaborators-only", + "sync-last-activity" ] } ] diff --git a/docs/connector.mdx b/docs/connector.mdx index cf147273..e7622e08 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -246,6 +246,8 @@ If you're using a personal access token to set up the connector: 1. **Optional.** If you do not want to include archived repos in syncs, click to enable **Omit archived repositories**. 1. **Optional.** For large organizations, click to enable **Optimize sync for large organizations**. This reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. + + 1. **Optional.** If you want to see when members were last active, click to enable **Sync last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. If you're using a GitHub app to set up the connector: @@ -265,6 +267,8 @@ If you're using a GitHub app to set up the connector: 1. **Optional.** If you do not want to include archived repos in syncs, click to enable **Omit archived repositories**. 1. **Optional.** For large organizations, click to enable **Optimize sync for large organizations**. This reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. + + 1. **Optional.** If you want to see when members were last active, click to enable **Sync last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. Click **Save**. @@ -323,6 +327,7 @@ baton-github-enterprise | `BATON_DIRECT_COLLABORATORS_ONLY` | Set to `true` to optimize sync for large organizations by reducing API calls (uses grant expansion for team-based repo access and skips per-team detail fetches). | | `BATON_PROVISIONING` | Set to `true` to enable provisioning and deprovisioning. | | `BATON_SYNC_SECRETS` | Set to `true` to sync secrets (API keys). | +| `BATON_SYNC_LAST_ACTIVITY` | Set to `true` to see when members were last active. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. | The connector connects to your C1 tenant and begins syncing. Verify it appears as **Connected** in **Integrations** > **Connectors**. @@ -359,6 +364,7 @@ baton-github-enterprise \ | `--enterprises` | Comma-separated list of enterprises to sync enterprise roles for. | | `--omit-archived-repositories` | Exclude archived repositories from syncs. | | `--direct-collaborators-only` | Optimize sync for large organizations by reducing API calls (uses grant expansion for team-based repo access and skips per-team detail fetches). | +| `--sync-last-activity` | See when members were last active. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. | | `--provisioning` | Enable provisioning and deprovisioning. | @@ -385,6 +391,23 @@ When deploying on Kubernetes with a minimal base image (for example, Debian slim +## Sync member last activity + +You can enable **Sync last activity** to have C1 show, for each member, the last time they were seen taking action in that organization (for example, commenting on an issue or pull request, reviewing code, or changing a setting). This appears on the member's profile in C1 alongside their other access details. + +**Requirements:** + +- The personal access token used by the connector must have the **read:audit_log** scope. +- The token must have permission to view each organization's audit log (typically an organization owner). + +**What "last activity" means:** This is not necessarily a login timestamp. GitHub's audit log only records actions members take, not every time someone signs in or browses the product. Because of this, C1 shows the most recent recorded action for a member, whatever that action was, as a best-effort signal of when they were last active. A member who only reads or browses without taking any recorded action won't show a last-activity date, even if they use GitHub regularly. + +To keep this fast on large organizations, raw Git operations (pushes, fetches, and clones done via `git` rather than the website or API) are not counted toward last activity — only web and API actions are. A member who exclusively interacts with GitHub through `git` and never uses the website or API (for example, a purely command-line workflow with no PR reviews, comments, or settings changes) won't show a last-activity date, even though they're actively using GitHub. + +If a member has never taken a recorded action, or if the connector can't reach an organization's audit log for any reason (for example, a missing scope or a plan without audit log access), C1 simply won't show a last-activity date for that member — this does not affect the rest of the sync. + +Activity is delivered as an ongoing background feed rather than as part of each full sync, so it may take a little time after enabling this option before last-activity dates first appear, and dates update continuously afterward rather than only at sync time. + ## Troubleshooting ### "Resource not accessible by integration" error diff --git a/go.mod b/go.mod index a4588916..912c5c90 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/conductorone/baton-github-enterprise go 1.25.2 +replace github.com/conductorone/baton-github => ../baton-github + require ( github.com/conductorone/baton-github v0.4.0 github.com/conductorone/baton-sdk v0.26.0 diff --git a/go.sum b/go.sum index 13ed7e34..dd8b6e78 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,6 @@ github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8 github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-github v0.4.0 h1:Hf72uDLoYrtmhII6BYyRs0qSG/vuhx6Po7QHHs4618E= -github.com/conductorone/baton-github v0.4.0/go.mod h1:SUpbLLRnO+fldsV/ALUd6JjONtUcsPya38psMNBIDCU= github.com/conductorone/baton-sdk v0.26.0 h1:aNKg81BhPAVGyYe+W4czZJnL9hzJEUkDinbt0klOeo4= github.com/conductorone/baton-sdk v0.26.0/go.mod h1:SKm95z4KkQ23Tufo2ys88lVzbwKb0AQEbKee5GE0Lig= github.com/conductorone/dpop v0.2.6 h1:fakwai/Xm2b/fcDUwJN41WtcSI/2UhQOyRIVvnnrrNA= diff --git a/pkg/config/conf.gen.go b/pkg/config/conf.gen.go index e20969e7..51e59484 100644 --- a/pkg/config/conf.gen.go +++ b/pkg/config/conf.gen.go @@ -14,6 +14,7 @@ type Githubenterprise struct { SyncSecrets bool `mapstructure:"sync-secrets"` OmitArchivedRepositories bool `mapstructure:"omit-archived-repositories"` DirectCollaboratorsOnly bool `mapstructure:"direct-collaborators-only"` + SyncLastActivity bool `mapstructure:"sync-last-activity"` } func (c *Githubenterprise) findFieldByTag(tagValue string) (any, bool) { diff --git a/pkg/config/config.go b/pkg/config/config.go index c0499124..390ee69f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -75,6 +75,12 @@ var ( "and skipping per-team detail fetches. Recommended for large orgs.", ), ) + + syncLastActivity = field.BoolField( + "sync-last-activity", + field.WithDisplayName("Sync user last activity"), + field.WithDescription("See when members were last active in your organizations."), + ) ) //go:generate go run ./gen @@ -90,6 +96,7 @@ var Config = field.NewConfiguration( syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, + syncLastActivity, }, field.WithConnectorDisplayName("GitHub Enterprise"), field.WithHelpUrl("/docs/baton/github-enterprise"), @@ -99,14 +106,14 @@ var Config = field.NewConfiguration( Name: GithubPersonalAccessTokenGroup, DisplayName: "Personal access token", HelpText: "Use a personal access token for authentication.", - Fields: []field.SchemaField{instanceUrlField, accessTokenField, orgsField, EnterprisesField, omitArchivedRepositories, directCollaboratorsOnly}, + Fields: []field.SchemaField{instanceUrlField, accessTokenField, orgsField, EnterprisesField, omitArchivedRepositories, directCollaboratorsOnly, syncLastActivity}, Default: true, }, { Name: GithubAppGroup, DisplayName: "GitHub app", HelpText: "Use a github app for authentication", - Fields: []field.SchemaField{instanceUrlField, appIDField, appPrivateKeyPath, orgField, EnterprisesField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly}, + Fields: []field.SchemaField{instanceUrlField, appIDField, appPrivateKeyPath, orgField, EnterprisesField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, syncLastActivity}, Default: false, }, }), diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index eaa505d2..30253ba4 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -24,6 +24,7 @@ func NewLambdaConnector(ctx context.Context, ghc *cfg.Githubenterprise, cliOpts AppPrivatekeyPath: ghc.AppPrivatekeyPath, Org: ghc.Org, DirectCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + SyncLastActivity: ghc.SyncLastActivity, }, cliOpts) } diff --git a/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go b/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go index 5ed99148..6541f0f2 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go +++ b/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go @@ -10,10 +10,12 @@ type Github struct { InstanceUrl string `mapstructure:"instance-url"` AppId string `mapstructure:"app-id"` AppPrivatekeyPath []byte `mapstructure:"app-privatekey-path"` + AppPrivatekey string `mapstructure:"app-privatekey"` Org string `mapstructure:"org"` SyncSecrets bool `mapstructure:"sync-secrets"` OmitArchivedRepositories bool `mapstructure:"omit-archived-repositories"` DirectCollaboratorsOnly bool `mapstructure:"direct-collaborators-only"` + SyncLastActivity bool `mapstructure:"sync-last-activity"` } func (c *Github) findFieldByTag(tagValue string) (any, bool) { diff --git a/vendor/github.com/conductorone/baton-github/pkg/config/config.go b/vendor/github.com/conductorone/baton-github/pkg/config/config.go index 2b9c5e63..5a142d66 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-github/pkg/config/config.go @@ -40,13 +40,25 @@ var ( field.WithRequired(true), ) + // appPrivateKeyPath and appPrivateKey are two ways to supply the same GitHub + // App private key. Neither is marked required individually: providing either + // one satisfies the "app private key required" check, which is enforced in the + // connector's GitHub App constructor (see pkg/connector.appPrivateKeyPEM). + // A framework-level constraint can't express "required only for the GitHub App + // auth group", since constraints are evaluated globally across auth methods. appPrivateKeyPath = field.FileUploadField( "app-privatekey-path", []string{".pem"}, field.WithDisplayName("GitHub App private key (.pem)"), - field.WithDescription("Path to private key that is used to connect to the GitHub App"), + field.WithDescription("Path to private key that is used to connect to the GitHub App. Ignored when app-privatekey is set."), + field.WithIsSecret(true), + ) + + appPrivateKey = field.StringField( + "app-privatekey", + field.WithDisplayName("GitHub App private key (PEM)"), + field.WithDescription("Raw PEM contents of the private key used to connect to the GitHub App. Takes precedence over app-privatekey-path when both are set."), field.WithIsSecret(true), - field.WithRequired(true), ) syncSecrets = field.BoolField( @@ -79,6 +91,18 @@ var ( field.WithDescription("Organization of your github app"), field.WithRequired(true), ) + + // syncLastActivity is hidden from this connector's GUI config and --help + // since it only applies to GitHub Enterprise audit-log access. + // baton-github-enterprise sets it directly on the shared Github struct, + // bypassing this CLI layer, so hiding it here doesn't affect that connector. + syncLastActivity = field.BoolField( + "sync-last-activity", + field.WithDisplayName("Sync users last activity"), + field.WithDescription("See when members were last active in your organizations."), + field.WithHidden(true), + field.WithExportTarget(field.ExportTargetCLIOnly), + ) ) //go:generate go run ./gen @@ -90,10 +114,12 @@ var Config = field.NewConfiguration( instanceUrlField, appIDField, appPrivateKeyPath, + appPrivateKey, orgField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, + syncLastActivity, }, field.WithConnectorDisplayName("GitHub v2"), field.WithHelpUrl("/docs/baton/github-v2"), @@ -110,7 +136,7 @@ var Config = field.NewConfiguration( Name: GithubAppGroup, DisplayName: "GitHub app", HelpText: "Use a github app for authentication", - Fields: []field.SchemaField{appIDField, appPrivateKeyPath, orgField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly}, + Fields: []field.SchemaField{appIDField, appPrivateKeyPath, appPrivateKey, orgField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly}, Default: false, }, }), diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go b/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go index 5f0852e8..7a6a7fea 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go @@ -26,8 +26,12 @@ func apiTokenResource(ctx context.Context, token *github.PersonalAccessToken) (* options = append(options, resourceSdk.WithSecretLastUsedAt(token.TokenLastUsedAt.Time)) } + // created_at has moved from SecretTrait to a Resource-level attribute, so + // set it via the resource-level option instead of the deprecated + // WithSecretCreatedAt trait option. + var resourceOpts []resourceSdk.ResourceOption if token.AccessGrantedAt != nil { - options = append(options, resourceSdk.WithSecretCreatedAt(token.AccessGrantedAt.Time)) + resourceOpts = append(resourceOpts, resourceSdk.WithResourceCreatedAt(token.AccessGrantedAt.Time)) } if token.TokenExpiresAt != nil { @@ -38,6 +42,7 @@ func apiTokenResource(ctx context.Context, token *github.PersonalAccessToken) (* resourceTypeApiToken, token.GetID(), options, + resourceOpts..., ) if err != nil { return nil, err diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/app.go b/vendor/github.com/conductorone/baton-github/pkg/connector/app.go index cafe9ed9..8f5cb7c1 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/app.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/app.go @@ -39,7 +39,9 @@ func appResource(ctx context.Context, installation *github.Installation, parentR opts := []resourceSdk.ResourceOption{ resourceSdk.WithParentResourceID(parentResourceID), - resourceSdk.WithAppTrait(resourceSdk.WithAppProfile(profile)), + resourceSdk.WithAppTrait(), + // profile has moved from AppTrait to a Resource-level attribute. + resourceSdk.WithResourceProfile(profile), resourceSdk.WithNHIType(v2.NonHumanIdentityTrait_NHI_TYPE_APP_REGISTRATION, "github.app"), } if installation.HTMLURL != nil { diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go index 2cdb5a62..72f0bf07 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go @@ -127,6 +127,7 @@ type GitHub struct { omitArchivedRepositories bool directCollaboratorsOnly bool enterprises []string + syncLastActivity bool } func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 { @@ -157,8 +158,18 @@ func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour return resourceSyncers } +func (gh *GitHub) EventFeeds(_ context.Context) []connectorbuilder.EventFeed { + if !gh.syncLastActivity { + return nil + } + + return []connectorbuilder.EventFeed{ + newUsageEventFeed(gh.client, gh.orgs), + } +} + // Metadata returns metadata about the connector. -func (gh *GitHub) Metadata(ctx context.Context) (*v2.ConnectorMetadata, error) { +func (gh *GitHub) Metadata(_ context.Context) (*v2.ConnectorMetadata, error) { return &v2.ConnectorMetadata{ DisplayName: "GitHub", AccountCreationSchema: &v2.ConnectorAccountCreationSchema{ @@ -346,11 +357,31 @@ func newWithGithubPAT(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + syncLastActivity: ghc.SyncLastActivity, }, nil } +// appPrivateKeyPEM returns the GitHub App private key PEM contents to use, +// preferring the in-memory app-privatekey flag over the on-disk +// app-privatekey-path. Providing either one satisfies the requirement; if +// neither is set an error is returned. +func appPrivateKeyPEM(ghc *cfg.Github) (string, error) { + if ghc.AppPrivatekey != "" { + return ghc.AppPrivatekey, nil + } + if len(ghc.AppPrivatekeyPath) > 0 { + return string(ghc.AppPrivatekeyPath), nil + } + return "", errors.New("github app authentication requires either --app-privatekey or --app-privatekey-path") +} + func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { - jwttoken, err := getJWTToken(ghc.AppId, string(ghc.AppPrivatekeyPath)) + privateKey, err := appPrivateKeyPEM(ghc) + if err != nil { + return nil, err + } + + jwttoken, err := getJWTToken(ghc.AppId, privateKey) if err != nil { return nil, err } @@ -382,7 +413,7 @@ func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { }, &appJWTTokenRefresher{ appID: ghc.AppId, - privateKey: string(ghc.AppPrivatekeyPath), + privateKey: privateKey, }, ) // Wrap the installation-token refresher in a refreshableTokenSource so the @@ -433,6 +464,7 @@ func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + syncLastActivity: ghc.SyncLastActivity, } return gh, nil } diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go b/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go index 52df44a2..fe49b5c6 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go @@ -64,10 +64,19 @@ func invitationToUserResource(invitation *github.Invitation, status string) (*v2 invitation.GetID(), []resourceSdk.UserTraitOption{ resourceSdk.WithEmail(invitation.GetEmail(), true), - resourceSdk.WithUserProfile(profile), + // An invitation is a pending/expired user that must not be + // reported as enabled. WithResourceStatus cannot express this: + // NewUserTrait force-defaults an unset trait status to ENABLED, so + // migrating this line would flip the emitted status from + // UNSPECIFIED to ENABLED. Keep the deprecated trait option (which + // also mirrors UNSPECIFIED to the resource level) to preserve the + // exact status semantics. + //nolint:staticcheck // deliberate: WithResourceStatus would force the trait status to ENABLED; UNSPECIFIED must be preserved for invitations. resourceSdk.WithStatus(v2.UserTrait_Status_STATUS_UNSPECIFIED), resourceSdk.WithUserLogin(login), }, + // profile has moved from UserTrait to a Resource-level attribute. + resourceSdk.WithResourceProfile(profile), ) if err != nil { return nil, err diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go b/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go index e78c30a0..32942686 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go @@ -53,9 +53,9 @@ func orgRoleResource( role.Name, resourceTypeOrgRole, role.ID, - []resourceSdk.RoleTraitOption{ - resourceSdk.WithRoleProfile(profile), - }, + []resourceSdk.RoleTraitOption{}, + // profile has moved from RoleTrait to a Resource-level attribute. + resourceSdk.WithResourceProfile(profile), resourceSdk.WithParentResourceID(org.Id), resourceSdk.WithAnnotation( &v2.V1Identifier{Id: fmt.Sprintf("org_role:%d", role.ID)}, diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/team.go b/vendor/github.com/conductorone/baton-github/pkg/connector/team.go index 8e1579ba..0dc2eb24 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/team.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/team.go @@ -45,7 +45,9 @@ func teamResource(team *github.Team, orgID int64, parentResourceID *v2.ResourceI team.GetName(), resourceTypeTeam, team.GetID(), - []rType.GroupTraitOption{rType.WithGroupProfile(profile)}, + []rType.GroupTraitOption{}, + // profile has moved from GroupTrait to a Resource-level attribute. + rType.WithResourceProfile(profile), rType.WithAnnotation( &v2.ExternalLink{Url: team.GetURL()}, &v2.V1Identifier{Id: fmt.Sprintf("team:%d", team.GetID())}, @@ -159,12 +161,10 @@ func (o *teamResourceType) Grants(ctx context.Context, resource *v2.Resource, op return nil, nil, err } - teamTrait, err := rType.GetGroupTrait(resource) - if err != nil { - return nil, nil, err - } - - orgID, ok := rType.GetProfileInt64Value(teamTrait.Profile, "orgID") + // profile has moved from GroupTrait to a Resource-level attribute; read it + // via GetProfile, which resolves the resource-level value (with a + // trait-level fallback for older data). + orgID, ok := rType.GetProfileInt64Value(rType.GetProfile(resource), "orgID") if !ok { return nil, nil, fmt.Errorf("error fetching orgID from team profile") } @@ -280,12 +280,9 @@ func (o *teamResourceType) Grant(ctx context.Context, principal *v2.Resource, en return nil, err } case resourceTypeTeam.Id: - groupTrait, err := rType.GetGroupTrait(entitlement.Resource) - if err != nil { - return nil, err - } - - orgID, ok := rType.GetProfileInt64Value(groupTrait.Profile, "orgID") + // profile has moved from GroupTrait to a Resource-level attribute; read + // it via GetProfile (resource-level value, with a trait-level fallback). + orgID, ok := rType.GetProfileInt64Value(rType.GetProfile(entitlement.Resource), "orgID") if !ok { return nil, fmt.Errorf("error fetching orgID from team profile") } diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go new file mode 100644 index 00000000..efd24be5 --- /dev/null +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go @@ -0,0 +1,296 @@ +package connector + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/google/go-github/v69/github" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// defaultActivityLookback bounds the very first poll when no earliest-event +// boundary is given yet; later polls advance via the feed's own cursor. +const defaultActivityLookback = 1 * time.Hour + +// maxAuditLogPagesPerCall caps total pages walked across all orgs per call +// so one very active org can't stall the feed; remaining pages resume via the cursor. +const maxAuditLogPagesPerCall = 20 + +// usageEventFeed streams member activity from each org's audit log as usage +// events, since GitHub has no per-user "last activity" field to sync directly. +type usageEventFeed struct { + client *github.Client + orgs []string +} + +func newUsageEventFeed(client *github.Client, orgs []string) *usageEventFeed { + return &usageEventFeed{client: client, orgs: orgs} +} + +func (f *usageEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFeedMetadata { + return &v2.EventFeedMetadata{ + Id: "github_usage_event_feed", + SupportedEventTypes: []v2.EventType{v2.EventType_EVENT_TYPE_USAGE}, + } +} + +// usageEventPageToken tracks progress through one pass over every configured +// org's audit log, walked newest-first until an entry at or before Since is +// reached (already seen in a previous pass). +type usageEventPageToken struct { + Orgs []string `json:"orgs,omitempty"` + OrgIndex int `json:"org_index"` + AuditLogCursor string `json:"audit_log_cursor,omitempty"` + Since string `json:"since,omitempty"` +} + +func unmarshalUsageEventPageToken(pToken *pagination.StreamToken) (*usageEventPageToken, error) { + pt := &usageEventPageToken{} + if pToken == nil || pToken.Cursor == "" { + return pt, nil + } + data, err := base64.StdEncoding.DecodeString(pToken.Cursor) + if err != nil { + return nil, fmt.Errorf("baton-github: failed to decode usage event feed cursor: %w", err) + } + if err := json.Unmarshal(data, pt); err != nil { + return nil, fmt.Errorf("baton-github: failed to unmarshal usage event feed cursor: %w", err) + } + return pt, nil +} + +func (pt *usageEventPageToken) marshal() (string, error) { + data, err := json.Marshal(pt) + if err != nil { + return "", fmt.Errorf("baton-github: failed to marshal usage event feed cursor: %w", err) + } + return base64.StdEncoding.EncodeToString(data), nil +} + +func (f *usageEventFeed) ListEvents( + ctx context.Context, + earliestEvent *timestamppb.Timestamp, + pToken *pagination.StreamToken, +) ([]*v2.Event, *pagination.StreamState, annotations.Annotations, error) { + l := ctxzap.Extract(ctx) + + if f.client == nil { + return nil, &pagination.StreamState{HasMore: false}, nil, nil + } + + cursor, err := unmarshalUsageEventPageToken(pToken) + if err != nil { + return nil, nil, nil, err + } + + if len(cursor.Orgs) == 0 { + // Snapshot the org list and "since" boundary once per pass, so + // mid-pass config changes don't shift what gets walked. + orgs, err := getOrgs(ctx, f.client, f.orgs) + if err != nil { + return nil, nil, nil, fmt.Errorf("baton-github: failed to list orgs for usage event feed: %w", err) + } + if len(orgs) == 0 { + return nil, &pagination.StreamState{HasMore: false}, nil, nil + } + + since := time.Now().Add(-defaultActivityLookback) + // Guard against a zero/degenerate earliestEvent producing a + // nonsensical "since year 1" query that GitHub's search parser rejects. + if earliestEvent != nil { + if t := earliestEvent.AsTime(); !t.IsZero() && t.After(time.Unix(0, 0)) { + since = t + } + } + + cursor = &usageEventPageToken{ + Orgs: orgs, + Since: since.Format(time.RFC3339Nano), + } + } + + if cursor.OrgIndex < 0 || cursor.OrgIndex >= len(cursor.Orgs) { + cursor.OrgIndex = 0 + cursor.AuditLogCursor = "" + } + + since, err := time.Parse(time.RFC3339Nano, cursor.Since) + if err != nil { + return nil, nil, nil, fmt.Errorf("baton-github: invalid usage event feed cursor timestamp: %w", err) + } + // created:>= is sent server-side so GitHub excludes already-seen + // entries; the check below stays as a safety net in case it's ignored. + sincePhrase := "created:>=" + since.UTC().Format("2006-01-02T15:04:05-07:00") + + var events []*v2.Event + // Tightest (lowest Remaining) rate limit seen across this call's requests. + var tightestRateLimit *v2.RateLimitDescription + + // TODO(jdc): Probably change this for loop for a series of requests that uses a more complex pagination cursor. + for page := 0; page < maxAuditLogPagesPerCall; page++ { + orgName := cursor.Orgs[cursor.OrgIndex] + + opts := &github.GetAuditLogOptions{ + Order: github.Ptr("desc"), + // "web" excludes raw git-protocol events (push/fetch/clone), + // which dominate audit-log volume without losing members who are + // otherwise covered by their web/API activity. + Include: github.Ptr("web"), + Phrase: github.Ptr(sincePhrase), + ListCursorOptions: github.ListCursorOptions{ + PerPage: maxPageSize, + Page: cursor.AuditLogCursor, + }, + } + + entries, resp, err := f.client.Organizations.GetAuditLog(ctx, orgName, opts) + // Read rate-limit headers before the error branch nils resp, since a + // 429 still carries them. + if resp != nil { + if rl, rlErr := extractRateLimitData(resp); rlErr == nil { + if tightestRateLimit == nil || rl.GetRemaining() < tightestRateLimit.GetRemaining() { + tightestRateLimit = rl + } + } + } + if err != nil { + // Skip-and-continue only for permanent per-org conditions (no + // audit-log access); anything else aborts instead of wasting the + // rest of the page budget. Rate-limit checks come first since + // GitHub can signal rate limiting via a 403. + var rateLimitErr *github.RateLimitError + var abuseRateLimitErr *github.AbuseRateLimitError + retryable := errors.As(err, &rateLimitErr) || errors.As(err, &abuseRateLimitErr) || + isRatelimited(resp) || isTemporarilyUnavailable(resp) + + switch { + case retryable: + return nil, nil, nil, wrapGitHubError(err, resp, + fmt.Sprintf("baton-github: failed to fetch audit log for org %s", orgName)) + case isNotFoundError(resp) || isPermissionError(resp): + l.Warn("org lacks audit-log access, skipping it for this pass", + zap.String("org", orgName), zap.Error(err)) + entries, resp = nil, nil + default: + return nil, nil, nil, wrapGitHubError(err, resp, + fmt.Sprintf("baton-github: failed to fetch audit log for org %s", orgName)) + } + } + + reachedBoundary := false + for _, entry := range entries { + // Check every entry's timestamp, even filtered ones, so an all-bot page still stops pagination. + if ts := entry.GetTimestamp().Time; !ts.IsZero() && !ts.After(since) { + reachedBoundary = true + break + } + + evt, ok := usageEventFromAuditEntry(orgName, entry) + if !ok { + continue + } + events = append(events, evt) + } + + if resp != nil && resp.NextPageToken != "" && !reachedBoundary { + cursor.AuditLogCursor = resp.NextPageToken + continue + } + + // Done with this org for this pass - advance to the next one. + cursor.OrgIndex++ + cursor.AuditLogCursor = "" + if cursor.OrgIndex >= len(cursor.Orgs) { + // Pass complete - the next call gets a fresh earliestEvent, so + // nothing needs to survive in the cursor. + tokenStr, err := (&usageEventPageToken{}).marshal() + if err != nil { + return nil, nil, nil, err + } + var annos annotations.Annotations + if tightestRateLimit != nil { + annos.WithRateLimiting(tightestRateLimit) + } + return events, &pagination.StreamState{Cursor: tokenStr, HasMore: false}, annos, nil + } + } + + tokenStr, err := cursor.marshal() + if err != nil { + return nil, nil, nil, err + } + var annos annotations.Annotations + if tightestRateLimit != nil { + annos.WithRateLimiting(tightestRateLimit) + } + return events, &pagination.StreamState{Cursor: tokenStr, HasMore: true}, annos, nil +} + +// usageEventFromAuditEntry converts one audit-log entry into a usage event +// tying the actor to the org they acted in. Returns ok=false when the entry +// can't be attributed to a synced user resource. +func usageEventFromAuditEntry(orgName string, entry *github.AuditEntry) (*v2.Event, bool) { + actor := entry.GetActor() + actorID := entry.GetActorID() + ts := entry.GetTimestamp().Time + if actorID == 0 || ts.IsZero() { + return nil, false + } + + // actor_is_bot is real but undocumented (only in AdditionalFields); trust + // it when present, else fall back to the "[bot]" login suffix. Bots + // aren't synced as users, so their events wouldn't correlate to anything. + if isBot, ok := entry.AdditionalFields["actor_is_bot"].(bool); ok { + if isBot { + return nil, false + } + } else if strings.HasSuffix(actor, "[bot]") { + return nil, false + } + + orgID := entry.GetOrgID() + if orgID == 0 { + return nil, false + } + + id := entry.GetDocumentID() + if id == "" { + // No stable ID from GitHub - synthesize one so dedup doesn't collapse + // every entry missing _document_id into one event. + id = fmt.Sprintf("%d:%d:%d:%s", orgID, actorID, ts.UnixNano(), entry.GetAction()) + } + + return &v2.Event{ + Id: id, + OccurredAt: timestamppb.New(ts), + Event: &v2.Event_UsageEvent{ + UsageEvent: &v2.UsageEvent{ + TargetResource: &v2.Resource{ + Id: &v2.ResourceId{ + ResourceType: resourceTypeOrg.Id, + Resource: strconv.FormatInt(orgID, 10), + }, + DisplayName: orgName, + }, + ActorResource: &v2.Resource{ + Id: &v2.ResourceId{ + ResourceType: resourceTypeUser.Id, + Resource: strconv.FormatInt(actorID, 10), + }, + DisplayName: actor, + }, + }, + }, + }, true +} diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/user.go b/vendor/github.com/conductorone/baton-github/pkg/connector/user.go index d5c6568f..705507a5 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/user.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/user.go @@ -48,8 +48,14 @@ func userResource(ctx context.Context, user *github.User, userEmail string, extr userTrait := []resource.UserTraitOption{ resource.WithEmail(userEmail, true), - resource.WithUserProfile(profile), - resource.WithStatus(v2.UserTrait_Status_STATUS_ENABLED), + } + + // profile, status, and icon have moved from UserTrait to Resource-level + // attributes. NewUserTrait still defaults the (unset) trait status to + // ENABLED, so the enabled semantics are preserved on both levels. + resourceOpts := []resource.ResourceOption{ + resource.WithResourceProfile(profile), + resource.WithResourceStatus(v2.Status_RESOURCE_STATUS_ENABLED, ""), } for _, email := range extraEmails { @@ -57,7 +63,7 @@ func userResource(ctx context.Context, user *github.User, userEmail string, extr } if user.GetAvatarURL() != "" { - userTrait = append(userTrait, resource.WithUserIcon(&v2.AssetRef{ + resourceOpts = append(resourceOpts, resource.WithResourceIcon(&v2.AssetRef{ Id: user.GetAvatarURL(), })) } @@ -70,15 +76,17 @@ func userResource(ctx context.Context, user *github.User, userEmail string, extr })) } + resourceOpts = append(resourceOpts, resource.WithAnnotation( + &v2.ExternalLink{Url: user.GetHTMLURL()}, + &v2.V1Identifier{Id: strconv.FormatInt(user.GetID(), 10)}, + )) + ret, err := resource.NewUserResource( displayName, resourceTypeUser, user.GetID(), userTrait, - resource.WithAnnotation( - &v2.ExternalLink{Url: user.GetHTMLURL()}, - &v2.V1Identifier{Id: strconv.FormatInt(user.GetID(), 10)}, - ), + resourceOpts..., ) if err != nil { return nil, err diff --git a/vendor/modules.txt b/vendor/modules.txt index d2c04853..7c7fe294 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -272,7 +272,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-github v0.4.0 +# github.com/conductorone/baton-github v0.4.0 => ../baton-github ## explicit; go 1.25.2 github.com/conductorone/baton-github/pkg/config github.com/conductorone/baton-github/pkg/connector @@ -1022,3 +1022,4 @@ modernc.org/memory modernc.org/sqlite modernc.org/sqlite/lib modernc.org/sqlite/vtab +# github.com/conductorone/baton-github => ../baton-github From 52c1fbed32d34b623bb5aa80afb655b01714b9ee Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 9 Sep 2026 12:53:31 -0300 Subject: [PATCH 2/9] chore: manually update vendored code for testing --- .../baton-github/pkg/connector/connector.go | 5 ++ .../baton-github/pkg/connector/usage_app.go | 64 +++++++++++++++++++ .../pkg/connector/usage_event_feed.go | 18 +++--- 3 files changed, 78 insertions(+), 9 deletions(-) create mode 100644 vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go index 72f0bf07..b7440df0 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go @@ -149,6 +149,11 @@ func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour resourceSyncers = append(resourceSyncers, APITokenBuilder(gh.client, gh.orgCache)) } + if gh.syncLastActivity { + // usageAppBuilder only exists to support usageEventFeed, so it's gated the same way. + resourceSyncers = append(resourceSyncers, newUsageAppBuilder()) + } + if len(gh.enterprises) > 0 { resourceSyncers = append(resourceSyncers, EnterpriseRoleBuilder(gh.client, gh.appClient, gh.customClient, gh.enterprises), diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go new file mode 100644 index 00000000..143da265 --- /dev/null +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go @@ -0,0 +1,64 @@ +package connector + +import ( + "context" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/types/entitlement" + resourceSdk "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +const ( + usageAppResourceID = "github" + usageAppDisplayName = "GitHub" + usageAppAccessEntitlement = "access" +) + +// resourceTypeUsageApp is a synthetic TRAIT_APP resource that usageEventFeed's +// UsageEvents target, since GitHub's real resource types carry no entitlement +// C1's usage uplift can key off of. Only synced when sync-last-activity is on. +var resourceTypeUsageApp = &v2.ResourceType{ + Id: "usage-app", + DisplayName: "GitHub", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_APP}, + Annotations: annotations.New(&v2.SkipGrants{}), +} + +// usageAppBuilder syncs a single static App resource for usageEventFeed's +// UsageEvents to target. +type usageAppBuilder struct{} + +func newUsageAppBuilder() *usageAppBuilder { + return &usageAppBuilder{} +} + +func (b *usageAppBuilder) ResourceType(_ context.Context) *v2.ResourceType { + return resourceTypeUsageApp +} + +func (b *usageAppBuilder) List(_ context.Context, _ *v2.ResourceId, _ resourceSdk.SyncOpAttrs) ([]*v2.Resource, *resourceSdk.SyncOpResults, error) { + res, err := resourceSdk.NewAppResource(usageAppDisplayName, resourceTypeUsageApp, usageAppResourceID, nil) + if err != nil { + return nil, nil, err + } + return []*v2.Resource{res}, &resourceSdk.SyncOpResults{}, nil +} + +func (b *usageAppBuilder) Entitlements(_ context.Context, resource *v2.Resource, _ resourceSdk.SyncOpAttrs) ([]*v2.Entitlement, *resourceSdk.SyncOpResults, error) { + return []*v2.Entitlement{ + entitlement.NewAssignmentEntitlement( + resource, + usageAppAccessEntitlement, + entitlement.WithGrantableTo(resourceTypeUser), + entitlement.WithDisplayName("GitHub Access"), + entitlement.WithDescription("Has access to GitHub"), + ), + }, &resourceSdk.SyncOpResults{}, nil +} + +func (b *usageAppBuilder) Grants(_ context.Context, _ *v2.Resource, _ resourceSdk.SyncOpAttrs) ([]*v2.Grant, *resourceSdk.SyncOpResults, error) { + // Grants are intentionally not emitted: the usage uplift maps the login + // actor directly to a synced app user, not via a grant. + return nil, &resourceSdk.SyncOpResults{}, nil +} diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go index efd24be5..71a38420 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go @@ -46,8 +46,8 @@ func (f *usageEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFeedMetad } // usageEventPageToken tracks progress through one pass over every configured -// org's audit log, walked newest-first until an entry at or before Since is -// reached (already seen in a previous pass). +// org's audit log, walked newest-first until an already-seen entry (at or +// before Since) is reached. type usageEventPageToken struct { Orgs []string `json:"orgs,omitempty"` OrgIndex int `json:"org_index"` @@ -196,7 +196,7 @@ func (f *usageEventFeed) ListEvents( break } - evt, ok := usageEventFromAuditEntry(orgName, entry) + evt, ok := usageEventFromAuditEntry(entry) if !ok { continue } @@ -238,9 +238,9 @@ func (f *usageEventFeed) ListEvents( } // usageEventFromAuditEntry converts one audit-log entry into a usage event -// tying the actor to the org they acted in. Returns ok=false when the entry -// can't be attributed to a synced user resource. -func usageEventFromAuditEntry(orgName string, entry *github.AuditEntry) (*v2.Event, bool) { +// targeting the usage-app resource (see usage_app.go). Returns ok=false when +// the entry can't be attributed to a synced user. +func usageEventFromAuditEntry(entry *github.AuditEntry) (*v2.Event, bool) { actor := entry.GetActor() actorID := entry.GetActorID() ts := entry.GetTimestamp().Time @@ -278,10 +278,10 @@ func usageEventFromAuditEntry(orgName string, entry *github.AuditEntry) (*v2.Eve UsageEvent: &v2.UsageEvent{ TargetResource: &v2.Resource{ Id: &v2.ResourceId{ - ResourceType: resourceTypeOrg.Id, - Resource: strconv.FormatInt(orgID, 10), + ResourceType: resourceTypeUsageApp.Id, + Resource: usageAppResourceID, }, - DisplayName: orgName, + DisplayName: usageAppDisplayName, }, ActorResource: &v2.Resource{ Id: &v2.ResourceId{ From 239b5873e263c51e3e67f3a702a970521bd1f38d Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 15 Sep 2026 04:02:12 -0300 Subject: [PATCH 3/9] docs: clarify on the docs the behavior of the new resource --- docs/connector.mdx | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/connector.mdx b/docs/connector.mdx index e7622e08..fb78aa65 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -400,6 +400,14 @@ You can enable **Sync last activity** to have C1 show, for each member, the last - The personal access token used by the connector must have the **read:audit_log** scope. - The token must have permission to view each organization's audit log (typically an organization owner). + +**Enabling the config option alone is not enough.** After you enable **Sync last activity** (or set `--sync-last-activity` / `BATON_SYNC_LAST_ACTIVITY=true`) and save the connector, go to the connector's **Capabilities & configuration** page in C1 and confirm the **GitHub Activity** resource type is enabled for sync. + +- On a **brand-new** connector, this resource type is included by default, so no extra step is needed. +- On an **existing, already-configured** connector, C1 does not automatically add a newly-available resource type to a previously saved sync selection — it will show up in the resource list as available, but disabled, until you manually enable it. + + + **What "last activity" means:** This is not necessarily a login timestamp. GitHub's audit log only records actions members take, not every time someone signs in or browses the product. Because of this, C1 shows the most recent recorded action for a member, whatever that action was, as a best-effort signal of when they were last active. A member who only reads or browses without taking any recorded action won't show a last-activity date, even if they use GitHub regularly. To keep this fast on large organizations, raw Git operations (pushes, fetches, and clones done via `git` rather than the website or API) are not counted toward last activity — only web and API actions are. A member who exclusively interacts with GitHub through `git` and never uses the website or API (for example, a purely command-line workflow with no PR reviews, comments, or settings changes) won't show a last-activity date, even though they're actively using GitHub. From 91480d02b8a1f88cca93a623692770ad1fdc7cc1 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 15 Sep 2026 04:10:31 -0300 Subject: [PATCH 4/9] feat: update baton-github and the sdk versions --- go.mod | 14 +- go.sum | 22 +- .../cockroachdb/swiss/runtime_go1.20.go | 6 +- .../baton-github/pkg/config/config.go | 5 +- .../baton-github/pkg/connector/connector.go | 7 +- .../baton-github/pkg/connector/invitation.go | 14 +- .../baton-github/pkg/connector/usage_app.go | 4 +- .../pkg/connector/usage_event_feed.go | 3 - .../baton-sdk/pb/c1/config/v1/config.pb.go | 12 +- .../pb/c1/config/v1/config_protoopaque.pb.go | 6 +- .../baton-sdk/pb/c1/connector/v2/action.pb.go | 151 +++-- .../pb/c1/connector/v2/action.pb.validate.go | 102 ++++ .../c1/connector/v2/action_protoopaque.pb.go | 172 ++++-- .../pb/c1/connectorapi/baton/v1/baton.pb.go | 86 +-- .../baton/v1/baton.pb.validate.go | 34 ++ .../baton/v1/baton_protoopaque.pb.go | 97 ++-- .../baton-sdk/pkg/actions/actions.go | 526 ++++++++++++++---- .../baton-sdk/pkg/connectorbuilder/actions.go | 61 +- .../baton-sdk/pkg/connectorrunner/runner.go | 35 +- .../pkg/dotc1z/engine/pebble/bulk_import.go | 400 +++++++++---- .../dotc1z/engine/pebble/deferred_index.go | 21 +- .../pkg/dotc1z/engine/pebble/digest.go | 10 +- .../pkg/dotc1z/engine/pebble/engine.go | 50 ++ .../pkg/dotc1z/engine/pebble/grant_digest.go | 130 ++++- .../engine/pebble/grant_digest_build.go | 57 +- .../engine/pebble/grant_digest_repair.go | 10 + .../pkg/dotc1z/engine/pebble/grants.go | 13 +- .../engine/pebble/id_index_migration.go | 49 +- .../dotc1z/engine/pebble/index_migrations.go | 9 + .../engine/pebble/internal/rawdb/keyspace.go | 43 +- .../engine/pebble/internal/rawdb/rawdb.go | 6 +- .../baton-sdk/pkg/dotc1z/source_cache.go | 5 + .../baton-sdk/pkg/dotc1z/to_pebble.go | 71 ++- .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../baton-sdk/pkg/sourcecache/sourcecache.go | 6 + .../conductorone/baton-sdk/pkg/sync/config.go | 72 +++ .../conductorone/baton-sdk/pkg/sync/hooks.go | 30 + .../baton-sdk/pkg/sync/ingest_filter.go | 6 +- .../baton-sdk/pkg/sync/ingest_invariants.go | 22 +- .../baton-sdk/pkg/sync/parallel_syncer.go | 22 +- .../baton-sdk/pkg/sync/queue_audit.go | 2 +- .../conductorone/baton-sdk/pkg/sync/state.go | 81 +-- .../baton-sdk/pkg/sync/store_caps.go | 142 +++++ .../conductorone/baton-sdk/pkg/sync/syncer.go | 526 +++++++++--------- .../baton-sdk/pkg/synccompactor/compactor.go | 4 +- .../pkg/synccompactor/compactor_pebble.go | 62 ++- .../baton-sdk/pkg/tasks/c1api/actions.go | 7 +- .../pkg/tasks/local/action_invoker.go | 47 +- .../baton-sdk/pkg/uhttp/errors.go | 89 +++ .../baton-sdk/pkg/uhttp/pagination.go | 47 ++ .../baton-sdk/pkg/uhttp/wrapper.go | 15 +- .../dpop_grpc/client_credential.go | 21 +- .../dpop/integrations/dpop_oauth2/retry.go | 115 ++++ .../dpop_oauth2/token_client_assertion.go | 169 ++++-- vendor/modules.txt | 13 +- 55 files changed, 2720 insertions(+), 1011 deletions(-) create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/config.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/hooks.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/store_caps.go create mode 100644 vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/retry.go diff --git a/go.mod b/go.mod index 912c5c90..21e3c56c 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,9 @@ module github.com/conductorone/baton-github-enterprise go 1.25.2 -replace github.com/conductorone/baton-github => ../baton-github - require ( - github.com/conductorone/baton-github v0.4.0 - github.com/conductorone/baton-sdk v0.26.0 + github.com/conductorone/baton-github v0.4.5 + github.com/conductorone/baton-sdk v0.30.0 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/quasilyte/go-ruleguard/dsl v0.3.23 ) @@ -45,11 +43,11 @@ require ( github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect github.com/cockroachdb/pebble/v2 v2.1.5 // indirect github.com/cockroachdb/redact v1.1.5 // indirect - github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b // indirect + github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/conductorone/dpop v0.2.6 // indirect - github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 // indirect - github.com/conductorone/dpop/integrations/dpop_oauth2 v0.2.5 // indirect + github.com/conductorone/dpop v0.3.0 // indirect + github.com/conductorone/dpop/integrations/dpop_grpc v0.3.0 // indirect + github.com/conductorone/dpop/integrations/dpop_oauth2 v0.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.9.0 // indirect github.com/doug-martin/goqu/v9 v9.19.0 // indirect diff --git a/go.sum b/go.sum index dd8b6e78..1c5ca11c 100644 --- a/go.sum +++ b/go.sum @@ -80,18 +80,20 @@ github.com/cockroachdb/pebble/v2 v2.1.5 h1:1ziHpaSau6qCXnFpQX3EBOH14yPHA8W66vKxs github.com/cockroachdb/pebble/v2 v2.1.5/go.mod h1:Reo1RTniv1UjVTAu/Fv74y5i3kJ5gmVrPhO9UtFiKn8= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8XTuY0PT9Ane9qZGul/p67vGYwl9BFI= -github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= +github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 h1:IJ+uNItEm0qx9FE2AgIc1PMsCUtk8nbSIzhQE1t5GWw= +github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-sdk v0.26.0 h1:aNKg81BhPAVGyYe+W4czZJnL9hzJEUkDinbt0klOeo4= -github.com/conductorone/baton-sdk v0.26.0/go.mod h1:SKm95z4KkQ23Tufo2ys88lVzbwKb0AQEbKee5GE0Lig= -github.com/conductorone/dpop v0.2.6 h1:fakwai/Xm2b/fcDUwJN41WtcSI/2UhQOyRIVvnnrrNA= -github.com/conductorone/dpop v0.2.6/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= -github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 h1:lYxYi9/WTSL9sE96CO0QF2BY3kehs8dTTApI134TGCA= -github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4/go.mod h1:LYNoUc1lkvozk9HBio+xI2w8YyfYy0v2cAJtIgrkj8o= -github.com/conductorone/dpop/integrations/dpop_oauth2 v0.2.5 h1:x/ZtD0YLNwlmoSv9SE4OBPJB9Hj2cpwyE5BAfia7aY8= -github.com/conductorone/dpop/integrations/dpop_oauth2 v0.2.5/go.mod h1:2eI0qv+XaEhoCw0GKFF1yH4X8Mp4KLVEVnQKRFEy4zs= +github.com/conductorone/baton-github v0.4.5 h1:+jvAgGMrWEhHJ9bdYoLfChrMK22pLoWvSzbsE6AncIU= +github.com/conductorone/baton-github v0.4.5/go.mod h1:VeeTzBFVAa9SIFjXPs1ETQ2y2jo4dEP0iXoTftRN0c0= +github.com/conductorone/baton-sdk v0.30.0 h1:KIzKnULYjl3/B36/HaW9G+F1EX0Hol+xFZnH00kVLEA= +github.com/conductorone/baton-sdk v0.30.0/go.mod h1:9iFpBag59RlwGkb4Tz6omYewqnj8obfbZzrtSaTLa5Q= +github.com/conductorone/dpop v0.3.0 h1:j5fZk0VqepGKYo+/NDikCOMsZcgs4HO4i0k56wRel5g= +github.com/conductorone/dpop v0.3.0/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= +github.com/conductorone/dpop/integrations/dpop_grpc v0.3.0 h1:R2uxHBtStgUn7cxbAnT3mAj6/e1akfP2Pj/hEPFB0D8= +github.com/conductorone/dpop/integrations/dpop_grpc v0.3.0/go.mod h1:f30gFNZHGkbPlufIDqzg5cr7llQQS0r5VQ+bJCnrDik= +github.com/conductorone/dpop/integrations/dpop_oauth2 v0.3.0 h1:g9OX0PW9DQyrGy/Tp2lesky/V1VH55cTY8z0Iq0ksOo= +github.com/conductorone/dpop/integrations/dpop_oauth2 v0.3.0/go.mod h1:RZqiSQdi4NXnoZtB3qG3XYN/6L4dHZy6jctEr5K5oPk= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/vendor/github.com/cockroachdb/swiss/runtime_go1.20.go b/vendor/github.com/cockroachdb/swiss/runtime_go1.20.go index f3149b83..79d530d1 100644 --- a/vendor/github.com/cockroachdb/swiss/runtime_go1.20.go +++ b/vendor/github.com/cockroachdb/swiss/runtime_go1.20.go @@ -17,12 +17,12 @@ // bumping of the go versions supported by adjusting the build tags below. The // way go version tags work the tag for goX.Y will be declared for every // subsequent release. So go1.20 will be defined for go1.21, go1.22, etc. The -// build tag "go1.20 && !go1.27" defines the range [go1.20, go1.27) (inclusive -// on go1.20, exclusive on go1.27). +// build tag "go1.20 && !go1.28" defines the range [go1.20, go1.28) (inclusive +// on go1.20, exclusive on go1.28). // The untested_go_version flag enables building on any go version, intended // to ease testing against Go at tip. -//go:build (go1.20 && !go1.27) || untested_go_version +//go:build (go1.20 && !go1.28) || untested_go_version package swiss diff --git a/vendor/github.com/conductorone/baton-github/pkg/config/config.go b/vendor/github.com/conductorone/baton-github/pkg/config/config.go index 5a142d66..60856c56 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-github/pkg/config/config.go @@ -57,7 +57,10 @@ var ( appPrivateKey = field.StringField( "app-privatekey", field.WithDisplayName("GitHub App private key (PEM)"), - field.WithDescription("Raw PEM contents of the private key used to connect to the GitHub App. Takes precedence over app-privatekey-path when both are set."), + field.WithDescription( + "Raw PEM contents of the private key used to connect to the GitHub App. Takes precedence over app-privatekey-path when both are set. "+ + `Literal \n escape sequences are also accepted and unescaped before use, since this field can't hold newlines when entered through a form.`, + ), field.WithIsSecret(true), ) diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go index b7440df0..b1bd1017 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go @@ -68,7 +68,7 @@ var ( resourceTypeInvitation = &v2.ResourceType{ Id: "invitation", DisplayName: "Invitation", - // Invitations emit TRAIT_USER with UserTrait_Status_STATUS_UNSPECIFIED. + // Invitations emit TRAIT_USER with STATUS_PENDING. // Accepted members from user.go emit STATUS_ENABLED. Traits: []v2.ResourceType_Trait{ v2.ResourceType_TRAIT_USER, @@ -502,7 +502,12 @@ func newGitHubGraphqlClient(ctx context.Context, instanceURL string, ts oauth2.T return githubv4.NewClient(tc), nil } +// escapedLineBreaks unescapes LF-, CRLF-, and CR-escaped line breaks (`\r\n`, +// `\n`, `\r`) to a real newline. +var escapedLineBreaks = strings.NewReplacer(`\r\n`, "\n", `\n`, "\n", `\r`, "\n") + func loadPrivateKeyFromString(p string) (*rsa.PrivateKey, error) { + p = escapedLineBreaks.Replace(p) block, _ := pem.Decode([]byte(p)) if block == nil || (block.Type != "PRIVATE KEY" && block.Type != "RSA PRIVATE KEY") { return nil, errors.New("invalid private key PEM format") diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go b/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go index fe49b5c6..38ef9575 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go @@ -64,19 +64,13 @@ func invitationToUserResource(invitation *github.Invitation, status string) (*v2 invitation.GetID(), []resourceSdk.UserTraitOption{ resourceSdk.WithEmail(invitation.GetEmail(), true), - // An invitation is a pending/expired user that must not be - // reported as enabled. WithResourceStatus cannot express this: - // NewUserTrait force-defaults an unset trait status to ENABLED, so - // migrating this line would flip the emitted status from - // UNSPECIFIED to ENABLED. Keep the deprecated trait option (which - // also mirrors UNSPECIFIED to the resource level) to preserve the - // exact status semantics. - //nolint:staticcheck // deliberate: WithResourceStatus would force the trait status to ENABLED; UNSPECIFIED must be preserved for invitations. - resourceSdk.WithStatus(v2.UserTrait_Status_STATUS_UNSPECIFIED), resourceSdk.WithUserLogin(login), }, - // profile has moved from UserTrait to a Resource-level attribute. + // profile and status have moved from UserTrait to Resource-level + // attributes. Expired invitations stay PENDING - they are still not a + // usable account - and carry the distinction in the status details. resourceSdk.WithResourceProfile(profile), + resourceSdk.WithResourceStatus(v2.Status_RESOURCE_STATUS_PENDING, status), ) if err != nil { return nil, err diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go index 143da265..e6f26888 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go @@ -11,7 +11,7 @@ import ( const ( usageAppResourceID = "github" - usageAppDisplayName = "GitHub" + usageAppDisplayName = "GitHub Activity" usageAppAccessEntitlement = "access" ) @@ -20,7 +20,7 @@ const ( // C1's usage uplift can key off of. Only synced when sync-last-activity is on. var resourceTypeUsageApp = &v2.ResourceType{ Id: "usage-app", - DisplayName: "GitHub", + DisplayName: "GitHub Activity", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_APP}, Annotations: annotations.New(&v2.SkipGrants{}), } diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go index 71a38420..d86dc02f 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go @@ -260,9 +260,6 @@ func usageEventFromAuditEntry(entry *github.AuditEntry) (*v2.Event, bool) { } orgID := entry.GetOrgID() - if orgID == 0 { - return nil, false - } id := entry.GetDocumentID() if id == "" { diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config.pb.go index 3280317f..7962c159 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config.pb.go @@ -554,7 +554,11 @@ type Field struct { Placeholder string `protobuf:"bytes,4,opt,name=placeholder,proto3" json:"placeholder,omitempty"` IsRequired bool `protobuf:"varint,5,opt,name=is_required,json=isRequired,proto3" json:"is_required,omitempty"` IsOps bool `protobuf:"varint,6,opt,name=is_ops,json=isOps,proto3" json:"is_ops,omitempty"` - IsSecret bool `protobuf:"varint,7,opt,name=is_secret,json=isSecret,proto3" json:"is_secret,omitempty"` + // For configuration and action arguments, the UI obscures this field. For + // action return types, the connector must return the value as PlaintextData + // through ActionHandlerWithSecrets; it is omitted from the public response + // and returned as EncryptedData. + IsSecret bool `protobuf:"varint,7,opt,name=is_secret,json=isSecret,proto3" json:"is_secret,omitempty"` // Types that are valid to be assigned to Field: // // *Field_StringField @@ -1088,7 +1092,11 @@ type Field_builder struct { Placeholder string IsRequired bool IsOps bool - IsSecret bool + // For configuration and action arguments, the UI obscures this field. For + // action return types, the connector must return the value as PlaintextData + // through ActionHandlerWithSecrets; it is omitted from the public response + // and returned as EncryptedData. + IsSecret bool // Fields of oneof Field: StringField *StringField IntField *IntField diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config_protoopaque.pb.go index e23183fe..0f4cde44 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/config/v1/config_protoopaque.pb.go @@ -1074,7 +1074,11 @@ type Field_builder struct { Placeholder string IsRequired bool IsOps bool - IsSecret bool + // For configuration and action arguments, the UI obscures this field. For + // action return types, the connector must return the value as PlaintextData + // through ActionHandlerWithSecrets; it is omitted from the public response + // and returned as EncryptedData. + IsSecret bool // Fields of oneof xxx_hidden_Field: StringField *StringField IntField *IntField diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go index 24b8ac51..556b6c79 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.go @@ -323,9 +323,11 @@ type InvokeActionRequest struct { // waits and waits beyond the validated ceiling are rejected as caller // bugs where the transport enforces validation rules; the server-side // cap is the backstop everywhere. - InlineWait *durationpb.Duration `protobuf:"bytes,5,opt,name=inline_wait,json=inlineWait,proto3" json:"inline_wait,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + InlineWait *durationpb.Duration `protobuf:"bytes,5,opt,name=inline_wait,json=inlineWait,proto3" json:"inline_wait,omitempty"` + // Public keys the connector uses to encrypt secret return values. + EncryptionConfigs []*EncryptionConfig `protobuf:"bytes,6,rep,name=encryption_configs,json=encryptionConfigs,proto3" json:"encryption_configs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InvokeActionRequest) Reset() { @@ -388,6 +390,13 @@ func (x *InvokeActionRequest) GetInlineWait() *durationpb.Duration { return nil } +func (x *InvokeActionRequest) GetEncryptionConfigs() []*EncryptionConfig { + if x != nil { + return x.EncryptionConfigs + } + return nil +} + func (x *InvokeActionRequest) SetName(v string) { x.Name = v } @@ -408,6 +417,10 @@ func (x *InvokeActionRequest) SetInlineWait(v *durationpb.Duration) { x.InlineWait = v } +func (x *InvokeActionRequest) SetEncryptionConfigs(v []*EncryptionConfig) { + x.EncryptionConfigs = v +} + func (x *InvokeActionRequest) HasArgs() bool { if x == nil { return false @@ -451,6 +464,8 @@ type InvokeActionRequest_builder struct { // bugs where the transport enforces validation rules; the server-side // cap is the backstop everywhere. InlineWait *durationpb.Duration + // Public keys the connector uses to encrypt secret return values. + EncryptionConfigs []*EncryptionConfig } func (b0 InvokeActionRequest_builder) Build() *InvokeActionRequest { @@ -462,16 +477,19 @@ func (b0 InvokeActionRequest_builder) Build() *InvokeActionRequest { x.Annotations = b.Annotations x.ResourceTypeId = b.ResourceTypeId x.InlineWait = b.InlineWait + x.EncryptionConfigs = b.EncryptionConfigs return m0 } type InvokeActionResponse struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` - Status BatonActionStatus `protobuf:"varint,2,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus" json:"status,omitempty"` - Annotations []*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3" json:"annotations,omitempty"` - Response *structpb.Struct `protobuf:"bytes,4,opt,name=response,proto3" json:"response,omitempty"` - Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Status BatonActionStatus `protobuf:"varint,2,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus" json:"status,omitempty"` + Annotations []*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3" json:"annotations,omitempty"` + Response *structpb.Struct `protobuf:"bytes,4,opt,name=response,proto3" json:"response,omitempty"` + Name string `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` + // Secret return values encrypted for every requested recipient. + EncryptedData []*EncryptedData `protobuf:"bytes,6,rep,name=encrypted_data,json=encryptedData,proto3" json:"encrypted_data,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -536,6 +554,13 @@ func (x *InvokeActionResponse) GetName() string { return "" } +func (x *InvokeActionResponse) GetEncryptedData() []*EncryptedData { + if x != nil { + return x.EncryptedData + } + return nil +} + func (x *InvokeActionResponse) SetId(v string) { x.Id = v } @@ -556,6 +581,10 @@ func (x *InvokeActionResponse) SetName(v string) { x.Name = v } +func (x *InvokeActionResponse) SetEncryptedData(v []*EncryptedData) { + x.EncryptedData = v +} + func (x *InvokeActionResponse) HasResponse() bool { if x == nil { return false @@ -575,6 +604,8 @@ type InvokeActionResponse_builder struct { Annotations []*anypb.Any Response *structpb.Struct Name string + // Secret return values encrypted for every requested recipient. + EncryptedData []*EncryptedData } func (b0 InvokeActionResponse_builder) Build() *InvokeActionResponse { @@ -586,6 +617,7 @@ func (b0 InvokeActionResponse_builder) Build() *InvokeActionResponse { x.Annotations = b.Annotations x.Response = b.Response x.Name = b.Name + x.EncryptedData = b.EncryptedData return m0 } @@ -679,12 +711,14 @@ func (b0 GetActionStatusRequest_builder) Build() *GetActionStatusRequest { } type GetActionStatusResponse struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` - Status BatonActionStatus `protobuf:"varint,3,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus" json:"status,omitempty"` - Annotations []*anypb.Any `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty"` - Response *structpb.Struct `protobuf:"bytes,5,opt,name=response,proto3" json:"response,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Id string `protobuf:"bytes,2,opt,name=id,proto3" json:"id,omitempty"` + Status BatonActionStatus `protobuf:"varint,3,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus" json:"status,omitempty"` + Annotations []*anypb.Any `protobuf:"bytes,4,rep,name=annotations,proto3" json:"annotations,omitempty"` + Response *structpb.Struct `protobuf:"bytes,5,opt,name=response,proto3" json:"response,omitempty"` + // Secret return values encrypted when the action settled. + EncryptedData []*EncryptedData `protobuf:"bytes,6,rep,name=encrypted_data,json=encryptedData,proto3" json:"encrypted_data,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -749,6 +783,13 @@ func (x *GetActionStatusResponse) GetResponse() *structpb.Struct { return nil } +func (x *GetActionStatusResponse) GetEncryptedData() []*EncryptedData { + if x != nil { + return x.EncryptedData + } + return nil +} + func (x *GetActionStatusResponse) SetName(v string) { x.Name = v } @@ -769,6 +810,10 @@ func (x *GetActionStatusResponse) SetResponse(v *structpb.Struct) { x.Response = v } +func (x *GetActionStatusResponse) SetEncryptedData(v []*EncryptedData) { + x.EncryptedData = v +} + func (x *GetActionStatusResponse) HasResponse() bool { if x == nil { return false @@ -788,6 +833,8 @@ type GetActionStatusResponse_builder struct { Status BatonActionStatus Annotations []*anypb.Any Response *structpb.Struct + // Secret return values encrypted when the action settled. + EncryptedData []*EncryptedData } func (b0 GetActionStatusResponse_builder) Build() *GetActionStatusResponse { @@ -799,6 +846,7 @@ func (b0 GetActionStatusResponse_builder) Build() *GetActionStatusResponse { x.Status = b.Status x.Annotations = b.Annotations x.Response = b.Response + x.EncryptedData = b.EncryptedData return m0 } @@ -1103,7 +1151,7 @@ var File_c1_connector_v2_action_proto protoreflect.FileDescriptor const file_c1_connector_v2_action_proto_rawDesc = "" + "\n" + - "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x17validate/validate.proto\"\xfb\x02\n" + + "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x17validate/validate.proto\"\xfb\x02\n" + "\x11BatonActionSchema\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x121\n" + "\targuments\x18\x02 \x03(\v2\x13.c1.config.v1.FieldR\targuments\x12:\n" + @@ -1113,30 +1161,33 @@ const file_c1_connector_v2_action_proto_rawDesc = "" + "\vdescription\x18\x06 \x01(\tR\vdescription\x12<\n" + "\vaction_type\x18\a \x03(\x0e2\x1b.c1.connector.v2.ActionTypeR\n" + "actionType\x12(\n" + - "\x10resource_type_id\x18\b \x01(\tR\x0eresourceTypeId\"\x84\x02\n" + + "\x10resource_type_id\x18\b \x01(\tR\x0eresourceTypeId\"\xd6\x02\n" + "\x13InvokeActionRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12+\n" + "\x04args\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04args\x126\n" + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12(\n" + "\x10resource_type_id\x18\x04 \x01(\tR\x0eresourceTypeId\x12J\n" + "\vinline_wait\x18\x05 \x01(\v2\x19.google.protobuf.DurationB\x0e\xfaB\v\xaa\x01\b\"\x04\b\x80\xa3\x052\x00R\n" + - "inlineWait\"\xe3\x01\n" + + "inlineWait\x12P\n" + + "\x12encryption_configs\x18\x06 \x03(\v2!.c1.connector.v2.EncryptionConfigR\x11encryptionConfigs\"\xaa\x02\n" + "\x14InvokeActionResponse\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12:\n" + "\x06status\x18\x02 \x01(\x0e2\".c1.connector.v2.BatonActionStatusR\x06status\x126\n" + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x123\n" + "\bresponse\x18\x04 \x01(\v2\x17.google.protobuf.StructR\bresponse\x12\x12\n" + - "\x04name\x18\x05 \x01(\tR\x04name\"x\n" + + "\x04name\x18\x05 \x01(\tR\x04name\x12E\n" + + "\x0eencrypted_data\x18\x06 \x03(\v2\x1e.c1.connector.v2.EncryptedDataR\rencryptedData\"x\n" + "\x16GetActionStatusRequest\x12\x16\n" + "\x04name\x18\x01 \x01(\tB\x02\x18\x01R\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x126\n" + - "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\"\xe6\x01\n" + + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\"\xad\x02\n" + "\x17GetActionStatusResponse\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x12:\n" + "\x06status\x18\x03 \x01(\x0e2\".c1.connector.v2.BatonActionStatusR\x06status\x126\n" + "\vannotations\x18\x04 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x123\n" + - "\bresponse\x18\x05 \x01(\v2\x17.google.protobuf.StructR\bresponse\"d\n" + + "\bresponse\x18\x05 \x01(\v2\x17.google.protobuf.StructR\bresponse\x12E\n" + + "\x0eencrypted_data\x18\x06 \x03(\v2\x1e.c1.connector.v2.EncryptedDataR\rencryptedData\"d\n" + "\x16GetActionSchemaRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\"\x8d\x01\n" + @@ -1193,6 +1244,8 @@ var file_c1_connector_v2_action_proto_goTypes = []any{ (*structpb.Struct)(nil), // 13: google.protobuf.Struct (*anypb.Any)(nil), // 14: google.protobuf.Any (*durationpb.Duration)(nil), // 15: google.protobuf.Duration + (*EncryptionConfig)(nil), // 16: c1.connector.v2.EncryptionConfig + (*EncryptedData)(nil), // 17: c1.connector.v2.EncryptedData } var file_c1_connector_v2_action_proto_depIdxs = []int32{ 11, // 0: c1.connector.v2.BatonActionSchema.arguments:type_name -> c1.config.v1.Field @@ -1202,32 +1255,35 @@ var file_c1_connector_v2_action_proto_depIdxs = []int32{ 13, // 4: c1.connector.v2.InvokeActionRequest.args:type_name -> google.protobuf.Struct 14, // 5: c1.connector.v2.InvokeActionRequest.annotations:type_name -> google.protobuf.Any 15, // 6: c1.connector.v2.InvokeActionRequest.inline_wait:type_name -> google.protobuf.Duration - 0, // 7: c1.connector.v2.InvokeActionResponse.status:type_name -> c1.connector.v2.BatonActionStatus - 14, // 8: c1.connector.v2.InvokeActionResponse.annotations:type_name -> google.protobuf.Any - 13, // 9: c1.connector.v2.InvokeActionResponse.response:type_name -> google.protobuf.Struct - 14, // 10: c1.connector.v2.GetActionStatusRequest.annotations:type_name -> google.protobuf.Any - 0, // 11: c1.connector.v2.GetActionStatusResponse.status:type_name -> c1.connector.v2.BatonActionStatus - 14, // 12: c1.connector.v2.GetActionStatusResponse.annotations:type_name -> google.protobuf.Any - 13, // 13: c1.connector.v2.GetActionStatusResponse.response:type_name -> google.protobuf.Struct - 14, // 14: c1.connector.v2.GetActionSchemaRequest.annotations:type_name -> google.protobuf.Any - 2, // 15: c1.connector.v2.GetActionSchemaResponse.schema:type_name -> c1.connector.v2.BatonActionSchema - 14, // 16: c1.connector.v2.GetActionSchemaResponse.annotations:type_name -> google.protobuf.Any - 14, // 17: c1.connector.v2.ListActionSchemasRequest.annotations:type_name -> google.protobuf.Any - 2, // 18: c1.connector.v2.ListActionSchemasResponse.schemas:type_name -> c1.connector.v2.BatonActionSchema - 14, // 19: c1.connector.v2.ListActionSchemasResponse.annotations:type_name -> google.protobuf.Any - 3, // 20: c1.connector.v2.ActionService.InvokeAction:input_type -> c1.connector.v2.InvokeActionRequest - 5, // 21: c1.connector.v2.ActionService.GetActionStatus:input_type -> c1.connector.v2.GetActionStatusRequest - 7, // 22: c1.connector.v2.ActionService.GetActionSchema:input_type -> c1.connector.v2.GetActionSchemaRequest - 9, // 23: c1.connector.v2.ActionService.ListActionSchemas:input_type -> c1.connector.v2.ListActionSchemasRequest - 4, // 24: c1.connector.v2.ActionService.InvokeAction:output_type -> c1.connector.v2.InvokeActionResponse - 6, // 25: c1.connector.v2.ActionService.GetActionStatus:output_type -> c1.connector.v2.GetActionStatusResponse - 8, // 26: c1.connector.v2.ActionService.GetActionSchema:output_type -> c1.connector.v2.GetActionSchemaResponse - 10, // 27: c1.connector.v2.ActionService.ListActionSchemas:output_type -> c1.connector.v2.ListActionSchemasResponse - 24, // [24:28] is the sub-list for method output_type - 20, // [20:24] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 16, // 7: c1.connector.v2.InvokeActionRequest.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 0, // 8: c1.connector.v2.InvokeActionResponse.status:type_name -> c1.connector.v2.BatonActionStatus + 14, // 9: c1.connector.v2.InvokeActionResponse.annotations:type_name -> google.protobuf.Any + 13, // 10: c1.connector.v2.InvokeActionResponse.response:type_name -> google.protobuf.Struct + 17, // 11: c1.connector.v2.InvokeActionResponse.encrypted_data:type_name -> c1.connector.v2.EncryptedData + 14, // 12: c1.connector.v2.GetActionStatusRequest.annotations:type_name -> google.protobuf.Any + 0, // 13: c1.connector.v2.GetActionStatusResponse.status:type_name -> c1.connector.v2.BatonActionStatus + 14, // 14: c1.connector.v2.GetActionStatusResponse.annotations:type_name -> google.protobuf.Any + 13, // 15: c1.connector.v2.GetActionStatusResponse.response:type_name -> google.protobuf.Struct + 17, // 16: c1.connector.v2.GetActionStatusResponse.encrypted_data:type_name -> c1.connector.v2.EncryptedData + 14, // 17: c1.connector.v2.GetActionSchemaRequest.annotations:type_name -> google.protobuf.Any + 2, // 18: c1.connector.v2.GetActionSchemaResponse.schema:type_name -> c1.connector.v2.BatonActionSchema + 14, // 19: c1.connector.v2.GetActionSchemaResponse.annotations:type_name -> google.protobuf.Any + 14, // 20: c1.connector.v2.ListActionSchemasRequest.annotations:type_name -> google.protobuf.Any + 2, // 21: c1.connector.v2.ListActionSchemasResponse.schemas:type_name -> c1.connector.v2.BatonActionSchema + 14, // 22: c1.connector.v2.ListActionSchemasResponse.annotations:type_name -> google.protobuf.Any + 3, // 23: c1.connector.v2.ActionService.InvokeAction:input_type -> c1.connector.v2.InvokeActionRequest + 5, // 24: c1.connector.v2.ActionService.GetActionStatus:input_type -> c1.connector.v2.GetActionStatusRequest + 7, // 25: c1.connector.v2.ActionService.GetActionSchema:input_type -> c1.connector.v2.GetActionSchemaRequest + 9, // 26: c1.connector.v2.ActionService.ListActionSchemas:input_type -> c1.connector.v2.ListActionSchemasRequest + 4, // 27: c1.connector.v2.ActionService.InvokeAction:output_type -> c1.connector.v2.InvokeActionResponse + 6, // 28: c1.connector.v2.ActionService.GetActionStatus:output_type -> c1.connector.v2.GetActionStatusResponse + 8, // 29: c1.connector.v2.ActionService.GetActionSchema:output_type -> c1.connector.v2.GetActionSchemaResponse + 10, // 30: c1.connector.v2.ActionService.ListActionSchemas:output_type -> c1.connector.v2.ListActionSchemasResponse + 27, // [27:31] is the sub-list for method output_type + 23, // [23:27] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { file_c1_connector_v2_action_proto_init() } @@ -1235,6 +1291,7 @@ func file_c1_connector_v2_action_proto_init() { if File_c1_connector_v2_action_proto != nil { return } + file_c1_connector_v2_resource_proto_init() type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go index 1dbd9fcb..d9407b57 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action.pb.validate.go @@ -367,6 +367,40 @@ func (m *InvokeActionRequest) validate(all bool) error { } } + for idx, item := range m.GetEncryptionConfigs() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, InvokeActionRequestValidationError{ + field: fmt.Sprintf("EncryptionConfigs[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, InvokeActionRequestValidationError{ + field: fmt.Sprintf("EncryptionConfigs[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return InvokeActionRequestValidationError{ + field: fmt.Sprintf("EncryptionConfigs[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + if len(errors) > 0 { return InvokeActionRequestMultiError(errors) } @@ -538,6 +572,40 @@ func (m *InvokeActionResponse) validate(all bool) error { // no validation rules for Name + for idx, item := range m.GetEncryptedData() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, InvokeActionResponseValidationError{ + field: fmt.Sprintf("EncryptedData[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, InvokeActionResponseValidationError{ + field: fmt.Sprintf("EncryptedData[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return InvokeActionResponseValidationError{ + field: fmt.Sprintf("EncryptedData[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + if len(errors) > 0 { return InvokeActionResponseMultiError(errors) } @@ -849,6 +917,40 @@ func (m *GetActionStatusResponse) validate(all bool) error { } } + for idx, item := range m.GetEncryptedData() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, GetActionStatusResponseValidationError{ + field: fmt.Sprintf("EncryptedData[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, GetActionStatusResponseValidationError{ + field: fmt.Sprintf("EncryptedData[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return GetActionStatusResponseValidationError{ + field: fmt.Sprintf("EncryptedData[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + if len(errors) > 0 { return GetActionStatusResponseMultiError(errors) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go index 9904d8af..ab8d9749 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/action_protoopaque.pb.go @@ -310,14 +310,15 @@ func (b0 BatonActionSchema_builder) Build() *BatonActionSchema { } type InvokeActionRequest struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` - xxx_hidden_Args *structpb.Struct `protobuf:"bytes,2,opt,name=args,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,4,opt,name=resource_type_id,json=resourceTypeId,proto3"` - xxx_hidden_InlineWait *durationpb.Duration `protobuf:"bytes,5,opt,name=inline_wait,json=inlineWait,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` + xxx_hidden_Args *structpb.Struct `protobuf:"bytes,2,opt,name=args,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,4,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_InlineWait *durationpb.Duration `protobuf:"bytes,5,opt,name=inline_wait,json=inlineWait,proto3"` + xxx_hidden_EncryptionConfigs *[]*EncryptionConfig `protobuf:"bytes,6,rep,name=encryption_configs,json=encryptionConfigs,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InvokeActionRequest) Reset() { @@ -382,6 +383,15 @@ func (x *InvokeActionRequest) GetInlineWait() *durationpb.Duration { return nil } +func (x *InvokeActionRequest) GetEncryptionConfigs() []*EncryptionConfig { + if x != nil { + if x.xxx_hidden_EncryptionConfigs != nil { + return *x.xxx_hidden_EncryptionConfigs + } + } + return nil +} + func (x *InvokeActionRequest) SetName(v string) { x.xxx_hidden_Name = v } @@ -402,6 +412,10 @@ func (x *InvokeActionRequest) SetInlineWait(v *durationpb.Duration) { x.xxx_hidden_InlineWait = v } +func (x *InvokeActionRequest) SetEncryptionConfigs(v []*EncryptionConfig) { + x.xxx_hidden_EncryptionConfigs = &v +} + func (x *InvokeActionRequest) HasArgs() bool { if x == nil { return false @@ -445,6 +459,8 @@ type InvokeActionRequest_builder struct { // bugs where the transport enforces validation rules; the server-side // cap is the backstop everywhere. InlineWait *durationpb.Duration + // Public keys the connector uses to encrypt secret return values. + EncryptionConfigs []*EncryptionConfig } func (b0 InvokeActionRequest_builder) Build() *InvokeActionRequest { @@ -456,18 +472,20 @@ func (b0 InvokeActionRequest_builder) Build() *InvokeActionRequest { x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_ResourceTypeId = b.ResourceTypeId x.xxx_hidden_InlineWait = b.InlineWait + x.xxx_hidden_EncryptionConfigs = &b.EncryptionConfigs return m0 } type InvokeActionResponse struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Id string `protobuf:"bytes,1,opt,name=id,proto3"` - xxx_hidden_Status BatonActionStatus `protobuf:"varint,2,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3"` - xxx_hidden_Response *structpb.Struct `protobuf:"bytes,4,opt,name=response,proto3"` - xxx_hidden_Name string `protobuf:"bytes,5,opt,name=name,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Id string `protobuf:"bytes,1,opt,name=id,proto3"` + xxx_hidden_Status BatonActionStatus `protobuf:"varint,2,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3"` + xxx_hidden_Response *structpb.Struct `protobuf:"bytes,4,opt,name=response,proto3"` + xxx_hidden_Name string `protobuf:"bytes,5,opt,name=name,proto3"` + xxx_hidden_EncryptedData *[]*EncryptedData `protobuf:"bytes,6,rep,name=encrypted_data,json=encryptedData,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *InvokeActionResponse) Reset() { @@ -532,6 +550,15 @@ func (x *InvokeActionResponse) GetName() string { return "" } +func (x *InvokeActionResponse) GetEncryptedData() []*EncryptedData { + if x != nil { + if x.xxx_hidden_EncryptedData != nil { + return *x.xxx_hidden_EncryptedData + } + } + return nil +} + func (x *InvokeActionResponse) SetId(v string) { x.xxx_hidden_Id = v } @@ -552,6 +579,10 @@ func (x *InvokeActionResponse) SetName(v string) { x.xxx_hidden_Name = v } +func (x *InvokeActionResponse) SetEncryptedData(v []*EncryptedData) { + x.xxx_hidden_EncryptedData = &v +} + func (x *InvokeActionResponse) HasResponse() bool { if x == nil { return false @@ -571,6 +602,8 @@ type InvokeActionResponse_builder struct { Annotations []*anypb.Any Response *structpb.Struct Name string + // Secret return values encrypted for every requested recipient. + EncryptedData []*EncryptedData } func (b0 InvokeActionResponse_builder) Build() *InvokeActionResponse { @@ -582,6 +615,7 @@ func (b0 InvokeActionResponse_builder) Build() *InvokeActionResponse { x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Response = b.Response x.xxx_hidden_Name = b.Name + x.xxx_hidden_EncryptedData = &b.EncryptedData return m0 } @@ -676,14 +710,15 @@ func (b0 GetActionStatusRequest_builder) Build() *GetActionStatusRequest { } type GetActionStatusResponse struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` - xxx_hidden_Id string `protobuf:"bytes,2,opt,name=id,proto3"` - xxx_hidden_Status BatonActionStatus `protobuf:"varint,3,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,4,rep,name=annotations,proto3"` - xxx_hidden_Response *structpb.Struct `protobuf:"bytes,5,opt,name=response,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` + xxx_hidden_Id string `protobuf:"bytes,2,opt,name=id,proto3"` + xxx_hidden_Status BatonActionStatus `protobuf:"varint,3,opt,name=status,proto3,enum=c1.connector.v2.BatonActionStatus"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,4,rep,name=annotations,proto3"` + xxx_hidden_Response *structpb.Struct `protobuf:"bytes,5,opt,name=response,proto3"` + xxx_hidden_EncryptedData *[]*EncryptedData `protobuf:"bytes,6,rep,name=encrypted_data,json=encryptedData,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetActionStatusResponse) Reset() { @@ -748,6 +783,15 @@ func (x *GetActionStatusResponse) GetResponse() *structpb.Struct { return nil } +func (x *GetActionStatusResponse) GetEncryptedData() []*EncryptedData { + if x != nil { + if x.xxx_hidden_EncryptedData != nil { + return *x.xxx_hidden_EncryptedData + } + } + return nil +} + func (x *GetActionStatusResponse) SetName(v string) { x.xxx_hidden_Name = v } @@ -768,6 +812,10 @@ func (x *GetActionStatusResponse) SetResponse(v *structpb.Struct) { x.xxx_hidden_Response = v } +func (x *GetActionStatusResponse) SetEncryptedData(v []*EncryptedData) { + x.xxx_hidden_EncryptedData = &v +} + func (x *GetActionStatusResponse) HasResponse() bool { if x == nil { return false @@ -787,6 +835,8 @@ type GetActionStatusResponse_builder struct { Status BatonActionStatus Annotations []*anypb.Any Response *structpb.Struct + // Secret return values encrypted when the action settled. + EncryptedData []*EncryptedData } func (b0 GetActionStatusResponse_builder) Build() *GetActionStatusResponse { @@ -798,6 +848,7 @@ func (b0 GetActionStatusResponse_builder) Build() *GetActionStatusResponse { x.xxx_hidden_Status = b.Status x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Response = b.Response + x.xxx_hidden_EncryptedData = &b.EncryptedData return m0 } @@ -1111,7 +1162,7 @@ var File_c1_connector_v2_action_proto protoreflect.FileDescriptor const file_c1_connector_v2_action_proto_rawDesc = "" + "\n" + - "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x17validate/validate.proto\"\xfb\x02\n" + + "\x1cc1/connector/v2/action.proto\x12\x0fc1.connector.v2\x1a\x19c1/config/v1/config.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x17validate/validate.proto\"\xfb\x02\n" + "\x11BatonActionSchema\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x121\n" + "\targuments\x18\x02 \x03(\v2\x13.c1.config.v1.FieldR\targuments\x12:\n" + @@ -1121,30 +1172,33 @@ const file_c1_connector_v2_action_proto_rawDesc = "" + "\vdescription\x18\x06 \x01(\tR\vdescription\x12<\n" + "\vaction_type\x18\a \x03(\x0e2\x1b.c1.connector.v2.ActionTypeR\n" + "actionType\x12(\n" + - "\x10resource_type_id\x18\b \x01(\tR\x0eresourceTypeId\"\x84\x02\n" + + "\x10resource_type_id\x18\b \x01(\tR\x0eresourceTypeId\"\xd6\x02\n" + "\x13InvokeActionRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12+\n" + "\x04args\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04args\x126\n" + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12(\n" + "\x10resource_type_id\x18\x04 \x01(\tR\x0eresourceTypeId\x12J\n" + "\vinline_wait\x18\x05 \x01(\v2\x19.google.protobuf.DurationB\x0e\xfaB\v\xaa\x01\b\"\x04\b\x80\xa3\x052\x00R\n" + - "inlineWait\"\xe3\x01\n" + + "inlineWait\x12P\n" + + "\x12encryption_configs\x18\x06 \x03(\v2!.c1.connector.v2.EncryptionConfigR\x11encryptionConfigs\"\xaa\x02\n" + "\x14InvokeActionResponse\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12:\n" + "\x06status\x18\x02 \x01(\x0e2\".c1.connector.v2.BatonActionStatusR\x06status\x126\n" + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x123\n" + "\bresponse\x18\x04 \x01(\v2\x17.google.protobuf.StructR\bresponse\x12\x12\n" + - "\x04name\x18\x05 \x01(\tR\x04name\"x\n" + + "\x04name\x18\x05 \x01(\tR\x04name\x12E\n" + + "\x0eencrypted_data\x18\x06 \x03(\v2\x1e.c1.connector.v2.EncryptedDataR\rencryptedData\"x\n" + "\x16GetActionStatusRequest\x12\x16\n" + "\x04name\x18\x01 \x01(\tB\x02\x18\x01R\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x126\n" + - "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\"\xe6\x01\n" + + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\"\xad\x02\n" + "\x17GetActionStatusResponse\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x12:\n" + "\x06status\x18\x03 \x01(\x0e2\".c1.connector.v2.BatonActionStatusR\x06status\x126\n" + "\vannotations\x18\x04 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x123\n" + - "\bresponse\x18\x05 \x01(\v2\x17.google.protobuf.StructR\bresponse\"d\n" + + "\bresponse\x18\x05 \x01(\v2\x17.google.protobuf.StructR\bresponse\x12E\n" + + "\x0eencrypted_data\x18\x06 \x03(\v2\x1e.c1.connector.v2.EncryptedDataR\rencryptedData\"d\n" + "\x16GetActionSchemaRequest\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\"\x8d\x01\n" + @@ -1201,6 +1255,8 @@ var file_c1_connector_v2_action_proto_goTypes = []any{ (*structpb.Struct)(nil), // 13: google.protobuf.Struct (*anypb.Any)(nil), // 14: google.protobuf.Any (*durationpb.Duration)(nil), // 15: google.protobuf.Duration + (*EncryptionConfig)(nil), // 16: c1.connector.v2.EncryptionConfig + (*EncryptedData)(nil), // 17: c1.connector.v2.EncryptedData } var file_c1_connector_v2_action_proto_depIdxs = []int32{ 11, // 0: c1.connector.v2.BatonActionSchema.arguments:type_name -> c1.config.v1.Field @@ -1210,32 +1266,35 @@ var file_c1_connector_v2_action_proto_depIdxs = []int32{ 13, // 4: c1.connector.v2.InvokeActionRequest.args:type_name -> google.protobuf.Struct 14, // 5: c1.connector.v2.InvokeActionRequest.annotations:type_name -> google.protobuf.Any 15, // 6: c1.connector.v2.InvokeActionRequest.inline_wait:type_name -> google.protobuf.Duration - 0, // 7: c1.connector.v2.InvokeActionResponse.status:type_name -> c1.connector.v2.BatonActionStatus - 14, // 8: c1.connector.v2.InvokeActionResponse.annotations:type_name -> google.protobuf.Any - 13, // 9: c1.connector.v2.InvokeActionResponse.response:type_name -> google.protobuf.Struct - 14, // 10: c1.connector.v2.GetActionStatusRequest.annotations:type_name -> google.protobuf.Any - 0, // 11: c1.connector.v2.GetActionStatusResponse.status:type_name -> c1.connector.v2.BatonActionStatus - 14, // 12: c1.connector.v2.GetActionStatusResponse.annotations:type_name -> google.protobuf.Any - 13, // 13: c1.connector.v2.GetActionStatusResponse.response:type_name -> google.protobuf.Struct - 14, // 14: c1.connector.v2.GetActionSchemaRequest.annotations:type_name -> google.protobuf.Any - 2, // 15: c1.connector.v2.GetActionSchemaResponse.schema:type_name -> c1.connector.v2.BatonActionSchema - 14, // 16: c1.connector.v2.GetActionSchemaResponse.annotations:type_name -> google.protobuf.Any - 14, // 17: c1.connector.v2.ListActionSchemasRequest.annotations:type_name -> google.protobuf.Any - 2, // 18: c1.connector.v2.ListActionSchemasResponse.schemas:type_name -> c1.connector.v2.BatonActionSchema - 14, // 19: c1.connector.v2.ListActionSchemasResponse.annotations:type_name -> google.protobuf.Any - 3, // 20: c1.connector.v2.ActionService.InvokeAction:input_type -> c1.connector.v2.InvokeActionRequest - 5, // 21: c1.connector.v2.ActionService.GetActionStatus:input_type -> c1.connector.v2.GetActionStatusRequest - 7, // 22: c1.connector.v2.ActionService.GetActionSchema:input_type -> c1.connector.v2.GetActionSchemaRequest - 9, // 23: c1.connector.v2.ActionService.ListActionSchemas:input_type -> c1.connector.v2.ListActionSchemasRequest - 4, // 24: c1.connector.v2.ActionService.InvokeAction:output_type -> c1.connector.v2.InvokeActionResponse - 6, // 25: c1.connector.v2.ActionService.GetActionStatus:output_type -> c1.connector.v2.GetActionStatusResponse - 8, // 26: c1.connector.v2.ActionService.GetActionSchema:output_type -> c1.connector.v2.GetActionSchemaResponse - 10, // 27: c1.connector.v2.ActionService.ListActionSchemas:output_type -> c1.connector.v2.ListActionSchemasResponse - 24, // [24:28] is the sub-list for method output_type - 20, // [20:24] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 16, // 7: c1.connector.v2.InvokeActionRequest.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 0, // 8: c1.connector.v2.InvokeActionResponse.status:type_name -> c1.connector.v2.BatonActionStatus + 14, // 9: c1.connector.v2.InvokeActionResponse.annotations:type_name -> google.protobuf.Any + 13, // 10: c1.connector.v2.InvokeActionResponse.response:type_name -> google.protobuf.Struct + 17, // 11: c1.connector.v2.InvokeActionResponse.encrypted_data:type_name -> c1.connector.v2.EncryptedData + 14, // 12: c1.connector.v2.GetActionStatusRequest.annotations:type_name -> google.protobuf.Any + 0, // 13: c1.connector.v2.GetActionStatusResponse.status:type_name -> c1.connector.v2.BatonActionStatus + 14, // 14: c1.connector.v2.GetActionStatusResponse.annotations:type_name -> google.protobuf.Any + 13, // 15: c1.connector.v2.GetActionStatusResponse.response:type_name -> google.protobuf.Struct + 17, // 16: c1.connector.v2.GetActionStatusResponse.encrypted_data:type_name -> c1.connector.v2.EncryptedData + 14, // 17: c1.connector.v2.GetActionSchemaRequest.annotations:type_name -> google.protobuf.Any + 2, // 18: c1.connector.v2.GetActionSchemaResponse.schema:type_name -> c1.connector.v2.BatonActionSchema + 14, // 19: c1.connector.v2.GetActionSchemaResponse.annotations:type_name -> google.protobuf.Any + 14, // 20: c1.connector.v2.ListActionSchemasRequest.annotations:type_name -> google.protobuf.Any + 2, // 21: c1.connector.v2.ListActionSchemasResponse.schemas:type_name -> c1.connector.v2.BatonActionSchema + 14, // 22: c1.connector.v2.ListActionSchemasResponse.annotations:type_name -> google.protobuf.Any + 3, // 23: c1.connector.v2.ActionService.InvokeAction:input_type -> c1.connector.v2.InvokeActionRequest + 5, // 24: c1.connector.v2.ActionService.GetActionStatus:input_type -> c1.connector.v2.GetActionStatusRequest + 7, // 25: c1.connector.v2.ActionService.GetActionSchema:input_type -> c1.connector.v2.GetActionSchemaRequest + 9, // 26: c1.connector.v2.ActionService.ListActionSchemas:input_type -> c1.connector.v2.ListActionSchemasRequest + 4, // 27: c1.connector.v2.ActionService.InvokeAction:output_type -> c1.connector.v2.InvokeActionResponse + 6, // 28: c1.connector.v2.ActionService.GetActionStatus:output_type -> c1.connector.v2.GetActionStatusResponse + 8, // 29: c1.connector.v2.ActionService.GetActionSchema:output_type -> c1.connector.v2.GetActionSchemaResponse + 10, // 30: c1.connector.v2.ActionService.ListActionSchemas:output_type -> c1.connector.v2.ListActionSchemasResponse + 27, // [27:31] is the sub-list for method output_type + 23, // [23:27] is the sub-list for method input_type + 23, // [23:23] is the sub-list for extension type_name + 23, // [23:23] is the sub-list for extension extendee + 0, // [0:23] is the sub-list for field type_name } func init() { file_c1_connector_v2_action_proto_init() } @@ -1243,6 +1302,7 @@ func file_c1_connector_v2_action_proto_init() { if File_c1_connector_v2_action_proto != nil { return } + file_c1_connector_v2_resource_proto_init() type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go index f2606742..62592f55 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go @@ -4528,8 +4528,10 @@ type Task_ActionInvokeTask struct { Annotations []*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3" json:"annotations,omitempty"` // Optional: if set, invokes a resource-scoped action ResourceTypeId string `protobuf:"bytes,4,opt,name=resource_type_id,json=resourceTypeId,proto3" json:"resource_type_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Public keys the connector uses to encrypt secret return values. + EncryptionConfigs []*v2.EncryptionConfig `protobuf:"bytes,5,rep,name=encryption_configs,json=encryptionConfigs,proto3" json:"encryption_configs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Task_ActionInvokeTask) Reset() { @@ -4585,6 +4587,13 @@ func (x *Task_ActionInvokeTask) GetResourceTypeId() string { return "" } +func (x *Task_ActionInvokeTask) GetEncryptionConfigs() []*v2.EncryptionConfig { + if x != nil { + return x.EncryptionConfigs + } + return nil +} + func (x *Task_ActionInvokeTask) SetName(v string) { x.Name = v } @@ -4601,6 +4610,10 @@ func (x *Task_ActionInvokeTask) SetResourceTypeId(v string) { x.ResourceTypeId = v } +func (x *Task_ActionInvokeTask) SetEncryptionConfigs(v []*v2.EncryptionConfig) { + x.EncryptionConfigs = v +} + func (x *Task_ActionInvokeTask) HasArgs() bool { if x == nil { return false @@ -4620,6 +4633,8 @@ type Task_ActionInvokeTask_builder struct { Annotations []*anypb.Any // Optional: if set, invokes a resource-scoped action ResourceTypeId string + // Public keys the connector uses to encrypt secret return values. + EncryptionConfigs []*v2.EncryptionConfig } func (b0 Task_ActionInvokeTask_builder) Build() *Task_ActionInvokeTask { @@ -4630,6 +4645,7 @@ func (b0 Task_ActionInvokeTask_builder) Build() *Task_ActionInvokeTask { x.Args = b.Args x.Annotations = b.Annotations x.ResourceTypeId = b.ResourceTypeId + x.EncryptionConfigs = b.EncryptionConfigs return m0 } @@ -5598,7 +5614,7 @@ var File_c1_connectorapi_baton_v1_baton_proto protoreflect.FileDescriptor const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\n" + - "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\x921\n" + + "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\xe41\n" + "\x04Task\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12=\n" + "\x06status\x18\x02 \x01(\x0e2%.c1.connectorapi.baton.v1.Task.StatusR\x06status\x12=\n" + @@ -5705,12 +5721,13 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x1aa\n" + "\x13ActionGetSchemaTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + - "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\xb5\x01\n" + + "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\x87\x02\n" + "\x10ActionInvokeTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12+\n" + "\x04args\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04args\x126\n" + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12(\n" + - "\x10resource_type_id\x18\x04 \x01(\tR\x0eresourceTypeId\x1an\n" + + "\x10resource_type_id\x18\x04 \x01(\tR\x0eresourceTypeId\x12P\n" + + "\x12encryption_configs\x18\x05 \x03(\v2!.c1.connector.v2.EncryptionConfigR\x11encryptionConfigs\x1an\n" + "\x10ActionStatusTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x126\n" + @@ -5994,35 +6011,36 @@ var file_c1_connectorapi_baton_v1_baton_proto_depIdxs = []int32{ 49, // 85: c1.connectorapi.baton.v1.Task.ActionGetSchemaTask.annotations:type_name -> google.protobuf.Any 63, // 86: c1.connectorapi.baton.v1.Task.ActionInvokeTask.args:type_name -> google.protobuf.Struct 49, // 87: c1.connectorapi.baton.v1.Task.ActionInvokeTask.annotations:type_name -> google.protobuf.Any - 49, // 88: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any - 49, // 89: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any - 40, // 90: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync - 49, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any - 49, // 92: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any - 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any - 49, // 94: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any - 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any - 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any - 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any - 2, // 98: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest - 4, // 99: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest - 5, // 100: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest - 8, // 101: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest - 12, // 102: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest - 10, // 103: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest - 14, // 104: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest - 3, // 105: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse - 7, // 106: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse - 6, // 107: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse - 9, // 108: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse - 13, // 109: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse - 11, // 110: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse - 15, // 111: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse - 105, // [105:112] is the sub-list for method output_type - 98, // [98:105] is the sub-list for method input_type - 98, // [98:98] is the sub-list for extension type_name - 98, // [98:98] is the sub-list for extension extendee - 0, // [0:98] is the sub-list for field type_name + 58, // 88: c1.connectorapi.baton.v1.Task.ActionInvokeTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 49, // 89: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any + 49, // 90: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any + 40, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync + 49, // 92: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any + 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any + 49, // 94: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any + 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any + 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any + 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any + 49, // 98: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any + 2, // 99: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest + 4, // 100: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest + 5, // 101: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest + 8, // 102: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest + 12, // 103: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest + 10, // 104: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest + 14, // 105: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest + 3, // 106: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse + 7, // 107: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse + 6, // 108: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse + 9, // 109: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse + 13, // 110: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse + 11, // 111: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse + 15, // 112: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse + 106, // [106:113] is the sub-list for method output_type + 99, // [99:106] is the sub-list for method input_type + 99, // [99:99] is the sub-list for extension type_name + 99, // [99:99] is the sub-list for extension extendee + 0, // [0:99] is the sub-list for field type_name } func init() { file_c1_connectorapi_baton_v1_baton_proto_init() } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go index a333caa7..c1a689c1 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go @@ -6962,6 +6962,40 @@ func (m *Task_ActionInvokeTask) validate(all bool) error { // no validation rules for ResourceTypeId + for idx, item := range m.GetEncryptionConfigs() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, Task_ActionInvokeTaskValidationError{ + field: fmt.Sprintf("EncryptionConfigs[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, Task_ActionInvokeTaskValidationError{ + field: fmt.Sprintf("EncryptionConfigs[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return Task_ActionInvokeTaskValidationError{ + field: fmt.Sprintf("EncryptionConfigs[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + if len(errors) > 0 { return Task_ActionInvokeTaskMultiError(errors) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go index 6a985011..7b394b78 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go @@ -4517,13 +4517,14 @@ func (b0 Task_ActionGetSchemaTask_builder) Build() *Task_ActionGetSchemaTask { } type Task_ActionInvokeTask struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` - xxx_hidden_Args *structpb.Struct `protobuf:"bytes,2,opt,name=args,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,4,opt,name=resource_type_id,json=resourceTypeId,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name,proto3"` + xxx_hidden_Args *structpb.Struct `protobuf:"bytes,2,opt,name=args,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,4,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_EncryptionConfigs *[]*v2.EncryptionConfig `protobuf:"bytes,5,rep,name=encryption_configs,json=encryptionConfigs,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *Task_ActionInvokeTask) Reset() { @@ -4581,6 +4582,15 @@ func (x *Task_ActionInvokeTask) GetResourceTypeId() string { return "" } +func (x *Task_ActionInvokeTask) GetEncryptionConfigs() []*v2.EncryptionConfig { + if x != nil { + if x.xxx_hidden_EncryptionConfigs != nil { + return *x.xxx_hidden_EncryptionConfigs + } + } + return nil +} + func (x *Task_ActionInvokeTask) SetName(v string) { x.xxx_hidden_Name = v } @@ -4597,6 +4607,10 @@ func (x *Task_ActionInvokeTask) SetResourceTypeId(v string) { x.xxx_hidden_ResourceTypeId = v } +func (x *Task_ActionInvokeTask) SetEncryptionConfigs(v []*v2.EncryptionConfig) { + x.xxx_hidden_EncryptionConfigs = &v +} + func (x *Task_ActionInvokeTask) HasArgs() bool { if x == nil { return false @@ -4616,6 +4630,8 @@ type Task_ActionInvokeTask_builder struct { Annotations []*anypb.Any // Optional: if set, invokes a resource-scoped action ResourceTypeId string + // Public keys the connector uses to encrypt secret return values. + EncryptionConfigs []*v2.EncryptionConfig } func (b0 Task_ActionInvokeTask_builder) Build() *Task_ActionInvokeTask { @@ -4626,6 +4642,7 @@ func (b0 Task_ActionInvokeTask_builder) Build() *Task_ActionInvokeTask { x.xxx_hidden_Args = b.Args x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_ResourceTypeId = b.ResourceTypeId + x.xxx_hidden_EncryptionConfigs = &b.EncryptionConfigs return m0 } @@ -5607,7 +5624,7 @@ var File_c1_connectorapi_baton_v1_baton_proto protoreflect.FileDescriptor const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\n" + - "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\x921\n" + + "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\xe41\n" + "\x04Task\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12=\n" + "\x06status\x18\x02 \x01(\x0e2%.c1.connectorapi.baton.v1.Task.StatusR\x06status\x12=\n" + @@ -5714,12 +5731,13 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x1aa\n" + "\x13ActionGetSchemaTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + - "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\xb5\x01\n" + + "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\x87\x02\n" + "\x10ActionInvokeTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12+\n" + "\x04args\x18\x02 \x01(\v2\x17.google.protobuf.StructR\x04args\x126\n" + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12(\n" + - "\x10resource_type_id\x18\x04 \x01(\tR\x0eresourceTypeId\x1an\n" + + "\x10resource_type_id\x18\x04 \x01(\tR\x0eresourceTypeId\x12P\n" + + "\x12encryption_configs\x18\x05 \x03(\v2!.c1.connector.v2.EncryptionConfigR\x11encryptionConfigs\x1an\n" + "\x10ActionStatusTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x126\n" + @@ -6003,35 +6021,36 @@ var file_c1_connectorapi_baton_v1_baton_proto_depIdxs = []int32{ 49, // 85: c1.connectorapi.baton.v1.Task.ActionGetSchemaTask.annotations:type_name -> google.protobuf.Any 63, // 86: c1.connectorapi.baton.v1.Task.ActionInvokeTask.args:type_name -> google.protobuf.Struct 49, // 87: c1.connectorapi.baton.v1.Task.ActionInvokeTask.annotations:type_name -> google.protobuf.Any - 49, // 88: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any - 49, // 89: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any - 40, // 90: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync - 49, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any - 49, // 92: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any - 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any - 49, // 94: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any - 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any - 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any - 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any - 2, // 98: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest - 4, // 99: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest - 5, // 100: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest - 8, // 101: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest - 12, // 102: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest - 10, // 103: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest - 14, // 104: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest - 3, // 105: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse - 7, // 106: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse - 6, // 107: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse - 9, // 108: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse - 13, // 109: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse - 11, // 110: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse - 15, // 111: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse - 105, // [105:112] is the sub-list for method output_type - 98, // [98:105] is the sub-list for method input_type - 98, // [98:98] is the sub-list for extension type_name - 98, // [98:98] is the sub-list for extension extendee - 0, // [0:98] is the sub-list for field type_name + 58, // 88: c1.connectorapi.baton.v1.Task.ActionInvokeTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 49, // 89: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any + 49, // 90: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any + 40, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync + 49, // 92: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any + 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any + 49, // 94: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any + 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any + 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any + 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any + 49, // 98: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any + 2, // 99: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest + 4, // 100: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest + 5, // 101: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest + 8, // 102: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest + 12, // 103: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest + 10, // 104: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest + 14, // 105: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest + 3, // 106: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse + 7, // 107: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse + 6, // 108: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse + 9, // 109: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse + 13, // 110: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse + 11, // 111: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse + 15, // 112: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse + 106, // [106:113] is the sub-list for method output_type + 99, // [99:106] is the sub-list for method input_type + 99, // [99:99] is the sub-list for extension type_name + 99, // [99:99] is the sub-list for extension extendee + 0, // [0:99] is the sub-list for field type_name } func init() { file_c1_connectorapi_baton_v1_baton_proto_init() } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/actions/actions.go b/vendor/github.com/conductorone/baton-sdk/pkg/actions/actions.go index 184a6d02..d6a8c296 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/actions/actions.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/actions/actions.go @@ -11,6 +11,7 @@ import ( config "github.com/conductorone/baton-sdk/pb/c1/config/v1" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/crypto" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/segmentio/ksuid" "go.opentelemetry.io/otel/trace" @@ -24,6 +25,29 @@ import ( type ActionHandler func(ctx context.Context, args *structpb.Struct) (*structpb.Struct, annotations.Annotations, error) +// ActionHandlerWithSecrets returns public values separately from plaintext +// secret values. The action manager encrypts PlaintextData before publishing +// the result or retaining it for a later status request. +type ActionHandlerWithSecrets func( + ctx context.Context, + args *structpb.Struct, +) (*structpb.Struct, []*v2.PlaintextData, annotations.Annotations, error) + +type actionHandlerResult struct { + response *structpb.Struct + plaintextData []*v2.PlaintextData + annotations annotations.Annotations + err error +} + +type registeredActionHandler struct { + invoke func(context.Context, *structpb.Struct) actionHandlerResult + secretReturnNames map[string]struct{} + requiredSecretReturnNames []string +} + +type actionEncryptFunc func(context.Context, *crypto.EncryptionManager, *v2.PlaintextData) ([]*v2.EncryptedData, error) + // IsInFlight reports whether the status describes an action // still executing: PENDING or RUNNING. func IsInFlight(s v2.BatonActionStatus) bool { @@ -38,13 +62,14 @@ func IsSettled(s v2.BatonActionStatus) bool { } type OutstandingAction struct { - Id string - Name string - Status v2.BatonActionStatus - Rv *structpb.Struct - Annos annotations.Annotations - Err error - StartedAt time.Time + Id string + Name string + Status v2.BatonActionStatus + Rv *structpb.Struct + EncryptedData []*v2.EncryptedData + Annos annotations.Annotations + Err error + StartedAt time.Time sync.Mutex // cancelled marks a FAILED status that came from request cancellation @@ -119,11 +144,13 @@ func (oa *OutstandingAction) SetError(ctx context.Context, err error) { oa.Lock() defer oa.Unlock() if oa.Status == v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED { + oa.EncryptedData = nil oa.setErrorLocked(err) oa.cancelled = false return } if oa.setStatusLocked(ctx, v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED) { + oa.EncryptedData = nil oa.setErrorLocked(err) } } @@ -169,9 +196,20 @@ func (oa *OutstandingAction) Result() (string, v2.BatonActionStatus, *structpb.S // result is the unexported form of Result. func (oa *OutstandingAction) result() (string, v2.BatonActionStatus, *structpb.Struct, annotations.Annotations) { + id, status, response, _, annos := oa.resultWithEncryptedData() + return id, status, response, annos +} + +// ResultWithEncryptedData returns the action's public and encrypted outcomes. +// The returned messages are owned by the action and must not be modified. +func (oa *OutstandingAction) ResultWithEncryptedData() (string, v2.BatonActionStatus, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations) { + return oa.resultWithEncryptedData() +} + +func (oa *OutstandingAction) resultWithEncryptedData() (string, v2.BatonActionStatus, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations) { oa.Lock() defer oa.Unlock() - return oa.Id, oa.Status, oa.Rv, oa.Annos + return oa.Id, oa.Status, oa.Rv, oa.EncryptedData, oa.Annos } // setOutcome publishes the handler's result and terminal status in one @@ -181,9 +219,28 @@ func (oa *OutstandingAction) result() (string, v2.BatonActionStatus, *structpb.S // exception: a cancellation-FAILED status is provisional, and the handler's // own outcome — success or failure — replaces it. func (oa *OutstandingAction) setOutcome(ctx context.Context, rv *structpb.Struct, annos annotations.Annotations, err error) { + oa.setOutcomeWithEncryptedData(ctx, rv, nil, annos, err) +} + +func (oa *OutstandingAction) setOutcomeWithEncryptedData( + ctx context.Context, + rv *structpb.Struct, + encryptedData []*v2.EncryptedData, + annos annotations.Annotations, + err error, +) { if rv != nil { rv = proto.Clone(rv).(*structpb.Struct) } + if encryptedData != nil { + encryptedDataCopy := make([]*v2.EncryptedData, len(encryptedData)) + for i, encrypted := range encryptedData { + if encrypted != nil { + encryptedDataCopy[i] = proto.Clone(encrypted).(*v2.EncryptedData) + } + } + encryptedData = encryptedDataCopy + } if annos != nil { annosCopy := make(annotations.Annotations, len(annos)) for i, a := range annos { @@ -200,6 +257,7 @@ func (oa *OutstandingAction) setOutcome(ctx context.Context, rv *structpb.Struct if err != nil { if oa.Status == v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED || oa.setStatusLocked(ctx, v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED) { oa.Rv = rv + oa.EncryptedData = nil oa.Annos = annos oa.setErrorLocked(err) oa.cancelled = false @@ -216,6 +274,7 @@ func (oa *OutstandingAction) setOutcome(ctx context.Context, rv *structpb.Struct oa.cancelled = false oa.Status = v2.BatonActionStatus_BATON_ACTION_STATUS_COMPLETE oa.Rv = rv + oa.EncryptedData = encryptedData oa.Annos = annos oa.Err = nil return @@ -223,6 +282,7 @@ func (oa *OutstandingAction) setOutcome(ctx context.Context, rv *structpb.Struct if oa.setStatusLocked(ctx, v2.BatonActionStatus_BATON_ACTION_STATUS_COMPLETE) { oa.Rv = rv + oa.EncryptedData = encryptedData oa.Annos = annos } } @@ -266,6 +326,26 @@ type ActionRegistry interface { RegisterAction(ctx context.Context, name string, schema *v2.BatonActionSchema, handler ActionHandler) error } +type actionRegistryWithSecrets interface { + RegisterWithSecrets(ctx context.Context, schema *v2.BatonActionSchema, handler ActionHandlerWithSecrets) error +} + +// RegisterWithSecrets registers an action whose handler can return plaintext +// secret values. The registry validates and encrypts those values before they +// are observable through InvokeAction or GetActionStatus. +func RegisterWithSecrets( + ctx context.Context, + registry ActionRegistry, + schema *v2.BatonActionSchema, + handler ActionHandlerWithSecrets, +) error { + secretRegistry, ok := registry.(actionRegistryWithSecrets) + if !ok { + return errors.New("action registry does not support secret results") + } + return secretRegistry.RegisterWithSecrets(ctx, schema, handler) +} + // Deprecated: Use ActionRegistry instead. // ResourceTypeActionRegistry is an alias for ActionRegistry for backwards compatibility. type ResourceTypeActionRegistry = ActionRegistry @@ -274,25 +354,29 @@ type ResourceTypeActionRegistry = ActionRegistry type ActionManager struct { // Global actions (no resource type) schemas map[string]*v2.BatonActionSchema // actionName -> schema - handlers map[string]ActionHandler // actionName -> handler + handlers map[string]registeredActionHandler // Resource-scoped actions (keyed by resource type) resourceSchemas map[string]map[string]*v2.BatonActionSchema // resourceTypeID -> actionName -> schema - resourceHandlers map[string]map[string]ActionHandler // resourceTypeID -> actionName -> handler + resourceHandlers map[string]map[string]registeredActionHandler // Outstanding actions (shared across global and resource-scoped) actions map[string]*OutstandingAction // actionID -> outstanding action - mu sync.RWMutex + encryptPlaintext actionEncryptFunc + mu sync.RWMutex } func NewActionManager(_ context.Context) *ActionManager { return &ActionManager{ schemas: make(map[string]*v2.BatonActionSchema), - handlers: make(map[string]ActionHandler), + handlers: make(map[string]registeredActionHandler), resourceSchemas: make(map[string]map[string]*v2.BatonActionSchema), - resourceHandlers: make(map[string]map[string]ActionHandler), + resourceHandlers: make(map[string]map[string]registeredActionHandler), actions: make(map[string]*OutstandingAction), + encryptPlaintext: func(ctx context.Context, manager *crypto.EncryptionManager, plaintext *v2.PlaintextData) ([]*v2.EncryptedData, error) { + return manager.Encrypt(ctx, plaintext) + }, } } @@ -367,10 +451,51 @@ func (a *ActionManager) Register(ctx context.Context, schema *v2.BatonActionSche // Deprecated: Use Register instead. // RegisterAction registers a global action (not scoped to a resource type). func (a *ActionManager) RegisterAction(ctx context.Context, name string, schema *v2.BatonActionSchema, handler ActionHandler) error { + if handler == nil { + return errors.New("action handler cannot be nil") + } + if hasSecretReturnTypes(schema) { + return errors.New("action schemas with secret return types must use RegisterWithSecrets") + } + return a.registerGlobalAction(ctx, name, schema, registeredActionHandler{ + invoke: func(ctx context.Context, args *structpb.Struct) actionHandlerResult { + response, annos, err := handler(ctx, args) + return actionHandlerResult{response: response, annotations: annos, err: err} + }, + }) +} + +// RegisterWithSecrets registers a global action that returns plaintext secret +// values separately from its public response. +func (a *ActionManager) RegisterWithSecrets(ctx context.Context, schema *v2.BatonActionSchema, handler ActionHandlerWithSecrets) error { + if schema == nil { + return errors.New("action schema cannot be nil") + } + if handler == nil { + return errors.New("action handler cannot be nil") + } + if !hasSecretReturnTypes(schema) { + return errors.New("secret action handler requires at least one secret return type") + } + secretReturnNames, requiredSecretReturnNames, err := validateSecretReturnTypes(schema.GetReturnTypes()) + if err != nil { + return err + } + return a.registerGlobalAction(ctx, schema.GetName(), schema, registeredActionHandler{ + invoke: func(ctx context.Context, args *structpb.Struct) actionHandlerResult { + response, plaintextData, annos, err := handler(ctx, args) + return actionHandlerResult{response: response, plaintextData: plaintextData, annotations: annos, err: err} + }, + secretReturnNames: secretReturnNames, + requiredSecretReturnNames: requiredSecretReturnNames, + }) +} + +func (a *ActionManager) registerGlobalAction(ctx context.Context, name string, schema *v2.BatonActionSchema, handler registeredActionHandler) error { a.mu.Lock() defer a.mu.Unlock() - if handler == nil { + if handler.invoke == nil { return errors.New("action handler cannot be nil") } err := a.registerActionSchema(ctx, name, schema) @@ -401,6 +526,57 @@ func (a *ActionManager) RegisterResourceAction( resourceTypeID string, schema *v2.BatonActionSchema, handler ActionHandler, +) error { + if handler == nil { + return errors.New("action handler cannot be nil") + } + if hasSecretReturnTypes(schema) { + return errors.New("action schemas with secret return types must use RegisterResourceActionWithSecrets") + } + return a.registerResourceAction(ctx, resourceTypeID, schema, registeredActionHandler{ + invoke: func(ctx context.Context, args *structpb.Struct) actionHandlerResult { + response, annos, err := handler(ctx, args) + return actionHandlerResult{response: response, annotations: annos, err: err} + }, + }) +} + +// RegisterResourceActionWithSecrets registers a resource-scoped action that +// returns plaintext secret values separately from its public response. +func (a *ActionManager) RegisterResourceActionWithSecrets( + ctx context.Context, + resourceTypeID string, + schema *v2.BatonActionSchema, + handler ActionHandlerWithSecrets, +) error { + if schema == nil { + return errors.New("action schema cannot be nil") + } + if handler == nil { + return errors.New("action handler cannot be nil") + } + if !hasSecretReturnTypes(schema) { + return errors.New("secret action handler requires at least one secret return type") + } + secretReturnNames, requiredSecretReturnNames, err := validateSecretReturnTypes(schema.GetReturnTypes()) + if err != nil { + return err + } + return a.registerResourceAction(ctx, resourceTypeID, schema, registeredActionHandler{ + invoke: func(ctx context.Context, args *structpb.Struct) actionHandlerResult { + response, plaintextData, annos, err := handler(ctx, args) + return actionHandlerResult{response: response, plaintextData: plaintextData, annotations: annos, err: err} + }, + secretReturnNames: secretReturnNames, + requiredSecretReturnNames: requiredSecretReturnNames, + }) +} + +func (a *ActionManager) registerResourceAction( + ctx context.Context, + resourceTypeID string, + schema *v2.BatonActionSchema, + handler registeredActionHandler, ) error { if resourceTypeID == "" { return errors.New("resource type ID cannot be empty") @@ -411,7 +587,7 @@ func (a *ActionManager) RegisterResourceAction( if schema.GetName() == "" { return errors.New("action schema name cannot be empty") } - if handler == nil { + if handler.invoke == nil { return fmt.Errorf("handler cannot be nil for action %s", schema.GetName()) } @@ -425,7 +601,7 @@ func (a *ActionManager) RegisterResourceAction( a.resourceSchemas[resourceTypeID] = make(map[string]*v2.BatonActionSchema) } if a.resourceHandlers[resourceTypeID] == nil { - a.resourceHandlers[resourceTypeID] = make(map[string]ActionHandler) + a.resourceHandlers[resourceTypeID] = make(map[string]registeredActionHandler) } actionName := schema.GetName() @@ -473,6 +649,10 @@ func (r *resourceTypeActionRegistry) Register(ctx context.Context, schema *v2.Ba return r.actionManager.RegisterResourceAction(ctx, r.resourceTypeID, schema, handler) } +func (r *resourceTypeActionRegistry) RegisterWithSecrets(ctx context.Context, schema *v2.BatonActionSchema, handler ActionHandlerWithSecrets) error { + return r.actionManager.RegisterResourceActionWithSecrets(ctx, r.resourceTypeID, schema, handler) +} + // Deprecated: Use Register instead. // RegisterAction registers a resource-scoped action. The name parameter is ignored; the name from schema is used. func (r *resourceTypeActionRegistry) RegisterAction(ctx context.Context, name string, schema *v2.BatonActionSchema, handler ActionHandler) error { @@ -557,19 +737,29 @@ func (a *ActionManager) GetActionSchema(_ context.Context, name string) (*v2.Bat return schema, nil, nil } -func (a *ActionManager) GetActionStatus(_ context.Context, actionId string) (v2.BatonActionStatus, string, *structpb.Struct, annotations.Annotations, error) { +func (a *ActionManager) GetActionStatus(ctx context.Context, actionId string) (v2.BatonActionStatus, string, *structpb.Struct, annotations.Annotations, error) { + status, name, response, _, annos, err := a.GetActionStatusWithEncryptedData(ctx, actionId) + return status, name, response, annos, err +} + +// GetActionStatusWithEncryptedData returns an outstanding action's public and +// encrypted results in one consistent snapshot. +func (a *ActionManager) GetActionStatusWithEncryptedData( + _ context.Context, + actionId string, +) (v2.BatonActionStatus, string, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations, error) { a.mu.RLock() defer a.mu.RUnlock() oa := a.actions[actionId] if oa == nil { - return v2.BatonActionStatus_BATON_ACTION_STATUS_UNKNOWN, "", nil, nil, status.Error(codes.NotFound, fmt.Sprintf("action id %s not found", actionId)) + return v2.BatonActionStatus_BATON_ACTION_STATUS_UNKNOWN, "", nil, nil, nil, status.Error(codes.NotFound, fmt.Sprintf("action id %s not found", actionId)) } // Don't return oa.Err here because error is for GetActionStatus, not the action itself. // oa.Rv contains any error. - _, st, rv, annos := oa.result() - return st, oa.Name, rv, annos, nil + _, st, rv, encryptedData, annos := oa.resultWithEncryptedData() + return st, oa.Name, rv, encryptedData, annos, nil } // InvokeAction invokes an action. If resourceTypeID is set, it invokes a resource-scoped action. @@ -593,6 +783,27 @@ func (a *ActionManager) InvokeActionWithWait( args *structpb.Struct, inlineWait time.Duration, ) (string, v2.BatonActionStatus, *structpb.Struct, annotations.Annotations, error) { + id, actionStatus, response, _, annos, err := a.InvokeActionWithWaitAndEncryption( + ctx, + name, + resourceTypeID, + args, + inlineWait, + nil, + ) + return id, actionStatus, response, annos, err +} + +// InvokeActionWithWaitAndEncryption invokes an action with recipients for any +// secret return values. +func (a *ActionManager) InvokeActionWithWaitAndEncryption( + ctx context.Context, + name string, + resourceTypeID string, + args *structpb.Struct, + inlineWait time.Duration, + encryptionConfigs []*v2.EncryptionConfig, +) (string, v2.BatonActionStatus, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations, error) { clamped := clampInlineWait(inlineWait) if clamped < inlineWait { ctxzap.Extract(ctx).Warn("capping requested inline wait", @@ -602,10 +813,10 @@ func (a *ActionManager) InvokeActionWithWait( inlineWait = clamped if resourceTypeID != "" { - return a.invokeResourceAction(ctx, resourceTypeID, name, args, inlineWait) + return a.invokeResourceAction(ctx, resourceTypeID, name, args, inlineWait, encryptionConfigs) } - return a.invokeGlobalAction(ctx, name, args, inlineWait) + return a.invokeGlobalAction(ctx, name, args, inlineWait, encryptionConfigs) } // invokeGlobalAction invokes a global (non-resource-scoped) action. @@ -614,80 +825,21 @@ func (a *ActionManager) invokeGlobalAction( name string, args *structpb.Struct, inlineWait time.Duration, -) (string, v2.BatonActionStatus, *structpb.Struct, annotations.Annotations, error) { + encryptionConfigs []*v2.EncryptionConfig, +) (string, v2.BatonActionStatus, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations, error) { a.mu.RLock() handler, ok := a.handlers[name] schema, schemaOk := a.schemas[name] a.mu.RUnlock() if !ok { - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.NotFound, fmt.Sprintf("handler for action %s not found", name)) + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.NotFound, fmt.Sprintf("handler for action %s not found", name)) } if !schemaOk || schema == nil { - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.Internal, fmt.Sprintf("schema for action %s not found", name)) - } - - // Validate constraints - if err := validateActionConstraints(schema.GetConstraints(), args); err != nil { - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.InvalidArgument, err.Error()) + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.Internal, fmt.Sprintf("schema for action %s not found", name)) } - oa := a.GetNewAction(name) - - done := make(chan struct{}) - - // The handler runs detached. Return its final result if it finishes - // within the inline wait; otherwise return the in-flight status. - go func() { // #nosec G118 -- action handlers intentionally outlive the request context and keep only trace/log metadata. - defer close(done) - defer func() { - if r := recover(); r != nil { - ctxzap.Extract(ctx).Error("panic in global action handler", - zap.String("action", name), - zap.Any("panic", r), - zap.Stack("stack")) - oa.SetError(ctx, fmt.Errorf("panic in action handler: %v", r)) - } - }() - oa.SetStatus(ctx, v2.BatonActionStatus_BATON_ACTION_STATUS_RUNNING) - bgCtx := trace.ContextWithSpanContext(context.Background(), trace.SpanContextFromContext(ctx)) - bgCtx = ctxzap.ToContext(bgCtx, ctxzap.Extract(ctx)) - handlerCtx, cancel := context.WithTimeoutCause(bgCtx, 1*time.Hour, errors.New("action handler timed out")) - defer cancel() - rv, annos, oaErr := handler(handlerCtx, args) - oa.setOutcome(ctx, rv, annos, oaErr) - }() - - // Stop releases the timer deterministically when the handler wins the - // select; an abandoned time.After timer would only be GC-eligible. - waitTimer := time.NewTimer(inlineWait) - defer waitTimer.Stop() - - select { - case <-done: - id, st, rv, annos := oa.result() - return id, st, rv, annos, nil - case <-waitTimer.C: - id, st, rv, annos := oa.result() - return id, st, rv, annos, nil - case <-ctx.Done(): - // The handler may have finished in the same instant; prefer its - // completed result over a spurious cancellation return. - select { - case <-done: - id, st, rv, annos := oa.result() - return id, st, rv, annos, nil - default: - } - oa.setCancelled(ctx, ctx.Err()) - id, st, rv, annos := oa.result() - if st == v2.BatonActionStatus_BATON_ACTION_STATUS_COMPLETE { - // The handler won the race to the lock; its completed result is - // the authoritative pairing, not the cancellation. - return id, st, rv, annos, nil - } - return id, st, rv, annos, ctx.Err() - } + return a.invokeRegisteredAction(ctx, name, "", args, inlineWait, encryptionConfigs, schema, handler) } // invokeResourceAction invokes a resource-scoped action. @@ -697,19 +849,20 @@ func (a *ActionManager) invokeResourceAction( actionName string, args *structpb.Struct, inlineWait time.Duration, -) (string, v2.BatonActionStatus, *structpb.Struct, annotations.Annotations, error) { + encryptionConfigs []*v2.EncryptionConfig, +) (string, v2.BatonActionStatus, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations, error) { if resourceTypeID == "" { - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.InvalidArgument, "resource type ID is required") + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.InvalidArgument, "resource type ID is required") } if actionName == "" { - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.InvalidArgument, "action name is required") + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.InvalidArgument, "action name is required") } a.mu.RLock() handlers, ok := a.resourceHandlers[resourceTypeID] if !ok { a.mu.RUnlock() - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.NotFound, fmt.Sprintf("no actions found for resource type %s", resourceTypeID)) + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.NotFound, fmt.Sprintf("no actions found for resource type %s", resourceTypeID)) } handler, ok := handlers[actionName] @@ -719,41 +872,79 @@ func (a *ActionManager) invokeResourceAction( v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, + nil, status.Error(codes.NotFound, fmt.Sprintf("handler for action %s not found for resource type %s", actionName, resourceTypeID)) } schemas, ok := a.resourceSchemas[resourceTypeID] if !ok { a.mu.RUnlock() - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.Internal, fmt.Sprintf("schemas not found for resource type %s", resourceTypeID)) + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.Internal, fmt.Sprintf("schemas not found for resource type %s", resourceTypeID)) } schema, ok := schemas[actionName] if !ok { a.mu.RUnlock() - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.Internal, fmt.Sprintf("schema not found for action %s", actionName)) + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.Internal, fmt.Sprintf("schema not found for action %s", actionName)) } a.mu.RUnlock() - // Validate constraints + return a.invokeRegisteredAction(ctx, actionName, resourceTypeID, args, inlineWait, encryptionConfigs, schema, handler) +} + +func (a *ActionManager) invokeRegisteredAction( + ctx context.Context, + actionName string, + resourceTypeID string, + args *structpb.Struct, + inlineWait time.Duration, + encryptionConfigs []*v2.EncryptionConfig, + schema *v2.BatonActionSchema, + handler registeredActionHandler, +) (string, v2.BatonActionStatus, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations, error) { if err := validateActionConstraints(schema.GetConstraints(), args); err != nil { - return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, status.Error(codes.InvalidArgument, err.Error()) + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.InvalidArgument, err.Error()) + } + + var encryptionManager *crypto.EncryptionManager + if len(handler.secretReturnNames) > 0 { + if len(encryptionConfigs) == 0 { + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Error(codes.InvalidArgument, "at least one encryption config is required for secret action results") + } + encryptionConfigs = cloneEncryptionConfigs(encryptionConfigs) + if err := crypto.ValidateEncryptionConfigs(encryptionConfigs); err != nil { + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, err + } + var err error + encryptionManager, err = crypto.NewEncryptionManager(nil, encryptionConfigs) + if err != nil { + return "", v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED, nil, nil, nil, status.Errorf(codes.InvalidArgument, "create encryption manager: %v", err) + } } oa := a.GetNewAction(actionName) done := make(chan struct{}) - // Invoke handler in goroutine + // The handler runs detached. Return its final result if it finishes + // within the inline wait; otherwise return the in-flight status. go func() { // #nosec G118 -- action handlers intentionally outlive the request context and keep only trace/log metadata. defer close(done) defer func() { if r := recover(); r != nil { - ctxzap.Extract(ctx).Error("panic in resource action handler", - zap.String("resource_type", resourceTypeID), - zap.String("action", actionName), - zap.Any("panic", r), - zap.Stack("stack")) - oa.SetError(ctx, fmt.Errorf("panic in action handler: %v", r)) + if len(handler.secretReturnNames) > 0 { + ctxzap.Extract(ctx).Error("panic in action handler", + zap.String("resource_type", resourceTypeID), + zap.String("action", actionName), + zap.Stack("stack")) + oa.SetError(ctx, errors.New("panic in action handler")) + } else { + ctxzap.Extract(ctx).Error("panic in action handler", + zap.String("resource_type", resourceTypeID), + zap.String("action", actionName), + zap.Any("panic", r), + zap.Stack("stack")) + oa.SetError(ctx, fmt.Errorf("panic in action handler: %v", r)) + } } }() oa.SetStatus(ctx, v2.BatonActionStatus_BATON_ACTION_STATUS_RUNNING) @@ -761,8 +952,21 @@ func (a *ActionManager) invokeResourceAction( bgCtx = ctxzap.ToContext(bgCtx, ctxzap.Extract(ctx)) handlerCtx, cancel := context.WithTimeoutCause(bgCtx, 1*time.Hour, errors.New("action handler timed out")) defer cancel() - rv, annos, oaErr := handler(handlerCtx, args) - oa.setOutcome(ctx, rv, annos, oaErr) + result := handler.invoke(handlerCtx, args) + encryptedData, resultErr := prepareActionResult( + handlerCtx, + handler, + encryptionManager, + a.encryptPlaintext, + result.response, + result.plaintextData, + result.err == nil, + ) + if resultErr != nil { + result.response = nil + result.err = errors.Join(result.err, resultErr) + } + oa.setOutcomeWithEncryptedData(ctx, result.response, encryptedData, result.annotations, result.err) }() // Stop releases the timer deterministically when the handler wins the @@ -772,29 +976,127 @@ func (a *ActionManager) invokeResourceAction( select { case <-done: - id, st, rv, annos := oa.result() - return id, st, rv, annos, nil + id, st, rv, encryptedData, annos := oa.resultWithEncryptedData() + return id, st, rv, encryptedData, annos, nil case <-waitTimer.C: - id, st, rv, annos := oa.result() - return id, st, rv, annos, nil + id, st, rv, encryptedData, annos := oa.resultWithEncryptedData() + return id, st, rv, encryptedData, annos, nil case <-ctx.Done(): // The handler may have finished in the same instant; prefer its // completed result over a spurious cancellation return. select { case <-done: - id, st, rv, annos := oa.result() - return id, st, rv, annos, nil + id, st, rv, encryptedData, annos := oa.resultWithEncryptedData() + return id, st, rv, encryptedData, annos, nil default: } oa.setCancelled(ctx, ctx.Err()) - id, st, rv, annos := oa.result() + id, st, rv, encryptedData, annos := oa.resultWithEncryptedData() if st == v2.BatonActionStatus_BATON_ACTION_STATUS_COMPLETE { // The handler won the race to the lock; its completed result is // the authoritative pairing, not the cancellation. - return id, st, rv, annos, nil + return id, st, rv, encryptedData, annos, nil + } + return id, st, rv, encryptedData, annos, ctx.Err() + } +} + +func hasSecretReturnTypes(schema *v2.BatonActionSchema) bool { + if schema == nil { + return false + } + for _, field := range schema.GetReturnTypes() { + if field.GetIsSecret() { + return true + } + } + return false +} + +func validateSecretReturnTypes(returnTypes []*config.Field) (map[string]struct{}, []string, error) { + seen := make(map[string]struct{}, len(returnTypes)) + requiredNames := make([]string, 0) + for _, field := range returnTypes { + if field == nil || !field.GetIsSecret() { + continue + } + name := field.GetName() + if name == "" { + return nil, nil, errors.New("secret return type name cannot be empty") + } + if _, ok := seen[name]; ok { + return nil, nil, fmt.Errorf("duplicate secret return type %q", name) + } + seen[name] = struct{}{} + if field.GetIsRequired() { + requiredNames = append(requiredNames, name) + } + } + return seen, requiredNames, nil +} + +func cloneEncryptionConfigs(configs []*v2.EncryptionConfig) []*v2.EncryptionConfig { + cloned := make([]*v2.EncryptionConfig, len(configs)) + for i, config := range configs { + if config != nil { + cloned[i] = proto.Clone(config).(*v2.EncryptionConfig) + } + } + return cloned +} + +func prepareActionResult( + ctx context.Context, + handler registeredActionHandler, + encryptionManager *crypto.EncryptionManager, + encryptPlaintext actionEncryptFunc, + response *structpb.Struct, + plaintextData []*v2.PlaintextData, + encrypt bool, +) ([]*v2.EncryptedData, error) { + if len(handler.secretReturnNames) == 0 { + return nil, nil + } + + for name := range response.GetFields() { + if _, ok := handler.secretReturnNames[name]; ok { + return nil, fmt.Errorf("secret return value %q must not be included in the public response", name) + } + } + + seen := make(map[string]struct{}, len(plaintextData)) + for i, plaintext := range plaintextData { + if plaintext == nil || plaintext.GetName() == "" || len(plaintext.GetBytes()) == 0 { + return nil, fmt.Errorf("plaintext return value %d must have a name and non-empty bytes", i) + } + if _, ok := handler.secretReturnNames[plaintext.GetName()]; !ok { + return nil, fmt.Errorf("plaintext return value %q is not declared as a secret return type", plaintext.GetName()) + } + if _, ok := seen[plaintext.GetName()]; ok { + return nil, fmt.Errorf("duplicate plaintext return value %q", plaintext.GetName()) + } + seen[plaintext.GetName()] = struct{}{} + } + + if !encrypt { + return nil, nil + } + + for _, name := range handler.requiredSecretReturnNames { + if _, ok := seen[name]; !ok { + return nil, fmt.Errorf("required secret return type %q is missing", name) + } + } + + var encryptedData []*v2.EncryptedData + for _, plaintext := range plaintextData { + encrypted, err := encryptPlaintext(ctx, encryptionManager, plaintext) + if err != nil { + return nil, fmt.Errorf("encrypt action return value %q: %w", plaintext.GetName(), err) } - return id, st, rv, annos, ctx.Err() + encryptedData = append(encryptedData, encrypted...) } + return encryptedData, nil } // validateActionConstraints validates that the provided args satisfy the schema constraints. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/actions.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/actions.go index 5527b0e6..47522624 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/actions.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/actions.go @@ -7,6 +7,8 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/structpb" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" @@ -79,6 +81,20 @@ type ActionManager interface { HasActions() bool } +type encryptedActionManager interface { + InvokeActionWithWaitAndEncryption( + ctx context.Context, + name string, + resourceTypeID string, + args *structpb.Struct, + inlineWait time.Duration, + encryptionConfigs []*v2.EncryptionConfig, + ) (string, v2.BatonActionStatus, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations, error) + GetActionStatusWithEncryptedData(ctx context.Context, id string) (v2.BatonActionStatus, string, *structpb.Struct, []*v2.EncryptedData, annotations.Annotations, error) +} + +var _ encryptedActionManager = (*actions.ActionManager)(nil) + // GlobalActionProvider allows connectors to register global (non-resource-scoped) actions. // This is the preferred method for registering global actions in new connectors. // Implement this interface instead of the deprecated CustomActionManager or RegisterActionManagerLimited. @@ -191,18 +207,32 @@ func (b *builder) InvokeAction(ctx context.Context, request *v2.InvokeActionRequ resourceTypeID := request.GetResourceTypeId() - id, actionStatus, resp, annos, err := b.actionManager.InvokeActionWithWait(ctx, request.GetName(), resourceTypeID, request.GetArgs(), request.GetInlineWait().AsDuration()) + encryptedManager, ok := b.actionManager.(encryptedActionManager) + if !ok { + err = status.Error(codes.Internal, "action manager does not support encrypted action results") + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + id, actionStatus, resp, encryptedData, annos, err := encryptedManager.InvokeActionWithWaitAndEncryption( + ctx, + request.GetName(), + resourceTypeID, + request.GetArgs(), + request.GetInlineWait().AsDuration(), + request.GetEncryptionConfigs(), + ) if err != nil { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) return nil, fmt.Errorf("error: invoking action failed: %w", err) } rv := v2.InvokeActionResponse_builder{ - Id: id, - Name: request.GetName(), - Status: actionStatus, - Annotations: annos, - Response: resp, + Id: id, + Name: request.GetName(), + Status: actionStatus, + Annotations: annos, + Response: resp, + EncryptedData: encryptedData, }.Build() b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) @@ -217,18 +247,25 @@ func (b *builder) GetActionStatus(ctx context.Context, request *v2.GetActionStat start := b.nowFunc() tt := tasks.ActionStatusType - actionStatus, name, rv, annos, err := b.actionManager.GetActionStatus(ctx, request.GetId()) + encryptedManager, ok := b.actionManager.(encryptedActionManager) + if !ok { + err = status.Error(codes.Internal, "action manager does not support encrypted action results") + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + actionStatus, name, rv, encryptedData, annos, err := encryptedManager.GetActionStatusWithEncryptedData(ctx, request.GetId()) if err != nil { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) return nil, fmt.Errorf("error: action status for id %s not found: %w", request.GetId(), err) } resp := v2.GetActionStatusResponse_builder{ - Id: request.GetId(), - Name: name, - Status: actionStatus, - Annotations: annos, - Response: rv, + Id: request.GetId(), + Name: name, + Status: actionStatus, + Annotations: annos, + Response: rv, + EncryptedData: encryptedData, }.Build() b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) return resp, nil diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go index be4fee2e..93e04e3a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go @@ -359,9 +359,10 @@ type createAccountConfig struct { } type invokeActionConfig struct { - action string - resourceTypeID string // Optional: if set, invokes a resource-scoped action - args *structpb.Struct + action string + resourceTypeID string // Optional: if set, invokes a resource-scoped action + args *structpb.Struct + encryptionConfigs []*v2.EncryptionConfig } type listActionSchemasConfig struct { @@ -567,13 +568,26 @@ func WithOnDemandCreateAccount(c1zPath string, login string, email string, profi // WithOnDemandInvokeAction creates an option for invoking an action. // If resourceTypeID is provided, it invokes a resource-scoped action. func WithOnDemandInvokeAction(c1zPath string, action string, resourceTypeID string, args *structpb.Struct) Option { + return WithOnDemandInvokeActionWithEncryption(c1zPath, action, resourceTypeID, args, nil) +} + +// WithOnDemandInvokeActionWithEncryption creates an option for invoking an +// action with recipients for encrypted results. +func WithOnDemandInvokeActionWithEncryption( + c1zPath string, + action string, + resourceTypeID string, + args *structpb.Struct, + encryptionConfigs []*v2.EncryptionConfig, +) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.onDemand = true cfg.c1zPath = c1zPath cfg.invokeActionConfig = &invokeActionConfig{ - action: action, - resourceTypeID: resourceTypeID, - args: args, + action: action, + resourceTypeID: resourceTypeID, + args: args, + encryptionConfigs: encryptionConfigs, } return nil } @@ -1040,7 +1054,14 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op tm = local.NewCreateAccountManager(ctx, cfg.c1zPath, cfg.createAccountConfig.login, cfg.createAccountConfig.email, cfg.createAccountConfig.profile, cfg.createAccountConfig.resourceTypeID) case cfg.invokeActionConfig != nil: - tm = local.NewActionInvoker(ctx, cfg.c1zPath, cfg.invokeActionConfig.action, cfg.invokeActionConfig.resourceTypeID, cfg.invokeActionConfig.args) + tm = local.NewActionInvokerWithEncryption( + ctx, + cfg.c1zPath, + cfg.invokeActionConfig.action, + cfg.invokeActionConfig.resourceTypeID, + cfg.invokeActionConfig.args, + cfg.invokeActionConfig.encryptionConfigs, + ) case cfg.listActionSchemasConfig != nil: tm = local.NewListActionSchemas(ctx, cfg.listActionSchemasConfig.resourceTypeID) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go index 03cb7d76..d0991991 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/bulk_import.go @@ -30,23 +30,60 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb" ) -// ErrBulkImportOutOfOrder is returned by BulkSyncImport's ordered add -// methods when a primary key arrives that does not sort strictly after -// the previous key in the same bucket. It means the caller's -// sorted-source contract was violated; the import cannot continue and -// must be Abort()ed. +// ErrBulkImportOutOfOrder is returned when a key arrives at an ordered +// SST writer without sorting strictly after its predecessor. Callers can +// only trigger it through AddResourceTypes, the one add method with a +// sorted-arrival contract; resources, entitlements, and grants re-sort +// internally, and their duplicate keys surface at Finish as corrupt +// input (ErrBulkImportDuplicateKey) instead. The import cannot continue +// past it and must be Abort()ed. var ErrBulkImportOutOfOrder = errors.New("bulk sync import: keys are not strictly increasing") -// errBulkImportDuplicateKey is the spill-merge guard against duplicate -// keys. Duplicates are impossible for well-formed sources (the importer -// requires global uniqueness); hitting this means corrupt input. -var errBulkImportDuplicateKey = errors.New("bulk sync import: duplicate key in spill merge") - -// bulkSpillKeyChunkBytes bounds the in-memory arena of one spill chunk. -// Chunks are sorted and written as sorted runs in the background while -// records keep streaming in, then k-way merged into SSTs at Finish. +// ErrBulkImportDuplicateKey is returned when a spill merge finds two +// records with the same key — from BulkSyncImport.Finish, and from the +// engine's other spill merges (the deferred index and digest builds at +// EndSync, the synth-grant layer, the open-time id-index migration). +// Duplicates are impossible for well-formed sources (every producer +// requires global uniqueness), so hitting this means corrupt input, +// not a caller ordering bug. +var ErrBulkImportDuplicateKey = errors.New("bulk sync import: duplicate key in spill merge") + +// bulkSpillKeyChunkBytes sizes the shared spill arena pool +// (getSpillArena) and the deferred build's translate-batch arenas. The +// engine's single-producer spill sorters cut chunks at +// deferredIndexSpillChunkBytes with explicit freelists (see that +// constant for the sizing trade-off); the bulk import derives its chunk +// size from bulkImportSortBudgetBytes instead, because its sorter count +// scales with the caller's scan fan-out. const bulkSpillKeyChunkBytes = 8 << 20 +// bulkImportSortBudgetBytes bounds the anonymous memory the bulk import's +// spill arenas can hold at once. Chunk size is derived from it, not +// hard-coded: chunkBytes = budget / liveArenas, where liveArenas counts +// every arena that can be committed simultaneously — one per sorter +// (resources, entitlements, the parent index, and per grant shard one +// primary plus one per index family) plus one per background sort slot. +// Adding an index family or raising the shard count therefore shrinks +// the chunks instead of silently raising peak RSS. +// +// Chunk size sets Finish's merge fan-in: every chunk stays open behind a +// bulkSpillBufferSize reader for the whole merge, so the derived value is +// clamped to [bulkImportMinChunkBytes, deferredIndexSpillChunkBytes] — +// the floor caps the run count at half what 8MiB chunks produce (a +// whale-sized ~17GB grant family is ~1,100 runs at the floor versus +// ~2,200 at 8MiB; grants.go records the synth-layer incident), and the +// ceiling is the size the single-producer builds already validated. +// +// Worked examples with three grant index families and 4 sort slots: the +// sanitizer (1 shard, 11 arenas) derives ~93MiB; the converter at 4 lanes +// (23 arenas) ~44MiB; at 8 lanes (39 arenas) ~26MiB. The freelist +// recycles arenas within the same budget, so idle arenas never add to it. +const bulkImportSortBudgetBytes = 1 << 30 + +// bulkImportMinChunkBytes is the floor on the derived bulk-import chunk +// size; see bulkImportSortBudgetBytes. +const bulkImportMinChunkBytes = 16 << 20 + // bulkSpillBufferSize is the bufio size for spill-chunk IO. const bulkSpillBufferSize = 1 << 20 @@ -124,20 +161,21 @@ func (w *bulkSSTWriter) finish() error { // is written once into its final table — no WAL append, no L0 flush, no // background compaction debt. // -// Resource types and resources stream straight into one SST per bucket -// and must arrive in strictly increasing encoded-key order (SQLite BINARY -// collation order on the key's tuple columns — the tuple key codec is -// order-preserving; violations fail with ErrBulkImportOutOfOrder). These -// add methods are single-threaded. +// Resource types stream straight into one SST and must arrive in strictly +// increasing encoded-key order (SQLite BINARY collation order on the key's +// tuple columns — the tuple key codec is order-preserving; violations fail +// with ErrBulkImportOutOfOrder). AddResourceTypes is single-threaded. // -// Entitlements and grants are keyed by structural identity, whose tuple -// order does NOT match the converter's external-id scan order, so they go -// through spill sorters instead (entitlements single-threaded on the -// parent; grants scale across goroutines, each scanning goroutine taking -// its own shard via NewGrantShard so the grant hot path acquires no -// shared lock at all). The secondary index families are derived -// internally from the translated records — the same nil-guards and key -// shapes as the engine's canonical writeXxxIndexes paths — and are +// Every other bucket is re-sorted on the way in and imposes no ordering +// requirement on its caller. Resources are keyed (resource_type_id, +// resource_id); entitlements and grants are keyed by structural identity, +// whose tuple order does NOT match a converter's external-id scan order. +// All three go through spill sorters (resources and entitlements +// single-threaded on the parent; grants scale across goroutines, each +// scanning goroutine taking its own shard via NewGrantShard so the grant +// hot path acquires no shared lock at all). The secondary index families +// are derived internally from the translated records — the same nil-guards +// and key shapes as the engine's canonical writeXxxIndexes paths — and are // key-only spill-sorted. Spill chunks sort and flush to disk in the // background while the scans keep streaming; Finish k-way merges each // family's sorted runs (across all shards) into one SST per family, in @@ -157,19 +195,50 @@ type BulkSyncImport struct { done bool resourceTypes *bulkSSTWriter - resources *bulkSSTWriter - // entitlements go through a spill sorter, not an ordered SST writer: - // the converter scans in external-id order, but the structural identity - // key does not sort the same way (tuple separators sort below printable - // bytes, and the flag component reorders stripped vs opaque ids), so the - // stream must be re-sorted before it can become an SST. + // resources and entitlements go through spill sorters, not ordered SST + // writers, so neither imposes an ordering precondition on its caller. + // For entitlements the mismatch is unconditional: the structural key + // never sorts the way an external-id scan does (tuple separators sort + // below printable bytes, and the flag component reorders stripped vs + // opaque ids), so the stream must be re-sorted before it can become an + // SST. + // + // For resources, keyed (resource_type_id, resource_id), it depends on + // the producer. A converter scanning SQLite with ORDER BY on that tuple + // already arrives sorted and paid nothing for an ordered writer, but a + // producer that rewrites resource ids — the c1z sanitizer HMACs them — + // emits an order unrelated to the destination key. Sorting here keeps + // every producer on one path rather than making sortedness a + // precondition each one has to re-establish, and it holds when a single + // resource type is too large to sort in memory. + // + // Spill-sorting stages each family transiently at ~2x (sorted runs + // plus the SST merged from them; the runs' ranges overlap uniformly, + // so release-on-exhaustion frees little before the merge tail). The + // ordered writer resources used before staged ~1x but pushed + // sortedness onto every producer. + resources *spillSorter entitlements *spillSorter // sortSem bounds concurrently running background chunk sorts // across all sorters and shards. sortSem chan struct{} + // Every sorter in the import — resources, entitlements, the parent + // index, and each shard's grant and index sorters — cuts chunks at + // chunkBytes and recycles arenas through this import-wide freelist + // (see newSorter). chunkBytes is derived from + // bulkImportSortBudgetBytes and the caller's expected shard count so + // the arenas' aggregate stays under the budget: sorts in flight are + // capped by sortSem, and idle arenas are capped by the freelist, so + // the live set is exactly the arena count the derivation counted. + // (A fresh arena's pages also commit only as it fills, so small + // imports stay well under the budget; recycled arenas are fully + // committed, which is why the budget counts them all.) + chunkBytes int + arenaFree *spillArenaFreeList + // Parent-level sorter for the (single-threaded) resource index keys. idxResourceByParent *spillSorter @@ -206,7 +275,13 @@ type BulkSyncImport struct { // the engine's current FRESH sync (see BulkSyncImport contract). Working // files are staged in a fresh directory under tmpDir ("" = system temp // dir) and removed by Finish/Abort. -func (e *Engine) StartBulkSyncImport(ctx context.Context, syncID string, tmpDir string) (*BulkSyncImport, error) { +// +// expectedShards is the number of grant shards the caller intends to open +// via NewGrantShard (values < 1 are treated as 1). It is a sizing hint, +// not a limit: the import derives its spill chunk size from it so the +// arenas' aggregate stays within bulkImportSortBudgetBytes. Opening more +// shards than declared still works but raises peak memory proportionally. +func (e *Engine) StartBulkSyncImport(ctx context.Context, syncID string, tmpDir string, expectedShards int) (*BulkSyncImport, error) { if !e.IsFreshSync() { return nil, errors.New("StartBulkSyncImport: sync is not fresh") } @@ -231,29 +306,45 @@ func (e *Engine) StartBulkSyncImport(ctx context.Context, syncID string, tmpDir syncID: syncID, dir: dir, sortSem: make(chan struct{}, sorters), + chunkBytes: bulkImportChunkBytes(expectedShards, sorters), resourcesByRT: map[string]int64{}, entitlementsByRT: map[string]int64{}, grantDupRowsByEntRT: map[string]int64{}, } - for _, w := range []struct { - slot **bulkSSTWriter - name string - }{ - {&b.resourceTypes, "resource-types"}, - {&b.resources, "resources"}, - } { - sw, err := newBulkSSTWriter(e.fs(), dir, w.name) - if err != nil { - b.Abort() - return nil, err - } - *w.slot = sw + b.arenaFree = newSpillArenaFreeList(b.chunkBytes, sorters+2) + sw, err := newBulkSSTWriter(e.fs(), dir, "resource-types") + if err != nil { + b.Abort() + return nil, err } - b.entitlements = newSpillSorter(dir, "entitlements", b.sortSem, bulkSpillKeyChunkBytes) - b.idxResourceByParent = newSpillSorter(dir, fmt.Sprintf("index-%02x-p", idxResourceByParent), b.sortSem, bulkSpillKeyChunkBytes) + b.resourceTypes = sw + b.resources = b.newSorter("resources") + b.entitlements = b.newSorter("entitlements") + b.idxResourceByParent = b.newSorter(fmt.Sprintf("index-%02x-p", idxResourceByParent)) return b, nil } +// bulkImportChunkBytes derives the spill chunk size that keeps the +// import's simultaneously-live arenas within bulkImportSortBudgetBytes. +// The live set is one arena per sorter — the three lane-independent +// sorters plus, per grant shard, one primary and one per index family — +// plus one per background sort slot (a sorter that just cut a chunk +// fills a fresh arena while the cut one sorts). See +// bulkImportSortBudgetBytes for the clamp rationale. +func bulkImportChunkBytes(expectedShards, sortSlots int) int { + expectedShards = max(1, expectedShards) + liveArenas := 3 + (1+len(grantIndexFamilies))*expectedShards + sortSlots + return min(deferredIndexSpillChunkBytes, max(bulkImportMinChunkBytes, bulkImportSortBudgetBytes/liveArenas)) +} + +// newSorter creates a spill sorter wired to the import's shared sort +// semaphore and arena freelist (see arenaFree for the sizing rationale). +func (b *BulkSyncImport) newSorter(name string) *spillSorter { + s := newSpillSorter(b.dir, name, b.sortSem, b.chunkBytes) + s.free = b.arenaFree + return s +} + // BulkGrantShard is one goroutine's private view of the grant import: // a primary-record spill sorter and per-family index sorters. Create // one per scanning goroutine via NewGrantShard; AddGrants on a shard @@ -287,12 +378,12 @@ func (b *BulkSyncImport) NewGrantShard() (*BulkGrantShard, error) { b.shardSeq++ s := &BulkGrantShard{ b: b, - grants: newSpillSorter(b.dir, fmt.Sprintf("grants-s%03d", id), b.sortSem, bulkSpillKeyChunkBytes), + grants: b.newSorter(fmt.Sprintf("grants-s%03d", id)), idx: map[byte]*spillSorter{}, entRT: map[string]int64{}, } for _, idx := range grantIndexFamilies { - s.idx[idx] = newSpillSorter(b.dir, fmt.Sprintf("index-%02x-s%03d", idx, id), b.sortSem, bulkSpillKeyChunkBytes) + s.idx[idx] = b.newSorter(fmt.Sprintf("index-%02x-s%03d", idx, id)) } b.shards = append(b.shards, s) return s, nil @@ -423,9 +514,12 @@ func (b *BulkSyncImport) AddResourceTypesWithDiscoveredAt(ctx context.Context, r return nil } -// AddResources translates and appends resources, which must arrive -// sorted by (resource_type_id, resource_id). by_parent index keys are -// derived and spilled with the same parent guard as writeResourceIndexes. +// AddResources translates and appends resources. Rows are keyed by +// (resource_type_id, resource_id) and re-sorted through a spill sorter, so +// arrival order does not matter; a duplicate key is corrupt input and +// surfaces at Finish from the spill merge rather than here. by_parent +// index keys are derived and spilled with the same parent guard as +// writeResourceIndexes. func (b *BulkSyncImport) AddResources(ctx context.Context, resources ...*v2.Resource) error { return b.AddResourcesWithDiscoveredAt(ctx, resources, nil) } @@ -540,7 +634,7 @@ func (b *BulkSyncImport) ComputedStats() *v3.SyncStatsRecord { rec := &v3.SyncStatsRecord{ SyncId: b.syncID, ResourceTypes: int64(b.resourceTypes.count), - Resources: int64(b.resources.count), + Resources: b.resources.count, Entitlements: b.entitlements.count, Grants: grants, ResourcesByResourceType: b.resourcesByRT, @@ -563,24 +657,21 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { } b.done = true // Full teardown, not a bare RemoveAll: Finish's error paths can return - // with the ordered SST writers still open (finish() failing on one - // leaves the other's file handle live) and with background chunk sorts - // still writing into the staging dir (nothing finalized the sorters - // yet). Removing the dir while a sort races its os.Create can strand - // the dir on disk, and Abort is a no-op once done is set — so this - // defer must do the closing and waiting itself. On success everything - // is already finished/finalized and teardown reduces to the RemoveAll. + // with background chunk sorts still writing into the staging dir + // (nothing finalized the sorters yet), and removing the dir while a + // sort races its os.Create can strand the dir on disk. Abort is a + // no-op once done is set, so this defer must do the waiting itself. + // On success everything is already finished/finalized and teardown + // reduces to the RemoveAll. defer b.teardown() start := time.Now() paths := make([]string, 0, 4+len(grantIndexFamilies)) - for _, w := range []*bulkSSTWriter{b.resourceTypes, b.resources} { - if err := w.finish(); err != nil { - return err - } - if w.count > 0 { - paths = append(paths, w.path) - } + if err := b.resourceTypes.finish(); err != nil { + return err + } + if b.resourceTypes.count > 0 { + paths = append(paths, b.resourceTypes.path) } b.mu.Lock() @@ -607,6 +698,7 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { } units := []mergeUnit{ {name: "grants", resolve: b.resolveDuplicateGrants}, + {name: "resources", sorters: []*spillSorter{b.resources}}, {name: "entitlements", sorters: []*spillSorter{b.entitlements}}, {name: fmt.Sprintf("index-%02x", idxResourceByParent), sorters: []*spillSorter{b.idxResourceByParent}}, } @@ -753,17 +845,15 @@ func (b *BulkSyncImport) Abort() { b.teardown() } -// teardown closes both ordered SST writers, waits out every spill +// teardown closes the ordered SST writer, waits out every spill // sorter's in-flight background chunk sorts, and then removes the // staging directory. The waits must precede the RemoveAll: a chunk // sort racing the removal can re-create a file mid-walk and strand the -// directory. Idempotent against already-finished writers and +// directory. Idempotent against an already-finished writer and // already-finalized sorters, so Finish can run it unconditionally. func (b *BulkSyncImport) teardown() { - for _, w := range []*bulkSSTWriter{b.resourceTypes, b.resources} { - if w != nil { - _ = w.finish() - } + if b.resourceTypes != nil { + _ = b.resourceTypes.finish() } b.mu.Lock() shards := b.shards @@ -775,7 +865,7 @@ func (b *BulkSyncImport) teardown() { w.abort() } } - for _, w := range []*spillSorter{b.entitlements, b.idxResourceByParent} { + for _, w := range []*spillSorter{b.resources, b.entitlements, b.idxResourceByParent} { if w != nil { w.abort() } @@ -902,6 +992,17 @@ func (s *spillSorter) add(key, val []byte) error { if err := s.takeErr(); err != nil { return err } + // Cut BEFORE an entry would overflow the arena's capacity, not after. + // Appending past cap makes Go reallocate the arena at ~1.25x with a + // full copy — a 128MiB memcpy per fresh arena — and the freelist then + // keeps the oversized copy, so every recycled arena would carry that + // growth for the rest of the build. Cutting early keeps arenas at + // exactly their allocated size and makes the budget arithmetic in + // bulkImportSortBudgetBytes true. (An arena holding nothing yet is + // left to grow: a single entry larger than a chunk is legal.) + if len(s.views) > 0 && len(s.arena)+len(key)+len(val) > cap(s.arena) { + s.cutAndDispatch() + } if s.arena == nil { if s.free != nil { s.arena = s.free.get() @@ -1126,6 +1227,91 @@ func readSpillEntry(r io.Reader, key, val *[]byte, lenBuf *[4]byte) (bool, error return true, nil } +// spillChunkCursors owns the open chunk files for one merge pass and +// unlinks each chunk as soon as its last entry has been read. The chunks +// and the SST being written hold the same entries, so a merge that keeps +// every chunk until teardown needs staging space for both copies at once; +// releasing at exhaustion bounds the overlap to the chunks still in +// flight. All of the engine's k-way merges read their chunks through this +// type (advance is readSpillEntry's only caller), so the bound holds for +// every spill merge, not just the bulk import's. +// +// Unlinking is deliberately confined to the exhausted-chunk path, where +// the file is provably fully consumed. A merge that fails partway closes +// its remaining descriptors and leaves those files for the staging-dir +// teardown that already owns them. +type spillChunkCursors struct { + paths []string + files []*os.File + bufs []*bufio.Reader + keyBufs [][]byte + valBufs [][]byte + lenBuf [4]byte +} + +func openSpillChunks(chunks []string) (*spillChunkCursors, error) { + c := &spillChunkCursors{ + paths: chunks, + files: make([]*os.File, len(chunks)), + bufs: make([]*bufio.Reader, len(chunks)), + keyBufs: make([][]byte, len(chunks)), + valBufs: make([][]byte, len(chunks)), + } + for i, chunk := range chunks { + f, err := os.Open(chunk) // #nosec G304 - engine-private scratch, staged under the caller's MkdirTemp dir. + if err != nil { + c.closeAll() + return nil, err + } + c.files[i] = f + c.bufs[i] = bufio.NewReaderSize(f, bulkSpillBufferSize) + } + return c, nil +} + +// advance reads the next entry of chunk i into that chunk's reusable +// buffers, reachable via key/val. It reports false once the chunk is +// exhausted, having already closed and unlinked it. +// +// Advancing an already-exhausted chunk keeps reporting false rather than +// faulting on the released reader. The merges re-push a chunk onto the +// heap only after a true, so none of them reach this today; the guard +// preserves the EOF idempotency of the inline readSpillEntry calls this +// replaced. +func (c *spillChunkCursors) advance(i int) (bool, error) { + if c.bufs[i] == nil { + return false, nil + } + ok, err := readSpillEntry(c.bufs[i], &c.keyBufs[i], &c.valBufs[i], &c.lenBuf) + if err != nil { + return false, err + } + if !ok { + c.closeChunk(i) + _ = os.Remove(c.paths[i]) + } + return ok, nil +} + +func (c *spillChunkCursors) key(i int) []byte { return c.keyBufs[i] } +func (c *spillChunkCursors) val(i int) []byte { return c.valBufs[i] } + +func (c *spillChunkCursors) closeChunk(i int) { + if c.files[i] == nil { + return + } + _ = c.files[i].Close() + c.files[i] = nil + c.bufs[i] = nil +} + +// closeAll releases descriptors without unlinking; see the type comment. +func (c *spillChunkCursors) closeAll() { + for i := range c.files { + c.closeChunk(i) + } +} + // mergeSortedSpillChunksToSST heap-merges the sorted chunk files into a // single SST. Duplicate keys are corruption (the importer requires // globally unique tuples) and fail the merge. fs is the engine FS the @@ -1133,30 +1319,19 @@ func readSpillEntry(r io.Reader, key, val *[]byte, lenBuf *[4]byte) (bool, error func mergeSortedSpillChunksToSST(ctx context.Context, fs vfs.FS, sstPath, name string, chunks []string) error { start := time.Now() l := ctxzap.Extract(ctx) - readers := make([]*os.File, 0, len(chunks)) - defer func() { - for _, r := range readers { - _ = r.Close() - } - }() - bufReaders := make([]*bufio.Reader, len(chunks)) - keyBufs := make([][]byte, len(chunks)) - valBufs := make([][]byte, len(chunks)) + cursors, err := openSpillChunks(chunks) + if err != nil { + return err + } + defer cursors.closeAll() h := &spillChunkHeap{} - var lenBuf [4]byte - for i, chunk := range chunks { - f, err := os.Open(chunk) // #nosec G304 - staged under the import's MkdirTemp dir. - if err != nil { - return err - } - readers = append(readers, f) - bufReaders[i] = bufio.NewReaderSize(f, bulkSpillBufferSize) - ok, err := readSpillEntry(bufReaders[i], &keyBufs[i], &valBufs[i], &lenBuf) + for i := range chunks { + ok, err := cursors.advance(i) if err != nil { return err } if ok { - h.push(spillChunkItem{chunkIdx: i, key: keyBufs[i], val: valBufs[i]}) + h.push(spillChunkItem{chunkIdx: i, key: cursors.key(i), val: cursors.val(i)}) } } @@ -1177,7 +1352,7 @@ func mergeSortedSpillChunksToSST(ctx context.Context, fs vfs.FS, sstPath, name s for len(*h) > 0 { item := h.pop() if bytes.Equal(item.key, last) { - return fmt.Errorf("%w: bucket %s key %x", errBulkImportDuplicateKey, name, item.key) + return fmt.Errorf("%w: bucket %s key %x", ErrBulkImportDuplicateKey, name, item.key) } var v []byte if len(item.val) > 0 { @@ -1205,12 +1380,12 @@ func mergeSortedSpillChunksToSST(ctx context.Context, fs vfs.FS, sstPath, name s } } last = append(last[:0], item.key...) - ok, err := readSpillEntry(bufReaders[item.chunkIdx], &keyBufs[item.chunkIdx], &valBufs[item.chunkIdx], &lenBuf) + ok, err := cursors.advance(item.chunkIdx) if err != nil { return err } if ok { - h.push(spillChunkItem{chunkIdx: item.chunkIdx, key: keyBufs[item.chunkIdx], val: valBufs[item.chunkIdx]}) + h.push(spillChunkItem{chunkIdx: item.chunkIdx, key: cursors.key(item.chunkIdx), val: cursors.val(item.chunkIdx)}) } } if err := writer.finish(); err != nil { @@ -1247,30 +1422,19 @@ func mergeSpillChunksToSSTResolvingDuplicates( ) (int64, error) { start := time.Now() l := ctxzap.Extract(ctx) - readers := make([]*os.File, 0, len(chunks)) - defer func() { - for _, r := range readers { - _ = r.Close() - } - }() - bufReaders := make([]*bufio.Reader, len(chunks)) - keyBufs := make([][]byte, len(chunks)) - valBufs := make([][]byte, len(chunks)) + cursors, err := openSpillChunks(chunks) + if err != nil { + return 0, err + } + defer cursors.closeAll() h := &spillChunkHeap{} - var lenBuf [4]byte - for i, chunk := range chunks { - f, err := os.Open(chunk) // #nosec G304 - staged under the import's MkdirTemp dir. - if err != nil { - return 0, err - } - readers = append(readers, f) - bufReaders[i] = bufio.NewReaderSize(f, bulkSpillBufferSize) - ok, err := readSpillEntry(bufReaders[i], &keyBufs[i], &valBufs[i], &lenBuf) + for i := range chunks { + ok, err := cursors.advance(i) if err != nil { return 0, err } if ok { - h.push(spillChunkItem{chunkIdx: i, key: keyBufs[i], val: valBufs[i]}) + h.push(spillChunkItem{chunkIdx: i, key: cursors.key(i), val: cursors.val(i)}) } } @@ -1359,12 +1523,12 @@ func mergeSpillChunksToSSTResolvingDuplicates( // Advance the popped chunk only AFTER the entry was consumed into // the group scratch: readSpillEntry overwrites the buffers item // aliases. - ok, err := readSpillEntry(bufReaders[item.chunkIdx], &keyBufs[item.chunkIdx], &valBufs[item.chunkIdx], &lenBuf) + ok, err := cursors.advance(item.chunkIdx) if err != nil { return dupGroups, err } if ok { - h.push(spillChunkItem{chunkIdx: item.chunkIdx, key: keyBufs[item.chunkIdx], val: valBufs[item.chunkIdx]}) + h.push(spillChunkItem{chunkIdx: item.chunkIdx, key: cursors.key(item.chunkIdx), val: cursors.val(item.chunkIdx)}) } } if err := flushCur(); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/deferred_index.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/deferred_index.go index 4baad169..3176184f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/deferred_index.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/deferred_index.go @@ -18,14 +18,19 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb" ) -// deferredIndexSpillChunkBytes is the spill-chunk arena size for the deferred -// index build's sorters. The shared bulkSpillKeyChunkBytes (8MiB) is sized -// for the bulk import, where lanes × index-families sorters are alive at once -// and small arenas bound aggregate memory. The deferred build is the opposite -// shape — one producer, at most two sorter families, nothing else running — -// and with 8MiB chunks a whale (57M+ index keys ≈ 6.4GB) produced an 801-way -// final merge: ~10 heap comparisons per entry plus 801 open chunk files with -// 1MiB readers (~800MB of buffers). 128MiB chunks cut that to ~50 runs. +// deferredIndexSpillChunkBytes is the spill-chunk arena size used by the +// engine's single-producer spill sorters (the deferred build, the synth +// layer, the digest build, the id-index migration), and the ceiling for +// the bulk import's budget-derived size (bulkImportSortBudgetBytes — its +// sorter count scales with the caller's fan-out, so it cannot use a fixed +// size). Chunk size sets the final merge's fan-in — every chunk stays +// open behind a 1MiB reader for the whole merge — and with 8MiB chunks a +// whale (57M+ index keys ≈ 6.4GB) produced an 801-way final merge: ~10 +// heap comparisons per entry plus 801 open chunk files (~800MB of +// buffers). 128MiB chunks cut that to ~50 runs. Large arenas stay +// affordable here because each of these builds has one or two producers, +// pairs them with a bounded spillArenaFreeList and a sort semaphore, and +// a fresh arena's pages commit only as it fills. // // Memory budget (revised for the second family): with the grant digest // index enabled the scan feeds TWO sorter families — by_principal diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/digest.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/digest.go index d05951a1..f72b0c5e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/digest.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/digest.go @@ -494,12 +494,10 @@ type DigestRoot struct { // computeBucketDigest would read that absence as "zero records" — the // false-clean trap dirtyPartitionBuckets' doc comment describes. func (e *Engine) getPartitionDigestRoot(spec digestIndexSpec, partition string) (DigestRoot, bool, error) { - if e.grantDigestBuildPending.Load() { - // An interrupted digest build's half-committed nodes may be - // durable while its hash index never ingested; until the pending - // state is consumed (a writable Open drops it; a read-only open - // cannot), no stored root may be trusted — report "never built", - // which every consumer already treats as "recalculate". + if e.grantDigestStateUntrusted() { + // See grantDigestStateUntrusted: no stored root may be trusted + // while either flag is set — report "never built", which every + // consumer already treats as "recalculate". return DigestRoot{}, false, nil } val, closer, err := e.db.Get(encodeDigestNodeKey(spec.indexID, partition, digestLevelRoot, nil)) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go index acf267ee..093a08f1 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go @@ -133,6 +133,17 @@ type Engine struct { // built" instead of trusting nodes a crashed build half-committed. grantDigestBuildPending atomic.Bool + // grantDigestAbiStale is the read-only-open counterpart of the ABI + // check in verifyGrantDigestABI: true when the file holds digest + // nodes whose stamp (rawdb.GrantDigestABIStampKey) does not name + // the current GrantDigestABIVersion — state built by different hash + // code, e.g. a file sealed by an older SDK. A writable Open drops + // such state instead of setting this, so on a writable engine it is + // always false; on a read-only engine it makes the digest root + // getters report "never built" (the same fail-safe shape as + // grantDigestBuildPending above), and consumers recalculate. + grantDigestAbiStale atomic.Bool + // test holds every test-only injection seam, sequestered on one // field so hooks don't accumulate on the production struct. All // zero in production; see testSeams (test_seams.go). @@ -307,6 +318,18 @@ func Open(ctx context.Context, dir string, opts ...Option) (*Engine, error) { _ = e.Close() return nil, err } + // Enforce the digest ABI contract: digest nodes not certified by a + // stamp naming the CURRENT GrantDigestABIVersion were computed by + // different hash code and must never be trusted or extended — a + // writable open drops them wholesale (the next EndSync's existing + // digests-absent path rebuilds everything at the current ABI); a + // read-only open flags them so the root getters report "never + // built". Runs after the probe so it sees post-marker-recovery + // presence, and its own drop re-falses the flag. + if err := e.verifyGrantDigestABI(ctx, o.readOnly); err != nil { + _ = e.Close() + return nil, err + } // Arm the mutation-path source-scope index obligations iff the file // actually holds by_source_scope entries (bounded seeks, same // contract as the digest probe): scope-free stores keep the exact @@ -572,6 +595,33 @@ func (e *Engine) IsFreshSync() bool { // See WithGrantDigestIndex. func (e *Engine) GrantDigestIndexEnabled() bool { return e.opts.grantDigestIndex } +// GrantDigestsPresent reports whether this engine currently holds ANY +// grant-digest state (nodes + the by_entitlement_principal_hash index +// beneath them) — the same Open-probed flag the record write paths +// gate their per-write invalidation obligation on. Exported for +// callers outside this package that need to tell "no digest state at +// all" apart from "digest state present but stale/invalidated" (e.g. +// the compactor's fold, deciding whether a byte-copied base needs a +// one-time digest build). +func (e *Engine) GrantDigestsPresent() bool { return e.db.GrantDigestsPresent() } + +// grantDigestStateUntrusted reports whether NO stored grant-digest +// state — digest nodes, the whole-file root, or the +// by_entitlement_principal_hash index beneath them — may be trusted +// right now. Both flags it OR's together mean this by construction: +// grantDigestBuildPending means an interrupted build may have left +// digest nodes durable while the hash index under them never finished +// ingesting; grantDigestAbiStale means the nodes and hash-index +// content hashes were computed by a different hash ABI (a read-only +// open of a file whose stamp doesn't name the current +// GrantDigestABIVersion). Either way, every getter and on-demand fold +// over that state must report "not built" rather than trust or +// recompute from it — see getPartitionDigestRoot, +// GetGrantDigestGlobalRoot, and ComputeEntitlementBucketDigest. +func (e *Engine) grantDigestStateUntrusted() bool { + return e.grantDigestBuildPending.Load() || e.grantDigestAbiStale.Load() +} + // takeFreshGrantsEmpty / takeFreshResourcesEmpty return true // exactly once per fresh sync, for the first PutXxxRecords call // of that type after diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go index c13e4eb5..f395eadf 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go @@ -9,6 +9,8 @@ import ( "github.com/cespare/xxhash/v2" "github.com/cockroachdb/pebble/v2" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" @@ -55,13 +57,102 @@ var grantDigestSpec = digestIndexSpec{ // ABI bump (a change to either hash's input framing) makes stored // manifest roots computed under different versions incomparable by // construction, rather than silently comparing unrelated hash schemes. -// Bump alongside any index-migration version bump that touches these -// hashes (see index_migrations.go). +// +// Enforcement on the stored state itself is the durable ABI stamp +// (rawdb.GrantDigestABIStampKey), written alongside every global-root +// write and checked once per Open (verifyGrantDigestABI): digest state +// whose stamp does not name this constant is dropped (writable open) +// or reported "never built" (read-only open), so a bump here is +// sufficient by itself to force every previously-sealed file's digest +// state to be rebuilt in full at the new ABI on its next writable use. +// A file with digest nodes but NO stamp was sealed by an SDK that +// predates the stamp; those builds all hashed at version 1, so absence +// reads as grantDigestABIVersionUnstamped and is current for as long +// as this constant stays 1 — introducing the stamp costs no rebuild. +// No index-migration entry is needed — see the note on digest-ABI +// handling in index_migrations.go. // // Exported so consumers of GrantContentHash / GrantDigestAccumulator // can check a stored root's abi_version before comparing. const GrantDigestABIVersion uint32 = 1 +// grantDigestABIVersionUnstamped is the ABI version a file with digest +// nodes but no stamp key is read as: every SDK build that predates the +// stamp hashed at version 1. Fixed forever — it describes shipped +// history, not the current ABI, and must not move when +// GrantDigestABIVersion does. +const grantDigestABIVersionUnstamped uint32 = 1 + +// grantDigestABIStampValue is the ABI stamp's stored value: the +// current GrantDigestABIVersion, uint32 BE (the index-migration +// applied-version encoding). +func grantDigestABIStampValue() []byte { + var buf [4]byte + binary.BigEndian.PutUint32(buf[:], GrantDigestABIVersion) + return buf[:] +} + +// verifyGrantDigestABI is the Open-time half of the ABI stamp contract +// (rawdb.GrantDigestABIStampKey; the write half is every global-root +// write site). If the file holds digest nodes (per the just-probed +// presence flag) whose stamped ABI version (readGrantDigestABIStamp) +// is not the current GrantDigestABIVersion, that state was computed by +// different hash code: a writable open restores the always-safe +// "digests absent" state — the next EndSync's existing digests-absent +// path (RepairMissingGrantDigests delegating to BuildGrantDigests) +// then rebuilds everything, hash rows and nodes and manifest root +// alike, at the current ABI. A read-only open cannot drop; it sets +// grantDigestAbiStale, which makes the digest root getters report +// "never built" (present-means-exact consumers recalculate — never a +// wrong answer, mirroring grantDigestBuildPending). +// +// A stale or orphaned stamp over an EMPTY node keyspace is left alone: +// with no nodes there is nothing to trust, and every build rewrites the +// stamp on its completion side (the fold's opening DeleteRange erases +// it first). +func (e *Engine) verifyGrantDigestABI(ctx context.Context, readOnly bool) error { + if !e.db.GrantDigestsPresent() { + return nil + } + stamped, err := e.readGrantDigestABIStamp() + if err != nil { + return err + } + if stamped == GrantDigestABIVersion { + return nil + } + if readOnly { + e.grantDigestAbiStale.Store(true) + return nil + } + ctxzap.Extract(ctx).Warn("pebble: grant digest state was built under a different hash ABI; dropping it — the next EndSync rebuilds it from scratch", + zap.Uint32("stamped_abi", stamped), + zap.Uint32("current_abi", GrantDigestABIVersion)) + return e.dropAllGrantDigestStateLocked() +} + +// readGrantDigestABIStamp returns the ABI version the file's digest +// state is stamped with. A missing stamp key reads as +// grantDigestABIVersionUnstamped (the pre-stamp SDKs all hashed at +// version 1); a malformed value reads as 0, which no real ABI version +// is, so it can never pass as current. Only meaningful when digest +// nodes are present — with none, there is no state for the stamp to +// describe. +func (e *Engine) readGrantDigestABIStamp() (uint32, error) { + val, closer, err := e.db.Get(rawdb.GrantDigestABIStampKey()) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return grantDigestABIVersionUnstamped, nil + } + return 0, err + } + defer closer.Close() + if len(val) != 4 { + return 0, nil + } + return binary.BigEndian.Uint32(val), nil +} + // The whole-file grant digest root's node-key level lives in // internal/keys (rawdb.DigestLevelGlobalRoot, consumed by // rawdb.GlobalGrantDigestNodeKey): the XOR fold of every @@ -85,7 +176,8 @@ func digestPartitionForEntitlement(id entitlementIdentity) string { // ABI: the two hash definitions below are part of the stored format. // Two SDK builds must hash identical grants identically or the digest // comparison reads "everything differs"; changing either input framing -// requires an index-migration bump (index_migrations.go). +// requires a GrantDigestABIVersion bump (which the durable ABI stamp +// then enforces at Open — no index migration is involved). // grantPrincipalBucketHash64 is the bucket address for a principal: // xxHash64 over the ENCODED principal segments @@ -110,7 +202,7 @@ func grantPrincipalBucketHash64(encodedPrincipalSegments []byte) uint64 { // ScanEntitlementGrantBucket). // // ABI: the stored truncation width, pinned to GrantDigestABIVersion. It may -// only grow, and only under an index-migration bump — which is why it is a +// only grow, and only under a GrantDigestABIVersion bump — which is why it is a // named constant rather than a literal in PrincipalBucketHash's signature: // widening the addressable bucket space must not change that signature. const DigestBucketHashBits = digestBucketHashLen * 8 @@ -146,7 +238,7 @@ const DigestBucketHashBits = digestBucketHashLen * 8 // // ABI: pinned to GrantDigestABIVersion alongside GrantContentHash. Two // SDK builds must place the same principal in the same bucket, so the -// input framing changes only under an index-migration bump. +// input framing changes only under a GrantDigestABIVersion bump. func PrincipalBucketHash(principalRT, principalID string) uint64 { enc := codec.AppendTupleStrings(make([]byte, 0, 64), principalRT, principalID) return grantPrincipalBucketHash64(enc) @@ -437,10 +529,12 @@ func (e *Engine) GetEntitlementDigestRoot(ctx context.Context, id entitlementIde // invalidation paths that drop any per-entitlement root — see // stageGrantDigestInvalidation and the Drop* functions below. func (e *Engine) GetGrantDigestGlobalRoot(ctx context.Context) (DigestRoot, bool, error) { - if e.grantDigestBuildPending.Load() { + if e.grantDigestStateUntrusted() { // Same guard as getPartitionDigestRoot: a global root committed // by an interrupted build must read as absent, not certify a - // hash index that was never ingested. + // hash index that was never ingested — and one computed under a + // different hash ABI (read-only open of an old file) must read + // as absent rather than compare hashes from another scheme. return DigestRoot{}, false, nil } val, closer, err := e.db.Get(rawdb.GlobalGrantDigestNodeKey()) @@ -470,7 +564,18 @@ func (e *Engine) GetGrantDigestGlobalRoot(ctx context.Context) (DigestRoot, bool // absent index range and returns {0, 0} — "zero grants", not "unknown". // Never use it as a fallback for a missing root; see // GetEntitlementDigestRoot and computeBucketDigest's precondition. +// +// Gated on grantDigestStateUntrusted: while either flag is set, the +// hash index this folds may be half-built (grantDigestBuildPending) or +// hold content hashes from a different ABI (grantDigestAbiStale), so +// folding it directly — unlike getPartitionDigestRoot, this method has +// no stored-root check of its own to lean on — would return digests +// derived from untrustworthy content. Report the same {0, 0} "not +// built / absent" shape a never-built partition already produces. func (e *Engine) ComputeEntitlementBucketDigest(ctx context.Context, id entitlementIdentity, bucket DigestBucket) ([]byte, int64, error) { + if e.grantDigestStateUntrusted() { + return make([]byte, hashLen), 0, nil + } return e.computeBucketDigest(ctx, grantDigestSpec, digestPartitionForEntitlement(id), bucket) } @@ -489,6 +594,17 @@ func (e *Engine) DirtyEntitlementBuckets(ctx context.Context, other *Engine, id // The primary key is reconstructed from each index key by byte splice // (no decode); the point Get per entry is the cost of MATERIALIZING a // changed grant, not of finding it. Orphan index entries are skipped. +// +// Deliberately NOT gated by grantDigestStateUntrusted, unlike +// getPartitionDigestRoot / GetGrantDigestGlobalRoot / +// ComputeEntitlementBucketDigest: a bucket's MEMBERSHIP is the +// principal bucket hash over the encoded principal identity, frozen by +// the v3 key encoding and untouched by any GrantDigestABIVersion bump +// (only the stored CONTENT hashes and digest nodes are ABI-dependent — +// see grantContentHash64 vs grantPrincipalBucketHash64). So a stale +// file's bucket placement is still exact, and this keeps yielding the +// grants a caller already knows to be dirty even on a read-only open +// over a stale ABI stamp. func (e *Engine) IterateGrantsByEntitlementBucket(ctx context.Context, id entitlementIdentity, bucket DigestBucket, yield func(*v3.GrantRecord) bool) error { lower, upper := grantDigestSpec.bucketBounds(digestPartitionForEntitlement(id), bucket) iter, err := e.db.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_build.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_build.go index 8043f53b..5ebc5d3e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_build.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_build.go @@ -1,13 +1,11 @@ package pebble import ( - "bufio" "bytes" "context" "encoding/binary" "errors" "fmt" - "os" "path/filepath" "runtime" "time" @@ -285,15 +283,20 @@ func (f *grantDigestFold) closePartition() error { // finish closes the last partition, writes the whole-file global root // (the fold of every partition this build touched — see globalXor/ -// globalTotal), and commits the tail batch. The global root lands in -// the same final batch as the last partition's nodes, so it is never -// visible without them: a crash between batches can only leave the -// global root ABSENT, never present ahead of a partition it should -// have folded in. +// globalTotal) plus the ABI stamp certifying which hash version +// computed it (rawdb.GrantDigestABIStampKey — the fold's opening +// DeleteRange erased any prior stamp), and commits the tail batch. The +// global root and stamp land in the same final batch as the last +// partition's nodes, so neither is ever visible without them: a crash +// between batches can only leave them ABSENT, never present ahead of a +// partition the root should have folded in. func (f *grantDigestFold) finish() error { if err := f.closePartition(); err != nil { return err } + if err := f.batch.Set(rawdb.GrantDigestABIStampKey(), grantDigestABIStampValue()); err != nil { + return err + } if err := f.batch.Set(rawdb.GlobalGrantDigestNodeKey(), packDigestLeaf(f.globalTotal, f.globalXor[:])); err != nil { return err } @@ -345,30 +348,19 @@ func splitGrantHashIndexKey(key []byte) ([]byte, uint16, bool) { func mergeGrantHashChunksToSST(ctx context.Context, fs vfs.FS, sstPath, name string, chunks []string, fold *grantDigestFold) error { start := time.Now() l := ctxzap.Extract(ctx) - readers := make([]*os.File, 0, len(chunks)) - defer func() { - for _, r := range readers { - _ = r.Close() - } - }() - bufReaders := make([]*bufio.Reader, len(chunks)) - keyBufs := make([][]byte, len(chunks)) - valBufs := make([][]byte, len(chunks)) + cursors, err := openSpillChunks(chunks) + if err != nil { + return err + } + defer cursors.closeAll() h := &spillChunkHeap{} - var lenBuf [4]byte - for i, chunk := range chunks { - f, err := os.Open(chunk) // #nosec G304 - staged under the build's MkdirTemp dir. - if err != nil { - return err - } - readers = append(readers, f) - bufReaders[i] = bufio.NewReaderSize(f, bulkSpillBufferSize) - ok, err := readSpillEntry(bufReaders[i], &keyBufs[i], &valBufs[i], &lenBuf) + for i := range chunks { + ok, err := cursors.advance(i) if err != nil { return err } if ok { - h.push(spillChunkItem{chunkIdx: i, key: keyBufs[i], val: valBufs[i]}) + h.push(spillChunkItem{chunkIdx: i, key: cursors.key(i), val: cursors.val(i)}) } } @@ -389,7 +381,7 @@ func mergeGrantHashChunksToSST(ctx context.Context, fs vfs.FS, sstPath, name str for len(*h) > 0 { item := h.pop() if bytes.Equal(item.key, last) { - return fmt.Errorf("%w: bucket %s key %x", errBulkImportDuplicateKey, name, item.key) + return fmt.Errorf("%w: bucket %s key %x", ErrBulkImportDuplicateKey, name, item.key) } partition, bucket, ok := splitGrantHashIndexKey(item.key) if !ok { @@ -419,12 +411,12 @@ func mergeGrantHashChunksToSST(ctx context.Context, fs vfs.FS, sstPath, name str } } last = append(last[:0], item.key...) - ok, err := readSpillEntry(bufReaders[item.chunkIdx], &keyBufs[item.chunkIdx], &valBufs[item.chunkIdx], &lenBuf) + ok, err := cursors.advance(item.chunkIdx) if err != nil { return err } if ok { - h.push(spillChunkItem{chunkIdx: item.chunkIdx, key: keyBufs[item.chunkIdx], val: valBufs[item.chunkIdx]}) + h.push(spillChunkItem{chunkIdx: item.chunkIdx, key: cursors.key(item.chunkIdx), val: cursors.val(item.chunkIdx)}) } } if err := writer.finish(); err != nil { @@ -487,7 +479,12 @@ func (e *Engine) buildGrantDigestsFromSpill(ctx context.Context, dir string, has } // Zero grants still means the digest WAS built (present-means- // exact — an absent global root would tell a manifest reader to - // recalculate instead of trusting "nothing to diff"). + // recalculate instead of trusting "nothing to diff"). The ABI + // stamp precedes the root: WAL prefix ordering then guarantees a + // durable root is never uncertified. + if err := e.db.DigestSet(rawdb.GrantDigestABIStampKey(), grantDigestABIStampValue(), opts); err != nil { + return err + } if err := e.db.DigestSet(rawdb.GlobalGrantDigestNodeKey(), packDigestLeaf(0, zeroDigest[:]), opts); err != nil { return err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_repair.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_repair.go index ef93fb07..365d813c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_repair.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest_repair.go @@ -583,6 +583,16 @@ func (e *Engine) recomputeGrantDigestGlobalRootLocked(ctx context.Context) error if e.IsFreshSync() { opts = pebble.NoSync } + // Re-stamp the ABI with the root. Redundant when the stamp survived + // (only full-range deletes remove it, and those remove the roots + // this recompute folds too), but writing both here keeps the + // invariant locally checkable: every global-root write site + // certifies the ABI that produced the state under it. Stamp first — + // WAL prefix ordering then guarantees a durable root is never + // uncertified. + if err := e.db.DigestSet(rawdb.GrantDigestABIStampKey(), grantDigestABIStampValue(), opts); err != nil { + return err + } if err := e.db.DigestSet(rawdb.GlobalGrantDigestNodeKey(), packDigestLeaf(total, xor[:]), opts); err != nil { return err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go index a4afeab7..fed31bf7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go @@ -46,8 +46,8 @@ func (e *Engine) PutGrantRecord(ctx context.Context, r *v3.GrantRecord) error { // the latest occurrence of each external_id and process only those — // earlier duplicates are dropped before any batch byte is written. // db.Get doesn't see in-batch writes either way, so this dedup pass is -// the load-bearing safety net that neither the old read-before-write -// path nor a pure skip-Get path provides. +// required: neither the old read-before-write path nor a pure skip-Get +// path drops earlier duplicates in the same batch. // // Read-before-write overwrite probe. On a NON-fresh sync the engine // must Get the prior primary key so StageGrantPutInline can stage the @@ -564,8 +564,10 @@ func (e *Engine) initSynthLayerSession(ctx context.Context, s *synthGrantLayerSe } // ingestSynthLayerSegment merges one segment's sorted chunks into an SST and -// ingests it. Chunk files are deleted once merged; the SST path is left for -// the session's final dir cleanup (Pebble links/copies it on ingest). +// ingests it. The merge unlinks each chunk as it drains it, so nothing is +// left to clean up here on success; a merge that fails partway leaves its +// remaining chunks to the session's final dir cleanup. The SST path is also +// left to that cleanup (Pebble links/copies it on ingest). // // Runs on the session's background worker, which deliberately bypasses the // engine write barrier (an Add holding writeMu can block on the bounded @@ -581,9 +583,6 @@ func (e *Engine) ingestSynthLayerSegment(ctx context.Context, dir string, seg sy if err := mergeSortedSpillChunksToSST(ctx, e.fs(), sstPath, seg.name, seg.chunks); err != nil { return err } - for _, chunk := range seg.chunks { - _ = os.Remove(chunk) - } e.checkpointMu.RLock() defer e.checkpointMu.RUnlock() if err := e.db.IngestSSTs(ctx, []string{sstPath}); err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go index acccd446..bb2c5fe5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/id_index_migration.go @@ -1,11 +1,9 @@ package pebble import ( - "bufio" "bytes" "context" "fmt" - "os" "path/filepath" "sort" "time" @@ -32,11 +30,11 @@ func (e *Engine) migrateIDIndexFormatToStructuredV1(ctx context.Context) error { defer e.removeStagingDir(dir) sortSem := make(chan struct{}, 4) - // 128MiB chunks (deferredIndexSpillChunkBytes), not the bulk import's - // 8MiB: the merge holds a 1MiB read buffer per chunk, so chunk size - // bounds the fan-in. At 8MiB a 150M-grant file would merge ~4,000 - // chunks (~4GB of buffers); at 128MiB it stays in the low hundreds. - // The arena freelist recycles the big chunks across all four sorters. + // 128MiB chunks (deferredIndexSpillChunkBytes): the merge holds a + // 1MiB read buffer per chunk, so chunk size bounds the fan-in. At + // 8MiB a 150M-grant file would merge ~4,000 chunks (~4GB of + // buffers); at 128MiB it stays in the low hundreds. The arena + // freelist recycles the big chunks across all four sorters. arenaFree := newSpillArenaFreeList(deferredIndexSpillChunkBytes, 6) grantPrimary := newSpillSorter(dir, "grant-primary", sortSem, deferredIndexSpillChunkBytes) entitlementPrimary := newSpillSorter(dir, "entitlement-primary", sortSem, deferredIndexSpillChunkBytes) @@ -281,30 +279,19 @@ func finalizeGrantPrimaryMigrationSorter(ctx context.Context, fs vfs.FS, dir, na } func mergeGrantPrimaryMigrationChunksToSST(ctx context.Context, fs vfs.FS, sstPath, name string, chunks []string, byPrincipal, byNeedsExpansion *spillSorter) error { - readers := make([]*os.File, 0, len(chunks)) - defer func() { - for _, r := range readers { - _ = r.Close() - } - }() - bufReaders := make([]*bufio.Reader, len(chunks)) - keyBufs := make([][]byte, len(chunks)) - valBufs := make([][]byte, len(chunks)) + cursors, err := openSpillChunks(chunks) + if err != nil { + return err + } + defer cursors.closeAll() h := &spillChunkHeap{} - var lenBuf [4]byte - for i, chunk := range chunks { - f, err := os.Open(chunk) // #nosec G304 - staged under migration temp dir. - if err != nil { - return err - } - readers = append(readers, f) - bufReaders[i] = bufio.NewReaderSize(f, bulkSpillBufferSize) - ok, err := readSpillEntry(bufReaders[i], &keyBufs[i], &valBufs[i], &lenBuf) + for i := range chunks { + ok, err := cursors.advance(i) if err != nil { return err } if ok { - h.push(spillChunkItem{chunkIdx: i, key: append([]byte(nil), keyBufs[i]...), val: append([]byte(nil), valBufs[i]...)}) + h.push(spillChunkItem{chunkIdx: i, key: append([]byte(nil), cursors.key(i)...), val: append([]byte(nil), cursors.val(i)...)}) } } @@ -336,13 +323,13 @@ func mergeGrantPrimaryMigrationChunksToSST(ctx context.Context, fs vfs.FS, sstPa first := h.pop() key := first.key values := [][]byte{first.val} - if err := advanceMigrationChunk(h, bufReaders, keyBufs, valBufs, &lenBuf, first.chunkIdx); err != nil { + if err := advanceMigrationChunk(h, cursors, first.chunkIdx); err != nil { return err } for len(*h) > 0 && bytes.Equal((*h)[0].key, key) { item := h.pop() values = append(values, item.val) - if err := advanceMigrationChunk(h, bufReaders, keyBufs, valBufs, &lenBuf, item.chunkIdx); err != nil { + if err := advanceMigrationChunk(h, cursors, item.chunkIdx); err != nil { return err } } @@ -406,13 +393,13 @@ func mergeGrantPrimaryMigrationChunksToSST(ctx context.Context, fs vfs.FS, sstPa return nil } -func advanceMigrationChunk(h *spillChunkHeap, readers []*bufio.Reader, keyBufs, valBufs [][]byte, lenBuf *[4]byte, idx int) error { - ok, err := readSpillEntry(readers[idx], &keyBufs[idx], &valBufs[idx], lenBuf) +func advanceMigrationChunk(h *spillChunkHeap, cursors *spillChunkCursors, idx int) error { + ok, err := cursors.advance(idx) if err != nil { return err } if ok { - h.push(spillChunkItem{chunkIdx: idx, key: append([]byte(nil), keyBufs[idx]...), val: append([]byte(nil), valBufs[idx]...)}) + h.push(spillChunkItem{chunkIdx: idx, key: append([]byte(nil), cursors.key(idx)...), val: append([]byte(nil), cursors.val(idx)...)}) } return nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go index 3f8bc812..743ff783 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go @@ -80,6 +80,15 @@ type indexMigration struct { // bait (unbounded latency and memory at Open on large files); prefer // seal-time derivation or explicit rebuild commands over registering // one here. +// +// GrantDigestABIVersion bumps in particular do NOT belong here: a +// migration records that it ran once, but old binaries can rewrite +// digest state afterwards without re-triggering it. The digest ABI is +// instead enforced by a stamp stored WITH the state +// (rawdb.GrantDigestABIStampKey, checked every Open by +// verifyGrantDigestABI), so re-polluted state is re-detected — and the +// remedy is again a cheap drop plus seal-time rebuild, never an +// Open-time backfill. var indexMigrations []indexMigration // applyIndexMigrations runs on engine Open (writable opens only — diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go index ab2d9226..470de6fa 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go @@ -77,6 +77,14 @@ const GrantPrimaryKeyPrefixLen = 3 // per-partition node regardless of partition bytes. const DigestLevelGlobalRoot byte = 2 +// DigestMetaIndexID is the reserved index-discriminator for +// engine-owned metadata keys inside the digest keyspace (today only +// GrantDigestABIStampKey). 0xFF sorts after every real digested +// index, so [v3|TypeDigest, v3|TypeDigest|DigestMetaIndexID) bounds +// exactly the digest NODES (see DigestNodeKeyspaceBounds). No +// digestIndexSpec may ever claim this byte. +const DigestMetaIndexID byte = 0xFF + // === grant primary-key splices === // SplitGrantPrimaryKey locates the partition/principal boundary of a @@ -490,9 +498,34 @@ func DeferredIdxPendingKey() []byte { return codec.AppendTupleStrings(buf, "deferred_grant_idx_pending") } -// DigestKeyspaceBounds bounds the entire digest keyspace (all digested -// indexes) — the presence-probe range for the digests-present flag. -func DigestKeyspaceBounds() ([]byte, []byte) { - lo := []byte{VersionV3, TypeDigest} - return lo, UpperBound(lo) +// GrantDigestABIStampKey is the durable record of which grant-digest +// hash ABI (the engine's GrantDigestABIVersion) this file's digest +// state — hash-index values and digest nodes — was computed under. +// Value: uint32 BE. Written only alongside the whole-file global root +// (the same present-means-exact certificate), read only at Open. +// +// It lives INSIDE the digest keyspace deliberately: every wholesale +// destroyer of digest state — the drop paths' full-range deletes, +// ResetForNewSync's excision, the fold build's opening DeleteRange — +// erases it without knowing it exists, INCLUDING the copies of those +// paths in already-shipped SDKs that predate the stamp. Absence with +// digest nodes present therefore always means "built by an SDK that +// predates the stamp" — every such build hashed at ABI version 1, so +// the engine reads a missing stamp as version 1 and compares that to +// its current ABI like any other stamp (drop and rebuild iff they +// differ). Under DigestMetaIndexID so no node scan or presence probe +// visits it. +func GrantDigestABIStampKey() []byte { + buf := make([]byte, 0, 3+len("grant_digest_abi")+2) + buf = append(buf, VersionV3, TypeDigest, DigestMetaIndexID) + return codec.AppendTupleStrings(buf, "grant_digest_abi") +} + +// DigestNodeKeyspaceBounds bounds the digest NODE keyspace: all +// digested indexes, excluding the DigestMetaIndexID metadata sub-range +// — the presence-probe range for the digests-present flag. The ABI +// stamp must not arm that flag: presence gates mutation-path +// invalidation and repair delegation, which are about nodes. +func DigestNodeKeyspaceBounds() ([]byte, []byte) { + return []byte{VersionV3, TypeDigest}, []byte{VersionV3, TypeDigest, DigestMetaIndexID} } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go index b46db6cc..5157fd7f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go @@ -290,9 +290,11 @@ func (d *DB) GrantDigestsPresent() bool { return d.grantDigestsPresent.Load() } func (d *DB) SetGrantDigestsPresent(present bool) { d.grantDigestsPresent.Store(present) } // ProbeGrantDigestsPresent initializes the presence flag with one -// bounded seek over the digest keyspace (the Open-time probe). +// bounded seek over the digest NODE keyspace (the Open-time probe). +// The ABI stamp's metadata sub-range is outside the bounds: a file +// holding only a leftover stamp has no digest state to invalidate. func (d *DB) ProbeGrantDigestsPresent() error { - lo, hi := DigestKeyspaceBounds() + lo, hi := DigestNodeKeyspaceBounds() iter, err := d.db.NewIter(&pebble.IterOptions{LowerBound: lo, UpperBound: hi}) if err != nil { return err diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go index fe1eb55d..90cca8ae 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go @@ -38,6 +38,11 @@ type sourceCacheStoreTestSeams struct { // implemented ONLY by the Pebble engine; the syncer type-asserts for it and // treats a store without it as "source cache unsupported" (no-op lookup, // no replay). It is deliberately NOT part of c1zstore.Store. +// +// Advanced: this interface exists for the SDK's replay orchestration, not +// for direct use. The correctness obligations (preflight, scope poisoning, +// compat validation) live in the callers; see pkg/sourcecache for the +// connector-facing contract. type SourceCacheStore interface { // LookupSourceCacheEntry returns this store's manifest entry for // (kind, scopeKey). Backs the connector-facing lookup when this diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go index ede01029..74708a21 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go @@ -108,6 +108,15 @@ func WithConvertTmpDir(dir string) ConvertOption { // goroutine). The default — min(4, GOMAXPROCS/2) — leaves headroom for // shared infrastructure; callers that own the machine can raise it, and // 1 fully serializes the grant scan. Values <= 0 are ignored. +// +// Sort memory is budgeted, not proportional to fan-out, up to a point: +// the bulk import sizes its spill chunks from a fixed budget divided by +// the arena count the lane count implies (see pebble's +// bulkImportSortBudgetBytes), so up to ~14 lanes more lanes mean smaller +// chunks and a wider final merge — more open chunk files and read buffers +// at Finish — rather than more RSS during the scan. Past that the chunk +// size hits its 16MiB floor and sort memory grows again, by roughly four +// 16MiB arenas per additional lane. func WithConvertParallelism(n int) ConvertOption { return func(c *convertConfig) { if n > 0 { @@ -171,17 +180,23 @@ type syncIDPreservingStarter interface { // .c1z written to outPath, which must not already exist. // // It uses the engine's BulkSyncImport SST fast path: each record table is -// streamed out of SQLite once, in primary-key order via `ORDER BY` on the -// key's tuple columns (SQLite BINARY collation is bytewise, and the engine's -// tuple key codec is order-preserving, so SQL order == encoded-key order — -// enforced at runtime by the importer's strictly-increasing check). Primary -// records stream straight into one sorted SST per bucket; secondary index -// keys are derived from the translated records and externally sorted into -// one index SST; everything is ingested in a single pebble Ingest. No -// memtable, no WAL, no L0 flush, no background compaction debt. +// streamed out of SQLite once. Resource types are scanned in primary-key +// order via `ORDER BY` on the key column (SQLite BINARY collation is +// bytewise and the engine's tuple key codec is order-preserving, so SQL +// order == encoded-key order — enforced at runtime by the importer's +// strictly-increasing check) and stream straight into one sorted SST. +// Resources, entitlements, and grants are keyed by tuples whose order the +// scan cannot cheaply reproduce, so the importer spill-sorts them (and the +// secondary index keys derived from them) into sorted runs and k-way +// merges each family into one SST at Finish; everything is ingested in a +// single pebble Ingest. No memtable, no WAL, no L0 flush, no background +// compaction debt. // // SQLite's UNIQUE(external_id, sync_id) indexes provide the no-duplicates -// guarantee the importer requires. +// guarantee the importer requires for resource types, resources, and +// entitlements. Grants are keyed by structural identity (entitlement + +// principal refs), which that index does not cover; legacy rows sharing an +// identity under distinct external ids fold at Finish with a warning. // // syncID selects the source sync to convert; the destination holds that one // sync and nothing else, so every other sync in the source is dropped. Those @@ -332,7 +347,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op if !ok { return nil, errors.New("to-pebble: destination store is not a pebble engine") } - bi, err := destEng.StartBulkSyncImport(ctx, destSyncID, cfg.tmpDir) + bi, err := destEng.StartBulkSyncImport(ctx, destSyncID, cfg.tmpDir, cfg.grantScanLanes()) if err != nil { return nil, fmt.Errorf("to-pebble: start bulk import: %w", err) } @@ -797,6 +812,20 @@ func (c *C1File) convertEntitlements(ctx context.Context, bi *pebble.BulkSyncImp // via WithConvertParallelism. const convertGrantScanLanes = 4 +// grantScanLanes returns the grant scan fan-out the conversion will +// attempt: the caller's WithConvertParallelism when set, otherwise half +// the available CPUs capped at convertGrantScanLanes, never below 1. +// convertGrants may still fall back to a single lane if the extra sqlite +// readers cannot attach; the value here is also what the bulk import +// sizes its spill arenas for, so it is computed once up front. +func (cfg *convertConfig) grantScanLanes() int { + lanes := min(convertGrantScanLanes, max(1, runtime.GOMAXPROCS(0)/2)) + if cfg.parallelism > 0 { + lanes = cfg.parallelism + } + return max(1, lanes) +} + // rawGrantRow is one grant row's raw column bytes, copied out of the // scan into a batch-owned arena so decoding can happen on another // goroutine after the scanner has moved on, plus the row's @@ -809,13 +838,13 @@ type rawGrantRow struct { // convertGrants streams the sync's grants into the bulk import. The // scan shards by EXTERNAL ID range over the UNIQUE(external_id, -// sync_id) index: each lane's ordered range scan yields rows already in -// the shard's final pebble key order, so grant primaries stream -// straight into one final SST per lane — no spill, no external sort, no -// merge (see BulkGrantShard). Range boundaries come from sampling -// external ids at random rowids and taking quantiles; uneven lanes only -// cost balance, never correctness, and pebble's Ingest rejects -// overlapping shard SSTs outright. +// sync_id) index purely to spread the SQLite read and decode work across +// lanes: grants are keyed by structural identity, whose order the +// external-id scan does not reproduce, so each shard spill-sorts its +// primaries and index keys and Finish k-way merges every shard's runs +// into one grants SST (see BulkGrantShard). Range boundaries come from +// sampling external ids at random rowids and taking quantiles; uneven +// lanes only cost balance, never correctness. // // Each lane is a two-stage pipeline: a reader goroutine does nothing // but step rows and memcpy the raw (data, expansion) column bytes into @@ -841,13 +870,7 @@ func (c *C1File) convertGrants(ctx context.Context, bi *pebble.BulkSyncImport, s return nil // no grants in this sync } - lanes := min(convertGrantScanLanes, max(1, runtime.GOMAXPROCS(0)/2)) - if cfg.parallelism > 0 { - lanes = cfg.parallelism - } - if lanes < 1 { - lanes = 1 - } + lanes := cfg.grantScanLanes() // The C1File's own pool is capped at one connection (WAL checkpoint // hygiene — see NewC1File) and defaults to locking_mode=EXCLUSIVE, // which holds its lock indefinitely once acquired and would starve a diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index 00d0d815..a50ee398 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.25.1" +const Version = "v0.29.0" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go index d3b3c5fd..191d4c77 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go @@ -1,6 +1,12 @@ // Package sourcecache defines the connector-facing surface of source-cache // replay (see proto/c1/connector/v2/annotation_source_cache.proto). // +// ADVANCED FUNCTIONALITY. Source-cache replay is an advanced, opt-in +// capability with strict correctness obligations on the connector (scope +// partitioning, validator lifetime — see the invariants below). Most +// connectors should not use this package; adopt it only in coordination +// with the SDK maintainers. +// // NOT YET WIRED. This package describes the intended contract, and the // storage and eligibility machinery beneath it is in place, but the syncer // does not install a Lookup or consume these annotations yet: SyncOpAttrs diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/config.go new file mode 100644 index 00000000..3aae8272 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/config.go @@ -0,0 +1,72 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/metrics" + "github.com/conductorone/baton-sdk/pkg/types/sessions" + "github.com/conductorone/baton-sdk/pkg/uotel" +) + +// syncConfig is everything the caller decides before a sync starts: the +// complete set of values written by the exported With* options (SyncOpt) +// and by nothing else. NewSyncer applies the options and then never writes +// to it again, so a syncConfig read during Sync — including after a resume, +// on any worker goroutine — is the caller's original request. +// +// Fields that an option seeds but the sync then mutates are NOT config and +// stay on syncer: store (WithConnectorStore, replaced by loadStore) and +// syncID (WithSyncID, replaced by startOrResumeSync). Keeping them out +// makes the immutable set trustworthy, which matters because a sync +// resumes across process boundaries: mutable run state has to survive in +// the store, config only has to be passed again. +type syncConfig struct { + c1zPath string + externalResourceC1ZPath string + externalResourceEntitlementIdFilter string + // externalResourceTraits are the resource type traits that this + // connector wants synced from the external resource source and made + // available to the External Identity Matcher (see externalMatchTraits, + // set via WithExternalResourceTraits). When left empty the matcher + // falls back to TRAIT_USER/TRAIT_GROUP, preserving pre-CE-975 behavior + // for callers that never opt in. + externalResourceTraits []v2.ResourceType_Trait + previousSyncC1ZPath string + previousSyncC1ZPathOptional bool + // failFastInvariants promotes every ingestion-invariant verdict + // (see ingest_invariants.go) to a hard, plainly-attributed sync + // failure — tolerated warns fail — and enables I4 (skipped + // entirely in default mode). Tests and equivalence harnesses set + // it; production default follows the per-invariant policy in the + // verdict table (ingestInvariants). + failFastInvariants bool + runDuration time.Duration + transitionHandler func(s Action) + progressHandler func(p *Progress) + tmpDir string + storageEngine c1zstore.Engine + skipFullSync bool + // compactionMergedStore marks the store as a pre-sealed artifact + // this process did not collect (WithCompactionMergedStore — the + // compactor's keep-newer merge and rollback-expansion's replay): + // invariant verdicts attribute merge-manufactured shapes to the + // merge and soften hard arms to aggregated warnings. Distinct from + // onlyExpandGrants, which changes WHAT syncs and carries no + // invariant policy on its own. + compactionMergedStore bool + targetedSyncResources []*v2.Resource + onlyExpandGrants bool + preserveEntitlementGraph bool + dontExpandGrants bool + skipEntitlementsAndGrants bool + skipGrants bool + syncType connectorstore.SyncType + setSessionStore sessions.SetSessionStore + syncResourceTypes []string + workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. + metricsHandler metrics.Handler + syncIdentity uotel.SyncIdentity +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/hooks.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/hooks.go new file mode 100644 index 00000000..bde39a53 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/hooks.go @@ -0,0 +1,30 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// syncTestHooks is the syncer's set of test seams: observation and +// fault-injection points the verification harnesses attach to in order to +// make an ordering-sensitive boundary reachable from a test. Every field is +// nil in production, so a seam costs one pointer comparison on the path it +// guards — the same cost these had as individual syncer fields. +// +// They live together so a reader can find every place a test can perturb a +// sync in one file, and so that adding a seam does not widen syncer. No +// production code path may set one of these; the only writers are in +// _test.go files, which is what the type name asserts. The fields therefore +// carry no `test` prefix of their own — any `test`-prefixed field elsewhere +// in the package is a seam that escaped this struct. +type syncTestHooks struct { + // ingestHaltHook, when non-nil, fires at named seams of the + // ingestion-invariant pass (see ingestInvariantHaltStages); + // returning an error fails the sync at exactly that boundary. The + // halt sweep uses it to prove crash/resume equivalence at every + // ordering-sensitive point. + ingestHaltHook func(stage string) error + // checkpointHook, when non-nil, observes every durably written + // checkpoint token. The cut harness uses it to count checkpoints + // and to simulate a crash immediately after a chosen one. + checkpointHook func(token string) + // queueAudit, when non-nil, records every parallelActionQueue + // event (seed/dequeue/commit/abort/done) for post-hoc verification + // of the queue contract. + queueAudit *queueAudit +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_filter.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_filter.go index 2cb2c394..97f96e13 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_filter.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_filter.go @@ -183,7 +183,7 @@ func (s *syncer) filterFreshEntitlements( ctx context.Context, entitlements []*v2.Entitlement, ) ([]*v2.Entitlement, error) { - if s.syncType != connectorstore.SyncTypeFull || len(entitlements) == 0 { + if s.cfg.syncType != connectorstore.SyncTypeFull || len(entitlements) == 0 { return entitlements, nil } out := make([]*v2.Entitlement, 0, len(entitlements)) @@ -284,7 +284,7 @@ func (s *syncer) filterGrantExpansionTypes( } func (s *syncer) filterFreshGrants(ctx context.Context, grants []*v2.Grant) ([]*v2.Grant, error) { - if s.syncType != connectorstore.SyncTypeFull || len(grants) == 0 { + if s.cfg.syncType != connectorstore.SyncTypeFull || len(grants) == 0 { return grants, nil } out := make([]*v2.Grant, 0, len(grants)) @@ -345,7 +345,7 @@ func (s *syncer) filterFreshGrants(ctx context.Context, grants []*v2.Grant) ([]* // resources whose type is absent from the sync's resource types, so storing // them is dead data even though the resource row itself could be written. func (s *syncer) filterFreshGrantResource(ctx context.Context, resource *v2.Resource) (bool, error) { - if s.syncType != connectorstore.SyncTypeFull || resource.GetId() == nil { + if s.cfg.syncType != connectorstore.SyncTypeFull || resource.GetId() == nil { return true, nil } exists, err := s.scheduledResourceTypeExists(ctx, resource.GetId().GetResourceType()) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go index 3bb9efbe..d23e9908 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go @@ -504,9 +504,7 @@ func runIngestInvariants( return nil, fmt.Errorf("ingest invariants: policy.SyncType is required (the zero value would silently skip the full-keyspace invariants; pass the sync's actual type)") } pass := &ingestInvariantsPass{store: store, p: &policy} - if facts, ok := store.(dotc1z.IngestInvariantStore); ok { - pass.facts = facts - } + pass.facts = resolveReaderCaps(store).ingestFacts var skippedNoStore []string coverage := make([]string, 0, len(ingestInvariants)) for i := range ingestInvariants { @@ -615,7 +613,7 @@ func storeCarriesTypeScopedTypes(ctx context.Context, store connectorstore.Reade // unfinished artifact the resume machinery will rewrite. func (s *syncer) runIngestionInvariants(ctx context.Context) error { s.pendingInvariantVerification = nil - if verificationWriter, ok := s.store.SyncMeta().(c1zstore.IngestInvariantVerificationWriter); ok { + if verificationWriter := s.caps.ingestVerification; verificationWriter != nil { // Invalidate any proof inherited from an earlier pass before // re-evaluating. A failed rerun must leave the sync unverified. if err := verificationWriter.ClearIngestInvariantVerification(ctx, s.syncID); err != nil { @@ -624,12 +622,12 @@ func (s *syncer) runIngestionInvariants(ctx context.Context) error { } policy := IngestInvariantsPolicy{ ActiveSyncID: s.getActiveSyncID(), - SyncType: s.syncType, - FailFast: s.failFastInvariants, - CompactionMerge: s.compactionMergedStore, + SyncType: s.cfg.syncType, + FailFast: s.cfg.failFastInvariants, + CompactionMerge: s.cfg.compactionMergedStore, childSchedule: &s.childSchedule, resourcesPhaseRan: s.resourcesPhaseRanHere, - syncResourceTypes: s.syncResourceTypes, + syncResourceTypes: s.cfg.syncResourceTypes, onRetainedInvalid: func() { s.ingestFilterStats.blockReplay(ingestQualityReasonRetainedInvalid) }, @@ -637,8 +635,8 @@ func (s *syncer) runIngestionInvariants(ctx context.Context) error { if st, ok := s.state.(*state); ok { policy.undrainedSpawned = st.UndrainedSpawnedCursors } - if s.testIngestHaltHook != nil { - policy.halt = s.testIngestHaltHook + if s.testHooks.ingestHaltHook != nil { + policy.halt = s.testHooks.ingestHaltHook } verification, err := RunIngestInvariantsWithVerification(ctx, s.store, policy) if err != nil { @@ -662,8 +660,8 @@ func (s *syncer) persistIngestInvariantVerification(ctx context.Context) error { } verification := *s.pendingInvariantVerification s.pendingInvariantVerification = nil - verificationWriter, ok := s.store.SyncMeta().(c1zstore.IngestInvariantVerificationWriter) - if !ok { + verificationWriter := s.caps.ingestVerification + if verificationWriter == nil { // SyncMeta predates verification metadata and is implemented outside // this repository. Preserve its established behavior; the absent // marker truthfully distinguishes it from built-in verified stores. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go index 35cbd737..a123ffc4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go @@ -211,10 +211,10 @@ func (s *syncer) parallelSync( case InitOp: s.state.FinishAction(ctx, stateAction) - if s.skipEntitlementsAndGrants { + if s.cfg.skipEntitlementsAndGrants { s.state.SetShouldSkipEntitlementsAndGrants() } - if s.skipGrants { + if s.cfg.skipGrants { s.state.SetShouldSkipGrants() } if len(targetedResources) > 0 { @@ -245,7 +245,7 @@ func (s *syncer) parallelSync( if s.externalResourceReader != nil { s.state.PushAction(ctx, Action{Op: SyncExternalResourcesOp}) } - if s.onlyExpandGrants { + if s.cfg.onlyExpandGrants { s.state.SetNeedsExpansion() err = s.Checkpoint(ctx, true) if err != nil { @@ -420,7 +420,7 @@ func (s *syncer) parallelSync( } } - if s.dontExpandGrants || !s.state.NeedsExpansion() { + if s.cfg.dontExpandGrants || !s.state.NeedsExpansion() { l.Debug("skipping grant expansion, no grants to expand") s.state.FinishAction(ctx, stateAction) continue @@ -725,7 +725,7 @@ func (s *syncer) setParallelActionTransitioner( func (s *syncer) syncParallel(ctx context.Context, retryer *retry.Retryer, actions []*Action, f func(ctx context.Context, action *Action) error) ([]error, error) { l := ctxzap.Extract(ctx) - l.Info("syncing in parallel", zap.Int("actions", len(actions)), zap.Int("workers", s.workerCount)) + l.Info("syncing in parallel", zap.Int("actions", len(actions)), zap.Int("workers", s.cfg.workerCount)) // One bounded summary span per fan-out batch. The per-action work (f) starts // its own linked-root span, so this stays a handful of spans per sync rather @@ -740,7 +740,7 @@ func (s *syncer) syncParallel(ctx context.Context, retryer *retry.Retryer, actio span.SetAttributes( attribute.String("sync.op", op), attribute.Int("sync.action_count", len(actions)), - attribute.Int("sync.worker_count", s.workerCount), + attribute.Int("sync.worker_count", s.cfg.workerCount), ) uotel.SetSyncIdentityAttrs(ctx, span) var batchErr error @@ -755,8 +755,8 @@ func (s *syncer) syncParallel(ctx context.Context, retryer *retry.Retryer, actio defer cancel(nil) queue := newParallelActionQueue(actions) - if s.testQueueAudit != nil { - queue.attachAudit(s.testQueueAudit, batchOp, s.testQueueAudit.newBatch()) + if s.testHooks.queueAudit != nil { + queue.attachAudit(s.testHooks.queueAudit, batchOp, s.testHooks.queueAudit.newBatch()) } s.setParallelActionTransitioner(func( transitionCtx context.Context, @@ -775,7 +775,7 @@ func (s *syncer) syncParallel(ctx context.Context, retryer *retry.Retryer, actio var errs []error var wg native_sync.WaitGroup - for i := 0; i < s.workerCount; i++ { + for i := 0; i < s.cfg.workerCount; i++ { wg.Go(func() { for { action, ok := queue.next() @@ -817,8 +817,8 @@ func (s *syncer) syncParallel(ctx context.Context, retryer *retry.Retryer, actio wg.Wait() batchErr = errors.Join(errs...) - if s.testQueueAudit != nil { - s.testQueueAudit.record(queueAuditEvent{kind: auditBatchEnd, batch: queue.auditBatch, clean: batchErr == nil}) + if s.testHooks.queueAudit != nil { + s.testHooks.queueAudit.record(queueAuditEvent{kind: auditBatchEnd, batch: queue.auditBatch, clean: batchErr == nil}) } return warnings, batchErr } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/queue_audit.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/queue_audit.go index d1c81a48..6738205f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/queue_audit.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/queue_audit.go @@ -14,7 +14,7 @@ import ( // the checker replays the log against the contract post-hoc. // // Production cost is one nil check per queue operation: the syncer only -// attaches an audit when a test sets testQueueAudit. +// attaches an audit when a test sets syncTestHooks.queueAudit. type queueAudit struct { mu native_sync.Mutex events []queueAuditEvent diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go index ecec42c4..5f2fc4e6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go @@ -90,11 +90,13 @@ func PrepareExpansionReplayToken(stateStr string) (string, error) { return "", err } st.SetNeedsExpansion() - // Clear any preserved entitlement graph. A graph preserved by - // WithPreserveEntitlementGraph has Loaded=true with every edge already - // marked expanded, so a replayed sync would skip graph loading and the - // expander would report done immediately — the replay would silently - // no-op. Clearing it makes the replay rebuild the graph from scratch. + // Clear any inline entitlement graph the token carried. Only a + // pre-omission SDK writes one — WithPreserveEntitlementGraph writes to the + // c1z sidecar. Such a graph has Loaded=true with every edge marked + // expanded, so a replayed sync would skip graph loading and the expander + // would report done immediately, making the replay a silent no-op. Marshal + // drops the graph from every token it writes, so this clear does not change + // the emitted bytes; it states the contract here instead of resting on that. st.ClearEntitlementGraph(context.Background()) if st.Current() == nil { // A finished sync deserializes with no action map, so seed one before @@ -134,11 +136,13 @@ type EntitlementGraphStore interface { // syncID. Returns nil (no error) when the store lacks the capability, no graph // was preserved, or the stored graph belongs to a different sync. func GraphFromStore(ctx context.Context, store c1zstore.Store, syncID string) (*expand.EntitlementGraph, error) { - gs, ok := store.(EntitlementGraphStore) - if !ok { + // Exported entry point: callers hand us a bare store, so this is one of + // the two places allowed to resolve capabilities (see store_caps.go). + caps := resolveReaderCaps(store) + if caps.entitlementGraph == nil { return nil, nil } - data, err := gs.GetEntitlementGraphBlob(ctx) + data, err := caps.entitlementGraph.GetEntitlementGraphBlob(ctx) if err != nil { return nil, err } @@ -152,11 +156,10 @@ func GraphFromStore(ctx context.Context, store c1zstore.Store, syncID string) (* if boundDigest == nil { return nil, nil } - digestReader, ok := store.(c1zstore.GrantGenerationDigestReader) - if !ok { + if caps.grantDigest == nil { return nil, nil } - currentDigest, found, err := digestReader.GrantGenerationDigest(ctx) + currentDigest, found, err := caps.grantDigest.GrantGenerationDigest(ctx) if err != nil { return nil, err } @@ -314,11 +317,6 @@ type state struct { // state so Unmarshal→Marshal round trips (e.g. expansion replay // tokens) preserve it. compaction *CompactionTokenStats - // checkpointEntitlementGraph opts back in to serializing the graph - // inline. Off by default because the encoding is what OOM-kills workers - // on large tenants; on, it trades that risk for cross-restart expansion - // progress. See Marshal. - checkpointEntitlementGraph bool // spawnedInFlight is the evidence set behind ingest invariant I10: // every spawned sibling cursor (EnqueuePageTokens) admitted to the // stack, keyed by action ID, removed only by the two legitimate @@ -352,21 +350,6 @@ type state struct { spawnedAdmitted map[parallelActionKey]string } -// stateOpt configures a state at construction. Not part of the State -// interface: these are process-level knobs from SyncOpt, not token contents, -// so they must not survive an Unmarshal from a token written elsewhere. -type stateOpt func(*state) - -// withCheckpointEntitlementGraph serializes the entitlement graph into every -// checkpoint, restoring the behavior from before it was dropped. Escape hatch -// for a tenant whose expansion cannot finish within one worker lifetime; costs -// O(graph) memory per checkpoint, which is what the default avoids. -func withCheckpointEntitlementGraph(enabled bool) stateOpt { - return func(st *state) { - st.checkpointEntitlementGraph = enabled - } -} - // ConnectorCallStat contains cumulative latency statistics for one connector method. type ConnectorCallStat struct { Count int64 `json:"count"` @@ -440,8 +423,8 @@ type serializedTokenV1 struct { Version uint64 `json:"version"` } -func newState(opts ...stateOpt) *state { - st := &state{ +func newState() *state { + return &state{ actions: make(map[string]Action), actionOrder: []string{}, currentActionID: 0, @@ -453,10 +436,6 @@ func newState(opts ...stateOpt) *state { spawnedInFlight: make(map[string]Action), spawnedAdmitted: make(map[parallelActionKey]string), } - for _, opt := range opts { - opt(st) - } - return st } // Current returns nil if there is no current action. Otherwise it returns a pointer to a copy of the current state. @@ -680,7 +659,7 @@ func (st *state) UndrainedSpawnedCursors() []string { // Marshal returns a string encoding of the state object. This is useful for datastores to checkpoint the current state. // -// By default the entitlement graph is NOT serialized. It is a projection of +// The entitlement graph is never serialized. It is a projection of // data already in the store (loadEntitlementGraph rebuilds it from // PendingExpansionPage, with no connector calls), and for large tenants its // JSON encoding multiplied the checkpoint's memory footprint several times @@ -699,30 +678,22 @@ func (st *state) UndrainedSpawnedCursors() []string { // safe for OLDER readers too — an old SDK resuming a graph-less token starts // a fresh graph and must restart the load from the first page. Only the // serialized copy is normalized; the live state keeps its page token. -// -// WithEntitlementGraphInCheckpoints restores the old inline-graph behavior. The -// graph and the expansion page token then travel together, as they must: the -// token is only resumable by a reader that also got the graph it indexes. func (st *state) Marshal() (string, error) { st.mtx.RLock() defer st.mtx.RUnlock() actions := st.actions - graph := st.entitlementGraph - if !st.checkpointEntitlementGraph { - graph = nil - for _, action := range st.actions { - if action.Op == SyncGrantExpansionOp && action.PageToken != "" { - actions = make(map[string]Action, len(st.actions)) - for id, a := range st.actions { - if a.Op == SyncGrantExpansionOp { - a.PageToken = "" - } - actions[id] = a + for _, action := range st.actions { + if action.Op == SyncGrantExpansionOp && action.PageToken != "" { + actions = make(map[string]Action, len(st.actions)) + for id, a := range st.actions { + if a.Op == SyncGrantExpansionOp { + a.PageToken = "" } - break + actions[id] = a } + break } } @@ -743,7 +714,7 @@ func (st *state) Marshal() (string, error) { ActionOrder: st.actionOrder, CurrentActionID: st.currentActionID, NeedsExpansion: st.needsExpansion, - EntitlementGraph: graph, + EntitlementGraph: nil, HasExternalResourceGrants: st.hasExternalResourceGrants, ShouldFetchRelatedResources: st.shouldFetchRelatedResources, ShouldSkipEntitlementsAndGrants: st.shouldSkipEntitlementsAndGrants, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/store_caps.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/store_caps.go new file mode 100644 index 00000000..0c08ecb3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/store_caps.go @@ -0,0 +1,142 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" +) + +// storeCaps is the set of optional store capabilities pkg/sync uses, +// resolved once when a store is attached (syncer.setStore) and read as +// plain fields afterwards. A nil field means "this engine does not +// implement it" — SQLite implements almost none of these; Pebble +// implements all of them. +// +// The point of resolving up front is that the store's shape is decided +// once, in one place a reader can enumerate, instead of being rediscovered +// by an `x.(Interface)` at each use site. Duck-typing at the use site +// hides how many capabilities the syncer actually depends on, and makes +// every new dependency a two-line addition nobody reviews as a dependency. +// +// The read-side subset is its own type, readerCaps, embedded here. The +// exported entry points that take a bare reader (GraphFromStore, +// runIngestInvariants) resolve a readerCaps, and a readerCaps is not +// assignable to a storeCaps, so a syncer cannot be handed a reader's +// capabilities in place of its store's — which would compile, and would +// silently switch every write-side path to its fallback. +type storeCaps struct { + readerCaps + // ingestVerification writes and clears the ingestion-invariant + // verification marker. It is a capability of the store's SyncMeta + // sub-store, not of the store itself. + ingestVerification c1zstore.IngestInvariantVerificationWriter + // resourceDeleter, entitlementDeleter and grantRefsDeleter are the + // record-level deletes that stale-external-principal reconciliation + // needs. All three or none: when any is nil the reconciliation pass + // warns which arms are missing and returns without deleting, leaving + // principals from an earlier attempt in place (SQLite implements none + // of them). + resourceDeleter resourceRecordDeleter + entitlementDeleter entitlementRecordDeleter + grantRefsDeleter grantByRefsDeleter + // grantBatchDeleter is the bulk form of grantRefsDeleter. When nil the + // external-principal delete path falls through to the per-grant loop, + // which commits once per grant. + grantBatchDeleter grantsByRefsBatchDeleter + // grantPrincipalKeys and principalSortedGrants are the store-level + // expansion fast paths. When nil the expander adapter lists whole + // grants and extracts principal keys itself, and reports grants as + // unsorted so the topological merge buffers and sorts per entitlement. + grantPrincipalKeys grantPrincipalKeyLister + principalSortedGrants principalSortedGrantLister + // newExpandedGrants and newExpandedContributions are the Grants + // sub-store's write fast paths for expansion output. When nil the + // adapter falls back to StoreExpandedGrants, materializing grants and + // paying read-before-write. + newExpandedGrants newExpandedGrantStorer + newExpandedContributions newExpandedGrantContributionStorer + // expandedGrantLayer is the layer-session surface for synthesized + // grants, and is a capability of the store's Grants sub-store rather + // than the store itself. When nil, BeginExpandedGrantLayer reports + // "no layer support" and the expander uses its unlayered path. + // + // Caching it is sound for the same reason as ingestVerification: both + // Grants() implementations (pebbleStore.Grants, C1File.Grants) return a + // fresh stateless wrapper over a pointer assigned once at construction + // and never reassigned. + expandedGrantLayer expandedGrantLayerStorer +} + +// readerCaps is the subset of storeCaps reachable on a bare +// connectorstore.Reader: everything a read-only consumer of a c1z (the +// graph loader, the ingest-invariant pass, the progress log) can use. +// A reader may be either engine indefinitely — external-resource and +// previous-sync artifacts stay legacy SQLite long after Pebble is the only +// sync target — so these are honestly optional in a way the write side is +// not. +type readerCaps struct { + // entitlementGraph is the c1z graph sidecar used to persist a + // preserved entitlement graph for later incremental expansion. + entitlementGraph EntitlementGraphStore + // grantDigest reads the exact whole-file grant digest written at seal + // time. A preserved graph is only reusable when it can be bound to + // one, so the graph sidecar is useless without this. + grantDigest c1zstore.GrantGenerationDigestReader + // ingestFacts is the referential-inspection surface the ingestion + // invariants query (ingest_invariants.go). Absent on engines without + // it, which is why the referential invariants degrade rather than + // fail there. + ingestFacts dotc1z.IngestInvariantStore + // dbSize reports the store's uncompressed working-set size, which the + // progress log folds into the periodic expansion line. + dbSize connectorstore.DBSizeProvider +} + +// resolveStoreCaps resolves every capability in storeCaps from a full +// store, including the ones that live on its SyncMeta and Grants +// sub-stores. The write-side capabilities are resolved only here: a bare +// reader cannot delete or open a grant layer. +// +// This and resolveReaderCaps are the only places in pkg/sync that may +// type-assert a store against an optional capability interface, whether +// the interface is exported from c1zstore/dotc1z/connectorstore or +// declared locally in this package. +func resolveStoreCaps(store c1zstore.Store) storeCaps { + if store == nil { + return storeCaps{} + } + caps := storeCaps{readerCaps: resolveReaderCaps(store)} + caps.resourceDeleter, _ = store.(resourceRecordDeleter) + caps.entitlementDeleter, _ = store.(entitlementRecordDeleter) + caps.grantRefsDeleter, _ = store.(grantByRefsDeleter) + caps.grantBatchDeleter, _ = store.(grantsByRefsBatchDeleter) + caps.grantPrincipalKeys, _ = store.(grantPrincipalKeyLister) + caps.principalSortedGrants, _ = store.(principalSortedGrantLister) + if meta := store.SyncMeta(); meta != nil { + caps.ingestVerification, _ = meta.(c1zstore.IngestInvariantVerificationWriter) + } + if grants := store.Grants(); grants != nil { + caps.expandedGrantLayer, _ = grants.(expandedGrantLayerStorer) + caps.newExpandedGrants, _ = grants.(newExpandedGrantStorer) + caps.newExpandedContributions, _ = grants.(newExpandedGrantContributionStorer) + } + return caps +} + +// resolveReaderCaps resolves the capabilities reachable on a bare reader, +// for the exported entry points that take one (GraphFromStore, +// runIngestInvariants). The write-side capabilities are not part of the +// result type: a reader cannot delete, open a grant layer, or write the +// verification marker, and returning the narrower type is what keeps a +// reader's capabilities from ever being assigned where a store's belong. +func resolveReaderCaps(store connectorstore.Reader) readerCaps { + if store == nil { + return readerCaps{} + } + var caps readerCaps + caps.entitlementGraph, _ = store.(EntitlementGraphStore) + caps.grantDigest, _ = store.(c1zstore.GrantGenerationDigestReader) + caps.ingestFacts, _ = store.(dotc1z.IngestInvariantStore) + caps.dbSize, _ = store.(connectorstore.DBSizeProvider) + return caps +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index cadd2fef..2c56b91f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -149,43 +149,28 @@ func (sm *syncMap[K, V]) Store(key K, val V) { // syncer orchestrates a connector sync and stores the results using the provided datasource.Writer. type syncer struct { - c1zPath string - externalResourceC1ZPath string - externalResourceEntitlementIdFilter string - // externalResourceTraits are the resource type traits that this - // connector wants synced from the external resource source and made - // available to the External Identity Matcher (see externalMatchTraits, - // set via WithExternalResourceTraits). When left empty the matcher - // falls back to TRAIT_USER/TRAIT_GROUP, preserving pre-CE-975 behavior - // for callers that never opt in. - externalResourceTraits []v2.ResourceType_Trait - previousSyncC1ZPath string - previousSyncC1ZPathOptional bool - store c1zstore.Store - externalResourceReader connectorstore.Reader - previousSyncReader connectorstore.Reader + // cfg is the caller's request: every value set by a With* option and + // nothing else, immutable once NewSyncer returns (see config.go). + cfg syncConfig + store c1zstore.Store + // caps are store's optional capabilities, resolved once by setStore + // (see store_caps.go). Always in step with store. + caps storeCaps + externalResourceReader connectorstore.Reader + previousSyncReader connectorstore.Reader // Ingestion-invariant state (see ingest_invariants.go): // childSchedule is the monotone record backing invariant I4; // resourcesPhaseRanHere gates I4 to processes that actually ran the - // resources phase; failFastInvariants promotes every invariant - // verdict to a hard, plainly-attributed sync failure — tolerated - // warns fail — and enables I4 (skipped entirely in default mode). - // Tests and equivalence harnesses set it; production default - // follows the per-invariant policy in the verdict table - // (ingestInvariants). + // resources phase. The fail-fast policy that promotes every verdict + // to a hard failure is cfg.failFastInvariants. childSchedule childScheduleSet resourcesPhaseRanHere bool - failFastInvariants bool // expandDropStats aggregates expansion edges dropped over missing // entitlements across the whole sync (see expand.DroppedEdgeStats); // summarized once when expansion completes. expandDropStats *expand.DroppedEdgeStats - // testIngestHaltHook, when non-nil, fires at named seams of the - // ingestion-invariant pass (see ingestInvariantHaltStages); - // returning an error fails the sync at exactly that boundary. The - // halt sweep uses it to prove crash/resume equivalence at every - // ordering-sensitive point. Nil in production: one pointer check. - testIngestHaltHook func(stage string) error + // testHooks are the test seams (see hooks.go). All nil in production. + testHooks syncTestHooks // pendingInvariantVerification is the verification a successful // runIngestionInvariants staged, awaiting persistence by // persistIngestInvariantVerification AFTER EndSync. Deferring the @@ -197,57 +182,21 @@ type syncer struct { // minCheckpointInterval). The checkpoint-cut verification harness // sets it to zero so every loop-top checkpoint durably commits, // making each one an enumerable crash-cut point. - checkpointInterval time.Duration - // testCheckpointHook, when non-nil, observes every durably written - // checkpoint token. The cut harness uses it to count checkpoints - // and to simulate a crash immediately after a chosen one. Nil in - // production: one pointer check. - testCheckpointHook func(token string) - // testQueueAudit, when non-nil, records every parallelActionQueue - // event (seed/dequeue/commit/abort/done) for post-hoc verification - // of the queue contract. Nil in production: one pointer check per - // queue operation. - testQueueAudit *queueAudit - connector types.ConnectorClient - state State - runDuration time.Duration - transitionHandler func(s Action) - progressHandler func(p *Progress) - tmpDir string - storageEngine c1zstore.Engine - skipFullSync bool - lastCheckPointTime time.Time - counts *progresslog.ProgressLog - targetedSyncResources []*v2.Resource - onlyExpandGrants bool - preserveEntitlementGraph bool - // compactionMergedStore marks the store as a pre-sealed artifact - // this process did not collect (WithCompactionMergedStore — the - // compactor's keep-newer merge and rollback-expansion's replay): - // invariant verdicts attribute merge-manufactured shapes to the - // merge and soften hard arms to aggregated warnings. Distinct from - // onlyExpandGrants, which changes WHAT syncs and carries no - // invariant policy on its own. - compactionMergedStore bool - dontExpandGrants bool - checkpointEntitlementGraph bool + checkpointInterval time.Duration + connector types.ConnectorClient + state State + lastCheckPointTime time.Time + counts *progresslog.ProgressLog syncID string skipEGForResourceType syncMap[string, bool] skipEntitlementsForResourceType syncMap[string, bool] + skipGrantsForResourceType syncMap[string, bool] typeScopedGrantsForResourceType syncMap[string, bool] typeScopedEntitlementsForResourceType syncMap[string, bool] scheduledResourceTypes syncMap[string, bool] ingestFilterStats ingestFilterStats - skipEntitlementsAndGrants bool - skipGrants bool resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] - syncType connectorstore.SyncType injectSyncIDAnnotation bool - setSessionStore sessions.SetSessionStore - syncResourceTypes []string - workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. - metricsHandler metrics.Handler - syncIdentity uotel.SyncIdentity recordStats bool // parallelActionTransitioner atomically commits parent pagination and // spawned work to state and the active worker pool. @@ -270,14 +219,34 @@ var _ Syncer = (*syncer)(nil) // a single narrow interface without knowing about C1ZStore. type expanderStoreAdapter struct { store c1zstore.Store + // caps are the store's optional fast paths, resolved once when the + // adapter is built (see storeCaps). A nil capability means the engine + // does not offer it, and each method below reports that exactly as the + // use-site assertion it replaced did. Holding the whole set rather than + // one field per capability keeps both construction sites from having to + // grow every time the expander learns about another fast path. + caps storeCaps } // NewExpanderStore adapts a c1zstore.Store into an expand.ExpanderStore, // bridging engine differences (Pebble exposes StoreExpandedGrants on its // Grants() sub-store, SQLite at top level). Use this instead of type-asserting -// the store, which is unsafe for Pebble. +// the store, which is unsafe for Pebble — and because this is where the +// store's optional expansion capabilities are resolved for callers who hand +// us a bare store. The syncer itself uses (*syncer).expanderStore, which +// reuses the capabilities it already resolved at attach time. func NewExpanderStore(store c1zstore.Store) expand.ExpanderStore { - return expanderStoreAdapter{store: store} + return &expanderStoreAdapter{store: store, caps: resolveStoreCaps(store)} +} + +// expanderStore is the syncer's one construction site for the expansion +// adapter. It exists so the production expansion path cannot be built with a +// zero storeCaps by accident: there is no argument to forget, the adapter +// takes whatever setStore resolved. The adapter is returned by pointer so +// calls through the expand.ExpanderStore interface do not copy the +// capability set on every page. +func (s *syncer) expanderStore() expand.ExpanderStore { + return &expanderStoreAdapter{store: s.store, caps: s.caps} } // persistEntitlementGraphToStore binds the preserved graph to the exact sealed @@ -286,15 +255,14 @@ func (s *syncer) persistEntitlementGraphToStore(ctx context.Context, syncID stri if g == nil { return } - gs, ok := s.store.(EntitlementGraphStore) - if !ok { + gs := s.caps.entitlementGraph + if gs == nil { return } - digestReader, ok := s.store.(c1zstore.GrantGenerationDigestReader) - if !ok { + if s.caps.grantDigest == nil { return } - digest, found, err := digestReader.GrantGenerationDigest(ctx) + digest, found, err := s.caps.grantDigest.GrantGenerationDigest(ctx) if err != nil || !found { ctxzap.Extract(ctx).Warn("preserve entitlement graph: sealed grant digest unavailable; graph will not be reusable", zap.Error(err)) return @@ -310,7 +278,7 @@ func (s *syncer) persistEntitlementGraphToStore(ctx context.Context, syncID stri } } -func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { +func (a *expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { return a.store.GetEntitlement(ctx, req) } @@ -331,7 +299,7 @@ func requireEntitlementRefs(ent *v2.Entitlement) error { return nil } -func (a expanderStoreAdapter) ListGrantsForEntitlement( +func (a *expanderStoreAdapter) ListGrantsForEntitlement( ctx context.Context, req *reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest, ) (*reader_v2.GrantsReaderServiceListGrantsForEntitlementResponse, error) { @@ -341,7 +309,7 @@ func (a expanderStoreAdapter) ListGrantsForEntitlement( return a.store.ListGrantsForEntitlement(ctx, req) } -func (a expanderStoreAdapter) ListGrantPrincipalKeysForEntitlement( +func (a *expanderStoreAdapter) ListGrantPrincipalKeysForEntitlement( ctx context.Context, entitlement *v2.Entitlement, pageToken string, @@ -352,9 +320,7 @@ func (a expanderStoreAdapter) ListGrantPrincipalKeysForEntitlement( } // Preserve Pebble's compact prefetch path through this wrapper. Non-Pebble // stores fall back to regular grant listing and local key extraction. - if store, ok := a.store.(interface { - ListGrantPrincipalKeysForEntitlement(context.Context, *v2.Entitlement, string, uint32) ([]string, string, error) - }); ok { + if store := a.caps.grantPrincipalKeys; store != nil { return store.ListGrantPrincipalKeysForEntitlement(ctx, entitlement, pageToken, pageSize) } resp, err := a.store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ @@ -376,23 +342,19 @@ func (a expanderStoreAdapter) ListGrantPrincipalKeysForEntitlement( return keys, resp.GetNextPageToken(), nil } -func (a expanderStoreAdapter) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { +func (a *expanderStoreAdapter) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { return a.store.Grants().StoreExpandedGrants(ctx, grants...) } -func (a expanderStoreAdapter) StoreNewExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { - if fast, ok := a.store.Grants().(interface { - StoreNewExpandedGrants(context.Context, ...*v2.Grant) error - }); ok { +func (a *expanderStoreAdapter) StoreNewExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { + if fast := a.caps.newExpandedGrants; fast != nil { return fast.StoreNewExpandedGrants(ctx, grants...) } return a.store.Grants().StoreExpandedGrants(ctx, grants...) } -func (a expanderStoreAdapter) StoreNewExpandedGrantContributions(ctx context.Context, dest *v2.Entitlement, principals []*storage_v3.PrincipalRef, sources []batonGrant.Sources) error { - if fast, ok := a.store.Grants().(interface { - StoreNewExpandedGrantContributions(context.Context, *v2.Entitlement, []*storage_v3.PrincipalRef, []batonGrant.Sources) error - }); ok { +func (a *expanderStoreAdapter) StoreNewExpandedGrantContributions(ctx context.Context, dest *v2.Entitlement, principals []*storage_v3.PrincipalRef, sources []batonGrant.Sources) error { + if fast := a.caps.newExpandedContributions; fast != nil { return fast.StoreNewExpandedGrantContributions(ctx, dest, principals, sources) } grants := make([]*v2.Grant, 0, len(principals)) @@ -407,6 +369,34 @@ func (a expanderStoreAdapter) StoreNewExpandedGrantContributions(ctx context.Con return a.store.Grants().StoreExpandedGrants(ctx, grants...) } +// grantPrincipalKeyLister is the store's compact principal-key prefetch for an +// entitlement (Pebble). Without it the adapter lists whole grants and extracts +// the keys locally. +type grantPrincipalKeyLister interface { + ListGrantPrincipalKeysForEntitlement(context.Context, *v2.Entitlement, string, uint32) ([]string, string, error) +} + +// principalSortedGrantLister reports whether the engine yields an +// entitlement's grants in principal order (Pebble's entitlement-first key +// does). Without it the topological merge buffers and sorts per entitlement. +type principalSortedGrantLister interface { + GrantsForEntitlementPrincipalSorted() bool +} + +// newExpandedGrantStorer persists caller-proven-new expanded grants without +// read-before-write, on the store's GrantStore (Pebble). Without it the +// adapter falls back to StoreExpandedGrants. +type newExpandedGrantStorer interface { + StoreNewExpandedGrants(context.Context, ...*v2.Grant) error +} + +// newExpandedGrantContributionStorer is the contribution-shaped form of +// newExpandedGrantStorer: it takes destination plus principal refs and skips +// materializing v2.Grants. Without it the adapter builds the grants itself. +type newExpandedGrantContributionStorer interface { + StoreNewExpandedGrantContributions(context.Context, *v2.Entitlement, []*storage_v3.PrincipalRef, []batonGrant.Sources) error +} + // expandedGrantLayerStorer is the layer-scoped synthesized-grant layer session // surface the store's GrantStore may implement (Pebble). Local interface so // the adapter can pass sessions through without importing engine internals. @@ -417,32 +407,30 @@ type expandedGrantLayerStorer interface { AbortExpandedGrantLayer(ctx context.Context) error } -func (a expanderStoreAdapter) BeginExpandedGrantLayer(ctx context.Context) (bool, error) { - if fast, ok := a.store.Grants().(expandedGrantLayerStorer); ok { - return fast.BeginExpandedGrantLayer(ctx) +func (a *expanderStoreAdapter) BeginExpandedGrantLayer(ctx context.Context) (bool, error) { + if a.caps.expandedGrantLayer != nil { + return a.caps.expandedGrantLayer.BeginExpandedGrantLayer(ctx) } return false, nil } -func (a expanderStoreAdapter) AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*storage_v3.PrincipalRef, sources []batonGrant.Sources) error { - fast, ok := a.store.Grants().(expandedGrantLayerStorer) - if !ok { +func (a *expanderStoreAdapter) AddExpandedGrantLayerContributions(ctx context.Context, dest *v2.Entitlement, principals []*storage_v3.PrincipalRef, sources []batonGrant.Sources) error { + if a.caps.expandedGrantLayer == nil { return errors.New("expanded grant layer: store does not support layer sessions") } - return fast.AddExpandedGrantLayerContributions(ctx, dest, principals, sources) + return a.caps.expandedGrantLayer.AddExpandedGrantLayerContributions(ctx, dest, principals, sources) } -func (a expanderStoreAdapter) FinishExpandedGrantLayer(ctx context.Context) error { - fast, ok := a.store.Grants().(expandedGrantLayerStorer) - if !ok { +func (a *expanderStoreAdapter) FinishExpandedGrantLayer(ctx context.Context) error { + if a.caps.expandedGrantLayer == nil { return errors.New("expanded grant layer: store does not support layer sessions") } - return fast.FinishExpandedGrantLayer(ctx) + return a.caps.expandedGrantLayer.FinishExpandedGrantLayer(ctx) } -func (a expanderStoreAdapter) AbortExpandedGrantLayer(ctx context.Context) error { - if fast, ok := a.store.Grants().(expandedGrantLayerStorer); ok { - return fast.AbortExpandedGrantLayer(ctx) +func (a *expanderStoreAdapter) AbortExpandedGrantLayer(ctx context.Context) error { + if a.caps.expandedGrantLayer != nil { + return a.caps.expandedGrantLayer.AbortExpandedGrantLayer(ctx) } return nil } @@ -471,11 +459,9 @@ func resourceFromPrincipalRef(ref *storage_v3.PrincipalRef) *v2.Resource { // principal-sort guarantee (Pebble) so the topological merge can stream grant // groups instead of buffering and sorting each entitlement. Engines that do not // implement it (SQLite) report false and get the buffering fallback. -func (a expanderStoreAdapter) GrantsForEntitlementPrincipalSorted() bool { - store, ok := a.store.(interface { - GrantsForEntitlementPrincipalSorted() bool - }) - return ok && store.GrantsForEntitlementPrincipalSorted() +func (a *expanderStoreAdapter) GrantsForEntitlementPrincipalSorted() bool { + store := a.caps.principalSortedGrants + return store != nil && store.GrantsForEntitlementPrincipalSorted() } const minCheckpointInterval = 10 * time.Second @@ -505,8 +491,8 @@ func (s *syncer) Checkpoint(ctx context.Context, force bool) error { if err != nil { return err } - if s.testCheckpointHook != nil { - s.testCheckpointHook(checkpoint) + if s.testHooks.checkpointHook != nil { + s.testHooks.checkpointHook(checkpoint) } return nil @@ -571,10 +557,10 @@ func (s *syncer) syncSummaryFields(span trace.Span) []zap.Field { } attrs := []attribute.KeyValue{ - attribute.String("sync.type", string(s.syncType)), + attribute.String("sync.type", string(s.cfg.syncType)), attribute.Int64("sync.steps.total_ms", stepsTotalMs), attribute.Int64("sync.completed_actions", int64(s.state.GetCompletedActionsCount())), //nolint:gosec // action counts fit int64 - attribute.Int("sync.worker_count", s.workerCount), + attribute.Int("sync.worker_count", s.cfg.workerCount), } for _, op := range timedSyncOps { bucket := op.String() @@ -608,17 +594,17 @@ func (s *syncer) syncSummaryFields(span trace.Span) []zap.Field { fields := []zap.Field{ zap.String("sync_id", s.syncID), - zap.String("sync_type", string(s.syncType)), + zap.String("sync_type", string(s.cfg.syncType)), zap.Any("sync_step_durations_ms", ops), zap.Int64("sync_steps_total_ms", stepsTotalMs), zap.Any("connector_call_stats", flatCalls), zap.Uint64("completed_actions", s.state.GetCompletedActionsCount()), - zap.Int("worker_count", s.workerCount), + zap.Int("worker_count", s.cfg.workerCount), } // Prefer identity from WithSyncIdentity so dashboards can group by // catalog_name (platform context often has catalog_id but not the name). - if s.syncIdentity.CatalogName != "" { - fields = append(fields, zap.String("catalog_name", s.syncIdentity.CatalogName)) + if s.cfg.syncIdentity.CatalogName != "" { + fields = append(fields, zap.String("catalog_name", s.cfg.syncIdentity.CatalogName)) } if len(waits) > 0 { fields = append(fields, zap.Any("sync_step_wait_ms", waits)) @@ -731,16 +717,16 @@ func (s *syncer) returnSyncError(l *zap.Logger, span trace.Span, err error) erro } func (s *syncer) handleInitialActionForStep(ctx context.Context, a Action) { - if s.transitionHandler != nil { - s.transitionHandler(a) + if s.cfg.transitionHandler != nil { + s.cfg.transitionHandler(a) } } func (s *syncer) handleProgress(ctx context.Context, a *Action, c int) { - if s.progressHandler != nil { + if s.cfg.progressHandler != nil { //nolint:gosec // No risk of overflow because `c` is a slice length. count := uint32(c) - s.progressHandler(NewProgress(a, count)) + s.cfg.progressHandler(NewProgress(a, count)) } } @@ -836,8 +822,8 @@ func (s *syncer) startOrResumeSync(ctx context.Context) (string, bool, error) { var syncID string var newSync bool var err error - if len(s.targetedSyncResources) == 0 { - syncID, newSync, err = s.store.StartOrResumeSync(ctx, s.syncType, "") + if len(s.cfg.targetedSyncResources) == 0 { + syncID, newSync, err = s.store.StartOrResumeSync(ctx, s.cfg.syncType, "") if err != nil { return "", false, err } @@ -881,8 +867,8 @@ func (s *syncer) Sync(ctx context.Context) error { // Propagate connector identity to every descendant span (sync + dotc1z). // An explicit WithSyncIdentity option wins; otherwise inherit whatever the // caller already set on ctx. - if !s.syncIdentity.IsZero() { - ctx = uotel.WithSyncIdentity(ctx, s.syncIdentity) + if !s.cfg.syncIdentity.IsZero() { + ctx = uotel.WithSyncIdentity(ctx, s.cfg.syncIdentity) } uotel.SetSyncIdentityAttrs(ctx, span) var err error @@ -894,7 +880,7 @@ func (s *syncer) Sync(ctx context.Context) error { // rate_limit_wait bucket alongside retry backoff. ctx = s.withRateLimitWaitObserver(ctx) - if s.skipFullSync { + if s.cfg.skipFullSync { return s.SkipSync(ctx) } @@ -903,8 +889,8 @@ func (s *syncer) Sync(ctx context.Context) error { runCtx := ctx var runCanc context.CancelFunc - if s.runDuration > 0 { - runCtx, runCanc = context.WithTimeout(ctx, s.runDuration) + if s.cfg.runDuration > 0 { + runCtx, runCanc = context.WithTimeout(ctx, s.cfg.runDuration) } if runCanc != nil { defer runCanc() @@ -935,8 +921,8 @@ func (s *syncer) Sync(ctx context.Context) error { } syncResourceTypeMap := make(map[string]bool) - if len(s.syncResourceTypes) > 0 { - for _, rt := range s.syncResourceTypes { + if len(s.cfg.syncResourceTypes) > 0 { + for _, rt := range s.cfg.syncResourceTypes { syncResourceTypeMap[rt] = true } } @@ -949,9 +935,9 @@ func (s *syncer) Sync(ctx context.Context) error { parentResourceTypeID string parentResourceID string } - seenTargetedResources := make(map[targetedResourceKey]struct{}, len(s.targetedSyncResources)) - for _, r := range s.targetedSyncResources { - if len(s.syncResourceTypes) > 0 { + seenTargetedResources := make(map[targetedResourceKey]struct{}, len(s.cfg.targetedSyncResources)) + for _, r := range s.cfg.targetedSyncResources { + if len(s.cfg.syncResourceTypes) > 0 { if _, ok := syncResourceTypeMap[r.GetId().GetResourceType()]; !ok { continue } @@ -999,7 +985,7 @@ func (s *syncer) Sync(ctx context.Context) error { // stale marker would read as verified while collection rewrites the // data underneath it. The invariant pass re-stages and the marker is // re-persisted after EndSync. - if w, ok := s.store.SyncMeta().(c1zstore.IngestInvariantVerificationWriter); ok { + if w := s.caps.ingestVerification; w != nil { if err := w.ClearIngestInvariantVerification(ctx, syncID); err != nil { return s.returnSyncError(l, span, fmt.Errorf("clear prior ingest invariant verification: %w", err)) } @@ -1010,7 +996,7 @@ func (s *syncer) Sync(ctx context.Context) error { return err } - state := newState(withCheckpointEntitlementGraph(s.checkpointEntitlementGraph)) + state := newState() err = state.Unmarshal(currentStep) if err != nil { return err @@ -1043,7 +1029,7 @@ func (s *syncer) Sync(ctx context.Context) error { entitlementGraph := s.state.EntitlementGraph(ctx) l.Info("resumed previous sync", zap.String("sync_id", syncID), - zap.String("sync_type", string(s.syncType)), + zap.String("sync_type", string(s.cfg.syncType)), zap.String("current_action_op", currentActionOp), zap.String("current_action_resource_id", currentActionResourceID), zap.String("current_action_resource_type_id", currentActionResourceTypeID), @@ -1087,27 +1073,27 @@ func (s *syncer) Sync(ctx context.Context) error { if err := s.runIngestionInvariants(ctx); err != nil { return s.returnSyncError(l, span, err) } - if s.testIngestHaltHook != nil { + if s.testHooks.ingestHaltHook != nil { // The seam AFTER the invariant pass and BEFORE the // checkpoint/EndSync below: a resumed sync re-runs the whole // invariant pass over the same state, so every check must be // idempotent. - if err := s.testIngestHaltHook(haltStageInvariantsComplete); err != nil { + if err := s.testHooks.ingestHaltHook(haltStageInvariantsComplete); err != nil { return s.returnSyncError(l, span, err) } } // Force a checkpoint to clear completed actions & entitlement graph in sync_token. // preserveEntitlementGraph keeps the graph for a later incremental - // expansion: written to the c1z sidecar when the store supports it (token - // stays skinny — a whale graph is megabytes), else kept in the final token. + // expansion by writing it to the c1z sidecar; the token never carries a + // graph. The only caller (synccompactor) sets the option solely for + // Pebble stores, which are the only implementers of both capabilities + // tested below, so a preserved graph always has a sidecar to go to. // Transient working state is stripped either way; a reload rebuilds it. var graphToPersist *expand.EntitlementGraph - if s.preserveEntitlementGraph { + if s.cfg.preserveEntitlementGraph { s.state.ClearEntitlementGraphTransientState(ctx) - _, hasGraphSidecar := s.store.(EntitlementGraphStore) - _, hasGrantDigest := s.store.(c1zstore.GrantGenerationDigestReader) - if hasGraphSidecar && hasGrantDigest { + if s.caps.entitlementGraph != nil && s.caps.grantDigest != nil { graphToPersist = s.state.PeekEntitlementGraph() s.state.ClearEntitlementGraph(ctx) } @@ -1194,8 +1180,8 @@ func (s *syncer) SkipSync(ctx context.Context) (err error) { runCtx := ctx var runCanc context.CancelFunc - if s.runDuration > 0 { - runCtx, runCanc = context.WithTimeout(ctx, s.runDuration) + if s.cfg.runDuration > 0 { + runCtx, runCanc = context.WithTimeout(ctx, s.cfg.runDuration) } if runCanc != nil { defer runCanc() @@ -1296,9 +1282,9 @@ func (s *syncer) SyncResourceTypes(ctx context.Context, action *Action) error { } var resourceTypes []*v2.ResourceType - if len(s.syncResourceTypes) > 0 { + if len(s.cfg.syncResourceTypes) > 0 { syncResourceTypeMap := make(map[string]bool) - for _, rt := range s.syncResourceTypes { + for _, rt := range s.cfg.syncResourceTypes { syncResourceTypeMap[rt] = true } for _, rt := range connectorResourceTypes { @@ -1321,7 +1307,7 @@ func (s *syncer) SyncResourceTypes(ctx context.Context, action *Action) error { if resp.GetNextPageToken() == "" { s.counts.LogResourceTypesProgress(ctx) - if len(s.syncResourceTypes) > 0 { + if len(s.cfg.syncResourceTypes) > 0 { validResourceTypesResp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ PageToken: action.PageToken, ActiveSyncId: s.getActiveSyncID(), @@ -1329,7 +1315,7 @@ func (s *syncer) SyncResourceTypes(ctx context.Context, action *Action) error { if err != nil { return err } - err = validateSyncResourceTypesFilter(s.syncResourceTypes, validResourceTypesResp.GetList()) + err = validateSyncResourceTypesFilter(s.cfg.syncResourceTypes, validResourceTypesResp.GetList()) if err != nil { return err } @@ -1410,7 +1396,7 @@ func (s *syncer) pushChildResourceActions(ctx context.Context, childTypeIDs []st func (s *syncer) childResourceActions(childTypeIDs []string, parentTypeID, parentID string) []Action { var actions []Action for _, childTypeID := range childTypeIDs { - if len(s.syncResourceTypes) > 0 && !slices.Contains(s.syncResourceTypes, childTypeID) { + if len(s.cfg.syncResourceTypes) > 0 && !slices.Contains(s.cfg.syncResourceTypes, childTypeID) { continue } // Monotone evidence for ingestion invariant I4 (see @@ -1435,7 +1421,7 @@ func (s *syncer) childResourceActions(childTypeIDs []string, parentTypeID, paren func (s *syncer) pendingChildResourceActions(childTypeIDs []string, parentTypeID, parentID string) []Action { var actions []Action for _, childTypeID := range childTypeIDs { - if len(s.syncResourceTypes) > 0 && !slices.Contains(s.syncResourceTypes, childTypeID) { + if len(s.cfg.syncResourceTypes) > 0 && !slices.Contains(s.cfg.syncResourceTypes, childTypeID) { continue } if s.childSchedule.has(childTypeID, parentTypeID, parentID) { @@ -1954,7 +1940,29 @@ func (s *syncer) shouldSkipGrants(ctx context.Context, r *v2.Resource) (bool, er return true, nil } - return s.shouldSkipEntitlementsAndGrants(ctx, r) + // Check SkipEntitlementsAndGrants at instance or resource-type scope. + ok, err := s.shouldSkipEntitlementsAndGrants(ctx, r) + if err != nil || ok { + return ok, err + } + + // Check SkipGrants at resource-type scope. + if skip, ok := s.skipGrantsForResourceType.Load(r.GetId().GetResourceType()); ok { + return skip, nil + } + + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: r.GetId().GetResourceType(), + }.Build()) + if err != nil { + return false, err + } + + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + skipGrants := rtAnnos.Contains(&v2.SkipGrants{}) + s.skipGrantsForResourceType.Store(r.GetId().GetResourceType(), skipGrants) + + return skipGrants, nil } // No span here: shouldSkipEntitlements is called per-resource and almost @@ -2762,7 +2770,7 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro for _, grant := range grants { grantAnnos := annotations.Annotations(grant.GetAnnotations()) - if !s.dontExpandGrants && grantAnnos.Contains(&v2.GrantExpandable{}) { + if !s.cfg.dontExpandGrants && grantAnnos.Contains(&v2.GrantExpandable{}) { s.state.SetNeedsExpansion() } if grantAnnos.ContainsAny(&v2.ExternalResourceMatchAll{}, &v2.ExternalResourceMatch{}, &v2.ExternalResourceMatchID{}) { @@ -2841,8 +2849,8 @@ func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) erro l := ctxzap.Extract(ctx) l.Info("Syncing external resources") - if s.externalResourceEntitlementIdFilter != "" { - err := s.SyncExternalResourcesWithGrantToEntitlement(ctx, s.externalResourceEntitlementIdFilter) + if s.cfg.externalResourceEntitlementIdFilter != "" { + err := s.SyncExternalResourcesWithGrantToEntitlement(ctx, s.cfg.externalResourceEntitlementIdFilter) if err != nil { return err } @@ -2864,14 +2872,14 @@ func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) erro // caller that still wants user/group matching alongside a new trait must // list all three. func (s *syncer) externalMatchTraits() map[v2.ResourceType_Trait]bool { - if len(s.externalResourceTraits) == 0 { + if len(s.cfg.externalResourceTraits) == 0 { return map[v2.ResourceType_Trait]bool{ v2.ResourceType_TRAIT_USER: true, v2.ResourceType_TRAIT_GROUP: true, } } - traits := make(map[v2.ResourceType_Trait]bool, len(s.externalResourceTraits)) - for _, t := range s.externalResourceTraits { + traits := make(map[v2.ResourceType_Trait]bool, len(s.cfg.externalResourceTraits)) + for _, t := range s.cfg.externalResourceTraits { traits[t] = true } return traits @@ -3181,9 +3189,12 @@ func (s *syncer) deleteStaleExternalPrincipals( currentIDs[id.GetResourceType()+"\x00"+id.GetResource()] = struct{}{} } - resourceDeleter, canDeleteResources := s.store.(resourceRecordDeleter) - entitlementDeleter, canDeleteEntitlements := s.store.(entitlementRecordDeleter) - grantDeleter, canDeleteGrants := s.store.(grantByRefsDeleter) + resourceDeleter := s.caps.resourceDeleter + entitlementDeleter := s.caps.entitlementDeleter + grantDeleter := s.caps.grantRefsDeleter + canDeleteResources := resourceDeleter != nil + canDeleteEntitlements := entitlementDeleter != nil + canDeleteGrants := grantDeleter != nil var staleIDs []*v2.ResourceId pageToken := "" for { @@ -3694,7 +3705,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi // network-attached storage this loop cost ~956s for ~90k grants. The // batch form amortizes the fsync without weakening durability. // err is the named value the deferred span reports, so assign it. - if batchDeleter, ok := s.store.(grantsByRefsBatchDeleter); ok { + if batchDeleter := s.caps.grantBatchDeleter; batchDeleter != nil { err = batchDeleter.DeleteGrantsByRefs(ctx, pendingDeletes...) return err } @@ -3702,7 +3713,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi // Prefer the refs-based delete (exact structural identity) when the // store supports it; external ids are a lossy external contract and // stores keyed by structural identity cannot always resolve them. - refsDeleter, _ := s.store.(grantByRefsDeleter) + refsDeleter := s.caps.grantRefsDeleter for _, grantToDelete := range pendingDeletes { if refsDeleter != nil { err = refsDeleter.DeleteGrantByRefs(ctx, grantToDelete) @@ -3793,12 +3804,13 @@ func (s *syncer) expandGrantsForEntitlements(ctx context.Context, action *Action // Create an expander and run a single step. // The expander needs Reader methods (on s.store) plus StoreExpandedGrants - // (on s.store.Grants()). An inline adapter composes them so expand - // stays decoupled from C1ZStore. + // (on s.store.Grants()). s.expanderStore composes them so expand stays + // decoupled from C1ZStore and so the adapter carries the capabilities + // setStore resolved. if s.expandDropStats == nil { s.expandDropStats = &expand.DroppedEdgeStats{} } - expander := expand.NewExpander(expanderStoreAdapter{s.store}, graph) + expander := expand.NewExpander(s.expanderStore(), graph) expander.SetDropStats(s.expandDropStats) err = expander.RunSingleStep(ctx) if err != nil { @@ -3831,24 +3843,24 @@ func (s *syncer) loadStore(ctx context.Context) error { return nil } - storeOpts := []dotc1z.C1ZOption{dotc1z.WithTmpDir(s.tmpDir)} - if s.storageEngine != "" { - storeOpts = append(storeOpts, dotc1z.WithEngine(s.storageEngine)) + storeOpts := []dotc1z.C1ZOption{dotc1z.WithTmpDir(s.cfg.tmpDir)} + if s.cfg.storageEngine != "" { + storeOpts = append(storeOpts, dotc1z.WithEngine(s.cfg.storageEngine)) } - store, err := dotc1z.NewStore(ctx, s.c1zPath, storeOpts...) + store, err := dotc1z.NewStore(ctx, s.cfg.c1zPath, storeOpts...) if err != nil { return err } - if s.setSessionStore != nil { + if s.cfg.setSessionStore != nil { // Instrumented so session-store cost is attributable in the sync // stats instead of vanishing into inflated connector-call latency // (e.g. a broken backend whose every request times out before the // connector falls back to real work). kind := "c1z_" + store.Metadata().Engine - s.setSessionStore.SetSessionStore(ctx, session.NewInstrumentedSessionStore(store.SessionStore(), kind, "", s.recordSessionOp)) + s.cfg.setSessionStore.SetSessionStore(ctx, session.NewInstrumentedSessionStore(store.SessionStore(), kind, "", s.recordSessionOp)) } - s.store = store + s.setStore(store) // Now that s.store is populated, wire the expand progress log's size // provider. NewSyncer could not do this when the caller used @@ -3858,17 +3870,25 @@ func (s *syncer) loadStore(ctx context.Context) error { return nil } +// setStore is the only way a store is attached to a syncer: it resolves the +// store's optional capabilities in the same statement, so s.caps can never +// describe a different store than s.store. Both attach points (NewSyncer for +// WithConnectorStore, loadStore for WithC1ZPath) go through here, as do the +// tests that drive a syncer without NewSyncer. +func (s *syncer) setStore(store c1zstore.Store) { + s.store = store + s.caps = resolveStoreCaps(store) +} + // wireCountsDBSizeProvider attaches the store's DBSizeProvider capability // (if implemented) to s.counts so LogExpandProgress emits decompressed_bytes // and growth delta during long expansions. Idempotent: may be called from // both NewSyncer (WithConnectorStore case) and loadStore (WithC1ZPath case). func (s *syncer) wireCountsDBSizeProvider() { - if s.counts == nil || s.store == nil { + if s.counts == nil || s.caps.dbSize == nil { return } - if sp, ok := s.store.(connectorstore.DBSizeProvider); ok { - s.counts.SetDBSizeProvider(sp) - } + s.counts.SetDBSizeProvider(s.caps.dbSize) } // Close closes the store so the c1z is flushed to disk. @@ -3943,7 +3963,7 @@ type SyncOpt func(s *syncer) func WithRunDuration(d time.Duration) SyncOpt { return func(s *syncer) { if d > 0 { - s.runDuration = d + s.cfg.runDuration = d } } } @@ -3952,7 +3972,7 @@ func WithRunDuration(d time.Duration) SyncOpt { func WithTransitionHandler(f func(s Action)) SyncOpt { return func(s *syncer) { if f != nil { - s.transitionHandler = f + s.cfg.transitionHandler = f } } } @@ -3963,7 +3983,7 @@ func WithTransitionHandler(f func(s Action)) SyncOpt { func WithProgressHandler(f func(s *Progress)) SyncOpt { return func(s *syncer) { if f != nil { - s.progressHandler = f + s.cfg.progressHandler = f } } } @@ -3972,7 +3992,7 @@ func WithProgressHandler(f func(s *Progress)) SyncOpt { // Either this or WithC1ZPath must be provided to create a new syncer. func WithConnectorStore(store c1zstore.Store) SyncOpt { return func(s *syncer) { - s.store = store + s.setStore(store) } } @@ -3980,13 +4000,13 @@ func WithConnectorStore(store c1zstore.Store) SyncOpt { // Either this or WithConnectorStore must be provided to create a new syncer. func WithC1ZPath(path string) SyncOpt { return func(s *syncer) { - s.c1zPath = path + s.cfg.c1zPath = path } } func WithTmpDir(path string) SyncOpt { return func(s *syncer) { - s.tmpDir = path + s.cfg.tmpDir = path } } @@ -3994,7 +4014,7 @@ func WithTmpDir(path string) SyncOpt { // file via WithC1ZPath. Empty uses the baton-sdk default. func WithStorageEngine(engine c1zstore.Engine) SyncOpt { return func(s *syncer) { - s.storageEngine = engine + s.cfg.storageEngine = engine } } @@ -4008,26 +4028,29 @@ func WithStorageEngine(engine c1zstore.Engine) SyncOpt { // InsertResourceGrants arm. func WithFailFastInvariants() SyncOpt { return func(s *syncer) { - s.failFastInvariants = true + s.cfg.failFastInvariants = true } } // WithSkipFullSync skips syncing entirely. func WithSkipFullSync() SyncOpt { return func(s *syncer) { - s.skipFullSync = true + s.cfg.skipFullSync = true } } func WithExternalResourceC1ZPath(path string) SyncOpt { return func(s *syncer) { - s.externalResourceC1ZPath = path + s.cfg.externalResourceC1ZPath = path } } // WithPreviousSyncC1ZPath registers a separate c1z holding the previous sync // for replay features. // +// Advanced: source-cache replay is advanced, opt-in functionality (see +// pkg/sourcecache); most callers should not set this option. +// // This is required for the single-sync v3 (Pebble) engine: a Pebble c1z // holds exactly one sync by contract, so there is no in-file "previous // sync" to replay from (StartNewSync replaces the prior sync). NewSyncer @@ -4045,29 +4068,30 @@ func WithExternalResourceC1ZPath(path string) SyncOpt { // do not carry the replay indexes and compaction provenance this gate requires. func WithPreviousSyncC1ZPath(path string) SyncOpt { return func(s *syncer) { - s.previousSyncC1ZPath = path - s.previousSyncC1ZPathOptional = false + s.cfg.previousSyncC1ZPath = path + s.cfg.previousSyncC1ZPathOptional = false } } // WithOptionalPreviousSyncC1ZPath is WithPreviousSyncC1ZPath with // best-effort semantics: if the file is missing, corrupt, or written by // an incompatible SDK, NewSyncer logs and proceeds WITHOUT replay -// instead of failing. Intended for cache-style replay sources the +// instead of failing. Advanced, opt-in functionality like its strict +// twin — see pkg/sourcecache. Intended for cache-style replay sources the // caller maintains automatically (the service-mode previous-sync spare) // — a bad cache file must never fail a sync. Callers that name a // specific file deliberately should use WithPreviousSyncC1ZPath, which // surfaces open, metadata-read, and close failures. func WithOptionalPreviousSyncC1ZPath(path string) SyncOpt { return func(s *syncer) { - s.previousSyncC1ZPath = path - s.previousSyncC1ZPathOptional = true + s.cfg.previousSyncC1ZPath = path + s.cfg.previousSyncC1ZPathOptional = true } } func WithExternalResourceEntitlementIdFilter(entitlementId string) SyncOpt { return func(s *syncer) { - s.externalResourceEntitlementIdFilter = entitlementId + s.cfg.externalResourceEntitlementIdFilter = entitlementId } } @@ -4083,25 +4107,25 @@ func WithExternalResourceEntitlementIdFilter(entitlementId string) SyncOpt { // would pass TRAIT_USER, TRAIT_GROUP, TRAIT_APP. func WithExternalResourceTraits(traits ...v2.ResourceType_Trait) SyncOpt { return func(s *syncer) { - s.externalResourceTraits = append(s.externalResourceTraits, traits...) + s.cfg.externalResourceTraits = append(s.cfg.externalResourceTraits, traits...) } } func WithTargetedSyncResources(resources []*v2.Resource) SyncOpt { return func(s *syncer) { - s.targetedSyncResources = resources + s.cfg.targetedSyncResources = resources if len(resources) > 0 { - s.syncType = connectorstore.SyncTypePartial + s.cfg.syncType = connectorstore.SyncTypePartial return } // No targeted resource IDs, so we need to update the sync type to either full or resources only. - WithSkipEntitlementsAndGrants(s.skipEntitlementsAndGrants)(s) + WithSkipEntitlementsAndGrants(s.cfg.skipEntitlementsAndGrants)(s) } } func WithSessionStore(sessionStore sessions.SetSessionStore) SyncOpt { return func(s *syncer) { - s.setSessionStore = sessionStore + s.cfg.setSessionStore = sessionStore } } @@ -4109,14 +4133,14 @@ func WithSessionStore(sessionStore sessions.SetSessionStore) SyncOpt { // If empty (the default), all resource types will be synced. func WithSyncResourceTypes(resourceTypeIDs []string) SyncOpt { return func(s *syncer) { - s.syncResourceTypes = resourceTypeIDs + s.cfg.syncResourceTypes = resourceTypeIDs } } // WithOnlyExpandGrants sets whether to skip syncing resources and only expand grants. func WithOnlyExpandGrants() SyncOpt { return func(s *syncer) { - s.onlyExpandGrants = true + s.cfg.onlyExpandGrants = true } } @@ -4134,16 +4158,17 @@ func WithOnlyExpandGrants() SyncOpt { // A normal connector sync must never set this. func WithCompactionMergedStore() SyncOpt { return func(s *syncer) { - s.compactionMergedStore = true + s.cfg.compactionMergedStore = true } } // WithPreserveEntitlementGraph preserves the entitlement graph for later -// incremental expansion. Pebble stores it in the c1z sidecar; stores without -// that capability retain it in the final sync token as a legacy fallback. +// incremental expansion by writing it to the c1z sidecar, which only Pebble +// stores implement. Set it only for a Pebble store: on any other engine the +// graph has nowhere to go, and checkpoints never carry one. func WithPreserveEntitlementGraph() SyncOpt { return func(s *syncer) { - s.preserveEntitlementGraph = true + s.cfg.preserveEntitlementGraph = true } } @@ -4152,7 +4177,7 @@ func WithPreserveEntitlementGraph() SyncOpt { // C1 will process the uploaded c1z and expand grants itself. func WithDontExpandGrants() SyncOpt { return func(s *syncer) { - s.dontExpandGrants = true + s.cfg.dontExpandGrants = true } } func WithSyncID(syncID string) SyncOpt { @@ -4165,15 +4190,15 @@ func WithSyncID(syncID string) SyncOpt { // If true, only resources will be synced. func WithSkipEntitlementsAndGrants(skip bool) SyncOpt { return func(s *syncer) { - s.skipEntitlementsAndGrants = skip + s.cfg.skipEntitlementsAndGrants = skip // Partial syncs can skip entitlements and grants, so don't update the sync type in that case. - if s.syncType == connectorstore.SyncTypePartial { + if s.cfg.syncType == connectorstore.SyncTypePartial { return } if skip { - s.syncType = connectorstore.SyncTypeResourcesOnly + s.cfg.syncType = connectorstore.SyncTypeResourcesOnly } else { - s.syncType = connectorstore.SyncTypeFull + s.cfg.syncType = connectorstore.SyncTypeFull } } } @@ -4182,24 +4207,7 @@ func WithSkipEntitlementsAndGrants(skip bool) SyncOpt { // Entitlements will still be synced. func WithSkipGrants(skip bool) SyncOpt { return func(s *syncer) { - s.skipGrants = skip - } -} - -// WithEntitlementGraphInCheckpoints serializes the entitlement graph into every -// checkpoint token. Off by default: the graph is a projection of data already in -// the store, and encoding it costs O(graph) memory several times over per -// checkpoint, which OOM-kills workers on large tenants. -// -// Enable it to keep expansion progress across restarts. That matters only for a -// tenant whose expansion cannot finish within one worker or activity lifetime — -// without it, such a sync re-runs the load and expansion phases on every resume -// and can fail to converge. Note the two failure modes trade off directly: the -// tenants large enough to need cross-restart progress are the ones whose graph -// is expensive enough to encode that checkpointing may OOM. -func WithEntitlementGraphInCheckpoints(enabled bool) SyncOpt { - return func(s *syncer) { - s.checkpointEntitlementGraph = enabled + s.cfg.skipGrants = skip } } @@ -4226,7 +4234,7 @@ func WithMetricsHandler(h metrics.Handler) SyncOpt { if h == nil { return } - s.metricsHandler = h + s.cfg.metricsHandler = h } } @@ -4236,7 +4244,7 @@ func WithMetricsHandler(h metrics.Handler) SyncOpt { // If < -1, 1 worker is used. (Nothing should do this, but there's no way to return an error in this option.) func WithWorkerCount(count int) SyncOpt { return func(s *syncer) { - s.workerCount = NormalizeWorkerCount(count) + s.cfg.workerCount = NormalizeWorkerCount(count) } } @@ -4247,16 +4255,18 @@ func WithWorkerCount(count int) SyncOpt { // via uotel.WithSyncIdentity, which is how it reaches dotc1z spans too. func WithSyncIdentity(id uotel.SyncIdentity) SyncOpt { return func(s *syncer) { - s.syncIdentity = id + s.cfg.syncIdentity = id } } // NewSyncer returns a new syncer object. func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (Syncer, error) { s := &syncer{ - connector: c, - syncType: connectorstore.SyncTypeFull, - workerCount: 1, + connector: c, + cfg: syncConfig{ + syncType: connectorstore.SyncTypeFull, + workerCount: 1, + }, checkpointInterval: minCheckpointInterval, } @@ -4264,13 +4274,13 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S o(s) } - if s.store == nil && s.c1zPath == "" { + if s.store == nil && s.cfg.c1zPath == "" { return nil, errors.New("a connector store writer or a db path must be provided") } progressLogOpts := []progresslog.Option{} - if s.metricsHandler != nil { - progressLogOpts = append(progressLogOpts, progresslog.WithMetricsHandler(s.metricsHandler)) + if s.cfg.metricsHandler != nil { + progressLogOpts = append(progressLogOpts, progresslog.WithMetricsHandler(s.cfg.metricsHandler)) } s.counts = progresslog.NewProgressCounts(ctx, progressLogOpts...) // Wire the DBSizeProvider now if the store is already set (WithConnectorStore @@ -4279,86 +4289,86 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S // would ship dead for every c1z-path caller — see syncer.loadStore. s.wireCountsDBSizeProvider() - if s.externalResourceC1ZPath != "" { - externalC1ZReader, err := dotc1z.NewStore(ctx, s.externalResourceC1ZPath, dotc1z.WithTmpDir(s.tmpDir), dotc1z.WithReadOnly(true)) + if s.cfg.externalResourceC1ZPath != "" { + externalC1ZReader, err := dotc1z.NewStore(ctx, s.cfg.externalResourceC1ZPath, dotc1z.WithTmpDir(s.cfg.tmpDir), dotc1z.WithReadOnly(true)) if err != nil { return nil, err } s.externalResourceReader = externalC1ZReader } - if s.previousSyncC1ZPath != "" { + if s.cfg.previousSyncC1ZPath != "" { // Open the previous-sync c1z read-only and engine-agnostically // (NewStore selects the engine from the file's magic byte), then // require Pebble: source-cache manifests and replay indexes are a // Pebble capability, so SQLite artifacts are cold inputs. - previousSyncStore, err := dotc1z.NewStore(ctx, s.previousSyncC1ZPath, + previousSyncStore, err := dotc1z.NewStore(ctx, s.cfg.previousSyncC1ZPath, dotc1z.WithReadOnly(true), - dotc1z.WithTmpDir(s.tmpDir), + dotc1z.WithTmpDir(s.cfg.tmpDir), ) switch { case err == nil: if _, ok := enginepkg.AsEngine(previousSyncStore); !ok { if closeErr := previousSyncStore.Close(ctx); closeErr != nil { - if s.previousSyncC1ZPathOptional { + if s.cfg.previousSyncC1ZPathOptional { ctxzap.Extract(ctx).Warn("non-Pebble previous-sync c1z could not close cleanly; syncing without source-cache replay", - zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.String("previous_sync_c1z_path", s.cfg.previousSyncC1ZPath), zap.Error(closeErr), ) break } - return nil, fmt.Errorf("error closing non-Pebble previous-sync c1z %q: %w", s.previousSyncC1ZPath, closeErr) + return nil, fmt.Errorf("error closing non-Pebble previous-sync c1z %q: %w", s.cfg.previousSyncC1ZPath, closeErr) } ctxzap.Extract(ctx).Warn("previous-sync c1z uses an engine that is not replay-eligible; syncing without source-cache replay", - zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.String("previous_sync_c1z_path", s.cfg.previousSyncC1ZPath), ) break } run, metaErr := previousSyncStore.SyncMeta().LatestFinishedSyncOfAnyType(ctx) if metaErr != nil { closeErr := previousSyncStore.Close(ctx) - if s.previousSyncC1ZPathOptional { + if s.cfg.previousSyncC1ZPathOptional { ctxzap.Extract(ctx).Warn("previous-sync c1z metadata unusable; syncing without source-cache replay", - zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.String("previous_sync_c1z_path", s.cfg.previousSyncC1ZPath), zap.Error(errors.Join(metaErr, closeErr)), ) break } return nil, fmt.Errorf( "error reading previous-sync c1z %q metadata: %w", - s.previousSyncC1ZPath, + s.cfg.previousSyncC1ZPath, errors.Join(metaErr, closeErr), ) } if run == nil || !run.UsableAsReplaySource() { if closeErr := previousSyncStore.Close(ctx); closeErr != nil { - if s.previousSyncC1ZPathOptional { + if s.cfg.previousSyncC1ZPathOptional { ctxzap.Extract(ctx).Warn("ineligible previous-sync c1z could not close cleanly; syncing without source-cache replay", - zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.String("previous_sync_c1z_path", s.cfg.previousSyncC1ZPath), zap.Error(closeErr), ) break } - return nil, fmt.Errorf("error closing ineligible previous-sync c1z %q: %w", s.previousSyncC1ZPath, closeErr) + return nil, fmt.Errorf("error closing ineligible previous-sync c1z %q: %w", s.cfg.previousSyncC1ZPath, closeErr) } ctxzap.Extract(ctx).Warn("previous-sync c1z is not replay-eligible; syncing without source-cache replay", - zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.String("previous_sync_c1z_path", s.cfg.previousSyncC1ZPath), ) break } s.previousSyncReader = previousSyncStore - case s.previousSyncC1ZPathOptional: + case s.cfg.previousSyncC1ZPathOptional: // Best-effort replay source (see WithOptionalPreviousSyncC1ZPath): // a missing/corrupt/incompatible cache file degrades to a sync // without ETag replay, never a failed sync. The caller that // maintains the cache replaces it after its next successful // upload, so a bad file self-heals. ctxzap.Extract(ctx).Warn("previous-sync c1z unusable; syncing without etag replay", - zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.String("previous_sync_c1z_path", s.cfg.previousSyncC1ZPath), zap.Error(err), ) default: - return nil, fmt.Errorf("error opening previous-sync c1z %q: %w", s.previousSyncC1ZPath, err) + return nil, fmt.Errorf("error opening previous-sync c1z %q: %w", s.cfg.previousSyncC1ZPath, err) } } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go index c6f9bdfc..aae05df9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go @@ -1321,8 +1321,8 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti // opted-in compactions preserve a fresh graph (so the incremental chain // heals after a fallback); otherwise drop any sidecar inherited from a // fold-copied base. Pebble-only: incremental expansion declines on other - // engines, and without a sidecar the preserved graph would only bloat - // the final sync token. + // engines, and without a sidecar a preserved graph has nowhere to go: + // no writer puts a graph in a sync token. if c.incrementalExpansion && c.resolvedEngine() == c1zstore.EnginePebble { syncOpts = append(syncOpts, sync.WithPreserveEntitlementGraph()) } else if gs, ok := c.compactedC1z.(sync.EntitlementGraphStore); ok { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go index 602f6df3..f098fb61 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go @@ -671,28 +671,48 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { // bucket copy up front, keeps the no-grant-write fold preserving // the base's still-exact digests for free even on a disabled-index // engine. See TestCompactPebbleFoldDigestIndexDisabledDropsDigests. - if len(foldStats.TouchedGrantPartitions) > 0 { - if !destEng.GrantDigestIndexEnabled() { - if err := destEng.DropAllGrantDigestState(ctx); err != nil { - return "", fmt.Errorf("compactPebbleFold: drop grant digest state (digest index disabled): %w", err) - } - l.Info("compactPebbleFold: grant writes with digest index disabled; dropped the base's copied digest state", - zap.Int("touched_partitions", len(foldStats.TouchedGrantPartitions))) - } else { - partitions := make([]string, 0, len(foldStats.TouchedGrantPartitions)) - for p := range foldStats.TouchedGrantPartitions { - partitions = append(partitions, p) - } - if err := destEng.InvalidateGrantDigestPartitions(ctx, partitions); err != nil { - return "", fmt.Errorf("compactPebbleFold: invalidate grant digest partitions: %w", err) - } - if err := destEng.RepairMissingGrantDigests(ctx); err != nil { - return "", fmt.Errorf("compactPebbleFold: repair grant digests: %w", err) - } - l.Info("compactPebbleFold: repaired grant digests for touched entitlements", - zap.Int("touched_partitions", len(partitions))) + // + // When the fold writes NO grants, the base's copied digest state is + // normally left exactly as it was (cheapest possible: zero touched + // partitions, nothing to repair). But that copy is only ever as + // good as what Open decided to keep: the dest's writable Open just + // dropped a base whose stamp named a different GrantDigestABIVersion, + // or the base was sealed with the digest index disabled, or a prior + // digest-build failure dropped it — either way the dest can carry NO + // digest state at all, and with no grant write to trigger the repair + // branch above, nothing else in this fold would ever fix that. So when + // the dest engine wants digests but GrantDigestsPresent() reports + // none, RepairMissingGrantDigests runs anyway: with nothing present + // it delegates straight to the full BuildGrantDigests, a one-time + // O(base) scan the first time a digest-less base is folded. Every + // later fold of the same lineage finds digests already present and + // stamped, and pays the normal O(partials) cost again. + switch { + case len(foldStats.TouchedGrantPartitions) > 0 && !destEng.GrantDigestIndexEnabled(): + if err := destEng.DropAllGrantDigestState(ctx); err != nil { + return "", fmt.Errorf("compactPebbleFold: drop grant digest state (digest index disabled): %w", err) } - } else { + l.Info("compactPebbleFold: grant writes with digest index disabled; dropped the base's copied digest state", + zap.Int("touched_partitions", len(foldStats.TouchedGrantPartitions))) + case len(foldStats.TouchedGrantPartitions) > 0: + partitions := make([]string, 0, len(foldStats.TouchedGrantPartitions)) + for p := range foldStats.TouchedGrantPartitions { + partitions = append(partitions, p) + } + if err := destEng.InvalidateGrantDigestPartitions(ctx, partitions); err != nil { + return "", fmt.Errorf("compactPebbleFold: invalidate grant digest partitions: %w", err) + } + if err := destEng.RepairMissingGrantDigests(ctx); err != nil { + return "", fmt.Errorf("compactPebbleFold: repair grant digests: %w", err) + } + l.Info("compactPebbleFold: repaired grant digests for touched entitlements", + zap.Int("touched_partitions", len(partitions))) + case destEng.GrantDigestIndexEnabled() && !destEng.GrantDigestsPresent(): + if err := destEng.RepairMissingGrantDigests(ctx); err != nil { + return "", fmt.Errorf("compactPebbleFold: build grant digests for a base with none: %w", err) + } + l.Info("compactPebbleFold: no grant writes, but the base carried no grant digest state; built it") + default: l.Info("compactPebbleFold: no grant writes; base grant digest state left untouched") } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/actions.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/actions.go index 6339cc31..4a6d0007 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/actions.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/c1api/actions.go @@ -128,9 +128,10 @@ func (c *actionInvokeTaskHandler) HandleTask(ctx context.Context) error { } reqBuilder := v2.InvokeActionRequest_builder{ - Name: t.GetName(), - Args: t.GetArgs(), - Annotations: t.GetAnnotations(), + Name: t.GetName(), + Args: t.GetArgs(), + Annotations: t.GetAnnotations(), + EncryptionConfigs: t.GetEncryptionConfigs(), } if resourceTypeID := t.GetResourceTypeId(); resourceTypeID != "" { reqBuilder.ResourceTypeId = resourceTypeID diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/action_invoker.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/action_invoker.go index 03932f75..7b53b610 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/action_invoker.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/action_invoker.go @@ -24,9 +24,10 @@ type localActionInvoker struct { dbPath string o sync.Once - action string - resourceTypeID string // Optional: if set, invokes a resource-scoped action - args *structpb.Struct + action string + resourceTypeID string // Optional: if set, invokes a resource-scoped action + args *structpb.Struct + encryptionConfigs []*v2.EncryptionConfig } func (m *localActionInvoker) GetTempDir() string { @@ -42,9 +43,10 @@ func (m *localActionInvoker) Next(ctx context.Context) (*v1.Task, time.Duration, m.o.Do(func() { task = v1.Task_builder{ ActionInvoke: v1.Task_ActionInvokeTask_builder{ - Name: m.action, - Args: m.args, - ResourceTypeId: m.resourceTypeID, + Name: m.action, + Args: m.args, + ResourceTypeId: m.resourceTypeID, + EncryptionConfigs: m.encryptionConfigs, }.Build(), }.Build() }) @@ -60,9 +62,10 @@ func (m *localActionInvoker) Process(ctx context.Context, task *v1.Task, cc type t := task.GetActionInvoke() reqBuilder := v2.InvokeActionRequest_builder{ - Name: t.GetName(), - Args: t.GetArgs(), - Annotations: t.GetAnnotations(), + Name: t.GetName(), + Args: t.GetArgs(), + Annotations: t.GetAnnotations(), + EncryptionConfigs: t.GetEncryptionConfigs(), } if resourceTypeID := t.GetResourceTypeId(); resourceTypeID != "" { reqBuilder.ResourceTypeId = resourceTypeID @@ -74,6 +77,7 @@ func (m *localActionInvoker) Process(ctx context.Context, task *v1.Task, cc type status := resp.GetStatus() finalResp := resp.GetResponse() + finalEncryptedData := resp.GetEncryptedData() l.Info("ActionInvoke response", zap.String("action_id", resp.GetId()), zap.String("name", resp.GetName()), @@ -97,10 +101,11 @@ func (m *localActionInvoker) Process(ctx context.Context, task *v1.Task, cc type } status = r.GetStatus() finalResp = r.GetResponse() + finalEncryptedData = r.GetEncryptedData() } } - l.Info("ActionInvoke response", zap.Any("resp", finalResp)) + l.Info("ActionInvoke response", zap.Any("resp", finalResp), zap.Any("encrypted_data", finalEncryptedData)) if status == v2.BatonActionStatus_BATON_ACTION_STATUS_FAILED { return fmt.Errorf("action invoke failed: %v", finalResp) @@ -112,10 +117,24 @@ func (m *localActionInvoker) Process(ctx context.Context, task *v1.Task, cc type // NewActionInvoker returns a task manager that queues an action invoke task. // If resourceTypeID is provided, it invokes a resource-scoped action. func NewActionInvoker(ctx context.Context, dbPath string, action string, resourceTypeID string, args *structpb.Struct) tasks.Manager { + return NewActionInvokerWithEncryption(ctx, dbPath, action, resourceTypeID, args, nil) +} + +// NewActionInvokerWithEncryption returns a task manager that supplies +// recipients for encrypted action results. +func NewActionInvokerWithEncryption( + _ context.Context, + dbPath string, + action string, + resourceTypeID string, + args *structpb.Struct, + encryptionConfigs []*v2.EncryptionConfig, +) tasks.Manager { return &localActionInvoker{ - dbPath: dbPath, - action: action, - resourceTypeID: resourceTypeID, - args: args, + dbPath: dbPath, + action: action, + resourceTypeID: resourceTypeID, + args: args, + encryptionConfigs: encryptionConfigs, } } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/errors.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/errors.go index ec719322..005b68a8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/errors.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/errors.go @@ -10,8 +10,11 @@ import ( "net/url" "strings" + "golang.org/x/oauth2" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + + "github.com/conductorone/baton-sdk/pkg/ratelimit" ) // wrapTransientNetworkError mirrors Baton HTTP retry classification for callers @@ -20,6 +23,29 @@ func wrapTransientNetworkError(err error) error { if err == nil { return nil } + + // A transient token-endpoint status (429/5xx) stays retryable even if + // the body also carries a recognized RFC 6749 error param; otherwise + // the error param takes priority over the HTTP status, since some + // servers report it on a 200 and 400 is the spec default for + // invalid_client/invalid_grant, both of which GrpcCodeFromHTTPStatus + // alone would misclassify. This branch is total once errors.As matches, + // so a RetrieveError is never run through the network-error checks below. + var retrieveErr *oauth2.RetrieveError + if errors.As(err, &retrieveErr) { + if retrieveErr.Response != nil && isTransientHTTPStatus(retrieveErr.Response.StatusCode) { + return wrapTransientOAuthTokenError(retrieveErr, err) + } + if code, ok := oauthTokenErrorCode(retrieveErr.ErrorCode); ok { + return WrapErrors(code, oauthTokenErrorMessage(retrieveErr), err) + } + code := codes.Unknown + if retrieveErr.Response != nil { + code = GrpcCodeFromHTTPStatus(retrieveErr.Response.StatusCode) + } + return WrapErrors(code, oauthTokenErrorMessage(retrieveErr), err) + } + if errors.Is(err, io.ErrUnexpectedEOF) { return WrapErrors(codes.Unavailable, "unexpected EOF", err) } @@ -88,6 +114,69 @@ func wrapTransientNetworkError(err error) error { return err } +// isTransientHTTPStatus reports whether GrpcCodeFromHTTPStatus maps +// statusCode to a code retry.Retryer.ShouldWaitAndRetry treats as retryable +// (Unavailable or DeadlineExceeded), so a transient token-endpoint failure +// stays retryable regardless of what error param the body also carries. +func isTransientHTTPStatus(statusCode int) bool { + switch GrpcCodeFromHTTPStatus(statusCode) { + case codes.Unavailable, codes.DeadlineExceeded: + return true + default: + return false + } +} + +// wrapTransientOAuthTokenError mirrors WrapErrorsWithRateLimitInfo's detail +// attachment (retry.Retryer reads it for rate-limit-aware backoff), while +// keeping ErrorDescription in the message the way oauthTokenErrorMessage +// does elsewhere in this file. +func wrapTransientOAuthTokenError(retrieveErr *oauth2.RetrieveError, err error) error { + msg := retrieveErr.Response.Status + if retrieveErr.ErrorDescription != "" { + msg = fmt.Sprintf("%s: %s", msg, retrieveErr.ErrorDescription) + } + st := status.New(GrpcCodeFromHTTPStatus(retrieveErr.Response.StatusCode), msg) + if description, rlErr := ratelimit.ExtractRateLimitData(retrieveErr.Response.StatusCode, &retrieveErr.Response.Header); rlErr == nil { + if withDetails, detailsErr := st.WithDetails(description); detailsErr == nil { + st = withDetails + } + } + return errors.Join(st.Err(), err) +} + +// oauthTokenErrorCode maps an RFC 6749 §5.2 token-error "error" parameter to +// a grpc code. ok is false when errCode is empty or unrecognized, signaling +// the caller to fall back to the HTTP status. +func oauthTokenErrorCode(errCode string) (codes.Code, bool) { + switch errCode { + case "invalid_client", "invalid_grant": + return codes.Unauthenticated, true + case "unauthorized_client", "access_denied": + return codes.PermissionDenied, true + case "invalid_scope", "invalid_request", "unsupported_grant_type", "unsupported_response_type": + return codes.InvalidArgument, true + default: + return codes.Unknown, false + } +} + +// oauthTokenErrorMessage prefers the RFC 6749 error/error_description pair +// the token endpoint sent, since that survives even when the HTTP status +// alone would be misleading (e.g. a 200 response carrying an error body). +func oauthTokenErrorMessage(retrieveErr *oauth2.RetrieveError) string { + switch { + case retrieveErr.ErrorCode != "" && retrieveErr.ErrorDescription != "": + return fmt.Sprintf("%s: %s", retrieveErr.ErrorCode, retrieveErr.ErrorDescription) + case retrieveErr.ErrorCode != "": + return retrieveErr.ErrorCode + case retrieveErr.Response != nil: + return retrieveErr.Response.Status + default: + return "oauth2 token request failed" + } +} + func isHTTP2ClientConnectionLost(err error) bool { return strings.Contains(err.Error(), "http2: client connection lost") } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/pagination.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/pagination.go index 397eb4b9..fdb54e28 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/pagination.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/pagination.go @@ -1,8 +1,14 @@ package uhttp import ( + "errors" + "fmt" + "reflect" "strings" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "github.com/conductorone/baton-sdk/pkg/pagination" ) @@ -97,3 +103,44 @@ func WithNextLinkPagination(bag *pagination.Bag, config *NextLinkConfig) DoOptio return nil } } + +// PaginatedResponse is implemented by response types that can report whether the API returned the pagination data the caller needs to fetch the next page. +type PaginatedResponse interface { + HasPaginationData() bool +} + +// ErrMissingPaginationData is the sentinel returned when a successful response decoded fine but carried no pagination data; match it with errors.Is. +var ErrMissingPaginationData = errors.New("uhttp: response is missing pagination data") + +// WithPaginationData decodes the body into response and fails the request if its pagination data is absent, so an API that silently drops its cursor errors instead of ending the sync after one page. +// response must be a non-nil pointer, since the body is decoded into it. +func WithPaginationData(response PaginatedResponse) DoOption { + return func(resp *WrapperResponse) error { + if response == nil { + return status.Error(codes.InvalidArgument, "WithPaginationData: response is nil") + } + + rv := reflect.ValueOf(response) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return status.Errorf(codes.InvalidArgument, "WithPaginationData: response must be a non-nil pointer, got %T", response) + } + + if !isSuccessStatusCode(resp.StatusCode) { + return nil + } + + if err := WithResponse(response)(resp); err != nil { + return err + } + + if !response.HasPaginationData() { + return WrapErrors( + codes.FailedPrecondition, + fmt.Sprintf("%T reported no pagination data. status code: %d", response, resp.StatusCode), + ErrMissingPaginationData, + ) + } + + return nil + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go index 22806159..a12ff18d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go @@ -259,7 +259,7 @@ func WithAlwaysXMLResponse(response any) DoOption { if resp.StatusCode == http.StatusNoContent { return nil } - if resp.StatusCode >= 200 && resp.StatusCode < 300 && len(resp.Body) == 0 { + if isSuccessStatusCode(resp.StatusCode) && len(resp.Body) == 0 { return nil } return unmarshalXMLToMap(genericResponse, resp) @@ -317,6 +317,13 @@ type ErrorResponse interface { Message() string } +// isSuccessStatusCode reports whether code is in the 2xx success class. +// http.StatusOK (200) is the inclusive lower bound and +// http.StatusMultipleChoices (300) the exclusive upper bound. +func isSuccessStatusCode(code int) bool { + return code >= http.StatusOK && code < http.StatusMultipleChoices +} + // GrpcCodeFromHTTPStatus maps an HTTP status code to the appropriate gRPC status code. func GrpcCodeFromHTTPStatus(httpStatus int) codes.Code { switch httpStatus { @@ -349,7 +356,7 @@ func GrpcCodeFromHTTPStatus(httpStatus int) codes.Code { func WithErrorResponse(resource ErrorResponse) DoOption { return func(resp *WrapperResponse) error { - if resp.StatusCode < 300 { + if resp.StatusCode < http.StatusMultipleChoices { return nil } @@ -425,7 +432,7 @@ func WithGenericResponse(response *map[string]any) DoOption { return nil } - if resp.StatusCode >= 200 && resp.StatusCode < 300 && len(resp.Body) == 0 { + if isSuccessStatusCode(resp.StatusCode) && len(resp.Body) == 0 { return nil } @@ -587,7 +594,7 @@ func (c *BaseHttpClient) Do(req *http.Request, options ...DoOption) (*http.Respo } } - if resp.StatusCode < 200 || resp.StatusCode >= 300 { + if !isSuccessStatusCode(resp.StatusCode) { grpcCode := GrpcCodeFromHTTPStatus(resp.StatusCode) return resp, WrapErrorsWithRateLimitInfo(grpcCode, resp, optErrs...) } diff --git a/vendor/github.com/conductorone/dpop/integrations/dpop_grpc/client_credential.go b/vendor/github.com/conductorone/dpop/integrations/dpop_grpc/client_credential.go index 4a8b4ce9..b0e229b9 100644 --- a/vendor/github.com/conductorone/dpop/integrations/dpop_grpc/client_credential.go +++ b/vendor/github.com/conductorone/dpop/integrations/dpop_grpc/client_credential.go @@ -5,9 +5,12 @@ import ( "errors" "net/url" + "github.com/conductorone/dpop/integrations/dpop_oauth2" "github.com/conductorone/dpop/pkg/dpop" "golang.org/x/oauth2" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" ) // DPoPCredentials implements the credentials.PerRPCCredentials interface @@ -54,7 +57,7 @@ func (d *DPoPCredentials) GetRequestMetadata(ctx context.Context, uri ...string) // Get the OAuth2 token token, err := d.tokenSource.Token() if err != nil { - return nil, err + return nil, tokenStatusError(err) } // Add access token to proof options @@ -78,3 +81,19 @@ func (d *DPoPCredentials) GetRequestMetadata(ctx context.Context, uri ...string) func (d *DPoPCredentials) RequireTransportSecurity() bool { return d.requireTLS } + +// tokenStatusError maps a token source failure onto a gRPC status so the +// transient/definitive classification survives the per-RPC credentials +// boundary — grpc-go flattens any non-status credentials error to +// codes.Unauthenticated. Transient failures (5xx responses, transport errors, +// timeouts; see dpop_oauth2.IsTransient) become codes.Unavailable so callers' +// retry policies treat them as retryable. Definitive failures (e.g. +// invalid_client, a disabled credential) become codes.Unauthenticated and +// fail fast. +func tokenStatusError(err error) error { + code := codes.Unauthenticated + if dpop_oauth2.IsTransient(err) { + code = codes.Unavailable + } + return status.Errorf(code, "dpop_grpc: failed to fetch token: %v", err) +} diff --git a/vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/retry.go b/vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/retry.go new file mode 100644 index 00000000..888be662 --- /dev/null +++ b/vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/retry.go @@ -0,0 +1,115 @@ +package dpop_oauth2 + +import ( + "context" + "errors" + "math/rand/v2" + "net/http" + "time" +) + +// Defaults chosen so a full retry cycle (attempts plus backoff) fits well +// within the 30 second budget Token() imposes on each call. +const ( + defaultRetryMaxAttempts = 3 + defaultRetryInitialDelay = 500 * time.Millisecond + defaultRetryMaxDelay = 2 * time.Second +) + +// RetryConfig controls how Token() retries transient token request failures: +// 5xx or 429 responses, transport-level errors, and timeouts. Every attempt +// re-runs the full token request with a freshly signed DPoP proof and client +// assertion; a proof's jti may be single-use, so an identical request is never +// replayed. Definitive OAuth protocol errors (e.g. invalid_client) are never +// retried. +type RetryConfig struct { + // MaxAttempts is the total number of attempts, including the first. + // Values below 1 are treated as 1 (retries disabled). + MaxAttempts int + // InitialDelay is the backoff before the first retry. It doubles on each + // subsequent retry, capped at MaxDelay, with jitter applied. + InitialDelay time.Duration + // MaxDelay caps the backoff between attempts. + MaxDelay time.Duration +} + +// DefaultRetryConfig returns the retry behavior used when no WithRetryConfig +// option is supplied. +func DefaultRetryConfig() RetryConfig { + return RetryConfig{ + MaxAttempts: defaultRetryMaxAttempts, + InitialDelay: defaultRetryInitialDelay, + MaxDelay: defaultRetryMaxDelay, + } +} + +func (c RetryConfig) normalized() RetryConfig { + if c.MaxAttempts < 1 { + c.MaxAttempts = 1 + } + if c.InitialDelay <= 0 { + c.InitialDelay = defaultRetryInitialDelay + } + if c.MaxDelay < c.InitialDelay { + c.MaxDelay = c.InitialDelay + } + return c +} + +// retryDelay computes the backoff preceding retry number `retry` (1-based): +// exponential doubling capped at MaxDelay, with equal jitter (half the delay +// is fixed, the other half randomized) so concurrent clients hitting the same +// outage don't retry in lockstep. +func (c RetryConfig) retryDelay(retry int) time.Duration { + delay := c.InitialDelay + for i := 1; i < retry; i++ { + delay *= 2 + if delay >= c.MaxDelay { + delay = c.MaxDelay + break + } + } + half := delay / 2 + return half + rand.N(half+1) +} + +// sleepBeforeRetry blocks for the backoff delay preceding the given retry. +// It returns false if ctx expires first. +func sleepBeforeRetry(ctx context.Context, cfg RetryConfig, retry int) bool { + timer := time.NewTimer(cfg.retryDelay(retry)) + defer timer.Stop() + select { + case <-ctx.Done(): + return false + case <-timer.C: + return true + } +} + +// IsTransient reports whether err is a token request failure that was +// classified as transient: the failure mode gives no indication the +// credential itself is bad, so retrying (with a fresh proof and assertion) +// may succeed. +func IsTransient(err error) bool { + return errors.Is(err, ErrTokenRequestTransient) +} + +// markTransient tags err as a transient token request failure. The result +// matches ErrTokenRequestTransient in addition to everything err already +// matched, and its message is unchanged. +func markTransient(err error) error { + return &transientError{error: err} +} + +type transientError struct{ error } + +func (e *transientError) Unwrap() []error { + return []error{e.error, ErrTokenRequestTransient} +} + +// isRetryableStatus reports whether an HTTP response status is worth +// retrying: any 5xx (upstream failure) or 429 (throttling). 4xx OAuth +// protocol rejections are definitive and must not be retried. +func isRetryableStatus(code int) bool { + return code >= 500 || code == http.StatusTooManyRequests +} diff --git a/vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/token_client_assertion.go b/vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/token_client_assertion.go index edb8588c..2ebaaace 100644 --- a/vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/token_client_assertion.go +++ b/vendor/github.com/conductorone/dpop/integrations/dpop_oauth2/token_client_assertion.go @@ -12,6 +12,7 @@ import ( "github.com/go-jose/go-jose/v4" "github.com/go-jose/go-jose/v4/jwt" + "github.com/google/uuid" "golang.org/x/oauth2" "github.com/conductorone/dpop/pkg/dpop" @@ -30,6 +31,14 @@ var ( // ErrTokenRequestFailed indicates the token request failed ErrTokenRequestFailed = errors.New("dpop_oauth2: token request failed") + // ErrTokenRequestTransient classifies a token request failure as likely + // transient: a 5xx or 429 response, a transport-level error, or a + // timeout. Errors matching this sentinel always also match + // ErrTokenRequestFailed; definitive OAuth protocol rejections (e.g. + // invalid_client) match only ErrTokenRequestFailed. Use IsTransient to + // test for it. + ErrTokenRequestTransient = errors.New("dpop_oauth2: transient token request failure") + // ErrProofCreationFailed indicates failure to create or sign DPoP proof ErrProofCreationFailed = errors.New("dpop_oauth2: failed to create or sign DPoP proof") ) @@ -88,6 +97,7 @@ type tokenSourceOptions struct { proofOptions []dpop.ProofOption nonceStore *NonceStore requestOptions []TokenRequestOption + retry RetryConfig } // WithBaseContext sets a custom base context for the token source @@ -125,6 +135,15 @@ func WithRequestOption(opt TokenRequestOption) TokenSourceOption { } } +// WithRetryConfig overrides how transient token request failures are retried. +// See RetryConfig for field semantics; set MaxAttempts to 1 to disable +// retries entirely. +func WithRetryConfig(cfg RetryConfig) TokenSourceOption { + return func(opts *tokenSourceOptions) { + opts.retry = cfg + } +} + func NewTokenSource(proofer *dpop.Proofer, tokenURL *url.URL, clientID string, clientSecret *jose.JSONWebKey, opts ...TokenSourceOption) (*tokenSource, error) { if proofer == nil { return nil, fmt.Errorf("%w: dpop-proofer", ErrMissingRequiredField) @@ -145,6 +164,7 @@ func NewTokenSource(proofer *dpop.Proofer, tokenURL *url.URL, clientID string, c options := &tokenSourceOptions{ baseCtx: context.Background(), httpClient: http.DefaultClient, + retry: DefaultRetryConfig(), } for _, opt := range opts { @@ -161,6 +181,7 @@ func NewTokenSource(proofer *dpop.Proofer, tokenURL *url.URL, clientID string, c requestOptions: options.requestOptions, proofOptions: options.proofOptions, nonceStore: options.nonceStore, + retry: options.retry.normalized(), }, nil } @@ -174,15 +195,66 @@ type tokenSource struct { requestOptions []TokenRequestOption proofOptions []dpop.ProofOption nonceStore *NonceStore + retry RetryConfig } func (c *tokenSource) Token() (*oauth2.Token, error) { ctx, done := context.WithTimeout(c.baseCtx, time.Second*30) defer done() - return c.tryToken(ctx, true) + + // Transient failures (5xx/429, transport errors, timeouts) are retried + // with capped exponential backoff + jitter. The retry re-enters tryToken, + // so every attempt signs a fresh DPoP proof and client assertion — both + // carry unique jtis, so an identical request is never replayed. + // Definitive failures (OAuth protocol rejections) return immediately. + // + // A nonce learned from a use_dpop_nonce challenge is carried across + // attempts so a bare consumer (no NonceStore) isn't re-challenged on + // every retry. + var lastErr error + retryNonce := "" + for attempt := 0; attempt < c.retry.MaxAttempts; attempt++ { + if attempt > 0 { + if !sleepBeforeRetry(ctx, c.retry, attempt) { + // The context died mid-backoff. A deadline expiry (the 30s + // Token() budget) is a timeout: surface the last transient + // failure so callers can still classify it. A caller cancel + // is not a timeout — strip the transient classification so + // nothing retries abandoned work. + if errors.Is(ctx.Err(), context.Canceled) { + // context.Cause preserves a WithCancelCause cause in the + // chain; for a plain cancel it is context.Canceled. + return nil, fmt.Errorf("%w: %w during retry backoff (last error: %v)", ErrTokenRequestFailed, context.Cause(ctx), lastErr) + } + break + } + } + + token, nonce, err := c.tryToken(ctx, true, retryNonce) + if err == nil { + return token, nil + } + if nonce != "" { + retryNonce = nonce + } + lastErr = err + if !IsTransient(err) { + return nil, err + } + } + return nil, lastErr } -func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool) (*oauth2.Token, error) { +// tryToken performs a single token request. retryNonce, when non-empty, is the +// nonce returned by a prior use_dpop_nonce challenge and is attached to this +// attempt's proof regardless of whether a NonceStore is configured. This is +// what makes a bare consumer (no NonceStore) nonce-aware: the challenge/retry +// is self-contained within a single Token() call. +// +// The second return value is the nonce in effect for this attempt (the +// carried retryNonce, a cached store nonce, or a newly challenged one), so +// the transient retry loop in Token() can carry it into the next attempt. +func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool, retryNonce string) (*oauth2.Token, string, error) { jsigner, err := jose.NewSigner( jose.SigningKey{ Algorithm: jose.EdDSA, @@ -190,7 +262,7 @@ func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool) (*oauth2. }, nil) if err != nil { - return nil, fmt.Errorf("%w: failed to create signer: %v", ErrProofCreationFailed, err) + return nil, retryNonce, fmt.Errorf("%w: failed to create signer: %v", ErrProofCreationFailed, err) } // Our token host may include a port, but the audience never expects a port @@ -198,6 +270,11 @@ func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool) (*oauth2. now := time.Now() claims := &jwt.Claims{ + // A unique jti makes every signed assertion distinct. Without it, + // second-precision timestamps plus deterministic Ed25519 signatures + // would make fast retries re-send a byte-identical assertion, which a + // server enforcing RFC 7523 single-use may reject. + ID: uuid.New().String(), Issuer: c.clientID, Subject: c.clientID, Audience: jwt.Audience{aud}, @@ -220,13 +297,13 @@ func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool) (*oauth2. for _, opt := range c.requestOptions { err = opt(tr) if err != nil { - return nil, fmt.Errorf("%w: failed to modify request: %v", ErrTokenRequestFailed, err) + return nil, retryNonce, fmt.Errorf("%w: failed to modify request: %v", ErrTokenRequestFailed, err) } } marshalledClaims, err := tr.Marshaler(claims) if err != nil { - return nil, fmt.Errorf("%w: failed to marshal claims: %v", ErrTokenRequestFailed, err) + return nil, retryNonce, fmt.Errorf("%w: failed to marshal claims: %v", ErrTokenRequestFailed, err) } method := http.MethodPost @@ -234,34 +311,37 @@ func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool) (*oauth2. proofOpts := make([]dpop.ProofOption, 0, len(c.proofOptions)+2) proofOpts = append(proofOpts, c.proofOptions...) - // Add nonce if available from store - if c.nonceStore != nil { - nonce := c.nonceStore.GetNonce() - if nonce != "" { - proofOpts = append(proofOpts, dpop.WithStaticNonce(nonce)) - } + // Attach a nonce when available. Prefer the nonce from a use_dpop_nonce + // challenge on this same Token() call (retryNonce); otherwise fall back to + // a cached nonce from the configured store for cross-call reuse. + nonce := retryNonce + if nonce == "" && c.nonceStore != nil { + nonce = c.nonceStore.GetNonce() + } + if nonce != "" { + proofOpts = append(proofOpts, dpop.WithStaticNonce(nonce)) } dpopProof, err := c.proofer.CreateProof(ctx, method, c.tokenURL.String(), proofOpts...) if err != nil { - return nil, fmt.Errorf("%w: failed to create proof: %v", ErrProofCreationFailed, err) + return nil, nonce, fmt.Errorf("%w: failed to create proof: %v", ErrProofCreationFailed, err) } rv, err := jsigner.Sign(marshalledClaims) if err != nil { - return nil, fmt.Errorf("%w: failed to sign proof: %v", ErrProofCreationFailed, err) + return nil, nonce, fmt.Errorf("%w: failed to sign proof: %v", ErrProofCreationFailed, err) } s, err := rv.CompactSerialize() if err != nil { - return nil, fmt.Errorf("%w: failed to serialize proof: %v", ErrProofCreationFailed, err) + return nil, nonce, fmt.Errorf("%w: failed to serialize proof: %v", ErrProofCreationFailed, err) } tr.Body["client_assertion"] = []string{s} req, err := http.NewRequestWithContext(ctx, method, c.tokenURL.String(), strings.NewReader(tr.Body.Encode())) if err != nil { - return nil, fmt.Errorf("%w: failed to create request: %v", ErrTokenRequestFailed, err) + return nil, nonce, fmt.Errorf("%w: failed to create request: %v", ErrTokenRequestFailed, err) } req.Header.Set(dpop.HeaderName, dpopProof) @@ -271,7 +351,23 @@ func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool) (*oauth2. resp, err := c.httpClient.Do(req) if err != nil { - return nil, fmt.Errorf("%w: failed to execute request: %v", ErrTokenRequestFailed, err) + // The transport error stays in the chain (%w) so callers can inspect + // the underlying cause (context.Canceled, net errors, ...). + reqErr := fmt.Errorf("%w: failed to execute request: %w", ErrTokenRequestFailed, err) + // A canceled context means the caller abandoned the call — that is + // not a transport failure, so don't classify it as retryable. Check + // the context as well as the returned error: when the context was + // canceled via context.WithCancelCause, Do returns the cause, which + // need not match context.Canceled. + if errors.Is(err, context.Canceled) || errors.Is(ctx.Err(), context.Canceled) { + return nil, nonce, reqErr + } + // Everything else that fails before an HTTP response (connection + // resets, proxy errors, timeouts — including a deadline expiry, which + // is exactly the timed-out token POST class) never reached the + // authorization server's OAuth logic: it carries no verdict about the + // credential, so it is safe to classify as retryable. + return nil, nonce, markTransient(reqErr) } defer resp.Body.Close() @@ -282,58 +378,67 @@ func (c *tokenSource) tryToken(ctx context.Context, firstAttempt bool) (*oauth2. ErrorDescription string `json:"error_description"` } if err := json.NewDecoder(resp.Body).Decode(&errorResp); err != nil { - return nil, fmt.Errorf("%w: failed to decode error response: %v", ErrTokenRequestFailed, err) + return nil, nonce, fmt.Errorf("%w: failed to decode error response: %v", ErrTokenRequestFailed, err) } if errorResp.Error == "use_dpop_nonce" { // Get the new nonce from header - nonce := resp.Header.Get(dpop.NonceHeaderName) - if nonce == "" { - return nil, ErrNonceMissing + challengeNonce := resp.Header.Get(dpop.NonceHeaderName) + if challengeNonce == "" { + return nil, nonce, ErrNonceMissing } - // Store the nonce if we have a store + // Store the nonce for cross-call reuse if we have a store if c.nonceStore != nil { - c.nonceStore.SetNonce(nonce) + c.nonceStore.SetNonce(challengeNonce) } // Only retry once on first attempt if !firstAttempt { - return nil, fmt.Errorf("%w: token request failed after retry: %s - %s", ErrTokenRequestFailed, errorResp.Error, errorResp.ErrorDescription) + return nil, challengeNonce, fmt.Errorf("%w: token request failed after retry: %s - %s", ErrTokenRequestFailed, errorResp.Error, errorResp.ErrorDescription) } - // Try again with the new nonce - return c.tryToken(ctx, false) + // Retry with the challenged nonce. Passing it explicitly means the + // retry is nonce-aware even with no NonceStore configured. + return c.tryToken(ctx, false, challengeNonce) } - return nil, fmt.Errorf("%w: %s - %s", ErrTokenRequestFailed, errorResp.Error, errorResp.ErrorDescription) + return nil, nonce, fmt.Errorf("%w: %s - %s", ErrTokenRequestFailed, errorResp.Error, errorResp.ErrorDescription) + } + + if isRetryableStatus(resp.StatusCode) { + return nil, nonce, markTransient(fmt.Errorf("%w: unexpected status code: %s", ErrTokenRequestFailed, resp.Status)) } if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%w: unexpected status code: %s", ErrTokenRequestFailed, resp.Status) + return nil, nonce, fmt.Errorf("%w: unexpected status code: %s", ErrTokenRequestFailed, resp.Status) } token := &oauth2.Token{} err = json.NewDecoder(resp.Body).Decode(token) if err != nil { - return nil, fmt.Errorf("%w: failed to decode token response: %v", ErrInvalidToken, err) + return nil, nonce, fmt.Errorf("%w: failed to decode token response: %v", ErrInvalidToken, err) } if token.AccessToken == "" { - return nil, fmt.Errorf("%w: empty access token", ErrInvalidToken) + return nil, nonce, fmt.Errorf("%w: empty access token", ErrInvalidToken) } if token.Expiry.IsZero() { token.Expiry = time.Now() if token.ExpiresIn > 0 { - token.Expiry = time.Now().Add(time.Duration(token.ExpiresIn-10) * time.Second) // 10 seconds before the token expires + expiresIn := token.ExpiresIn - 10 // 10 seconds before the token expires + if expiresIn < 0 { + expiresIn = 0 + } + token.Expiry = time.Now().Add(time.Duration(expiresIn) * time.Second) } } // Accept both DPoP and Bearer tokens // If we sent a DPoP proof but got a Bearer token, that means the AS doesn't support DPoP if !strings.EqualFold(token.TokenType, "DPoP") && !strings.EqualFold(token.TokenType, "Bearer") { - return nil, fmt.Errorf("%w: invalid token type: %s", ErrInvalidToken, token.TokenType) + return nil, nonce, fmt.Errorf("%w: invalid token type: %s", ErrInvalidToken, token.TokenType) } - return token, nil + return token, nonce, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 7c7fe294..179a3b57 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -266,18 +266,18 @@ github.com/cockroachdb/redact/internal/markers github.com/cockroachdb/redact/internal/redact github.com/cockroachdb/redact/internal/rfmt github.com/cockroachdb/redact/internal/rfmt/fmtsort -# github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b +# github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 ## explicit; go 1.21 github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-github v0.4.0 => ../baton-github +# github.com/conductorone/baton-github v0.4.5 ## explicit; go 1.25.2 github.com/conductorone/baton-github/pkg/config github.com/conductorone/baton-github/pkg/connector github.com/conductorone/baton-github/pkg/customclient -# github.com/conductorone/baton-sdk v0.26.0 +# github.com/conductorone/baton-sdk v0.30.0 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 @@ -350,13 +350,13 @@ github.com/conductorone/baton-sdk/pkg/uotel github.com/conductorone/baton-sdk/pkg/uotel/uotelzap github.com/conductorone/baton-sdk/pkg/ustrings github.com/conductorone/baton-sdk/pkg/utls -# github.com/conductorone/dpop v0.2.6 +# github.com/conductorone/dpop v0.3.0 ## explicit; go 1.23.4 github.com/conductorone/dpop/pkg/dpop -# github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 +# github.com/conductorone/dpop/integrations/dpop_grpc v0.3.0 ## explicit; go 1.23.4 github.com/conductorone/dpop/integrations/dpop_grpc -# github.com/conductorone/dpop/integrations/dpop_oauth2 v0.2.5 +# github.com/conductorone/dpop/integrations/dpop_oauth2 v0.3.0 ## explicit; go 1.23.4 github.com/conductorone/dpop/integrations/dpop_oauth2 # github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc @@ -1022,4 +1022,3 @@ modernc.org/memory modernc.org/sqlite modernc.org/sqlite/lib modernc.org/sqlite/vtab -# github.com/conductorone/baton-github => ../baton-github From 9f95071b6590dd7592f6b4ffc937bd0d0557e832 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 15 Sep 2026 04:49:25 -0300 Subject: [PATCH 5/9] fix(config): wrap long field list to satisfy line-length lint Also document the GitHub App permission required for sync-last-activity alongside the existing PAT scope requirement. Co-Authored-By: Claude Sonnet 5 --- docs/connector.mdx | 4 ++-- pkg/config/config.go | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/connector.mdx b/docs/connector.mdx index 4b6ecff2..73b7e093 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -394,8 +394,8 @@ You can enable **Sync last activity** to have C1 show, for each member, the last **Requirements:** -- The personal access token used by the connector must have the **read:audit_log** scope. -- The token must have permission to view each organization's audit log (typically an organization owner). +- **Personal access token:** the token must have the **read:audit_log** scope, and permission to view each organization's audit log (typically an organization owner). +- **GitHub App:** the app must have the **Organization permissions > Administration: Read-only access** permission. If you followed the GitHub App setup instructions above, this is already granted (it's also required for SAML/SSO detection), so no extra permission needs to be added. **Enabling the config option alone is not enough.** After you enable **Sync last activity** (or set `--sync-last-activity` / `BATON_SYNC_LAST_ACTIVITY=true`) and save the connector, go to the connector's **Capabilities & configuration** page in C1 and confirm the **GitHub Activity** resource type is enabled for sync. diff --git a/pkg/config/config.go b/pkg/config/config.go index 390ee69f..f52282bc 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -113,8 +113,11 @@ var Config = field.NewConfiguration( Name: GithubAppGroup, DisplayName: "GitHub app", HelpText: "Use a github app for authentication", - Fields: []field.SchemaField{instanceUrlField, appIDField, appPrivateKeyPath, orgField, EnterprisesField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, syncLastActivity}, - Default: false, + Fields: []field.SchemaField{ + instanceUrlField, appIDField, appPrivateKeyPath, orgField, EnterprisesField, + syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, syncLastActivity, + }, + Default: false, }, }), ) From 59fca5f6dbdb97d786085412b1ea4da8b428d95f Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 15 Sep 2026 05:57:16 -0300 Subject: [PATCH 6/9] docs: align "Sync last activity" toggle label with actual field name The config field's display name is "Sync user last activity" (pkg/config/config.go:81), but the docs referred to it as "Sync last activity" in four places, which wouldn't match what users see in the C1 config UI. Co-Authored-By: Claude Sonnet 5 --- docs/connector.mdx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/connector.mdx b/docs/connector.mdx index 73b7e093..fa1e8342 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -244,7 +244,7 @@ If you're using a personal access token to set up the connector: 1. **Optional.** For large organizations, click to enable **Optimize sync for large organizations**. This reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. - 1. **Optional.** If you want to see when members were last active, click to enable **Sync last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. + 1. **Optional.** If you want to see when members were last active, click to enable **Sync user last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. If you're using a GitHub app to set up the connector: @@ -265,7 +265,7 @@ If you're using a GitHub app to set up the connector: 1. **Optional.** For large organizations, click to enable **Optimize sync for large organizations**. This reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. - 1. **Optional.** If you want to see when members were last active, click to enable **Sync last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. + 1. **Optional.** If you want to see when members were last active, click to enable **Sync user last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. Click **Save**. @@ -390,7 +390,7 @@ When deploying on Kubernetes with a minimal base image (for example, Debian slim ## Sync member last activity -You can enable **Sync last activity** to have C1 show, for each member, the last time they were seen taking action in that organization (for example, commenting on an issue or pull request, reviewing code, or changing a setting). This appears on the member's profile in C1 alongside their other access details. +You can enable **Sync user last activity** to have C1 show, for each member, the last time they were seen taking action in that organization (for example, commenting on an issue or pull request, reviewing code, or changing a setting). This appears on the member's profile in C1 alongside their other access details. **Requirements:** @@ -398,7 +398,7 @@ You can enable **Sync last activity** to have C1 show, for each member, the last - **GitHub App:** the app must have the **Organization permissions > Administration: Read-only access** permission. If you followed the GitHub App setup instructions above, this is already granted (it's also required for SAML/SSO detection), so no extra permission needs to be added. -**Enabling the config option alone is not enough.** After you enable **Sync last activity** (or set `--sync-last-activity` / `BATON_SYNC_LAST_ACTIVITY=true`) and save the connector, go to the connector's **Capabilities & configuration** page in C1 and confirm the **GitHub Activity** resource type is enabled for sync. +**Enabling the config option alone is not enough.** After you enable **Sync user last activity** (or set `--sync-last-activity` / `BATON_SYNC_LAST_ACTIVITY=true`) and save the connector, go to the connector's **Capabilities & configuration** page in C1 and confirm the **GitHub Activity** resource type is enabled for sync. - On a **brand-new** connector, this resource type is included by default, so no extra step is needed. - On an **existing, already-configured** connector, C1 does not automatically add a newly-available resource type to a previously saved sync selection — it will show up in the resource list as available, but disabled, until you manually enable it. From b53c38203798d1229c3477109fde28f259be0684 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 15 Sep 2026 12:06:53 -0300 Subject: [PATCH 7/9] docs: address FeliLucero1 review findings on last-activity docs - Reword last-activity docs to reflect that attribution is global across all synced orgs, not per-organization - Drop the ambiguous "included by default on brand-new connectors" claim for the GitHub Activity resource type - Fix stale invitation status doc (Unspecified -> Pending, per the v0.4.5 vendor bump already in this PR) - Add missing GitHub Activity row to the capabilities table - Clarify that sync-last-activity is based on audit-log activity in the config field description (regenerated config_schema.json and README accordingly) Co-Authored-By: Claude Sonnet 5 --- README.md | 2 +- config_schema.json | 2 +- docs/connector.mdx | 12 ++++++------ pkg/config/config.go | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f2aa6c4d..76b5655f 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ Flags: --skip-full-sync This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC) --ticketing This must be set to enable ticketing support ($BATON_TICKETING) --token string required: The GitHub access token used to connect to the GitHub API. ($BATON_TOKEN) - --sync-last-activity See when members were last active in your organizations. ($BATON_SYNC_LAST_ACTIVITY) + --sync-last-activity See when members were last active in your organizations, based on audit-log activity. ($BATON_SYNC_LAST_ACTIVITY) --sync-secrets Whether to sync secrets or not ($BATON_SYNC_SECRETS) -v, --version version for baton-github-enterprise diff --git a/config_schema.json b/config_schema.json index dae6b7ee..2acf3e5e 100644 --- a/config_schema.json +++ b/config_schema.json @@ -187,7 +187,7 @@ { "name": "sync-last-activity", "displayName": "Sync user last activity", - "description": "See when members were last active in your organizations.", + "description": "See when members were last active in your organizations, based on audit-log activity.", "boolField": {} } ], diff --git a/docs/connector.mdx b/docs/connector.mdx index fa1e8342..48d5eec9 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -25,13 +25,16 @@ If you access GitHub at `github.com`, go to the [GitHub](/baton/github) integrat | Licenses | | | | Secrets - API keys | | | | GitHub App | | | +| GitHub Activity\*\*\* | | | -The GitHub Enterprise connector supports [automatic account provisioning and deprovisioning](/product/admin/account-provisioning). New accounts will send an invitation to the account owner; if an invitation is pending, the account status will be shown as **Unspecified**. +The GitHub Enterprise connector supports [automatic account provisioning and deprovisioning](/product/admin/account-provisioning). New accounts will send an invitation to the account owner; if an invitation is pending, the account status will be shown as **Pending**. \* Due to limitations on the GitHub API, syncing multiple orgs requires a personal access token (PAT). A separate connector and GitHub app is required when using the GitHub App integration. \*\* Organization roles are also referred to as "enterprise licenses" in GitHub's documentation. +\*\*\* Only synced when **Sync user last activity** is enabled. See [Sync member last activity](#sync-member-last-activity) below. + [This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. @@ -390,7 +393,7 @@ When deploying on Kubernetes with a minimal base image (for example, Debian slim ## Sync member last activity -You can enable **Sync user last activity** to have C1 show, for each member, the last time they were seen taking action in that organization (for example, commenting on an issue or pull request, reviewing code, or changing a setting). This appears on the member's profile in C1 alongside their other access details. +You can enable **Sync user last activity** to have C1 show, for each member, the most recent recorded action across all synced organizations (for example, commenting on an issue or pull request, reviewing code, or changing a setting). This appears on the member's profile in C1 alongside their other access details. **Requirements:** @@ -398,10 +401,7 @@ You can enable **Sync user last activity** to have C1 show, for each member, the - **GitHub App:** the app must have the **Organization permissions > Administration: Read-only access** permission. If you followed the GitHub App setup instructions above, this is already granted (it's also required for SAML/SSO detection), so no extra permission needs to be added. -**Enabling the config option alone is not enough.** After you enable **Sync user last activity** (or set `--sync-last-activity` / `BATON_SYNC_LAST_ACTIVITY=true`) and save the connector, go to the connector's **Capabilities & configuration** page in C1 and confirm the **GitHub Activity** resource type is enabled for sync. - -- On a **brand-new** connector, this resource type is included by default, so no extra step is needed. -- On an **existing, already-configured** connector, C1 does not automatically add a newly-available resource type to a previously saved sync selection — it will show up in the resource list as available, but disabled, until you manually enable it. +**Enabling the config option alone is not enough.** After you enable **Sync user last activity** (or set `--sync-last-activity` / `BATON_SYNC_LAST_ACTIVITY=true`) and save the connector, go to the connector's **Capabilities & configuration** page in C1 and confirm the **GitHub Activity** resource type is enabled for sync — it will show up in the resource list as available, but disabled, until you manually enable it. diff --git a/pkg/config/config.go b/pkg/config/config.go index f52282bc..9bcddc1d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -79,7 +79,7 @@ var ( syncLastActivity = field.BoolField( "sync-last-activity", field.WithDisplayName("Sync user last activity"), - field.WithDescription("See when members were last active in your organizations."), + field.WithDescription("See when members were last active in your organizations, based on audit-log activity."), ) ) From 862dc468bbe3b4c17de71362454a72c91c9ea32a Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 16 Sep 2026 13:36:24 -0300 Subject: [PATCH 8/9] chore: update baton-github version --- go.mod | 2 +- go.sum | 4 +-- .../pkg/connector/usage_event_feed.go | 25 ++++++++++++++++--- vendor/modules.txt | 2 +- 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index f90a487f..0a7bb8a2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/conductorone/baton-github-enterprise go 1.25.2 require ( - github.com/conductorone/baton-github v0.4.5 + github.com/conductorone/baton-github v0.4.6 github.com/conductorone/baton-sdk v0.30.1 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index 3261f358..62a78f2d 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 h1:IJ+uNItEm0qx9 github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-github v0.4.5 h1:+jvAgGMrWEhHJ9bdYoLfChrMK22pLoWvSzbsE6AncIU= -github.com/conductorone/baton-github v0.4.5/go.mod h1:VeeTzBFVAa9SIFjXPs1ETQ2y2jo4dEP0iXoTftRN0c0= +github.com/conductorone/baton-github v0.4.6 h1:Arz7eZ7fJVrLOfK6gs2kw04teGuQY/vQkgtrrL3pgEA= +github.com/conductorone/baton-github v0.4.6/go.mod h1:rooHPNjHMlUi/5WTG7obpFwwLEqJQr9CCyLzgF2IGus= github.com/conductorone/baton-sdk v0.30.1 h1:NfDk6KCrOhQUMOGU8ZAgHK898rE2h8wvDEt1fXylr0M= github.com/conductorone/baton-sdk v0.30.1/go.mod h1:9iFpBag59RlwGkb4Tz6omYewqnj8obfbZzrtSaTLa5Q= github.com/conductorone/dpop v0.3.0 h1:j5fZk0VqepGKYo+/NDikCOMsZcgs4HO4i0k56wRel5g= diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go index d86dc02f..01be314c 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go @@ -203,9 +203,11 @@ func (f *usageEventFeed) ListEvents( events = append(events, evt) } - if resp != nil && resp.NextPageToken != "" && !reachedBoundary { - cursor.AuditLogCursor = resp.NextPageToken - continue + if resp != nil && !reachedBoundary { + if nextPage := nextAuditLogPage(resp); nextPage != "" { + cursor.AuditLogCursor = nextPage + continue + } } // Done with this org for this pass - advance to the next one. @@ -237,6 +239,23 @@ func (f *usageEventFeed) ListEvents( return events, &pagination.StreamState{Cursor: tokenStr, HasMore: true}, annos, nil } +// nextAuditLogPage returns the token to request the next audit-log page, or +// "" if there isn't one. GitHub's org audit-log endpoint returns opaque +// cursor pagination on github.com/GHEC (go-github parses the Link header's +// non-numeric "page" value into Response.NextPageToken), but GHES-style +// numeric "page=N" Link headers parse into Response.NextPage (int) instead, +// leaving NextPageToken empty. Checking only NextPageToken silently truncates +// GHES audit logs to a single page. +func nextAuditLogPage(resp *github.Response) string { + if resp.NextPageToken != "" { + return resp.NextPageToken + } + if resp.NextPage != 0 { + return strconv.Itoa(resp.NextPage) + } + return "" +} + // usageEventFromAuditEntry converts one audit-log entry into a usage event // targeting the usage-app resource (see usage_app.go). Returns ok=false when // the entry can't be attributed to a synced user. diff --git a/vendor/modules.txt b/vendor/modules.txt index b63e384f..010eae91 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -272,7 +272,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-github v0.4.5 +# github.com/conductorone/baton-github v0.4.6 ## explicit; go 1.25.2 github.com/conductorone/baton-github/pkg/config github.com/conductorone/baton-github/pkg/connector From c05473e849c1a7c1ff9f424bbb11292191ea0f74 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Fri, 18 Sep 2026 13:44:14 -0300 Subject: [PATCH 9/9] chore: bump baton-github to v0.4.7 Picks up the audit-log pagination fix (resp.After checked first for the org audit-log endpoint's cursor, fixing silent truncation on GHEC and most GHES orgs) and per-org sampled warnings for orgs that permanently lack audit-log access (baton-github#193). Co-Authored-By: Claude Sonnet 5 --- go.mod | 2 +- go.sum | 4 +- .../baton-github/pkg/connector/logging.go | 41 +++++++++++ .../pkg/connector/usage_event_feed.go | 72 +++++++++++++------ vendor/modules.txt | 2 +- 5 files changed, 95 insertions(+), 26 deletions(-) create mode 100644 vendor/github.com/conductorone/baton-github/pkg/connector/logging.go diff --git a/go.mod b/go.mod index 0a7bb8a2..34090db8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/conductorone/baton-github-enterprise go 1.25.2 require ( - github.com/conductorone/baton-github v0.4.6 + github.com/conductorone/baton-github v0.4.7 github.com/conductorone/baton-sdk v0.30.1 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index 62a78f2d..2b82195b 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 h1:IJ+uNItEm0qx9 github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-github v0.4.6 h1:Arz7eZ7fJVrLOfK6gs2kw04teGuQY/vQkgtrrL3pgEA= -github.com/conductorone/baton-github v0.4.6/go.mod h1:rooHPNjHMlUi/5WTG7obpFwwLEqJQr9CCyLzgF2IGus= +github.com/conductorone/baton-github v0.4.7 h1:a/YRA4Iy4ey9wk2Azrg0v+E/AWmltnvPzVkPGJIQP6c= +github.com/conductorone/baton-github v0.4.7/go.mod h1:rooHPNjHMlUi/5WTG7obpFwwLEqJQr9CCyLzgF2IGus= github.com/conductorone/baton-sdk v0.30.1 h1:NfDk6KCrOhQUMOGU8ZAgHK898rE2h8wvDEt1fXylr0M= github.com/conductorone/baton-sdk v0.30.1/go.mod h1:9iFpBag59RlwGkb4Tz6omYewqnj8obfbZzrtSaTLa5Q= github.com/conductorone/dpop v0.3.0 h1:j5fZk0VqepGKYo+/NDikCOMsZcgs4HO4i0k56wRel5g= diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/logging.go b/vendor/github.com/conductorone/baton-github/pkg/connector/logging.go new file mode 100644 index 00000000..416ca117 --- /dev/null +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/logging.go @@ -0,0 +1,41 @@ +package connector + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" +) + +type sampledWarn struct { + n atomic.Uint64 +} + +func (s *sampledWarn) log(ctx context.Context, msg string, fields ...zap.Field) { + n := s.n.Add(1) + if !shouldLogSample(n) { + return + } + ctxzap.Extract(ctx).Warn(msg, append(fields, zap.Uint64("total_occurrences", n))...) +} + +type perKeySampledWarn struct { + m sync.Map // key -> *sampledWarn +} + +func (p *perKeySampledWarn) log(ctx context.Context, key, msg string, fields ...zap.Field) { + actual, _ := p.m.LoadOrStore(key, &sampledWarn{}) + actual.(*sampledWarn).log(ctx, msg, fields...) +} + +// shouldLogSample reports whether the nth occurrence should be logged. +func shouldLogSample(n uint64) bool { + switch { + case n <= 1, n == 10, n == 100: + return true + default: + return n%1000 == 0 + } +} diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go index 01be314c..f0905c35 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go @@ -14,7 +14,6 @@ import ( "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/google/go-github/v69/github" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -32,6 +31,10 @@ const maxAuditLogPagesPerCall = 20 type usageEventFeed struct { client *github.Client orgs []string + + // used to handle log Warn prints for skipped orgs, sampled per org so + // one noisy org can't starve another org's first occurrence out of the log. + skippedOrgs perKeySampledWarn } func newUsageEventFeed(client *github.Client, orgs []string) *usageEventFeed { @@ -49,10 +52,14 @@ func (f *usageEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFeedMetad // org's audit log, walked newest-first until an already-seen entry (at or // before Since) is reached. type usageEventPageToken struct { - Orgs []string `json:"orgs,omitempty"` - OrgIndex int `json:"org_index"` - AuditLogCursor string `json:"audit_log_cursor,omitempty"` - Since string `json:"since,omitempty"` + Orgs []string `json:"orgs,omitempty"` + OrgIndex int `json:"org_index"` + // AuditLogCursor is the opaque value to resume from. Whether it goes into + // the request's After or Page field depends on AuditLogCursorIsPage - see + // nextAuditLogPage. + AuditLogCursor string `json:"audit_log_cursor,omitempty"` + AuditLogCursorIsPage bool `json:"audit_log_cursor_is_page,omitempty"` + Since string `json:"since,omitempty"` } func unmarshalUsageEventPageToken(pToken *pagination.StreamToken) (*usageEventPageToken, error) { @@ -83,8 +90,6 @@ func (f *usageEventFeed) ListEvents( earliestEvent *timestamppb.Timestamp, pToken *pagination.StreamToken, ) ([]*v2.Event, *pagination.StreamState, annotations.Annotations, error) { - l := ctxzap.Extract(ctx) - if f.client == nil { return nil, &pagination.StreamState{HasMore: false}, nil, nil } @@ -123,6 +128,7 @@ func (f *usageEventFeed) ListEvents( if cursor.OrgIndex < 0 || cursor.OrgIndex >= len(cursor.Orgs) { cursor.OrgIndex = 0 cursor.AuditLogCursor = "" + cursor.AuditLogCursorIsPage = false } since, err := time.Parse(time.RFC3339Nano, cursor.Since) @@ -150,9 +156,13 @@ func (f *usageEventFeed) ListEvents( Phrase: github.Ptr(sincePhrase), ListCursorOptions: github.ListCursorOptions{ PerPage: maxPageSize, - Page: cursor.AuditLogCursor, }, } + if cursor.AuditLogCursorIsPage { + opts.Page = cursor.AuditLogCursor + } else { + opts.After = cursor.AuditLogCursor + } entries, resp, err := f.client.Organizations.GetAuditLog(ctx, orgName, opts) // Read rate-limit headers before the error branch nils resp, since a @@ -179,8 +189,10 @@ func (f *usageEventFeed) ListEvents( return nil, nil, nil, wrapGitHubError(err, resp, fmt.Sprintf("baton-github: failed to fetch audit log for org %s", orgName)) case isNotFoundError(resp) || isPermissionError(resp): - l.Warn("org lacks audit-log access, skipping it for this pass", - zap.String("org", orgName), zap.Error(err)) + f.skippedOrgs.log(ctx, orgName, "org lacks audit-log access, skipping it for this pass", + zap.String("org", orgName), zap.Error(err), + ) + entries, resp = nil, nil default: return nil, nil, nil, wrapGitHubError(err, resp, @@ -204,8 +216,9 @@ func (f *usageEventFeed) ListEvents( } if resp != nil && !reachedBoundary { - if nextPage := nextAuditLogPage(resp); nextPage != "" { + if nextPage, isPage := nextAuditLogPage(resp); nextPage != "" { cursor.AuditLogCursor = nextPage + cursor.AuditLogCursorIsPage = isPage continue } } @@ -213,6 +226,7 @@ func (f *usageEventFeed) ListEvents( // Done with this org for this pass - advance to the next one. cursor.OrgIndex++ cursor.AuditLogCursor = "" + cursor.AuditLogCursorIsPage = false if cursor.OrgIndex >= len(cursor.Orgs) { // Pass complete - the next call gets a fresh earliestEvent, so // nothing needs to survive in the cursor. @@ -239,21 +253,35 @@ func (f *usageEventFeed) ListEvents( return events, &pagination.StreamState{Cursor: tokenStr, HasMore: true}, annos, nil } -// nextAuditLogPage returns the token to request the next audit-log page, or -// "" if there isn't one. GitHub's org audit-log endpoint returns opaque -// cursor pagination on github.com/GHEC (go-github parses the Link header's -// non-numeric "page" value into Response.NextPageToken), but GHES-style -// numeric "page=N" Link headers parse into Response.NextPage (int) instead, -// leaving NextPageToken empty. Checking only NextPageToken silently truncates -// GHES audit logs to a single page. -func nextAuditLogPage(resp *github.Response) string { +// nextAuditLogPage returns the cursor to request the next audit-log page +// (empty if there isn't one), and whether that cursor belongs in the +// request's Page field (true) or its After field (false). +// +// The org audit-log endpoint documents three pagination shapes depending on +// what the server returns in the Link header's rel="next" entry: +// - after= (GHEC and GHES): the documented, primary mechanism - +// go-github parses this into Response.After. Checked first since it's +// what both github.com and GHES actually return in practice. +// - page= (legacy fallback some GHES versions may still +// emit): go-github can't parse a non-numeric page value as an int, so it +// lands in Response.NextPageToken instead. +// - page= (numeric, classic GHES offset pagination): parses into +// Response.NextPage. +// +// Checking only NextPageToken/NextPage (as earlier code did) misses the +// after= case entirely, silently truncating every GHEC org - and most GHES +// orgs - to a single page. +func nextAuditLogPage(resp *github.Response) (string, bool) { + if resp.After != "" { + return resp.After, false + } if resp.NextPageToken != "" { - return resp.NextPageToken + return resp.NextPageToken, true } if resp.NextPage != 0 { - return strconv.Itoa(resp.NextPage) + return strconv.Itoa(resp.NextPage), true } - return "" + return "", false } // usageEventFromAuditEntry converts one audit-log entry into a usage event diff --git a/vendor/modules.txt b/vendor/modules.txt index 010eae91..7e8599bc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -272,7 +272,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-github v0.4.6 +# github.com/conductorone/baton-github v0.4.7 ## explicit; go 1.25.2 github.com/conductorone/baton-github/pkg/config github.com/conductorone/baton-github/pkg/connector