diff --git a/README.md b/README.md index 005c30b0..76b5655f 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,7 @@ Flags: --skip-full-sync This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC) --ticketing This must be set to enable ticketing support ($BATON_TICKETING) --token string required: The GitHub access token used to connect to the GitHub API. ($BATON_TOKEN) + --sync-last-activity See when members were last active in your organizations, based on audit-log activity. ($BATON_SYNC_LAST_ACTIVITY) --sync-secrets Whether to sync secrets or not ($BATON_SYNC_SECRETS) -v, --version version for baton-github-enterprise diff --git a/config_schema.json b/config_schema.json index 51cf1358..2acf3e5e 100644 --- a/config_schema.json +++ b/config_schema.json @@ -183,6 +183,12 @@ "displayName": "Optimize sync for large organizations", "description": "Reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. Recommended for large orgs.", "boolField": {} + }, + { + "name": "sync-last-activity", + "displayName": "Sync user last activity", + "description": "See when members were last active in your organizations, based on audit-log activity.", + "boolField": {} } ], "displayName": "GitHub Enterprise", @@ -199,7 +205,8 @@ "orgs", "enterprises", "omit-archived-repositories", - "direct-collaborators-only" + "direct-collaborators-only", + "sync-last-activity" ], "default": true }, @@ -215,7 +222,8 @@ "enterprises", "sync-secrets", "omit-archived-repositories", - "direct-collaborators-only" + "direct-collaborators-only", + "sync-last-activity" ] } ] diff --git a/docs/connector.mdx b/docs/connector.mdx index 035b6038..48d5eec9 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -25,13 +25,16 @@ If you access GitHub at `github.com`, go to the [GitHub](/baton/github) integrat | Licenses | | | | Secrets - API keys | | | | GitHub App | | | +| GitHub Activity\*\*\* | | | -The GitHub Enterprise connector supports [automatic account provisioning and deprovisioning](/product/admin/account-provisioning). New accounts will send an invitation to the account owner; if an invitation is pending, the account status will be shown as **Unspecified**. +The GitHub Enterprise connector supports [automatic account provisioning and deprovisioning](/product/admin/account-provisioning). New accounts will send an invitation to the account owner; if an invitation is pending, the account status will be shown as **Pending**. \* Due to limitations on the GitHub API, syncing multiple orgs requires a personal access token (PAT). A separate connector and GitHub app is required when using the GitHub App integration. \*\* Organization roles are also referred to as "enterprise licenses" in GitHub's documentation. +\*\*\* Only synced when **Sync user last activity** is enabled. See [Sync member last activity](#sync-member-last-activity) below. + [This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. @@ -243,6 +246,8 @@ If you're using a personal access token to set up the connector: 1. **Optional.** If you do not want to include archived repos in syncs, click to enable **Omit archived repositories**. 1. **Optional.** For large organizations, click to enable **Optimize sync for large organizations**. This reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. + + 1. **Optional.** If you want to see when members were last active, click to enable **Sync user last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. If you're using a GitHub app to set up the connector: @@ -262,6 +267,8 @@ If you're using a GitHub app to set up the connector: 1. **Optional.** If you do not want to include archived repos in syncs, click to enable **Omit archived repositories**. 1. **Optional.** For large organizations, click to enable **Optimize sync for large organizations**. This reduces API calls by using grant expansion for team-based repo access and skipping per-team detail fetches. + + 1. **Optional.** If you want to see when members were last active, click to enable **Sync user last activity**. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. Click **Save**. @@ -320,6 +327,7 @@ baton-github-enterprise | `BATON_DIRECT_COLLABORATORS_ONLY` | Set to `true` to optimize sync for large organizations by reducing API calls (uses grant expansion for team-based repo access and skips per-team detail fetches). | | `BATON_PROVISIONING` | Set to `true` to enable provisioning and deprovisioning. | | `BATON_SYNC_SECRETS` | Set to `true` to sync secrets (API keys). | +| `BATON_SYNC_LAST_ACTIVITY` | Set to `true` to see when members were last active. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. | The connector connects to your C1 tenant and begins syncing. Verify it appears as **Connected** in **Apps** > **Connectors**. @@ -356,6 +364,7 @@ baton-github-enterprise \ | `--enterprises` | Comma-separated list of enterprises to sync enterprise roles for. | | `--omit-archived-repositories` | Exclude archived repositories from syncs. | | `--direct-collaborators-only` | Optimize sync for large organizations by reducing API calls (uses grant expansion for team-based repo access and skips per-team detail fetches). | +| `--sync-last-activity` | See when members were last active. See [Sync member last activity](#sync-member-last-activity) for requirements and limitations. | | `--provisioning` | Enable provisioning and deprovisioning. | @@ -382,6 +391,28 @@ When deploying on Kubernetes with a minimal base image (for example, Debian slim +## Sync member last activity + +You can enable **Sync user last activity** to have C1 show, for each member, the most recent recorded action across all synced organizations (for example, commenting on an issue or pull request, reviewing code, or changing a setting). This appears on the member's profile in C1 alongside their other access details. + +**Requirements:** + +- **Personal access token:** the token must have the **read:audit_log** scope, and permission to view each organization's audit log (typically an organization owner). +- **GitHub App:** the app must have the **Organization permissions > Administration: Read-only access** permission. If you followed the GitHub App setup instructions above, this is already granted (it's also required for SAML/SSO detection), so no extra permission needs to be added. + + +**Enabling the config option alone is not enough.** After you enable **Sync user last activity** (or set `--sync-last-activity` / `BATON_SYNC_LAST_ACTIVITY=true`) and save the connector, go to the connector's **Capabilities & configuration** page in C1 and confirm the **GitHub Activity** resource type is enabled for sync — it will show up in the resource list as available, but disabled, until you manually enable it. + + + +**What "last activity" means:** This is not necessarily a login timestamp. GitHub's audit log only records actions members take, not every time someone signs in or browses the product. Because of this, C1 shows the most recent recorded action for a member, whatever that action was, as a best-effort signal of when they were last active. A member who only reads or browses without taking any recorded action won't show a last-activity date, even if they use GitHub regularly. + +To keep this fast on large organizations, raw Git operations (pushes, fetches, and clones done via `git` rather than the website or API) are not counted toward last activity — only web and API actions are. A member who exclusively interacts with GitHub through `git` and never uses the website or API (for example, a purely command-line workflow with no PR reviews, comments, or settings changes) won't show a last-activity date, even though they're actively using GitHub. + +If a member has never taken a recorded action, or if the connector can't reach an organization's audit log for any reason (for example, a missing scope or a plan without audit log access), C1 simply won't show a last-activity date for that member — this does not affect the rest of the sync. + +Activity is delivered as an ongoing background feed rather than as part of each full sync, so it may take a little time after enabling this option before last-activity dates first appear, and dates update continuously afterward rather than only at sync time. + ## Troubleshooting ### "Resource not accessible by integration" error diff --git a/go.mod b/go.mod index e096d427..34090db8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/conductorone/baton-github-enterprise go 1.25.2 require ( - github.com/conductorone/baton-github v0.4.0 + github.com/conductorone/baton-github v0.4.7 github.com/conductorone/baton-sdk v0.30.1 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index 559660f8..2b82195b 100644 --- a/go.sum +++ b/go.sum @@ -84,8 +84,8 @@ github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258 h1:IJ+uNItEm0qx9 github.com/cockroachdb/swiss v0.0.0-20260820225851-333444432258/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-github v0.4.0 h1:Hf72uDLoYrtmhII6BYyRs0qSG/vuhx6Po7QHHs4618E= -github.com/conductorone/baton-github v0.4.0/go.mod h1:SUpbLLRnO+fldsV/ALUd6JjONtUcsPya38psMNBIDCU= +github.com/conductorone/baton-github v0.4.7 h1:a/YRA4Iy4ey9wk2Azrg0v+E/AWmltnvPzVkPGJIQP6c= +github.com/conductorone/baton-github v0.4.7/go.mod h1:rooHPNjHMlUi/5WTG7obpFwwLEqJQr9CCyLzgF2IGus= github.com/conductorone/baton-sdk v0.30.1 h1:NfDk6KCrOhQUMOGU8ZAgHK898rE2h8wvDEt1fXylr0M= github.com/conductorone/baton-sdk v0.30.1/go.mod h1:9iFpBag59RlwGkb4Tz6omYewqnj8obfbZzrtSaTLa5Q= github.com/conductorone/dpop v0.3.0 h1:j5fZk0VqepGKYo+/NDikCOMsZcgs4HO4i0k56wRel5g= diff --git a/pkg/config/conf.gen.go b/pkg/config/conf.gen.go index e20969e7..51e59484 100644 --- a/pkg/config/conf.gen.go +++ b/pkg/config/conf.gen.go @@ -14,6 +14,7 @@ type Githubenterprise struct { SyncSecrets bool `mapstructure:"sync-secrets"` OmitArchivedRepositories bool `mapstructure:"omit-archived-repositories"` DirectCollaboratorsOnly bool `mapstructure:"direct-collaborators-only"` + SyncLastActivity bool `mapstructure:"sync-last-activity"` } func (c *Githubenterprise) findFieldByTag(tagValue string) (any, bool) { diff --git a/pkg/config/config.go b/pkg/config/config.go index c0499124..9bcddc1d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -75,6 +75,12 @@ var ( "and skipping per-team detail fetches. Recommended for large orgs.", ), ) + + syncLastActivity = field.BoolField( + "sync-last-activity", + field.WithDisplayName("Sync user last activity"), + field.WithDescription("See when members were last active in your organizations, based on audit-log activity."), + ) ) //go:generate go run ./gen @@ -90,6 +96,7 @@ var Config = field.NewConfiguration( syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, + syncLastActivity, }, field.WithConnectorDisplayName("GitHub Enterprise"), field.WithHelpUrl("/docs/baton/github-enterprise"), @@ -99,15 +106,18 @@ var Config = field.NewConfiguration( Name: GithubPersonalAccessTokenGroup, DisplayName: "Personal access token", HelpText: "Use a personal access token for authentication.", - Fields: []field.SchemaField{instanceUrlField, accessTokenField, orgsField, EnterprisesField, omitArchivedRepositories, directCollaboratorsOnly}, + Fields: []field.SchemaField{instanceUrlField, accessTokenField, orgsField, EnterprisesField, omitArchivedRepositories, directCollaboratorsOnly, syncLastActivity}, Default: true, }, { Name: GithubAppGroup, DisplayName: "GitHub app", HelpText: "Use a github app for authentication", - Fields: []field.SchemaField{instanceUrlField, appIDField, appPrivateKeyPath, orgField, EnterprisesField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly}, - Default: false, + Fields: []field.SchemaField{ + instanceUrlField, appIDField, appPrivateKeyPath, orgField, EnterprisesField, + syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, syncLastActivity, + }, + Default: false, }, }), ) diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index eaa505d2..30253ba4 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -24,6 +24,7 @@ func NewLambdaConnector(ctx context.Context, ghc *cfg.Githubenterprise, cliOpts AppPrivatekeyPath: ghc.AppPrivatekeyPath, Org: ghc.Org, DirectCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + SyncLastActivity: ghc.SyncLastActivity, }, cliOpts) } diff --git a/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go b/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go index 5ed99148..6541f0f2 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go +++ b/vendor/github.com/conductorone/baton-github/pkg/config/conf.gen.go @@ -10,10 +10,12 @@ type Github struct { InstanceUrl string `mapstructure:"instance-url"` AppId string `mapstructure:"app-id"` AppPrivatekeyPath []byte `mapstructure:"app-privatekey-path"` + AppPrivatekey string `mapstructure:"app-privatekey"` Org string `mapstructure:"org"` SyncSecrets bool `mapstructure:"sync-secrets"` OmitArchivedRepositories bool `mapstructure:"omit-archived-repositories"` DirectCollaboratorsOnly bool `mapstructure:"direct-collaborators-only"` + SyncLastActivity bool `mapstructure:"sync-last-activity"` } func (c *Github) findFieldByTag(tagValue string) (any, bool) { diff --git a/vendor/github.com/conductorone/baton-github/pkg/config/config.go b/vendor/github.com/conductorone/baton-github/pkg/config/config.go index 2b9c5e63..60856c56 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-github/pkg/config/config.go @@ -40,13 +40,28 @@ var ( field.WithRequired(true), ) + // appPrivateKeyPath and appPrivateKey are two ways to supply the same GitHub + // App private key. Neither is marked required individually: providing either + // one satisfies the "app private key required" check, which is enforced in the + // connector's GitHub App constructor (see pkg/connector.appPrivateKeyPEM). + // A framework-level constraint can't express "required only for the GitHub App + // auth group", since constraints are evaluated globally across auth methods. appPrivateKeyPath = field.FileUploadField( "app-privatekey-path", []string{".pem"}, field.WithDisplayName("GitHub App private key (.pem)"), - field.WithDescription("Path to private key that is used to connect to the GitHub App"), + field.WithDescription("Path to private key that is used to connect to the GitHub App. Ignored when app-privatekey is set."), + field.WithIsSecret(true), + ) + + appPrivateKey = field.StringField( + "app-privatekey", + field.WithDisplayName("GitHub App private key (PEM)"), + field.WithDescription( + "Raw PEM contents of the private key used to connect to the GitHub App. Takes precedence over app-privatekey-path when both are set. "+ + `Literal \n escape sequences are also accepted and unescaped before use, since this field can't hold newlines when entered through a form.`, + ), field.WithIsSecret(true), - field.WithRequired(true), ) syncSecrets = field.BoolField( @@ -79,6 +94,18 @@ var ( field.WithDescription("Organization of your github app"), field.WithRequired(true), ) + + // syncLastActivity is hidden from this connector's GUI config and --help + // since it only applies to GitHub Enterprise audit-log access. + // baton-github-enterprise sets it directly on the shared Github struct, + // bypassing this CLI layer, so hiding it here doesn't affect that connector. + syncLastActivity = field.BoolField( + "sync-last-activity", + field.WithDisplayName("Sync users last activity"), + field.WithDescription("See when members were last active in your organizations."), + field.WithHidden(true), + field.WithExportTarget(field.ExportTargetCLIOnly), + ) ) //go:generate go run ./gen @@ -90,10 +117,12 @@ var Config = field.NewConfiguration( instanceUrlField, appIDField, appPrivateKeyPath, + appPrivateKey, orgField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly, + syncLastActivity, }, field.WithConnectorDisplayName("GitHub v2"), field.WithHelpUrl("/docs/baton/github-v2"), @@ -110,7 +139,7 @@ var Config = field.NewConfiguration( Name: GithubAppGroup, DisplayName: "GitHub app", HelpText: "Use a github app for authentication", - Fields: []field.SchemaField{appIDField, appPrivateKeyPath, orgField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly}, + Fields: []field.SchemaField{appIDField, appPrivateKeyPath, appPrivateKey, orgField, syncSecrets, omitArchivedRepositories, directCollaboratorsOnly}, Default: false, }, }), diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go b/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go index 5f0852e8..7a6a7fea 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/api_token.go @@ -26,8 +26,12 @@ func apiTokenResource(ctx context.Context, token *github.PersonalAccessToken) (* options = append(options, resourceSdk.WithSecretLastUsedAt(token.TokenLastUsedAt.Time)) } + // created_at has moved from SecretTrait to a Resource-level attribute, so + // set it via the resource-level option instead of the deprecated + // WithSecretCreatedAt trait option. + var resourceOpts []resourceSdk.ResourceOption if token.AccessGrantedAt != nil { - options = append(options, resourceSdk.WithSecretCreatedAt(token.AccessGrantedAt.Time)) + resourceOpts = append(resourceOpts, resourceSdk.WithResourceCreatedAt(token.AccessGrantedAt.Time)) } if token.TokenExpiresAt != nil { @@ -38,6 +42,7 @@ func apiTokenResource(ctx context.Context, token *github.PersonalAccessToken) (* resourceTypeApiToken, token.GetID(), options, + resourceOpts..., ) if err != nil { return nil, err diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/app.go b/vendor/github.com/conductorone/baton-github/pkg/connector/app.go index cafe9ed9..8f5cb7c1 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/app.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/app.go @@ -39,7 +39,9 @@ func appResource(ctx context.Context, installation *github.Installation, parentR opts := []resourceSdk.ResourceOption{ resourceSdk.WithParentResourceID(parentResourceID), - resourceSdk.WithAppTrait(resourceSdk.WithAppProfile(profile)), + resourceSdk.WithAppTrait(), + // profile has moved from AppTrait to a Resource-level attribute. + resourceSdk.WithResourceProfile(profile), resourceSdk.WithNHIType(v2.NonHumanIdentityTrait_NHI_TYPE_APP_REGISTRATION, "github.app"), } if installation.HTMLURL != nil { diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go index 2cdb5a62..b1bd1017 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/connector.go @@ -68,7 +68,7 @@ var ( resourceTypeInvitation = &v2.ResourceType{ Id: "invitation", DisplayName: "Invitation", - // Invitations emit TRAIT_USER with UserTrait_Status_STATUS_UNSPECIFIED. + // Invitations emit TRAIT_USER with STATUS_PENDING. // Accepted members from user.go emit STATUS_ENABLED. Traits: []v2.ResourceType_Trait{ v2.ResourceType_TRAIT_USER, @@ -127,6 +127,7 @@ type GitHub struct { omitArchivedRepositories bool directCollaboratorsOnly bool enterprises []string + syncLastActivity bool } func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 { @@ -148,6 +149,11 @@ func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour resourceSyncers = append(resourceSyncers, APITokenBuilder(gh.client, gh.orgCache)) } + if gh.syncLastActivity { + // usageAppBuilder only exists to support usageEventFeed, so it's gated the same way. + resourceSyncers = append(resourceSyncers, newUsageAppBuilder()) + } + if len(gh.enterprises) > 0 { resourceSyncers = append(resourceSyncers, EnterpriseRoleBuilder(gh.client, gh.appClient, gh.customClient, gh.enterprises), @@ -157,8 +163,18 @@ func (gh *GitHub) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour return resourceSyncers } +func (gh *GitHub) EventFeeds(_ context.Context) []connectorbuilder.EventFeed { + if !gh.syncLastActivity { + return nil + } + + return []connectorbuilder.EventFeed{ + newUsageEventFeed(gh.client, gh.orgs), + } +} + // Metadata returns metadata about the connector. -func (gh *GitHub) Metadata(ctx context.Context) (*v2.ConnectorMetadata, error) { +func (gh *GitHub) Metadata(_ context.Context) (*v2.ConnectorMetadata, error) { return &v2.ConnectorMetadata{ DisplayName: "GitHub", AccountCreationSchema: &v2.ConnectorAccountCreationSchema{ @@ -346,11 +362,31 @@ func newWithGithubPAT(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + syncLastActivity: ghc.SyncLastActivity, }, nil } +// appPrivateKeyPEM returns the GitHub App private key PEM contents to use, +// preferring the in-memory app-privatekey flag over the on-disk +// app-privatekey-path. Providing either one satisfies the requirement; if +// neither is set an error is returned. +func appPrivateKeyPEM(ghc *cfg.Github) (string, error) { + if ghc.AppPrivatekey != "" { + return ghc.AppPrivatekey, nil + } + if len(ghc.AppPrivatekeyPath) > 0 { + return string(ghc.AppPrivatekeyPath), nil + } + return "", errors.New("github app authentication requires either --app-privatekey or --app-privatekey-path") +} + func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { - jwttoken, err := getJWTToken(ghc.AppId, string(ghc.AppPrivatekeyPath)) + privateKey, err := appPrivateKeyPEM(ghc) + if err != nil { + return nil, err + } + + jwttoken, err := getJWTToken(ghc.AppId, privateKey) if err != nil { return nil, err } @@ -382,7 +418,7 @@ func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { }, &appJWTTokenRefresher{ appID: ghc.AppId, - privateKey: string(ghc.AppPrivatekeyPath), + privateKey: privateKey, }, ) // Wrap the installation-token refresher in a refreshableTokenSource so the @@ -433,6 +469,7 @@ func newWithGithubApp(ctx context.Context, ghc *cfg.Github) (*GitHub, error) { syncSecrets: ghc.SyncSecrets, omitArchivedRepositories: ghc.OmitArchivedRepositories, directCollaboratorsOnly: ghc.DirectCollaboratorsOnly, + syncLastActivity: ghc.SyncLastActivity, } return gh, nil } @@ -465,7 +502,12 @@ func newGitHubGraphqlClient(ctx context.Context, instanceURL string, ts oauth2.T return githubv4.NewClient(tc), nil } +// escapedLineBreaks unescapes LF-, CRLF-, and CR-escaped line breaks (`\r\n`, +// `\n`, `\r`) to a real newline. +var escapedLineBreaks = strings.NewReplacer(`\r\n`, "\n", `\n`, "\n", `\r`, "\n") + func loadPrivateKeyFromString(p string) (*rsa.PrivateKey, error) { + p = escapedLineBreaks.Replace(p) block, _ := pem.Decode([]byte(p)) if block == nil || (block.Type != "PRIVATE KEY" && block.Type != "RSA PRIVATE KEY") { return nil, errors.New("invalid private key PEM format") diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go b/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go index 52df44a2..38ef9575 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/invitation.go @@ -64,10 +64,13 @@ func invitationToUserResource(invitation *github.Invitation, status string) (*v2 invitation.GetID(), []resourceSdk.UserTraitOption{ resourceSdk.WithEmail(invitation.GetEmail(), true), - resourceSdk.WithUserProfile(profile), - resourceSdk.WithStatus(v2.UserTrait_Status_STATUS_UNSPECIFIED), resourceSdk.WithUserLogin(login), }, + // profile and status have moved from UserTrait to Resource-level + // attributes. Expired invitations stay PENDING - they are still not a + // usable account - and carry the distinction in the status details. + resourceSdk.WithResourceProfile(profile), + resourceSdk.WithResourceStatus(v2.Status_RESOURCE_STATUS_PENDING, status), ) if err != nil { return nil, err diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/logging.go b/vendor/github.com/conductorone/baton-github/pkg/connector/logging.go new file mode 100644 index 00000000..416ca117 --- /dev/null +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/logging.go @@ -0,0 +1,41 @@ +package connector + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" +) + +type sampledWarn struct { + n atomic.Uint64 +} + +func (s *sampledWarn) log(ctx context.Context, msg string, fields ...zap.Field) { + n := s.n.Add(1) + if !shouldLogSample(n) { + return + } + ctxzap.Extract(ctx).Warn(msg, append(fields, zap.Uint64("total_occurrences", n))...) +} + +type perKeySampledWarn struct { + m sync.Map // key -> *sampledWarn +} + +func (p *perKeySampledWarn) log(ctx context.Context, key, msg string, fields ...zap.Field) { + actual, _ := p.m.LoadOrStore(key, &sampledWarn{}) + actual.(*sampledWarn).log(ctx, msg, fields...) +} + +// shouldLogSample reports whether the nth occurrence should be logged. +func shouldLogSample(n uint64) bool { + switch { + case n <= 1, n == 10, n == 100: + return true + default: + return n%1000 == 0 + } +} diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go b/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go index e78c30a0..32942686 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/org_role.go @@ -53,9 +53,9 @@ func orgRoleResource( role.Name, resourceTypeOrgRole, role.ID, - []resourceSdk.RoleTraitOption{ - resourceSdk.WithRoleProfile(profile), - }, + []resourceSdk.RoleTraitOption{}, + // profile has moved from RoleTrait to a Resource-level attribute. + resourceSdk.WithResourceProfile(profile), resourceSdk.WithParentResourceID(org.Id), resourceSdk.WithAnnotation( &v2.V1Identifier{Id: fmt.Sprintf("org_role:%d", role.ID)}, diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/team.go b/vendor/github.com/conductorone/baton-github/pkg/connector/team.go index 8e1579ba..0dc2eb24 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/team.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/team.go @@ -45,7 +45,9 @@ func teamResource(team *github.Team, orgID int64, parentResourceID *v2.ResourceI team.GetName(), resourceTypeTeam, team.GetID(), - []rType.GroupTraitOption{rType.WithGroupProfile(profile)}, + []rType.GroupTraitOption{}, + // profile has moved from GroupTrait to a Resource-level attribute. + rType.WithResourceProfile(profile), rType.WithAnnotation( &v2.ExternalLink{Url: team.GetURL()}, &v2.V1Identifier{Id: fmt.Sprintf("team:%d", team.GetID())}, @@ -159,12 +161,10 @@ func (o *teamResourceType) Grants(ctx context.Context, resource *v2.Resource, op return nil, nil, err } - teamTrait, err := rType.GetGroupTrait(resource) - if err != nil { - return nil, nil, err - } - - orgID, ok := rType.GetProfileInt64Value(teamTrait.Profile, "orgID") + // profile has moved from GroupTrait to a Resource-level attribute; read it + // via GetProfile, which resolves the resource-level value (with a + // trait-level fallback for older data). + orgID, ok := rType.GetProfileInt64Value(rType.GetProfile(resource), "orgID") if !ok { return nil, nil, fmt.Errorf("error fetching orgID from team profile") } @@ -280,12 +280,9 @@ func (o *teamResourceType) Grant(ctx context.Context, principal *v2.Resource, en return nil, err } case resourceTypeTeam.Id: - groupTrait, err := rType.GetGroupTrait(entitlement.Resource) - if err != nil { - return nil, err - } - - orgID, ok := rType.GetProfileInt64Value(groupTrait.Profile, "orgID") + // profile has moved from GroupTrait to a Resource-level attribute; read + // it via GetProfile (resource-level value, with a trait-level fallback). + orgID, ok := rType.GetProfileInt64Value(rType.GetProfile(entitlement.Resource), "orgID") if !ok { return nil, fmt.Errorf("error fetching orgID from team profile") } diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go new file mode 100644 index 00000000..e6f26888 --- /dev/null +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_app.go @@ -0,0 +1,64 @@ +package connector + +import ( + "context" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/types/entitlement" + resourceSdk "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +const ( + usageAppResourceID = "github" + usageAppDisplayName = "GitHub Activity" + usageAppAccessEntitlement = "access" +) + +// resourceTypeUsageApp is a synthetic TRAIT_APP resource that usageEventFeed's +// UsageEvents target, since GitHub's real resource types carry no entitlement +// C1's usage uplift can key off of. Only synced when sync-last-activity is on. +var resourceTypeUsageApp = &v2.ResourceType{ + Id: "usage-app", + DisplayName: "GitHub Activity", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_APP}, + Annotations: annotations.New(&v2.SkipGrants{}), +} + +// usageAppBuilder syncs a single static App resource for usageEventFeed's +// UsageEvents to target. +type usageAppBuilder struct{} + +func newUsageAppBuilder() *usageAppBuilder { + return &usageAppBuilder{} +} + +func (b *usageAppBuilder) ResourceType(_ context.Context) *v2.ResourceType { + return resourceTypeUsageApp +} + +func (b *usageAppBuilder) List(_ context.Context, _ *v2.ResourceId, _ resourceSdk.SyncOpAttrs) ([]*v2.Resource, *resourceSdk.SyncOpResults, error) { + res, err := resourceSdk.NewAppResource(usageAppDisplayName, resourceTypeUsageApp, usageAppResourceID, nil) + if err != nil { + return nil, nil, err + } + return []*v2.Resource{res}, &resourceSdk.SyncOpResults{}, nil +} + +func (b *usageAppBuilder) Entitlements(_ context.Context, resource *v2.Resource, _ resourceSdk.SyncOpAttrs) ([]*v2.Entitlement, *resourceSdk.SyncOpResults, error) { + return []*v2.Entitlement{ + entitlement.NewAssignmentEntitlement( + resource, + usageAppAccessEntitlement, + entitlement.WithGrantableTo(resourceTypeUser), + entitlement.WithDisplayName("GitHub Access"), + entitlement.WithDescription("Has access to GitHub"), + ), + }, &resourceSdk.SyncOpResults{}, nil +} + +func (b *usageAppBuilder) Grants(_ context.Context, _ *v2.Resource, _ resourceSdk.SyncOpAttrs) ([]*v2.Grant, *resourceSdk.SyncOpResults, error) { + // Grants are intentionally not emitted: the usage uplift maps the login + // actor directly to a synced app user, not via a grant. + return nil, &resourceSdk.SyncOpResults{}, nil +} diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go new file mode 100644 index 00000000..f0905c35 --- /dev/null +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/usage_event_feed.go @@ -0,0 +1,340 @@ +package connector + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/google/go-github/v69/github" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// defaultActivityLookback bounds the very first poll when no earliest-event +// boundary is given yet; later polls advance via the feed's own cursor. +const defaultActivityLookback = 1 * time.Hour + +// maxAuditLogPagesPerCall caps total pages walked across all orgs per call +// so one very active org can't stall the feed; remaining pages resume via the cursor. +const maxAuditLogPagesPerCall = 20 + +// usageEventFeed streams member activity from each org's audit log as usage +// events, since GitHub has no per-user "last activity" field to sync directly. +type usageEventFeed struct { + client *github.Client + orgs []string + + // used to handle log Warn prints for skipped orgs, sampled per org so + // one noisy org can't starve another org's first occurrence out of the log. + skippedOrgs perKeySampledWarn +} + +func newUsageEventFeed(client *github.Client, orgs []string) *usageEventFeed { + return &usageEventFeed{client: client, orgs: orgs} +} + +func (f *usageEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFeedMetadata { + return &v2.EventFeedMetadata{ + Id: "github_usage_event_feed", + SupportedEventTypes: []v2.EventType{v2.EventType_EVENT_TYPE_USAGE}, + } +} + +// usageEventPageToken tracks progress through one pass over every configured +// org's audit log, walked newest-first until an already-seen entry (at or +// before Since) is reached. +type usageEventPageToken struct { + Orgs []string `json:"orgs,omitempty"` + OrgIndex int `json:"org_index"` + // AuditLogCursor is the opaque value to resume from. Whether it goes into + // the request's After or Page field depends on AuditLogCursorIsPage - see + // nextAuditLogPage. + AuditLogCursor string `json:"audit_log_cursor,omitempty"` + AuditLogCursorIsPage bool `json:"audit_log_cursor_is_page,omitempty"` + Since string `json:"since,omitempty"` +} + +func unmarshalUsageEventPageToken(pToken *pagination.StreamToken) (*usageEventPageToken, error) { + pt := &usageEventPageToken{} + if pToken == nil || pToken.Cursor == "" { + return pt, nil + } + data, err := base64.StdEncoding.DecodeString(pToken.Cursor) + if err != nil { + return nil, fmt.Errorf("baton-github: failed to decode usage event feed cursor: %w", err) + } + if err := json.Unmarshal(data, pt); err != nil { + return nil, fmt.Errorf("baton-github: failed to unmarshal usage event feed cursor: %w", err) + } + return pt, nil +} + +func (pt *usageEventPageToken) marshal() (string, error) { + data, err := json.Marshal(pt) + if err != nil { + return "", fmt.Errorf("baton-github: failed to marshal usage event feed cursor: %w", err) + } + return base64.StdEncoding.EncodeToString(data), nil +} + +func (f *usageEventFeed) ListEvents( + ctx context.Context, + earliestEvent *timestamppb.Timestamp, + pToken *pagination.StreamToken, +) ([]*v2.Event, *pagination.StreamState, annotations.Annotations, error) { + if f.client == nil { + return nil, &pagination.StreamState{HasMore: false}, nil, nil + } + + cursor, err := unmarshalUsageEventPageToken(pToken) + if err != nil { + return nil, nil, nil, err + } + + if len(cursor.Orgs) == 0 { + // Snapshot the org list and "since" boundary once per pass, so + // mid-pass config changes don't shift what gets walked. + orgs, err := getOrgs(ctx, f.client, f.orgs) + if err != nil { + return nil, nil, nil, fmt.Errorf("baton-github: failed to list orgs for usage event feed: %w", err) + } + if len(orgs) == 0 { + return nil, &pagination.StreamState{HasMore: false}, nil, nil + } + + since := time.Now().Add(-defaultActivityLookback) + // Guard against a zero/degenerate earliestEvent producing a + // nonsensical "since year 1" query that GitHub's search parser rejects. + if earliestEvent != nil { + if t := earliestEvent.AsTime(); !t.IsZero() && t.After(time.Unix(0, 0)) { + since = t + } + } + + cursor = &usageEventPageToken{ + Orgs: orgs, + Since: since.Format(time.RFC3339Nano), + } + } + + if cursor.OrgIndex < 0 || cursor.OrgIndex >= len(cursor.Orgs) { + cursor.OrgIndex = 0 + cursor.AuditLogCursor = "" + cursor.AuditLogCursorIsPage = false + } + + since, err := time.Parse(time.RFC3339Nano, cursor.Since) + if err != nil { + return nil, nil, nil, fmt.Errorf("baton-github: invalid usage event feed cursor timestamp: %w", err) + } + // created:>= is sent server-side so GitHub excludes already-seen + // entries; the check below stays as a safety net in case it's ignored. + sincePhrase := "created:>=" + since.UTC().Format("2006-01-02T15:04:05-07:00") + + var events []*v2.Event + // Tightest (lowest Remaining) rate limit seen across this call's requests. + var tightestRateLimit *v2.RateLimitDescription + + // TODO(jdc): Probably change this for loop for a series of requests that uses a more complex pagination cursor. + for page := 0; page < maxAuditLogPagesPerCall; page++ { + orgName := cursor.Orgs[cursor.OrgIndex] + + opts := &github.GetAuditLogOptions{ + Order: github.Ptr("desc"), + // "web" excludes raw git-protocol events (push/fetch/clone), + // which dominate audit-log volume without losing members who are + // otherwise covered by their web/API activity. + Include: github.Ptr("web"), + Phrase: github.Ptr(sincePhrase), + ListCursorOptions: github.ListCursorOptions{ + PerPage: maxPageSize, + }, + } + if cursor.AuditLogCursorIsPage { + opts.Page = cursor.AuditLogCursor + } else { + opts.After = cursor.AuditLogCursor + } + + entries, resp, err := f.client.Organizations.GetAuditLog(ctx, orgName, opts) + // Read rate-limit headers before the error branch nils resp, since a + // 429 still carries them. + if resp != nil { + if rl, rlErr := extractRateLimitData(resp); rlErr == nil { + if tightestRateLimit == nil || rl.GetRemaining() < tightestRateLimit.GetRemaining() { + tightestRateLimit = rl + } + } + } + if err != nil { + // Skip-and-continue only for permanent per-org conditions (no + // audit-log access); anything else aborts instead of wasting the + // rest of the page budget. Rate-limit checks come first since + // GitHub can signal rate limiting via a 403. + var rateLimitErr *github.RateLimitError + var abuseRateLimitErr *github.AbuseRateLimitError + retryable := errors.As(err, &rateLimitErr) || errors.As(err, &abuseRateLimitErr) || + isRatelimited(resp) || isTemporarilyUnavailable(resp) + + switch { + case retryable: + return nil, nil, nil, wrapGitHubError(err, resp, + fmt.Sprintf("baton-github: failed to fetch audit log for org %s", orgName)) + case isNotFoundError(resp) || isPermissionError(resp): + f.skippedOrgs.log(ctx, orgName, "org lacks audit-log access, skipping it for this pass", + zap.String("org", orgName), zap.Error(err), + ) + + entries, resp = nil, nil + default: + return nil, nil, nil, wrapGitHubError(err, resp, + fmt.Sprintf("baton-github: failed to fetch audit log for org %s", orgName)) + } + } + + reachedBoundary := false + for _, entry := range entries { + // Check every entry's timestamp, even filtered ones, so an all-bot page still stops pagination. + if ts := entry.GetTimestamp().Time; !ts.IsZero() && !ts.After(since) { + reachedBoundary = true + break + } + + evt, ok := usageEventFromAuditEntry(entry) + if !ok { + continue + } + events = append(events, evt) + } + + if resp != nil && !reachedBoundary { + if nextPage, isPage := nextAuditLogPage(resp); nextPage != "" { + cursor.AuditLogCursor = nextPage + cursor.AuditLogCursorIsPage = isPage + continue + } + } + + // Done with this org for this pass - advance to the next one. + cursor.OrgIndex++ + cursor.AuditLogCursor = "" + cursor.AuditLogCursorIsPage = false + if cursor.OrgIndex >= len(cursor.Orgs) { + // Pass complete - the next call gets a fresh earliestEvent, so + // nothing needs to survive in the cursor. + tokenStr, err := (&usageEventPageToken{}).marshal() + if err != nil { + return nil, nil, nil, err + } + var annos annotations.Annotations + if tightestRateLimit != nil { + annos.WithRateLimiting(tightestRateLimit) + } + return events, &pagination.StreamState{Cursor: tokenStr, HasMore: false}, annos, nil + } + } + + tokenStr, err := cursor.marshal() + if err != nil { + return nil, nil, nil, err + } + var annos annotations.Annotations + if tightestRateLimit != nil { + annos.WithRateLimiting(tightestRateLimit) + } + return events, &pagination.StreamState{Cursor: tokenStr, HasMore: true}, annos, nil +} + +// nextAuditLogPage returns the cursor to request the next audit-log page +// (empty if there isn't one), and whether that cursor belongs in the +// request's Page field (true) or its After field (false). +// +// The org audit-log endpoint documents three pagination shapes depending on +// what the server returns in the Link header's rel="next" entry: +// - after= (GHEC and GHES): the documented, primary mechanism - +// go-github parses this into Response.After. Checked first since it's +// what both github.com and GHES actually return in practice. +// - page= (legacy fallback some GHES versions may still +// emit): go-github can't parse a non-numeric page value as an int, so it +// lands in Response.NextPageToken instead. +// - page= (numeric, classic GHES offset pagination): parses into +// Response.NextPage. +// +// Checking only NextPageToken/NextPage (as earlier code did) misses the +// after= case entirely, silently truncating every GHEC org - and most GHES +// orgs - to a single page. +func nextAuditLogPage(resp *github.Response) (string, bool) { + if resp.After != "" { + return resp.After, false + } + if resp.NextPageToken != "" { + return resp.NextPageToken, true + } + if resp.NextPage != 0 { + return strconv.Itoa(resp.NextPage), true + } + return "", false +} + +// usageEventFromAuditEntry converts one audit-log entry into a usage event +// targeting the usage-app resource (see usage_app.go). Returns ok=false when +// the entry can't be attributed to a synced user. +func usageEventFromAuditEntry(entry *github.AuditEntry) (*v2.Event, bool) { + actor := entry.GetActor() + actorID := entry.GetActorID() + ts := entry.GetTimestamp().Time + if actorID == 0 || ts.IsZero() { + return nil, false + } + + // actor_is_bot is real but undocumented (only in AdditionalFields); trust + // it when present, else fall back to the "[bot]" login suffix. Bots + // aren't synced as users, so their events wouldn't correlate to anything. + if isBot, ok := entry.AdditionalFields["actor_is_bot"].(bool); ok { + if isBot { + return nil, false + } + } else if strings.HasSuffix(actor, "[bot]") { + return nil, false + } + + orgID := entry.GetOrgID() + + id := entry.GetDocumentID() + if id == "" { + // No stable ID from GitHub - synthesize one so dedup doesn't collapse + // every entry missing _document_id into one event. + id = fmt.Sprintf("%d:%d:%d:%s", orgID, actorID, ts.UnixNano(), entry.GetAction()) + } + + return &v2.Event{ + Id: id, + OccurredAt: timestamppb.New(ts), + Event: &v2.Event_UsageEvent{ + UsageEvent: &v2.UsageEvent{ + TargetResource: &v2.Resource{ + Id: &v2.ResourceId{ + ResourceType: resourceTypeUsageApp.Id, + Resource: usageAppResourceID, + }, + DisplayName: usageAppDisplayName, + }, + ActorResource: &v2.Resource{ + Id: &v2.ResourceId{ + ResourceType: resourceTypeUser.Id, + Resource: strconv.FormatInt(actorID, 10), + }, + DisplayName: actor, + }, + }, + }, + }, true +} diff --git a/vendor/github.com/conductorone/baton-github/pkg/connector/user.go b/vendor/github.com/conductorone/baton-github/pkg/connector/user.go index d5c6568f..705507a5 100644 --- a/vendor/github.com/conductorone/baton-github/pkg/connector/user.go +++ b/vendor/github.com/conductorone/baton-github/pkg/connector/user.go @@ -48,8 +48,14 @@ func userResource(ctx context.Context, user *github.User, userEmail string, extr userTrait := []resource.UserTraitOption{ resource.WithEmail(userEmail, true), - resource.WithUserProfile(profile), - resource.WithStatus(v2.UserTrait_Status_STATUS_ENABLED), + } + + // profile, status, and icon have moved from UserTrait to Resource-level + // attributes. NewUserTrait still defaults the (unset) trait status to + // ENABLED, so the enabled semantics are preserved on both levels. + resourceOpts := []resource.ResourceOption{ + resource.WithResourceProfile(profile), + resource.WithResourceStatus(v2.Status_RESOURCE_STATUS_ENABLED, ""), } for _, email := range extraEmails { @@ -57,7 +63,7 @@ func userResource(ctx context.Context, user *github.User, userEmail string, extr } if user.GetAvatarURL() != "" { - userTrait = append(userTrait, resource.WithUserIcon(&v2.AssetRef{ + resourceOpts = append(resourceOpts, resource.WithResourceIcon(&v2.AssetRef{ Id: user.GetAvatarURL(), })) } @@ -70,15 +76,17 @@ func userResource(ctx context.Context, user *github.User, userEmail string, extr })) } + resourceOpts = append(resourceOpts, resource.WithAnnotation( + &v2.ExternalLink{Url: user.GetHTMLURL()}, + &v2.V1Identifier{Id: strconv.FormatInt(user.GetID(), 10)}, + )) + ret, err := resource.NewUserResource( displayName, resourceTypeUser, user.GetID(), userTrait, - resource.WithAnnotation( - &v2.ExternalLink{Url: user.GetHTMLURL()}, - &v2.V1Identifier{Id: strconv.FormatInt(user.GetID(), 10)}, - ), + resourceOpts..., ) if err != nil { return nil, err diff --git a/vendor/modules.txt b/vendor/modules.txt index 72953cfb..7e8599bc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -272,7 +272,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-github v0.4.0 +# github.com/conductorone/baton-github v0.4.7 ## explicit; go 1.25.2 github.com/conductorone/baton-github/pkg/config github.com/conductorone/baton-github/pkg/connector