diff --git a/.github/workflows/notify-non-employee-comments.yml b/.github/workflows/notify-non-employee-comments.yml index e1f0738368fd..6d4e7b77bc1e 100644 --- a/.github/workflows/notify-non-employee-comments.yml +++ b/.github/workflows/notify-non-employee-comments.yml @@ -51,7 +51,7 @@ jobs: core.info(`Building digest for ${since.toISOString()} to ${until.toISOString()}.`); const authors = new Map(); - function addActivity({ author, createdAt, url, conversation, kind }) { + function addActivity({ author, createdAt, url, kind }) { const timestamp = Date.parse(createdAt); if (!(timestamp >= since.getTime() && timestamp < until.getTime())) { return; @@ -68,7 +68,7 @@ jobs: if (!authors.has(login)) { authors.set(login, { login: author.login, activities: [] }); } - authors.get(login).activities.push({ url, conversation, kind }); + authors.get(login).activities.push({ url, kind }); } let page = 0; @@ -94,8 +94,6 @@ jobs: author: comment.user, createdAt: comment.created_at, url: comment.html_url, - conversation: comment.issue_url ?? - comment.pull_request_url.replace('/pulls/', '/issues/'), kind: 'comment', }); } @@ -125,7 +123,7 @@ jobs: repository(owner: "github", name: "docs") { pullRequests(first: 50, after: $pullRequestsCursor, orderBy: { field: UPDATED_AT, direction: DESC }) { - nodes { id number updatedAt ...digestReviews } + nodes { id updatedAt ...digestReviews } pageInfo { hasNextPage endCursor } } } @@ -144,7 +142,6 @@ jobs: author: review.author, createdAt: review.submittedAt, url: review.url, - conversation: `https://api.github.com/repos/github/docs/issues/${pullRequest.number}`, kind: 'review', }); } @@ -194,35 +191,82 @@ jobs: return; } - let message = `Daily non-employee comment and review digest for github/docs\n` + - `${since.toISOString()} to ${until.toISOString()} (UTC)\n` + - `Non-employee users: ${nonEmployees.length}; comments: ${totalComments}; reviews: ${totalReviews}\n` + - 'Comment and review bodies omitted; follow the links to read them on GitHub.'; - const shortened = '\n\n[Digest shortened to fit one Slack message; some authors or activity links are omitted.]'; - const maximumLength = 35000 - shortened.length; - const lines = nonEmployees.flatMap((author) => [ - `\n\n@${author.login} (https://github.com/${author.login})\n` + - `Comments: ${author.activities.filter((activity) => activity.kind === 'comment').length}; ` + - `reviews: ${author.activities.filter((activity) => activity.kind === 'review').length}; conversations: ` + - `${new Set(author.activities.map((activity) => activity.conversation)).size}`, - ...author.activities.map((activity) => - `\n ${activity.kind === 'review' ? 'Review: ' : ''}${activity.url}`), - ]); - for (const line of lines) { - if (message.length + line.length > maximumLength) { - message += shortened; - core.warning('Digest shortened to fit one Slack message.'); + const title = 'Daily non-employee comment and review digest for github/docs'; + // Slack caps a table at 100 rows and 10,000 characters across its cells. + // Count link URLs as well as link text, because Slack does not document whether URLs count. + const maximumRows = 100; + let remainingCharacters = 10000; + let shortened = false; + const rawText = (text) => { + remainingCharacters -= text.length; + return { type: 'raw_text', text }; + }; + const richText = (elements) => ({ + type: 'rich_text', + elements: [{ type: 'rich_text_section', elements }], + }); + const rows = [[rawText('User'), rawText('Comments'), rawText('Reviews')]]; + for (const author of nonEmployees) { + const profileUrl = `https://github.com/${author.login}`; + const rowCharacters = author.login.length + profileUrl.length + 2; + if (rows.length >= maximumRows || rowCharacters > remainingCharacters) { + shortened = true; + break; + } + remainingCharacters -= rowCharacters; + const row = [richText([{ type: 'link', url: profileUrl, text: author.login }])]; + for (const kind of ['comment', 'review']) { + const urls = author.activities + .filter((activity) => activity.kind === kind) + .map((activity) => activity.url); + const links = []; + for (const [index, url] of urls.entries()) { + const text = `${index + 1}`; + const characters = text.length + url.length + (index ? 1 : 0); + if (shortened || characters > remainingCharacters) { + shortened = true; + break; + } + remainingCharacters -= characters; + if (index) { + links.push({ type: 'text', text: ' ' }); + } + links.push({ type: 'link', url, text }); + } + // The row reserves characters for this placeholder. + row.push(links.length ? richText(links) : { type: 'raw_text', text: '-' }); + } + rows.push(row); + if (shortened) { break; } - message += line; } + if (shortened) { + core.warning('Digest shortened to fit one Slack table.'); + } + const blocks = [ + { + type: 'section', + text: { + type: 'mrkdwn', + text: `*${title}*\nComment and review bodies omitted. Follow the links to read them on GitHub.` + + (shortened ? '\n_Digest shortened to fit one Slack table. Some authors or links are omitted._' : ''), + }, + }, + { + type: 'table', + column_settings: [{ is_wrapped: false }, { is_wrapped: true }, { is_wrapped: true }], + rows, + }, + ]; for (const author of nonEmployees) { core.setSecret(author.login); } - core.setOutput('slack_message', message); + core.setOutput('slack_text', title); + core.setOutput('slack_blocks', JSON.stringify(blocks)); - name: Notify the docs-open-source channel - if: ${{ steps.digest.outputs.slack_message != '' }} + if: ${{ steps.digest.outputs.slack_blocks != '' }} uses: slackapi/slack-github-action@dcb1066f776dd043e64d0e8ba94ca15cc7e1875d # v4.0.0 with: method: chat.postMessage @@ -231,8 +275,8 @@ jobs: # Slack channel ID for docs-open-source. payload: | channel: C017M6T23GA - text: ${{ toJSON(steps.digest.outputs.slack_message) }} - mrkdwn: false + text: ${{ toJSON(steps.digest.outputs.slack_text) }} + blocks: ${{ steps.digest.outputs.slack_blocks }} link_names: false unfurl_links: false unfurl_media: false diff --git a/content/admin/troubleshooting-your-instance/index.md b/content/admin/troubleshooting-your-instance/index.md index 47db7fa84b55..b6f49cca5a94 100644 --- a/content/admin/troubleshooting-your-instance/index.md +++ b/content/admin/troubleshooting-your-instance/index.md @@ -9,5 +9,6 @@ children: - /troubleshooting-upgrades-and-migrations - /troubleshooting-storage-and-disks - /troubleshooting-github-actions-data-and-database-health + - /troubleshooting-search-webhooks-and-background-jobs shortTitle: Troubleshoot your instance --- diff --git a/content/admin/troubleshooting-your-instance/troubleshooting-search-webhooks-and-background-jobs/index.md b/content/admin/troubleshooting-your-instance/troubleshooting-search-webhooks-and-background-jobs/index.md new file mode 100644 index 000000000000..71464b5001d0 --- /dev/null +++ b/content/admin/troubleshooting-your-instance/troubleshooting-search-webhooks-and-background-jobs/index.md @@ -0,0 +1,9 @@ +--- +title: Troubleshooting search, webhooks, and background jobs +intro: 'Resolve problems with the services that index content, deliver webhooks, and process background work on {% data variables.location.product_location %}.' +versions: + ghes: '*' +children: + - /webhook-deliveries-stuck-in-the-job-queue +shortTitle: Search, webhooks, and jobs +--- diff --git a/content/admin/troubleshooting-your-instance/troubleshooting-search-webhooks-and-background-jobs/webhook-deliveries-stuck-in-the-job-queue.md b/content/admin/troubleshooting-your-instance/troubleshooting-search-webhooks-and-background-jobs/webhook-deliveries-stuck-in-the-job-queue.md new file mode 100644 index 000000000000..8267feeb338d --- /dev/null +++ b/content/admin/troubleshooting-your-instance/troubleshooting-search-webhooks-and-background-jobs/webhook-deliveries-stuck-in-the-job-queue.md @@ -0,0 +1,196 @@ +--- +title: Webhook deliveries stuck in the job queue +intro: 'If {% data variables.location.product_location %} stops sending webhooks and jobs build up in the internal job queue, the webhook delivery service may have lost its connection to the queue. You can confirm the problem, restore deliveries, and prevent it from happening again.' +versions: + ghes: '*' +shortTitle: Webhook deliveries stuck +contentType: how-tos +category: + - Troubleshoot your instance +--- + +## Symptoms + +You may see any of the following. + +* Pushes and other events that should trigger webhooks do not produce any delivery attempts. For more information, see [AUTOTITLE](/webhooks/testing-and-troubleshooting-webhooks/viewing-webhook-deliveries). +* In the {% data variables.enterprise.management_console %} monitor dashboard, the `Aqueduct queue depth` graph shows `hookshot-production.hookshot` jobs building up. For more information, see [AUTOTITLE](/admin/monitoring-and-managing-your-instance/monitoring-your-instance/about-the-monitor-dashboards). +* The system logs contain `hookshot-go` errors that say `Failed to receive job from queue`, with `connection refused` on `aqueduct-lite.sock`. + +## Causes + +`hookshot-go` is the service that sends webhooks. It gets each delivery as a job from `aqueduct-lite`, the internal job queue. + +On older patch releases, `hookshot-go` connects to `aqueduct-lite` over a UNIX socket served by `haproxy-cluster-proxy`, the instance's internal proxy. If that connection is refused, `hookshot-go` cannot receive jobs, so no webhooks are sent and the jobs wait in the queue. + +Restarting `hookshot-go` or `aqueduct-lite`, or running a configuration apply, may not restore the connection on its own. + +Newer patch releases connect over TCP by default instead, which has proven more reliable. See [Resolving the problem](#resolving-the-problem) for the releases that include this change. + +## Confirming the problem + +1. SSH into the administrative shell. For more information, see [AUTOTITLE](/admin/administering-your-instance/administering-your-instance-from-the-command-line/accessing-the-administrative-shell-ssh). +1. Search the system logs for the error. + + ```shell + rg -z 'hookshot-go.*Failed to receive job from queue' /var/log/syslog* + ``` + + If you are affected, you will see entries like this. + + ```text + Dec 20 17:28:04 github-example-com-primary hookshot-go[2598]: Timestamp=2024-12-20T17:28:04.923986779Z SeverityText=ERROR Body="Failed to receive job from queue" host.name=github-example-com-primary exception.message="receive failed: twirp error internal: failed to do request: Post \"http://unix/twirp/aqueduct.api.v1.JobQueueService/Receive\": dial unix /go/src/app/sockets/aqueduct-lite.sock: connect: connection refused" exception.type=*errors.withStack gh.hookshot.queue=hookshot-redeliveries + ``` + +1. If you are running a patch release older than those listed in [Resolving the problem](#resolving-the-problem), check whether the socket accepts connections. + + ```shell + echo "GET /" | sudo socat - UNIX-CONNECT:/data/hookshot/shared/sockets/aqueduct-lite.sock + ``` + + If the connection is refused, you are affected. + + ```text + 2025/03/20 23:21:51 socat[4164899] E connect(5, AF=1 "/data/hookshot/shared/sockets/aqueduct-lite.sock", 50): Connection refused + ``` + + If you see an `HTTP/1.1 400 Bad request` response instead, the socket is up and accepting connections. + +## Resolving the problem + +> [!WARNING] +> Every fix in this article interrupts service on {% data variables.location.product_location %}. Restarting the internal proxy causes a brief interruption. Switching to TCP, rebooting, and upgrading disrupt services for longer. Plan a maintenance window before you start. + +The permanent fix is to upgrade to a patch release where `hookshot-go` connects to `aqueduct-lite` over TCP by default. + +The interim steps below apply only while `hookshot-go` uses the UNIX socket, and the socket check in [Confirming the problem](#confirming-the-problem) reports `Connection refused`. If you are on a patch release that connects over TCP by default, or you have already switched to TCP, do not run them. Contact {% data variables.contact.github_support %} instead. See [Contacting {% data variables.contact.github_support %}](#contacting-github-support). + +Until you can upgrade, work through the following in order, and stop when webhooks are delivered again. + +1. [Restart the internal proxy](#restarting-the-internal-proxy). This works on every release. +1. If your patch release supports TCP but does not use it by default, [switch to TCP](#switching-to-tcp-on-earlier-patch-releases). If webhooks are still not delivered after you switch, contact {% data variables.contact.github_support %} rather than rebooting. +1. If you cannot switch to TCP and the problem continues, [reboot the instance](#rebooting-the-instance). + +### Upgrading to a patch release with TCP by default + +TCP is the default from the following patch releases onward. Upgrade to the latest patch release for your feature release, which includes the change. + +| Feature release | First patch release with TCP by default | +| :- | :- | +| 3.19 | 3.19.8 | +| 3.20 | 3.20.4 | +| 3.21 | 3.21.5 | +| 3.22 and later | All releases | + +No configuration change is needed after you upgrade. For more information, see [AUTOTITLE](/admin/upgrading-your-instance/preparing-to-upgrade/overview-of-the-upgrade-process) and [AUTOTITLE](/admin/upgrading-your-instance/performing-an-upgrade/upgrading-with-a-hotpatch). + +### Restarting the internal proxy + +Restarting `haproxy-cluster-proxy` recreates the socket. It causes a brief service interruption. + +1. SSH into the administrative shell. For more information, see [AUTOTITLE](/admin/administering-your-instance/administering-your-instance-from-the-command-line/accessing-the-administrative-shell-ssh). +1. Stop and remove the proxy job, then start it again. + + ```shell + nomad job stop --purge haproxy-cluster-proxy + sudo nomad job run /etc/nomad-jobs/haproxy/haproxy-cluster-proxy.hcl + ``` + +1. Check the socket again. You should see an `HTTP/1.1 400 Bad request` response. + + ```shell + echo "GET /" | sudo socat - UNIX-CONNECT:/data/hookshot/shared/sockets/aqueduct-lite.sock + ``` + +1. Confirm that `hookshot-production.hookshot` jobs in the `Aqueduct queue depth` graph start to fall, and that new events produce webhook deliveries. + +### Switching to TCP on earlier patch releases + +Every supported feature release has a setting that switches `hookshot-go` to TCP. Use it if you are running one of the following patch releases, where the setting exists but is not on by default. + +* 3.19.0 to 3.19.7 +* 3.20.0 to 3.20.3 +* 3.21.0 to 3.21.4 + +> [!WARNING] +> Running `ghe-config-apply` disrupts services. + +1. Enable maintenance mode. For more information, see [AUTOTITLE](/admin/administering-your-instance/configuring-maintenance-mode/enabling-and-scheduling-maintenance-mode). +1. SSH into the administrative shell, then restart the internal proxy. + + ```shell + nomad job stop --purge haproxy-cluster-proxy + sudo nomad job run /etc/nomad-jobs/haproxy/haproxy-cluster-proxy.hcl + ``` + +1. Turn on the TCP connection, then apply the configuration. + + ```shell + ghe-config app.hookshot-go.aqueduct-tcp-connection-enabled true + ghe-config-apply + ``` + + For more information, see [AUTOTITLE](/admin/administering-your-instance/administering-your-instance-from-the-command-line/command-line-utilities#ghe-config). + +1. Check that `aqueduct-lite` accepts connections over TCP. You should see an `HTTP/1.1 400 Bad request` response. + + ```shell + echo "GET /" | sudo socat - TCP:localhost:9096 + ``` + +1. Disable maintenance mode. +1. Confirm that `hookshot-production.hookshot` jobs in the `Aqueduct queue depth` graph start to fall, and that new events produce webhook deliveries. + +If you run into an unexpected problem with TCP, switch back to the UNIX socket. + +1. Enable maintenance mode. +1. Turn off the TCP connection, then apply the configuration. + + ```shell + ghe-config app.hookshot-go.aqueduct-tcp-connection-enabled false + ghe-config-apply + ``` + +1. Check the socket. You should see an `HTTP/1.1 400 Bad request` response. + + ```shell + echo "GET /" | sudo socat - UNIX-CONNECT:/data/hookshot/shared/sockets/aqueduct-lite.sock + ``` + +1. Disable maintenance mode. + +### Rebooting the instance + +If you cannot switch to TCP and webhooks are still not delivered, reboot the instance. A reboot restores the connection, but the problem can return until you upgrade. + +1. Schedule a maintenance window with your users. +1. Enable maintenance mode, then wait for active processes to complete. For more information, see [AUTOTITLE](/admin/administering-your-instance/configuring-maintenance-mode/enabling-and-scheduling-maintenance-mode). +1. Reboot the instance. + + ```shell + sudo reboot + ``` + +1. When the instance is back, SSH into the administrative shell and check the socket again. You should see an `HTTP/1.1 400 Bad request` response. + + ```shell + echo "GET /" | sudo socat - UNIX-CONNECT:/data/hookshot/shared/sockets/aqueduct-lite.sock + ``` + +1. Disable maintenance mode. +1. Confirm that `hookshot-production.hookshot` jobs in the `Aqueduct queue depth` graph start to fall, and that new events produce webhook deliveries. + +## Preventing recurrence + +Keep {% data variables.location.product_location %} on the latest patch release for its feature release. Patch releases from those listed above onward connect over TCP by default, which avoids this problem. If you cannot upgrade yet, switch to TCP manually. + +## Contacting {% data variables.contact.github_support %} + +If webhooks are still not delivered after you reboot, if you see these errors after switching to TCP or on a patch release that connects over TCP by default, or if you are unsure about any of these steps, contact {% data variables.contact.github_support %} rather than continuing. For more information, see [AUTOTITLE](/support/contacting-github-support/creating-a-support-ticket). + +If you have already opened a ticket, attaching a support bundle helps {% data variables.contact.github_support %} diagnose the problem faster. For more information, see [AUTOTITLE](/support/contacting-github-support/providing-data-to-github-support). + +## Further reading + +* [AUTOTITLE](/webhooks/testing-and-troubleshooting-webhooks/troubleshooting-webhooks) +* [AUTOTITLE](/admin/monitoring-and-managing-your-instance/monitoring-your-instance/about-system-logs) diff --git a/content/billing/concepts/product-billing/cloud-and-local-sandboxes.md b/content/billing/concepts/product-billing/cloud-and-local-sandboxes.md index ed34d6cc7938..2faf13804407 100644 --- a/content/billing/concepts/product-billing/cloud-and-local-sandboxes.md +++ b/content/billing/concepts/product-billing/cloud-and-local-sandboxes.md @@ -24,7 +24,7 @@ Usage is measured from the moment a session starts until it is stopped or delete ### Compute -The compute meter tracks the time a cloud sandbox is running. Compute is not metered while a sandbox is stopped. +The compute meter tracks the time a cloud sandbox is running, measured in vCPU seconds. A vCPU second is one virtual CPU running for one second, so a sandbox with 2 vCPUs uses 2 vCPU seconds for each second it runs. Compute is not metered while a sandbox is stopped. ### Memory @@ -40,13 +40,13 @@ For more information about cloud and local sandboxes, see [AUTOTITLE](/copilot/c {% data reusables.billing.compute-allowance-model %} -For example, suppose a cloud sandbox runs for 600 seconds with 4 GiB of memory. At the published rates below: +For example, suppose a cloud sandbox with 2 vCPUs and 4 GiB of memory runs for 600 seconds. At the published rates below: -* The compute cost is 600 compute seconds × $0.000024 USD, or $0.0144 USD. -* The memory cost is 2,400 GiB-seconds (4 GiB × 600 seconds) × $0.000003 USD, or $0.0072 USD. -* A total of $0.0216 USD is deducted from the account's included compute usage. +* The compute cost is 1,200 vCPU seconds (2 vCPUs × 600 seconds) × $0.000024 USD, or $0.0288 USD. +* The memory cost is 2,400 GiB seconds (4 GiB × 600 seconds) × $0.000003 USD, or $0.0072 USD. +* A total of $0.036 USD is deducted from the account's included compute usage. -This example shows how usage measured in seconds and GiB-seconds draws from the same monetary value as runner usage measured in minutes. For the amount included with each plan, see [AUTOTITLE](/billing/reference/product-usage-included). +This example shows how usage measured in vCPU seconds and GiB seconds draws from the same monetary value as runner usage measured in minutes. For the amount included with each plan, see [AUTOTITLE](/billing/reference/product-usage-included). ## Paying for use @@ -56,7 +56,7 @@ You pay for cloud sandboxes using the payment method set up for your {% data var | Meter | Description | Unit | Price (USD) | | --- | --- | --- |------------| -| Compute | Time that a cloud sandbox session is running. | Compute second | $0.000024 | +| Compute | Time that a cloud sandbox session is running. | vCPU second | $0.000024 | | Memory | Memory allocated to a cloud sandbox session while it is running. | GiB second | $0.000003 | | Storage | Snapshot storage for stopped sessions. | GiB month | $0.005 | diff --git a/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/managing-access-to-agentic-autofix-for-code-scanning.md b/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/managing-access-to-agentic-autofix-for-code-scanning.md index a5f981fe5710..6caa7ab307b6 100644 --- a/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/managing-access-to-agentic-autofix-for-code-scanning.md +++ b/content/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/managing-access-to-agentic-autofix-for-code-scanning.md @@ -22,6 +22,8 @@ Access controls for {% data variables.copilot.agentic_autofix %} cascade from th {% data variables.copilot.agentic_autofix_caps %} is not available for user-owned repositories in an enterprise that uses {% data variables.product.prodname_emus %}. +Enabling {% data variables.copilot.agentic_autofix %} does not change the authentication requirements for assigning alerts through the REST API. GitHub App installation access tokens cannot be used to assign an alert to {% data variables.product.prodname_copilot_short %} or start an agentic autofix session. For information about assigning an alert to {% data variables.product.prodname_copilot_short %}, see [AUTOTITLE](/code-security/how-tos/manage-security-alerts/manage-code-scanning-alerts/resolve-alerts#fixing-alerts-with-copilot). + If your organization used {% data variables.copilot.agentic_autofix %} during the public preview, you must enable it again. Before you enable the feature, review how the use of {% data variables.product.prodname_ai_credits_short %} and {% data variables.product.prodname_actions %} minutes affects your costs.