From b46fd32694b55a45e943055c4512dcc42e7b5107 Mon Sep 17 00:00:00 2001 From: Josh Johanning Date: Thu, 6 Aug 2026 19:46:15 -0500 Subject: [PATCH 1/2] feat: add Copilot usage reporting scripts --- gh-cli/README.md | 18 ++ ...pilot-ai-credit-usage-by-user-and-model.sh | 129 ++++++++++ scripts/README.md | 4 + .../copilot-enterprise-usage-report/README.md | 48 ++++ .../copilot-enterprise-usage-report.sh | 223 ++++++++++++++++++ 5 files changed, 422 insertions(+) create mode 100755 gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh create mode 100644 scripts/copilot-enterprise-usage-report/README.md create mode 100755 scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh diff --git a/gh-cli/README.md b/gh-cli/README.md index c42cd0a..7bf8a73 100644 --- a/gh-cli/README.md +++ b/gh-cli/README.md @@ -812,6 +812,24 @@ joshjohanning-org/.github, no code scanning results Gets the commits of since a certain date - date should be in [ISO 8601](https://en.wikipedia.org/wiki/ISO_8601) format, ie: `since=2022-03-28T16:00:49Z` +### get-copilot-ai-credit-usage-by-user-and-model.sh + +Exports monthly Copilot AI-credit usage by user and model for an enterprise. +The script first uses daily per-user metrics to identify users with AI-credit +usage, then makes one billing API call per identified user. + +```shell +./get-copilot-ai-credit-usage-by-user-and-model.sh [year] [month] [output.csv] [max-users] +``` + +The script uses the current `gh auth` credential. Classic PAT authentication +may require `read:enterprise` and `manage_billing:copilot`. + +> [!WARNING] +> The model breakdown endpoint only accepts one user filter at a time. Large +> enterprises can require thousands of API calls and may approach the API rate +> limit. Use the optional `max-users` argument for testing. + ### get-dependencies-in-repository.sh Gets dependencies used in the repository, including the ecosystem and version number. diff --git a/gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh b/gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh new file mode 100755 index 0000000..7b31db7 --- /dev/null +++ b/gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "Usage: $0 [year] [month] [output.csv] [max-users]" >&2 +} + +fail() { + echo "Error: $*" >&2 + exit 1 +} + +api() { + local error_file output + error_file=$(mktemp) + TEMP_FILES+=("$error_file") + + if ! output=$(gh api "$@" 2>"$error_file"); then + cat "$error_file" >&2 + if grep -q "HTTP 404" "$error_file"; then + cat >&2 <<'EOF' + +The API returned 404. Confirm the enterprise slug and credential access. +For classic PAT authentication, try: + gh auth refresh -s read:enterprise -s manage_billing:copilot + +The metrics API also requires the enterprise Copilot usage metrics policy. +The enterprise AI credit endpoint may require an enterprise owner or billing +manager using a classic PAT. +EOF + fi + return 1 + fi + + printf '%s' "$output" +} + +days_in_month() { + case "$1" in + 1|3|5|7|8|10|12) echo 31 ;; + 4|6|9|11) echo 30 ;; + 2) + if (( YEAR % 400 == 0 || (YEAR % 4 == 0 && YEAR % 100 != 0) )); then + echo 29 + else + echo 28 + fi + ;; + *) fail "Month must be between 1 and 12." ;; + esac +} + +[[ $# -ge 1 && $# -le 5 ]] || { usage; exit 1; } +for command_name in gh jq curl; do + command -v "$command_name" >/dev/null 2>&1 || + fail "Required command not found: $command_name" +done +gh auth status >/dev/null 2>&1 || + fail "GitHub CLI is not authenticated. Run: gh auth login" + +ENTERPRISE="$1" +YEAR="${2:-$(date +%Y)}" +MONTH=$((10#${3:-$(date +%m)})) +[[ "$YEAR" =~ ^[0-9]{4}$ ]] || fail "Year must use YYYY format." + +MONTH_PADDED=$(printf '%02d' "$MONTH") +OUTPUT="${4:-${ENTERPRISE}-copilot-ai-credit-usage-${YEAR}-${MONTH_PADDED}.csv}" +MAX_USERS="${5:-0}" +[[ "$MAX_USERS" =~ ^[0-9]+$ ]] || fail "Max users must be a non-negative integer." + +ACTIVE_USERS=$(mktemp) +TEMP_FILES=("$ACTIVE_USERS") +trap 'rm -f "${TEMP_FILES[@]}"' EXIT + +LAST_DAY=$(days_in_month "$MONTH") +echo "Finding users with AI-credit usage in $YEAR-$MONTH_PADDED..." >&2 + +for ((day = 1; day <= LAST_DAY; day++)); do + report_date=$(printf '%04d-%02d-%02d' "$YEAR" "$MONTH" "$day") + echo "Reading $report_date..." >&2 + api -H "X-GitHub-Api-Version: 2026-03-10" \ + "/enterprises/$ENTERPRISE/copilot/metrics/reports/users-1-day?day=$report_date" | + jq -r 'if type == "object" then (.download_links // [])[] else empty end' | + while IFS= read -r url; do + [[ -n "$url" ]] || continue + curl -fsSL "$url" | + jq -r 'select(type == "object" and (.ai_credits_used // 0) > 0) | .user_login' + done +done | sort -u >"$ACTIVE_USERS" + +user_count=$(wc -l <"$ACTIVE_USERS" | tr -d ' ') +if (( MAX_USERS > 0 && user_count > MAX_USERS )); then + head -n "$MAX_USERS" "$ACTIVE_USERS" >"${ACTIVE_USERS}.limited" + mv "${ACTIVE_USERS}.limited" "$ACTIVE_USERS" + user_count="$MAX_USERS" +fi + +echo "Found $user_count users. This next phase makes one billing API call per user." >&2 +if (( user_count > 1000 )); then + echo "Warning: this may approach the 5,000 requests/hour classic PAT limit." >&2 +fi + +{ + echo '"user","model","gross_ai_credits","included_ai_credits","net_ai_credits","gross_amount","net_spend"' + while IFS= read -r login; do + [[ -n "$login" ]] || continue + echo "Processing $login..." >&2 + api --method GET -H "X-GitHub-Api-Version: 2026-03-10" \ + "/enterprises/$ENTERPRISE/settings/billing/ai_credit/usage" \ + -f year="$YEAR" -f month="$MONTH" -f user="$login" | + jq -r --arg user "$login" ' + .usageItems + | select(type == "array" and length > 0) + | group_by(.model)[] + | [ + $user, + .[0].model, + (map(.grossQuantity) | add), + (map(.discountQuantity) | add), + (map(.netQuantity) | add), + (map(.grossAmount) | add), + (map(.netAmount) | add) + ] + | @csv + ' + done <"$ACTIVE_USERS" +} >"$OUTPUT" + +echo "Created $OUTPUT" >&2 diff --git a/scripts/README.md b/scripts/README.md index 67b5718..f12e408 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -44,6 +44,10 @@ Migrate work items from Azure DevOps to GitHub issues - this just links out to a See: [code-scanning-coverage-report](./code-scanning-coverage-report/README.md) +## copilot-enterprise-usage-report + +See: [copilot-enterprise-usage-report](./copilot-enterprise-usage-report/README.md) + ## create-app-jwt.py This script will generate a JWT for a GitHub App. It will use the private key and app ID from the GitHub App's settings page. diff --git a/scripts/copilot-enterprise-usage-report/README.md b/scripts/copilot-enterprise-usage-report/README.md new file mode 100644 index 0000000..2b3a568 --- /dev/null +++ b/scripts/copilot-enterprise-usage-report/README.md @@ -0,0 +1,48 @@ +# copilot-enterprise-usage-report + +Generate monthly GitHub Copilot enterprise usage reports from the current +`enterprise-1-day` Usage Metrics API. + +## Outputs + +- Weekly surface activity CSV +- Standalone HTML report with embedded data + +The report distinguishes unlike metrics instead of treating them as equivalent: + +- IDEs expose user-initiated interactions and generation activity. +- Copilot CLI and Copilot App expose prompt and request counts. +- Copilot Coding Agent exposes Copilot-created pull requests. +- Copilot Code Review exposes Copilot-reviewed pull requests. + +## Prerequisites + +- An authenticated GitHub CLI session: `gh auth status` +- `jq`, `curl`, and `base64` +- Enterprise access to Copilot usage metrics + +Classic PATs need `read:enterprise` or `manage_billing:copilot`. Fine-grained +credentials need the **View Enterprise Copilot Metrics** permission. The +enterprise **Copilot usage metrics** policy must also be enabled. + +## Usage + +```shell +./copilot-enterprise-usage-report.sh [year] [month] [output-prefix] +``` + +Example: + +```shell +./copilot-enterprise-usage-report.sh avocado-corp 2026 7 +``` + +This creates: + +```text +avocado-corp-copilot-usage-2026-07-weekly.csv +avocado-corp-copilot-usage-2026-07.html +``` + +The API provides up to one year of daily history beginning October 10, 2025. +Recent data can take several UTC days to finalize. diff --git a/scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh b/scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh new file mode 100755 index 0000000..7c6963a --- /dev/null +++ b/scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + echo "Usage: $0 [year] [month] [output-prefix]" >&2 +} + +fail() { + echo "Error: $*" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "Required command not found: $1" +} + +api() { + local error_file output + error_file=$(mktemp) + TEMP_FILES+=("$error_file") + + if ! output=$(gh api "$@" 2>"$error_file"); then + cat "$error_file" >&2 + if grep -q "HTTP 404" "$error_file"; then + cat >&2 <<'EOF' + +The API returned 404. Confirm: +- The enterprise slug is correct. +- Copilot usage metrics are enabled in enterprise AI Controls. +- Your credential is an enterprise owner, billing manager, or has the + "View Enterprise Copilot Metrics" permission. +- A classic PAT has read:enterprise or manage_billing:copilot. + +For a classic PAT authenticated through gh: + gh auth refresh -s read:enterprise -s manage_billing:copilot +EOF + fi + return 1 + fi + + printf '%s' "$output" +} + +days_in_month() { + case "$1" in + 1|3|5|7|8|10|12) echo 31 ;; + 4|6|9|11) echo 30 ;; + 2) + if (( YEAR % 400 == 0 || (YEAR % 4 == 0 && YEAR % 100 != 0) )); then + echo 29 + else + echo 28 + fi + ;; + *) fail "Month must be between 1 and 12." ;; + esac +} + +[[ $# -ge 1 && $# -le 4 ]] || { usage; exit 1; } + +for command_name in gh jq curl base64; do + require_command "$command_name" +done + +gh auth status >/dev/null 2>&1 || + fail "GitHub CLI is not authenticated. Run: gh auth login" + +ENTERPRISE="$1" +YEAR="${2:-$(date +%Y)}" +MONTH=$((10#${3:-$(date +%m)})) +[[ "$YEAR" =~ ^[0-9]{4}$ ]] || fail "Year must use YYYY format." + +MONTH_PADDED=$(printf '%02d' "$MONTH") +PREFIX="${4:-${ENTERPRISE}-copilot-usage-${YEAR}-${MONTH_PADDED}}" +WEEKLY_OUTPUT="${PREFIX}-weekly.csv" +HTML_OUTPUT="${PREFIX}.html" +RAW_ROWS=$(mktemp) +TEMP_FILES=("$RAW_ROWS") + +cleanup() { + rm -f "${TEMP_FILES[@]}" +} +trap cleanup EXIT + +LAST_DAY=$(days_in_month "$MONTH") + +echo "Downloading $ENTERPRISE Copilot metrics for $YEAR-$MONTH_PADDED..." >&2 + +for ((day = 1; day <= LAST_DAY; day++)); do + report_date=$(printf '%04d-%02d-%02d' "$YEAR" "$MONTH" "$day") + echo "Reading $report_date..." >&2 + + api -H "X-GitHub-Api-Version: 2026-03-10" \ + "/enterprises/$ENTERPRISE/copilot/metrics/reports/enterprise-1-day?day=$report_date" | + jq -r 'if type == "object" then (.download_links // [])[] else empty end' | + while IFS= read -r url; do + [[ -n "$url" ]] || continue + curl -fsSL "$url" | + jq -c ' + select(type == "object") as $r + | ( + $r.totals_by_ide[]? + | { + day: $r.day, surface: .ide, kind: "ide", + interactions: (.user_initiated_interaction_count // 0), + generations: (.code_generation_activity_count // 0), + acceptances: (.code_acceptance_activity_count // 0), + requests: 0, outcomes: 0, outcome_label: "", + loc_suggested: (.loc_suggested_to_add_sum // 0), + loc_added: (.loc_added_sum // 0) + } + ), + ( + $r.totals_by_feature[]? + | { + day: $r.day, surface: .feature, kind: "feature", + interactions: (.user_initiated_interaction_count // 0), + generations: (.code_generation_activity_count // 0), + acceptances: (.code_acceptance_activity_count // 0), + requests: 0, outcomes: 0, outcome_label: "", + loc_suggested: (.loc_suggested_to_add_sum // 0), + loc_added: (.loc_added_sum // 0) + } + ), + ( + select(($r.totals_by_cli? | type) == "object") + | { + day: $r.day, surface: "copilot_cli", kind: "surface", + interactions: ($r.totals_by_cli.prompt_count // 0), + generations: 0, acceptances: 0, + requests: ($r.totals_by_cli.request_count // 0), + outcomes: 0, outcome_label: "", loc_suggested: 0, loc_added: 0 + } + ), + ( + select(($r.totals_by_copilot_app? | type) == "object") + | { + day: $r.day, surface: "copilot_app", kind: "surface", + interactions: ($r.totals_by_copilot_app.prompt_count // 0), + generations: 0, acceptances: 0, + requests: ($r.totals_by_copilot_app.request_count // 0), + outcomes: 0, outcome_label: "", loc_suggested: 0, loc_added: 0 + } + ), + ( + select(($r.pull_requests.total_created_by_copilot // 0) > 0) + | { + day: $r.day, surface: "copilot_coding_agent", kind: "surface", + interactions: 0, generations: 0, acceptances: 0, requests: 0, + outcomes: ($r.pull_requests.total_created_by_copilot // 0), + outcome_label: "Copilot-created PRs", loc_suggested: 0, loc_added: 0 + } + ), + ( + select(($r.pull_requests.total_reviewed_by_copilot // 0) > 0) + | { + day: $r.day, surface: "copilot_code_review", kind: "surface", + interactions: 0, generations: 0, acceptances: 0, requests: 0, + outcomes: ($r.pull_requests.total_reviewed_by_copilot // 0), + outcome_label: "Copilot-reviewed PRs", loc_suggested: 0, loc_added: 0 + } + ) + ' >>"$RAW_ROWS" + done +done + +jq -rs ' + def week_start: + (.day + "T00:00:00Z" | fromdateiso8601) as $ts + | ($ts | strftime("%w") | tonumber) as $weekday + | ($ts - ($weekday * 86400) | strftime("%Y-%m-%d")); + map(select(.kind != "feature") + {week_start: week_start}) + | sort_by(.week_start, .surface) + | group_by([.week_start, .surface]) + | map({ + week_start: .[0].week_start, + surface: .[0].surface, + surface_type: .[0].kind, + interactions: (map(.interactions) | add), + generations: (map(.generations) | add), + requests: (map(.requests) | add), + outcomes: (map(.outcomes) | add), + outcome_label: (map(.outcome_label) | map(select(length > 0)) | first // "") + }) + | ( + ["week_start","surface","surface_type","interactions","generations","requests","outcomes","outcome_label"], + (.[] | [.week_start,.surface,.surface_type,.interactions,.generations,.requests,.outcomes,.outcome_label]) + ) + | @csv +' "$RAW_ROWS" >"$WEEKLY_OUTPUT" + +CSV_BASE64=$(base64 <"$WEEKLY_OUTPUT" | tr -d '\n') +cat >"$HTML_OUTPUT" < + +${ENTERPRISE} Copilot usage ${YEAR}-${MONTH_PADDED} + +
+

${ENTERPRISE} Copilot usage

${YEAR}-${MONTH_PADDED}, generated from enterprise daily usage metrics

0Interactions
0Reported requests
0Agent outcomes
+

Weekly interactions

Interactions are user-initiated prompts. CLI and Copilot App request counts include agentic follow-up calls. CCA and CCR are reported as pull request outcomes.
+

Surface summary

SurfaceInteractionsGenerationsRequestsOutcomes
+
+ +EOF + +echo "Created $WEEKLY_OUTPUT" >&2 +echo "Created $HTML_OUTPUT" >&2 From 82097f9237ce6d557f1ff49330ea51c329ed33fe Mon Sep 17 00:00:00 2001 From: Josh Johanning Date: Thu, 6 Aug 2026 19:56:29 -0500 Subject: [PATCH 2/2] fix: address Copilot reporting review --- ...copilot-ai-credit-usage-by-user-and-model.sh | 3 +++ .../copilot-enterprise-usage-report/README.md | 8 ++++---- .../copilot-enterprise-usage-report.sh | 17 ++++------------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh b/gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh index 7b31db7..ec1c29a 100755 --- a/gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh +++ b/gh-cli/get-copilot-ai-credit-usage-by-user-and-model.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Export monthly enterprise AI credit usage grouped by user and model. +# Usage: get-copilot-ai-credit-usage-by-user-and-model.sh [year] [month] [output.csv] [max-users] +# Requires authenticated gh access to Copilot metrics and billing plus jq and curl. set -euo pipefail usage() { diff --git a/scripts/copilot-enterprise-usage-report/README.md b/scripts/copilot-enterprise-usage-report/README.md index 2b3a568..26550f5 100644 --- a/scripts/copilot-enterprise-usage-report/README.md +++ b/scripts/copilot-enterprise-usage-report/README.md @@ -10,10 +10,10 @@ Generate monthly GitHub Copilot enterprise usage reports from the current The report distinguishes unlike metrics instead of treating them as equivalent: -- IDEs expose user-initiated interactions and generation activity. -- Copilot CLI and Copilot App expose prompt and request counts. -- Copilot Coding Agent exposes Copilot-created pull requests. -- Copilot Code Review exposes Copilot-reviewed pull requests. +- IDEs expose user-initiated interactions and generation activity +- Copilot CLI and Copilot App expose prompt and request counts +- Copilot Coding Agent exposes Copilot-created pull requests +- Copilot Code Review exposes Copilot-reviewed pull requests ## Prerequisites diff --git a/scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh b/scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh index 7c6963a..cbaa965 100755 --- a/scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh +++ b/scripts/copilot-enterprise-usage-report/copilot-enterprise-usage-report.sh @@ -1,4 +1,7 @@ #!/usr/bin/env bash +# Generate monthly enterprise Copilot usage as a weekly CSV and standalone HTML report. +# Usage: copilot-enterprise-usage-report.sh [year] [month] [output-prefix] +# Requires authenticated gh access to enterprise Copilot metrics plus jq, curl, and base64. set -euo pipefail usage() { @@ -110,18 +113,6 @@ for ((day = 1; day <= LAST_DAY; day++)); do loc_added: (.loc_added_sum // 0) } ), - ( - $r.totals_by_feature[]? - | { - day: $r.day, surface: .feature, kind: "feature", - interactions: (.user_initiated_interaction_count // 0), - generations: (.code_generation_activity_count // 0), - acceptances: (.code_acceptance_activity_count // 0), - requests: 0, outcomes: 0, outcome_label: "", - loc_suggested: (.loc_suggested_to_add_sum // 0), - loc_added: (.loc_added_sum // 0) - } - ), ( select(($r.totals_by_cli? | type) == "object") | { @@ -169,7 +160,7 @@ jq -rs ' (.day + "T00:00:00Z" | fromdateiso8601) as $ts | ($ts | strftime("%w") | tonumber) as $weekday | ($ts - ($weekday * 86400) | strftime("%Y-%m-%d")); - map(select(.kind != "feature") + {week_start: week_start}) + map(. + {week_start: week_start}) | sort_by(.week_start, .surface) | group_by([.week_start, .surface]) | map({