diff --git a/.prettierignore b/.prettierignore index 1f17cccbc8d..23dfbd0f6a9 100644 --- a/.prettierignore +++ b/.prettierignore @@ -12,6 +12,8 @@ build/ # Generated files *.lock.yml +schemas/audit.schema.json +schemas/logs.schema.json # DevContainer configuration (should not be reformatted) .devcontainer/devcontainer.json diff --git a/Makefile b/Makefile index cd69f494db7..1cc2d098a9d 100644 --- a/Makefile +++ b/Makefile @@ -1256,9 +1256,22 @@ sync-install-script-hashes: @bash scripts/update-install-script-hashes.sh @echo "✓ Install script hashes synced successfully" +# Generate audit and logs JSON schemas +.PHONY: schemas +schemas: build + @set -eu; \ + for schema in audit logs; do \ + target="schemas/$$schema.schema.json"; \ + tmp=$$(mktemp "$$target.tmp.XXXXXX"); \ + trap 'rm -f "$$tmp"' EXIT HUP INT TERM; \ + ./$(BINARY_NAME) json-schema "$$schema" > "$$tmp"; \ + mv "$$tmp" "$$target"; \ + trap - EXIT HUP INT TERM; \ + done + # Recompile all workflow files .PHONY: recompile -recompile: build +recompile: schemas ./$(BINARY_NAME) init --codespaces "" ./$(BINARY_NAME) compile --validate --verbose --purge --schedule-seed github/gh-aw # ./$(BINARY_NAME) compile --dir pkg/cli/workflows --validate --verbose --purge diff --git a/cmd/gh-aw/argument_syntax_test.go b/cmd/gh-aw/argument_syntax_test.go index 7dbd9c4fd5a..36d9ab6bb46 100644 --- a/cmd/gh-aw/argument_syntax_test.go +++ b/cmd/gh-aw/argument_syntax_test.go @@ -31,6 +31,13 @@ func TestArgumentSyntaxConsistency(t *testing.T) { argsValidator: "MinimumNArgs(1)", shouldValidate: func(cmd *cobra.Command) error { return cmd.Args(cmd, []string{"123456"}) }, }, + { + name: "json-schema command requires schema", + command: cli.NewJSONSchemaCommand(), + expectedUse: "json-schema ", + argsValidator: "ExactArgs(1)", + shouldValidate: func(cmd *cobra.Command) error { return cmd.Args(cmd, []string{"audit"}) }, + }, { name: "trial command requires workflow-spec", command: cli.NewTrialCommand(validateEngine), diff --git a/cmd/gh-aw/command_groups_test.go b/cmd/gh-aw/command_groups_test.go index 6ef34040420..e08336b18d8 100644 --- a/cmd/gh-aw/command_groups_test.go +++ b/cmd/gh-aw/command_groups_test.go @@ -61,6 +61,7 @@ func TestCommandGroupAssignments(t *testing.T) { {name: "completion command in utilities group", commandName: "completion", expectedGroup: "utilities", shouldHaveGroup: true}, {name: "hash-frontmatter command in utilities group", commandName: "hash-frontmatter", expectedGroup: "utilities", shouldHaveGroup: true}, {name: "project command in utilities group", commandName: "project", expectedGroup: "utilities", shouldHaveGroup: true}, + {name: "json-schema command in utilities group", commandName: "json-schema", expectedGroup: "utilities", shouldHaveGroup: true}, // Commands without groups (intentionally) {name: "version command without group", commandName: "version", expectedGroup: "", shouldHaveGroup: false}, diff --git a/cmd/gh-aw/main.go b/cmd/gh-aw/main.go index ced2c19665d..739f2ca6623 100644 --- a/cmd/gh-aw/main.go +++ b/cmd/gh-aw/main.go @@ -560,7 +560,7 @@ type commandSet struct { addCmd, addWizardCmd, editCmd, updateCmd, deployCmd, trialCmd, initCmd, statusCmd, listCmd *cobra.Command mcpCmd, logsCmd, auditCmd, viewCmd, healthCmd, outcomesCmd, mcpServerCmd, prCmd, secretsCmd *cobra.Command fixCmd, upgradeCmd, completionCmd, hashCmd, projectCmd, doctorCmd, checksCmd, validateCmd, lintCmd *cobra.Command - domainsCmd, experimentsCmd, forecastCmd, gradersCmd, modelsCmd, envCmd *cobra.Command + domainsCmd, experimentsCmd, forecastCmd, gradersCmd, modelsCmd, envCmd, jsonSchemaCmd *cobra.Command } func fixPathForCommand(s string) string { @@ -743,6 +743,7 @@ func createCommandSet() commandSet { gradersCmd: cli.NewGradersCommand(), envCmd: cli.NewEnvCommand(), modelsCmd: cli.NewModelsCommand(), + jsonSchemaCmd: cli.NewJSONSchemaCommand(), } cli.RegisterEngineFlagCompletion(cmds.initCmd) return cmds @@ -861,6 +862,7 @@ func assignCommandGroups(cmds commandSet) { cmds.statusCmd.GroupID, cmds.listCmd.GroupID, cmds.experimentsCmd.GroupID, cmds.forecastCmd.GroupID, cmds.modelsCmd.GroupID = "analysis", "analysis", "analysis", "analysis", "analysis" cmds.gradersCmd.GroupID = "analysis" cmds.mcpServerCmd.GroupID, cmds.prCmd.GroupID, cmds.completionCmd.GroupID, cmds.hashCmd.GroupID, cmds.projectCmd.GroupID = "utilities", "utilities", "utilities", "utilities", "utilities" + cmds.jsonSchemaCmd.GroupID = "utilities" } func addCommandsToRoot(cmds commandSet) { @@ -869,7 +871,7 @@ func addCommandsToRoot(cmds commandSet) { runCmd, removeCmd, cmds.statusCmd, cmds.listCmd, enableCmd, disableCmd, cmds.logsCmd, cmds.auditCmd, cmds.viewCmd, cmds.healthCmd, cmds.outcomesCmd, cmds.checksCmd, cmds.mcpCmd, cmds.mcpServerCmd, cmds.prCmd, versionCmd, cmds.secretsCmd, cmds.fixCmd, cmds.validateCmd, cmds.lintCmd, cmds.completionCmd, cmds.hashCmd, cmds.projectCmd, cmds.doctorCmd, - cmds.domainsCmd, cmds.experimentsCmd, cmds.forecastCmd, cmds.gradersCmd, cmds.modelsCmd, cmds.envCmd, + cmds.domainsCmd, cmds.experimentsCmd, cmds.forecastCmd, cmds.gradersCmd, cmds.modelsCmd, cmds.envCmd, cmds.jsonSchemaCmd, ) } diff --git a/docs/adr/59720-expose-json-schemas-for-cli-output.md b/docs/adr/59720-expose-json-schemas-for-cli-output.md new file mode 100644 index 00000000000..57eecf21b86 --- /dev/null +++ b/docs/adr/59720-expose-json-schemas-for-cli-output.md @@ -0,0 +1,50 @@ +# ADR-59720: Expose JSON schemas for CLI output + +**Date**: 2026-09-09 +**Status**: Draft +**Deciders**: gh-aw maintainers + +--- + +### Context + +`gh aw audit --json` and `gh aw logs --json` already emit structured data, but the repository did not provide a supported way for users or downstream tooling to discover those payload shapes. This PR adds a `json-schema` CLI command, checked-in schema artifacts, Makefile regeneration, tests, and documentation centered on the audit and logs outputs. The change also keeps the Go output types as the source of truth by generating schemas directly from `AuditData` and `LogsData`. The repository needs an explicit decision on whether structured CLI outputs should expose versioned machine-readable schemas as part of the developer-facing interface. + +### Decision + +We will expose JSON schemas for the structured `audit` and `logs` CLI outputs through a new `gh aw json-schema ` command and commit regenerated schema artifacts under `schemas/`. We will generate those schemas directly from the existing Go output types and share the same serialization path between CLI output and checked-in artifacts so the published schemas stay deterministic and consistent. We will also regenerate the artifacts during `make recompile` so schema freshness becomes part of the normal repository regeneration flow. + +### Alternatives Considered + +#### Alternative 1: Keep structured JSON output undocumented and schema-less + +The project could continue emitting `audit --json` and `logs --json` without publishing schemas. This was considered because it avoids adding a new command, generated files, and regeneration logic. It was not chosen because downstream automation would still need to reverse-engineer payloads, and the PR explicitly adds tests and docs to make the output contract inspectable. + +#### Alternative 2: Hand-maintain static schema files separately from the Go types + +Another option would be to write `audit.schema.json` and `logs.schema.json` manually and update them when the output types change. This was considered because it could avoid adding schema-generation entry points to the CLI. It was not chosen because the PR evidence shows a stronger preference for using `GenerateOutputSchema[...]()` and shared marshaling so JSON tags and type definitions remain the single source of truth. + +#### Alternative 3: Expose schemas only in repository files, without a CLI command + +The project could check in generated schema artifacts but omit a user-facing command. This was considered because consumers could read the committed files directly from the repository. It was not chosen because the PR intentionally adds `gh aw json-schema audit` and `gh aw json-schema logs`, making schema discovery available from the installed CLI and not only from the source tree. + +### Consequences + +#### Positive +- Downstream tools gain a supported machine-readable contract for `gh aw audit --json` and `gh aw logs --json`. +- Generating schemas from `AuditData` and `LogsData` reduces drift between implementation and published schema. +- Deterministic CLI output and checked-in artifacts make schema changes easier to test and review. + +#### Negative +- The repository now carries large generated schema artifacts that must be regenerated when output types change. +- `make recompile` takes on additional responsibility, so schema generation failures can block broader regeneration workflows. +- Exposing schemas makes output-shape changes more visible and may increase compatibility expectations for future changes. + +#### Neutral +- A new `json-schema` command is added to the CLI utilities group. +- Tests now verify command registration, argument handling, schema validity, determinism, and artifact freshness. +- `.prettierignore` is extended to exclude the generated schema files from formatting. + +--- + +*ADR created by [adr-writer agent]. Review and finalize before changing status from Draft to Accepted.* diff --git a/docs/src/content/docs/reference/audit.md b/docs/src/content/docs/reference/audit.md index 941783451b0..1fb76525923 100644 --- a/docs/src/content/docs/reference/audit.md +++ b/docs/src/content/docs/reference/audit.md @@ -89,6 +89,17 @@ With multiple comparisons, `--json` emits a single object for one comparison or When artifacts are present, audit processing also persists extracted skill-activation data into `run_summary.json`, which downstream automation can consume alongside the rendered report. +## JSON output schemas + +Use `gh aw json-schema` to generate a JSON Schema for structured audit or logs output. The `audit` schema describes `gh aw audit --json`, while the `logs` schema describes `gh aw logs --json`. The schema is written to stdout and can be redirected to a file: + +```bash +gh aw json-schema audit > audit.schema.json +gh aw json-schema logs > logs.schema.json +``` + +`make recompile` regenerates the checked-in `schemas/audit.schema.json` and `schemas/logs.schema.json` files. These schemas derive directly from the Go `AuditData` and `LogsData` types, so changes to either type may change its generated schema. + ## `gh aw logs --format ` Generate a cross-run security and performance audit report across multiple recent workflow runs. diff --git a/pkg/cli/json_schema_command.go b/pkg/cli/json_schema_command.go new file mode 100644 index 00000000000..f508c197645 --- /dev/null +++ b/pkg/cli/json_schema_command.go @@ -0,0 +1,39 @@ +package cli + +import ( + "fmt" + "io" + "os" + + "github.com/github/gh-aw/pkg/constants" + "github.com/spf13/cobra" +) + +// NewJSONSchemaCommand creates the json-schema command. +func NewJSONSchemaCommand() *cobra.Command { + return newJSONSchemaCommand(os.Stdout) +} + +func newJSONSchemaCommand(output io.Writer) *cobra.Command { + cmd := &cobra.Command{ + Use: "json-schema ", + Short: "Generate JSON Schemas for structured command output", + Long: "Generate the JSON Schema for audit or logs JSON output.", + Example: ` ` + string(constants.CLIExtensionPrefix) + ` json-schema audit + ` + string(constants.CLIExtensionPrefix) + ` json-schema logs`, + Args: cobra.ExactArgs(1), + ValidArgs: []string{"audit", "logs"}, + RunE: func(cmd *cobra.Command, args []string) error { + data, err := GenerateNamedOutputSchema(args[0]) + if err != nil { + return err + } + if _, err := output.Write(data); err != nil { + return fmt.Errorf("failed to write %s schema: %w", args[0], err) + } + return nil + }, + SilenceUsage: true, + } + return cmd +} diff --git a/pkg/cli/json_schema_command_test.go b/pkg/cli/json_schema_command_test.go new file mode 100644 index 00000000000..c58b8e23b70 --- /dev/null +++ b/pkg/cli/json_schema_command_test.go @@ -0,0 +1,227 @@ +//go:build !integration + +package cli + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/google/jsonschema-go/jsonschema" +) + +func TestJSONSchemaCommand(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + schemaName string + oneOfCount int + validOutput []any + }{ + { + name: "audit", + schemaName: "audit", + oneOfCount: 3, + validOutput: []any{ + AuditData{ + Overview: OverviewData{}, + Metrics: MetricsData{}, + DownloadedFiles: []FileInfo{}, + MCPServerHealth: &MCPServerHealth{ + Servers: []MCPServerHealthDetail{{ + MCPServerStatsBase: MCPServerStatsBase{ + ServerName: "github", + ToolCallCount: 3, + ErrorCount: 1, + }, + RequestCount: 3, + }}, + }, + }, + AuditDiff{}, + []AuditDiff{{}}, + }, + }, + { + name: "logs", + schemaName: "logs", + oneOfCount: 2, + validOutput: []any{ + LogsData{ + ToolUsage: []ToolUsageSummary{{ + ToolUsageStatsBase: ToolUsageStatsBase{ + ToolName: "github_get_issue", + CallCount: 3, + }, + Runs: 1, + }}, + MCPFailures: []MCPFailureSummary{{ + ServerName: "github", + AggregatedSummaryBase: AggregatedSummaryBase{ + Count: 1, + Workflows: []string{"example"}, + RunIDs: []int64{1}, + }, + }}, + AccessLog: &AccessLogSummary{ + ByWorkflow: map[string]*DomainAnalysis{ + "example": { + AnalysisBase: AnalysisBase{ + TotalRequests: 3, + AllowedRequests: 2, + BlockedRequests: 1, + }, + }, + }, + }, + }, + CrossRunAuditReport{ + MCPHealth: []MCPServerCrossRunHealth{{ + MCPServerStatsBase: MCPServerStatsBase{ + ServerName: "github", + ToolCallCount: 3, + ErrorCount: 1, + }, + RunsConnected: 1, + TotalRuns: 1, + }}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + first, stderr, err := executeJSONSchemaCommand(tt.schemaName) + if err != nil { + t.Fatalf("json-schema %s failed: %v", tt.schemaName, err) + } + if stderr != "" { + t.Fatalf("json-schema %s wrote to stderr: %q", tt.schemaName, stderr) + } + if !json.Valid([]byte(first)) { + t.Fatalf("json-schema %s returned invalid JSON", tt.schemaName) + } + if !strings.HasSuffix(first, "\n") { + t.Fatalf("json-schema %s output does not end with a newline", tt.schemaName) + } + if strings.Contains(first, "\x1b[") { + t.Fatalf("json-schema %s output contains ANSI escape sequences", tt.schemaName) + } + + var schema jsonschema.Schema + if err := json.Unmarshal([]byte(first), &schema); err != nil { + t.Fatalf("unmarshal %s schema: %v", tt.schemaName, err) + } + resolved, err := schema.Resolve(&jsonschema.ResolveOptions{}) + if err != nil { + t.Fatalf("resolve %s schema: %v", tt.schemaName, err) + } + if len(schema.OneOf) != tt.oneOfCount { + t.Errorf("%s schema has %d output shapes, want %d", tt.schemaName, len(schema.OneOf), tt.oneOfCount) + } + for _, output := range tt.validOutput { + data, err := json.Marshal(output) + if err != nil { + t.Fatalf("marshal %T: %v", output, err) + } + var value any + if err := json.Unmarshal(data, &value); err != nil { + t.Fatalf("unmarshal %T: %v", output, err) + } + if err := resolved.Validate(value); err != nil { + t.Errorf("%s schema does not validate %T output: %v", tt.schemaName, output, err) + } + } + + expected, err := GenerateNamedOutputSchema(tt.schemaName) + if err != nil { + t.Fatalf("generate expected %s schema: %v", tt.schemaName, err) + } + if first != string(expected) { + t.Errorf("json-schema %s output differs from shared generator", tt.schemaName) + } + + second, secondStderr, err := executeJSONSchemaCommand(tt.schemaName) + if err != nil { + t.Fatalf("second json-schema %s invocation failed: %v", tt.schemaName, err) + } + if secondStderr != "" { + t.Fatalf("second json-schema %s invocation wrote to stderr: %q", tt.schemaName, secondStderr) + } + if first != second { + t.Errorf("json-schema %s output is not deterministic", tt.schemaName) + } + }) + } +} + +func TestJSONSchemaCommandRejectsInvalidArguments(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + args []string + want string + }{ + {name: "missing", want: "accepts 1 arg(s), received 0"}, + {name: "unknown", args: []string{"unknown"}, want: "unsupported schema: unknown"}, + {name: "extra", args: []string{"audit", "extra-argument"}, want: "accepts 1 arg(s), received 2"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + stdout, _, err := executeJSONSchemaCommand(tt.args...) + if err == nil { + t.Fatal("expected json-schema command to fail") + } + if !strings.Contains(err.Error(), tt.want) { + t.Fatalf("error %q does not contain %q", err, tt.want) + } + if stdout != "" { + t.Fatalf("failed command wrote partial output to stdout: %q", stdout) + } + }) + } +} + +func TestGeneratedOutputSchemasAreCurrent(t *testing.T) { + t.Parallel() + + for _, schemaName := range []string{"audit", "logs"} { + t.Run(schemaName, func(t *testing.T) { + t.Parallel() + + expected, err := GenerateNamedOutputSchema(schemaName) + if err != nil { + t.Fatalf("generate %s schema: %v", schemaName, err) + } + path := filepath.Join("..", "..", "schemas", schemaName+".schema.json") + actual, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read generated schema %s: %v", path, err) + } + if !bytes.Equal(actual, expected) { + t.Errorf("%s is stale; run make recompile", path) + } + }) + } +} + +func executeJSONSchemaCommand(args ...string) (stdout string, stderr string, err error) { + var stdoutBuffer bytes.Buffer + var stderrBuffer bytes.Buffer + cmd := newJSONSchemaCommand(&stdoutBuffer) + cmd.SetErr(&stderrBuffer) + cmd.SetArgs(args) + err = cmd.Execute() + return stdoutBuffer.String(), stderrBuffer.String(), err +} diff --git a/pkg/cli/mcp_schema.go b/pkg/cli/mcp_schema.go index 00f59a2fc98..fd5bd5c5611 100644 --- a/pkg/cli/mcp_schema.go +++ b/pkg/cli/mcp_schema.go @@ -38,6 +38,205 @@ func GenerateSchema[T any]() (*jsonschema.Schema, error) { return jsonschema.For[T](nil) } +// GenerateOutputSchema generates a JSON schema for a structured command output type. +func GenerateOutputSchema[T any]() (*jsonschema.Schema, error) { + return GenerateSchema[T]() +} + +// MarshalOutputSchema serializes a generated output schema using the CLI's +// standard indented JSON format. +func MarshalOutputSchema(schema *jsonschema.Schema) ([]byte, error) { + data, err := json.MarshalIndent(schema, "", " ") + if err != nil { + return nil, fmt.Errorf("failed to marshal output schema to JSON: %w", err) + } + return append(data, '\n'), nil +} + +// GenerateNamedOutputSchema generates and serializes a known CLI output schema. +func GenerateNamedOutputSchema(name string) ([]byte, error) { + var ( + schema *jsonschema.Schema + err error + ) + switch name { + case "audit": + schema, err = generateAuditOutputSchema() + case "logs": + schema, err = generateLogsOutputSchema() + default: + return nil, fmt.Errorf("unsupported schema: %s", name) + } + if err != nil { + return nil, fmt.Errorf("failed to generate %s schema: %w", name, err) + } + return MarshalOutputSchema(schema) +} + +func generateAuditOutputSchema() (*jsonschema.Schema, error) { + auditData, err := generateAuditDataOutputSchema() + if err != nil { + return nil, err + } + auditDiff, err := GenerateOutputSchema[AuditDiff]() + if err != nil { + return nil, err + } + auditDiffs, err := GenerateOutputSchema[[]AuditDiff]() + if err != nil { + return nil, err + } + return &jsonschema.Schema{OneOf: []*jsonschema.Schema{auditData, auditDiff, auditDiffs}}, nil +} + +func generateLogsOutputSchema() (*jsonschema.Schema, error) { + logsData, err := generateLogsDataOutputSchema() + if err != nil { + return nil, err + } + crossRun, err := generateCrossRunAuditOutputSchema() + if err != nil { + return nil, err + } + return &jsonschema.Schema{OneOf: []*jsonschema.Schema{logsData, crossRun}}, nil +} + +// These wire types mirror custom MarshalJSON output, which reflection cannot infer. +type toolUsageSummaryWire struct { + Name string `json:"name"` + TotalCalls int `json:"total_calls"` + Runs int `json:"runs"` + MaxOutputSize int `json:"max_output_size,omitempty"` + MaxDuration string `json:"max_duration,omitempty"` +} + +type mcpServerHealthDetailWire struct { + ServerName string `json:"server_name"` + RequestCount int `json:"request_count"` + ToolCalls int `json:"tool_calls"` + ErrorCount int `json:"error_count"` + ErrorRate float64 `json:"error_rate"` + ErrorRateStr string `json:"error_rate_str"` + AvgLatency string `json:"avg_latency"` + Status string `json:"status"` +} + +type mcpServerCrossRunHealthWire struct { + ServerName string `json:"server_name"` + RunsConnected int `json:"runs_connected"` + TotalRuns int `json:"total_runs"` + TotalCalls int `json:"total_calls"` + TotalErrors int `json:"total_errors"` + ErrorRate float64 `json:"error_rate"` + Unreliable bool `json:"unreliable"` +} + +type mcpFailureSummaryWire struct { + ServerName string `json:"server_name"` + Count int `json:"count"` + Workflows []string `json:"workflows"` + RunIDs []int64 `json:"run_ids"` +} + +type domainAnalysisWireSchema struct { + TotalRequests int `json:"total_requests"` + AllowedCount int `json:"allowed_count"` + BlockedCount int `json:"blocked_count"` + AllowedDomains []string `json:"allowed_domains,omitempty"` + BlockedDomains []string `json:"blocked_domains,omitempty"` +} + +func generateAuditDataOutputSchema() (*jsonschema.Schema, error) { + schema, err := GenerateOutputSchema[AuditData]() + if err != nil { + return nil, err + } + wire, err := GenerateOutputSchema[mcpServerHealthDetailWire]() + if err != nil { + return nil, err + } + if err := replaceArrayItemSchema(schema, []string{"mcp_server_health", "servers"}, wire); err != nil { + return nil, err + } + return schema, nil +} + +func generateLogsDataOutputSchema() (*jsonschema.Schema, error) { + schema, err := GenerateOutputSchema[LogsData]() + if err != nil { + return nil, err + } + wire, err := GenerateOutputSchema[toolUsageSummaryWire]() + if err != nil { + return nil, err + } + if err := replaceArrayItemSchema(schema, []string{"tool_usage"}, wire); err != nil { + return nil, err + } + mcpFailures, err := GenerateOutputSchema[mcpFailureSummaryWire]() + if err != nil { + return nil, err + } + if err := replaceArrayItemSchema(schema, []string{"mcp_failures"}, mcpFailures); err != nil { + return nil, err + } + accessAnalysis, err := GenerateOutputSchema[domainAnalysisWireSchema]() + if err != nil { + return nil, err + } + if err := replaceMapValueSchema(schema, []string{"access_log", "by_workflow"}, accessAnalysis); err != nil { + return nil, err + } + return schema, nil +} + +func generateCrossRunAuditOutputSchema() (*jsonschema.Schema, error) { + schema, err := GenerateOutputSchema[CrossRunAuditReport]() + if err != nil { + return nil, err + } + wire, err := GenerateOutputSchema[mcpServerCrossRunHealthWire]() + if err != nil { + return nil, err + } + if err := replaceArrayItemSchema(schema, []string{"mcp_health"}, wire); err != nil { + return nil, err + } + return schema, nil +} + +func replaceArrayItemSchema(schema *jsonschema.Schema, path []string, replacement *jsonschema.Schema) error { + target := schema + for _, property := range path { + var ok bool + target, ok = target.Properties[property] + if !ok { + return fmt.Errorf("schema property %q not found", property) + } + } + if target.Items == nil { + return fmt.Errorf("schema property %q is not an array", path[len(path)-1]) + } + target.Items = replacement + return nil +} + +func replaceMapValueSchema(schema *jsonschema.Schema, path []string, replacement *jsonschema.Schema) error { + target := schema + for _, property := range path { + var ok bool + target, ok = target.Properties[property] + if !ok { + return fmt.Errorf("schema property %q not found", property) + } + } + if target.AdditionalProperties == nil { + return fmt.Errorf("schema property %q is not a map", path[len(path)-1]) + } + target.AdditionalProperties = replacement + return nil +} + func generateSchemaWithDefaults[T any](defaults map[string]any) (*jsonschema.Schema, error) { schema, err := GenerateSchema[T]() if err != nil { diff --git a/schemas/audit.schema.json b/schemas/audit.schema.json new file mode 100644 index 00000000000..46f665ba02e --- /dev/null +++ b/schemas/audit.schema.json @@ -0,0 +1,4425 @@ +{ + "oneOf": [ + { + "type": "object", + "properties": { + "cache_source": { + "type": "string" + }, + "overview": { + "type": "object", + "properties": { + "run_id": { + "type": "integer" + }, + "workflow_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "conclusion": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "started_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "event": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "url": { + "type": "string" + }, + "logs_path": { + "type": "string" + }, + "experiment": { + "type": "string" + }, + "context": { + "type": [ + "null", + "object" + ], + "properties": { + "repo": { + "type": "string" + }, + "run_id": { + "type": "string" + }, + "workflow_id": { + "type": "string" + }, + "workflow_call_id": { + "type": "string" + }, + "time": { + "type": "string" + }, + "actor": { + "type": "string" + }, + "event_type": { + "type": "string" + }, + "item_type": { + "type": "string" + }, + "item_number": { + "type": "string" + }, + "comment_id": { + "type": "string" + } + }, + "required": [ + "repo", + "run_id", + "workflow_id" + ], + "additionalProperties": false + } + }, + "required": [ + "run_id", + "workflow_name", + "status", + "created_at", + "event", + "branch", + "url" + ], + "additionalProperties": false + }, + "comparison": { + "type": [ + "null", + "object" + ], + "properties": { + "baseline_found": { + "type": "boolean" + }, + "baseline": { + "type": [ + "null", + "object" + ], + "properties": { + "run_id": { + "type": "integer" + }, + "workflow_name": { + "type": "string" + }, + "conclusion": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "selection": { + "type": "string" + }, + "matched_on": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "run_id" + ], + "additionalProperties": false + }, + "delta": { + "type": [ + "null", + "object" + ], + "properties": { + "turns": { + "type": "object", + "properties": { + "before": { + "type": "integer" + }, + "after": { + "type": "integer" + }, + "changed": { + "type": "boolean" + } + }, + "required": [ + "before", + "after", + "changed" + ], + "additionalProperties": false + }, + "posture": { + "type": "object", + "properties": { + "before": { + "type": "string" + }, + "after": { + "type": "string" + }, + "changed": { + "type": "boolean" + } + }, + "required": [ + "before", + "after", + "changed" + ], + "additionalProperties": false + }, + "blocked_requests": { + "type": "object", + "properties": { + "before": { + "type": "integer" + }, + "after": { + "type": "integer" + }, + "changed": { + "type": "boolean" + } + }, + "required": [ + "before", + "after", + "changed" + ], + "additionalProperties": false + }, + "mcp_failure": { + "type": [ + "null", + "object" + ], + "properties": { + "before": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "after": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "newly_present": { + "type": "boolean" + } + }, + "required": [ + "newly_present" + ], + "additionalProperties": false + } + }, + "required": [ + "turns", + "posture", + "blocked_requests" + ], + "additionalProperties": false + }, + "classification": { + "type": [ + "null", + "object" + ], + "properties": { + "label": { + "type": "string" + }, + "reason_codes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "label" + ], + "additionalProperties": false + }, + "recommendation": { + "type": [ + "null", + "object" + ], + "properties": { + "action": { + "type": "string" + } + }, + "required": [ + "action" + ], + "additionalProperties": false + } + }, + "required": [ + "baseline_found" + ], + "additionalProperties": false + }, + "task_domain": { + "type": [ + "null", + "object" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "name", + "label" + ], + "additionalProperties": false + }, + "behavior_fingerprint": { + "type": [ + "null", + "object" + ], + "properties": { + "execution_style": { + "type": "string" + }, + "tool_breadth": { + "type": "string" + }, + "actuation_style": { + "type": "string" + }, + "resource_profile": { + "type": "string" + }, + "dispatch_mode": { + "type": "string" + }, + "agentic_fraction": { + "type": "number" + } + }, + "required": [ + "execution_style", + "tool_breadth", + "actuation_style", + "resource_profile", + "dispatch_mode", + "agentic_fraction" + ], + "additionalProperties": false + }, + "agentic_assessments": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "evidence": { + "type": "string" + }, + "recommendation": { + "type": "string" + } + }, + "required": [ + "kind", + "severity", + "summary" + ], + "additionalProperties": false + } + }, + "metrics": { + "type": "object", + "properties": { + "token_usage": { + "type": "integer" + }, + "aic": { + "type": "number" + }, + "ambient_context": { + "type": [ + "null", + "object" + ], + "properties": { + "input_tokens": { + "type": "integer" + }, + "cached_tokens": { + "type": "integer" + }, + "effective_tokens": { + "type": "integer" + } + }, + "required": [ + "input_tokens", + "cached_tokens" + ], + "additionalProperties": false + }, + "working_set": { + "type": [ + "null", + "object" + ], + "properties": { + "measurement_state": { + "type": "string" + }, + "rebuild_factor": { + "type": [ + "null", + "number" + ] + }, + "cumulative_input_tokens": { + "type": "integer" + }, + "peak_input_tokens": { + "type": "integer" + }, + "rebuild_excess_tokens": { + "type": "integer" + }, + "invocations": { + "type": "integer" + } + }, + "required": [ + "measurement_state", + "cumulative_input_tokens", + "peak_input_tokens", + "rebuild_excess_tokens", + "invocations" + ], + "additionalProperties": false + }, + "action_minutes": { + "type": "number" + }, + "turns": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "warning_count": { + "type": "integer" + } + }, + "required": [ + "error_count", + "warning_count" + ], + "additionalProperties": false + }, + "key_findings": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "category": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "impact": { + "type": "string" + } + }, + "required": [ + "category", + "severity", + "title", + "description" + ], + "additionalProperties": false + } + }, + "recommendations": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "priority": { + "type": "string" + }, + "action": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "example": { + "type": "string" + } + }, + "required": [ + "priority", + "action", + "reason" + ], + "additionalProperties": false + } + }, + "observability_insights": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "category": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "title": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "evidence": { + "type": "string" + } + }, + "required": [ + "category", + "severity", + "title", + "summary" + ], + "additionalProperties": false + } + }, + "performance_metrics": { + "type": [ + "null", + "object" + ], + "properties": { + "tokens_per_minute": { + "type": "number" + }, + "avg_tool_duration": { + "type": "string" + }, + "most_used_tool": { + "type": "string" + }, + "network_requests": { + "type": "integer" + } + }, + "additionalProperties": false + }, + "engine_config": { + "type": [ + "null", + "object" + ], + "properties": { + "engine_id": { + "type": "string" + }, + "engine_name": { + "type": "string" + }, + "model": { + "type": "string" + }, + "version": { + "type": "string" + }, + "cli_version": { + "type": "string" + }, + "firewall_version": { + "type": "string" + }, + "mcp_servers": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "trigger_event": { + "type": "string" + }, + "repository": { + "type": "string" + } + }, + "required": [ + "engine_id" + ], + "additionalProperties": false + }, + "prompt_analysis": { + "type": [ + "null", + "object" + ], + "properties": { + "prompt_size": { + "type": "integer" + }, + "prompt_file": { + "type": "string" + } + }, + "required": [ + "prompt_size" + ], + "additionalProperties": false + }, + "session_analysis": { + "type": [ + "null", + "object" + ], + "properties": { + "wall_time": { + "type": "string" + }, + "turn_count": { + "type": "integer" + }, + "avg_turn_duration": { + "type": "string" + }, + "avg_time_between_turns": { + "type": "string" + }, + "max_time_between_turns": { + "type": "string" + }, + "tokens_per_minute": { + "type": "number" + }, + "timeout_detected": { + "type": "boolean" + }, + "noop_count": { + "type": "integer" + }, + "agent_active_ratio": { + "type": "number" + }, + "cache_warning": { + "type": "string" + } + }, + "required": [ + "timeout_detected" + ], + "additionalProperties": false + }, + "safe_output_summary": { + "type": [ + "null", + "object" + ], + "properties": { + "total_items": { + "type": "integer" + }, + "items_by_type": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "summary": { + "type": "string" + }, + "temporary_id_map_status": { + "type": "string" + }, + "temporary_id_mappings": { + "type": "integer" + }, + "chained_target_count": { + "type": "integer" + }, + "chained_followup_action_count": { + "type": "integer" + }, + "delegated_temp_target_count": { + "type": "integer" + }, + "closed_temp_target_count": { + "type": "integer" + }, + "type_details": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "count": { + "type": "integer" + } + }, + "required": [ + "type", + "count" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_items", + "items_by_type", + "summary" + ], + "additionalProperties": false + }, + "mcp_server_health": { + "type": [ + "null", + "object" + ], + "properties": { + "total_servers": { + "type": "integer" + }, + "healthy_servers": { + "type": "integer" + }, + "degraded_servers": { + "type": "integer" + }, + "failed_servers": { + "type": "integer" + }, + "summary": { + "type": "string" + }, + "total_requests": { + "type": "integer" + }, + "total_errors": { + "type": "integer" + }, + "error_rate": { + "type": "number" + }, + "servers": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "request_count": { + "type": "integer" + }, + "tool_calls": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "error_rate": { + "type": "number" + }, + "error_rate_str": { + "type": "string" + }, + "avg_latency": { + "type": "string" + }, + "status": { + "type": "string" + } + }, + "required": [ + "server_name", + "request_count", + "tool_calls", + "error_count", + "error_rate", + "error_rate_str", + "avg_latency", + "status" + ], + "additionalProperties": false + } + }, + "slowest_calls": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "duration": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_name", + "duration" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_servers", + "healthy_servers", + "degraded_servers", + "failed_servers", + "summary", + "total_requests", + "total_errors", + "error_rate" + ], + "additionalProperties": false + }, + "jobs": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "conclusion": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "steps": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "conclusion": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "started_at": { + "type": "string" + }, + "completed_at": { + "type": "string" + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + } + }, + "required": [ + "name", + "status" + ], + "additionalProperties": false + } + }, + "downloaded_files": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "description": { + "type": "string" + } + }, + "required": [ + "path", + "size", + "description" + ], + "additionalProperties": false + } + }, + "missing_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "tool": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "alternatives": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "run_id": { + "type": "integer" + }, + "experiment_name": { + "type": "string" + }, + "variant": { + "type": "string" + } + }, + "required": [ + "tool", + "reason", + "timestamp" + ], + "additionalProperties": false + } + }, + "missing_data": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "data_type": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "context": { + "type": "string" + }, + "alternatives": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "run_id": { + "type": "integer" + }, + "experiment_name": { + "type": "string" + }, + "variant": { + "type": "string" + } + }, + "required": [ + "data_type", + "reason", + "timestamp" + ], + "additionalProperties": false + } + }, + "noops": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "run_id": { + "type": "integer" + }, + "experiment_name": { + "type": "string" + }, + "variant": { + "type": "string" + } + }, + "required": [ + "message", + "timestamp" + ], + "additionalProperties": false + } + }, + "mcp_failures": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "run_id": { + "type": "integer" + }, + "experiment_name": { + "type": "string" + }, + "variant": { + "type": "string" + } + }, + "required": [ + "server_name", + "status", + "timestamp" + ], + "additionalProperties": false + } + }, + "skill_activations": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "source": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "run_id": { + "type": "integer" + }, + "experiment_name": { + "type": "string" + }, + "variant": { + "type": "string" + } + }, + "required": [ + "name", + "status", + "timestamp" + ], + "additionalProperties": false + } + }, + "firewall_token_usage": { + "type": [ + "null", + "object" + ], + "properties": { + "total_input_tokens": { + "type": "integer" + }, + "total_output_tokens": { + "type": "integer" + }, + "total_cache_read_tokens": { + "type": "integer" + }, + "total_cache_write_tokens": { + "type": "integer" + }, + "total_requests": { + "type": "integer" + }, + "total_steering_events": { + "type": "integer" + }, + "total_duration_ms": { + "type": "integer" + }, + "total_response_bytes": { + "type": "integer" + }, + "cache_efficiency": { + "type": "number" + }, + "total_effective_tokens": { + "type": "integer" + }, + "total_aic": { + "type": "number" + }, + "ambient_context": { + "type": [ + "null", + "object" + ], + "properties": { + "input_tokens": { + "type": "integer" + }, + "cached_tokens": { + "type": "integer" + }, + "effective_tokens": { + "type": "integer" + } + }, + "required": [ + "input_tokens", + "cached_tokens" + ], + "additionalProperties": false + }, + "by_model": { + "type": "object", + "additionalProperties": { + "type": [ + "null", + "object" + ], + "properties": { + "provider": { + "type": "string" + }, + "input_tokens": { + "type": "integer" + }, + "output_tokens": { + "type": "integer" + }, + "cache_read_tokens": { + "type": "integer" + }, + "cache_write_tokens": { + "type": "integer" + }, + "reasoning_tokens": { + "type": "integer" + }, + "effective_tokens": { + "type": "integer" + }, + "requests": { + "type": "integer" + }, + "duration_ms": { + "type": "integer" + }, + "response_bytes": { + "type": "integer" + }, + "aic": { + "type": "number" + } + }, + "required": [ + "provider", + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "requests", + "duration_ms", + "response_bytes" + ], + "additionalProperties": false + } + }, + "subagent_model_requests": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "agent_name": { + "type": "string" + }, + "requested_model": { + "type": "string" + }, + "invocation_count": { + "type": "integer" + }, + "effective_model": { + "type": "string" + }, + "reason_code": { + "type": "string" + } + }, + "required": [ + "agent_name", + "requested_model", + "invocation_count" + ], + "additionalProperties": false + } + }, + "subagent_model_actuals": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "requests": { + "type": "integer" + } + }, + "required": [ + "model", + "requests" + ], + "additionalProperties": false + } + }, + "mismatch_count": { + "type": "integer" + }, + "warnings": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "total_input_tokens", + "total_output_tokens", + "total_cache_read_tokens", + "total_cache_write_tokens", + "total_requests", + "total_duration_ms", + "total_response_bytes", + "cache_efficiency", + "by_model" + ], + "additionalProperties": false + }, + "github_rate_limit_usage": { + "type": [ + "null", + "object" + ], + "properties": { + "total_requests_made": { + "type": "integer" + }, + "core_consumed": { + "type": "integer" + }, + "core_consumed_source": { + "type": "string" + }, + "core_remaining": { + "type": "integer" + }, + "core_limit": { + "type": "integer" + }, + "resources": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "resource": { + "type": "string" + }, + "requests_made": { + "type": "integer" + }, + "quota_consumed": { + "type": "integer" + }, + "final_remaining": { + "type": "integer" + }, + "limit": { + "type": "integer" + } + }, + "required": [ + "resource", + "requests_made", + "quota_consumed", + "final_remaining", + "limit" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_requests_made", + "core_consumed", + "core_remaining", + "core_limit" + ], + "additionalProperties": false + }, + "firewall_analysis": { + "type": [ + "null", + "object" + ], + "properties": { + "allowed_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "blocked_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "total_requests": { + "type": "integer" + }, + "allowed_requests": { + "type": "integer" + }, + "blocked_requests": { + "type": "integer" + }, + "requests_by_domain": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "allowed": { + "type": "integer" + }, + "blocked": { + "type": "integer" + } + }, + "required": [ + "allowed", + "blocked" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_requests", + "allowed_requests", + "blocked_requests" + ], + "additionalProperties": false + }, + "policy_analysis": { + "type": [ + "null", + "object" + ], + "properties": { + "policy_summary": { + "type": "string" + }, + "rule_hits": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "rule": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "order": { + "type": "integer" + }, + "action": { + "type": "string" + }, + "aclName": { + "type": "string" + }, + "protocol": { + "type": "string" + }, + "domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "description": { + "type": "string" + } + }, + "required": [ + "id", + "order", + "action", + "aclName", + "protocol", + "domains", + "description" + ], + "additionalProperties": false + }, + "hits": { + "type": "integer" + } + }, + "required": [ + "rule", + "hits" + ], + "additionalProperties": false + } + }, + "denied_requests": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "ts": { + "type": "number" + }, + "host": { + "type": "string" + }, + "status": { + "type": "integer" + }, + "rule_id": { + "type": "string" + }, + "action": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "ts", + "host", + "status", + "rule_id", + "action" + ], + "additionalProperties": false + } + }, + "total_requests": { + "type": "integer" + }, + "allowed_count": { + "type": "integer" + }, + "denied_count": { + "type": "integer" + }, + "unique_domains": { + "type": "integer" + } + }, + "required": [ + "policy_summary", + "rule_hits", + "total_requests", + "allowed_count", + "denied_count", + "unique_domains" + ], + "additionalProperties": false + }, + "redacted_domains_analysis": { + "type": [ + "null", + "object" + ], + "properties": { + "total_domains": { + "type": "integer" + }, + "domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "total_domains", + "domains" + ], + "additionalProperties": false + }, + "errors": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "message": { + "type": "string" + }, + "line": { + "type": "integer" + }, + "file": { + "type": "string" + } + }, + "required": [ + "type", + "message" + ], + "additionalProperties": false + } + }, + "warnings": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "message": { + "type": "string" + }, + "line": { + "type": "integer" + }, + "file": { + "type": "string" + } + }, + "required": [ + "type", + "message" + ], + "additionalProperties": false + } + }, + "tool_usage": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "call_count": { + "type": "integer" + }, + "max_input_size": { + "type": "integer" + }, + "max_output_size": { + "type": "integer" + }, + "max_duration": { + "type": "string" + }, + "output_sample": { + "type": "string" + } + }, + "required": [ + "name", + "call_count" + ], + "additionalProperties": false + } + }, + "mcp_tool_usage": { + "type": [ + "null", + "object" + ], + "properties": { + "summary": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "call_count": { + "type": "integer" + }, + "total_input_size": { + "type": "integer" + }, + "total_output_size": { + "type": "integer" + }, + "max_input_size": { + "type": "integer" + }, + "max_output_size": { + "type": "integer" + }, + "avg_duration": { + "type": "string" + }, + "max_duration": { + "type": "string" + }, + "error_count": { + "type": "integer" + } + }, + "required": [ + "server_name", + "tool_name", + "call_count", + "total_input_size", + "total_output_size", + "max_input_size", + "max_output_size" + ], + "additionalProperties": false + } + }, + "tool_calls": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "tool_call_id": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "method": { + "type": "string" + }, + "input_size": { + "type": "integer" + }, + "output_size": { + "type": "integer" + }, + "duration": { + "type": "string" + }, + "status": { + "type": "string" + }, + "error": { + "type": "string" + }, + "effective_token_delta": { + "type": "integer" + } + }, + "required": [ + "timestamp", + "server_name", + "tool_name", + "input_size", + "output_size", + "status" + ], + "additionalProperties": false + } + }, + "servers": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_call_count": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "request_count": { + "type": "integer" + }, + "total_input_size": { + "type": "integer" + }, + "total_output_size": { + "type": "integer" + }, + "avg_duration": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_call_count", + "request_count", + "total_input_size", + "total_output_size" + ], + "additionalProperties": false + } + }, + "filtered_events": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "server_id": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "secrecy_tags": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "integrity_tags": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "author_association": { + "type": "string" + }, + "author_login": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "number": { + "type": "string" + } + }, + "required": [ + "timestamp", + "server_id", + "tool_name", + "reason" + ], + "additionalProperties": false + } + }, + "integrity": { + "type": [ + "null", + "object" + ], + "properties": { + "total_filtered": { + "type": "integer" + }, + "runs_with_filtered_events": { + "type": "integer" + }, + "filtered_server_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "filtered_tool_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "filtered_reason_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + }, + "required": [ + "total_filtered" + ], + "additionalProperties": false + }, + "guard_policy_summary": { + "type": [ + "null", + "object" + ], + "properties": { + "total_blocked": { + "type": "integer" + }, + "integrity_blocked": { + "type": "integer" + }, + "repo_scope_blocked": { + "type": "integer" + }, + "access_denied": { + "type": "integer" + }, + "blocked_user_denied": { + "type": "integer" + }, + "permission_denied": { + "type": "integer" + }, + "private_repo_denied": { + "type": "integer" + }, + "events": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "server_id": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "error_code": { + "type": "integer" + }, + "reason": { + "type": "string" + }, + "message": { + "type": "string" + }, + "details": { + "type": "string" + }, + "repository": { + "type": "string" + } + }, + "required": [ + "timestamp", + "server_id", + "tool_name", + "error_code", + "reason", + "message" + ], + "additionalProperties": false + } + }, + "blocked_tool_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "blocked_server_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + }, + "required": [ + "total_blocked", + "integrity_blocked", + "repo_scope_blocked", + "access_denied", + "events" + ], + "additionalProperties": false + } + }, + "required": [ + "summary", + "tool_calls" + ], + "additionalProperties": false + }, + "created_items": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "url": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "repo": { + "type": "string" + }, + "temporaryId": { + "type": "string" + }, + "metadata": { + "type": "object", + "additionalProperties": true + }, + "before_state": { + "type": "object", + "additionalProperties": true + }, + "after_state": { + "type": "object", + "additionalProperties": true + }, + "timestamp": { + "type": "string" + } + }, + "required": [ + "type", + "timestamp" + ], + "additionalProperties": false + } + }, + "outcomes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "outcome_status": { + "type": "string" + }, + "evidence_strength": { + "type": "string" + }, + "signal": { + "type": "string" + }, + "type": { + "type": "string" + }, + "object_url": { + "type": "string" + }, + "object_number": { + "type": "integer" + }, + "traced_root_url": { + "type": "string" + }, + "attribution_status": { + "type": "string" + }, + "attribution_source": { + "type": "string" + }, + "repo": { + "type": "string" + }, + "detail": { + "type": "string" + }, + "time_to_outcome_hours": { + "type": "number" + }, + "human_comments": { + "type": "integer" + }, + "human_edits": { + "type": "integer" + }, + "human_reviews": { + "type": "integer" + }, + "zero_touch": { + "type": "boolean" + }, + "objective_value": { + "type": "integer" + }, + "objective_labels": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "created_at": { + "type": "string" + }, + "checked_at": { + "type": "string" + }, + "eval_error": { + "type": "string" + } + }, + "required": [ + "outcome_status", + "evidence_strength", + "type", + "created_at", + "checked_at" + ], + "additionalProperties": false + } + }, + "outcome_summary": { + "type": [ + "null", + "object" + ], + "properties": { + "total": { + "type": "integer" + }, + "accepted": { + "type": "integer" + }, + "rejected": { + "type": "integer" + }, + "ignored": { + "type": "integer" + }, + "pending": { + "type": "integer" + }, + "accepted_strong": { + "type": "integer" + }, + "accepted_medium": { + "type": "integer" + }, + "accepted_weak": { + "type": "integer" + }, + "fallback_exists_only_count": { + "type": "integer" + }, + "lifecycle": { + "type": "integer" + }, + "errors": { + "type": "integer" + }, + "zero_touch": { + "type": "integer" + }, + "acceptance_rate": { + "type": "number" + }, + "waste_rate": { + "type": "number" + }, + "zero_touch_rate": { + "type": "number" + }, + "median_time_to_outcome_hours": { + "type": "number" + }, + "cost_per_accepted_outcome": { + "type": "number" + }, + "total_objective_value": { + "type": "integer" + }, + "accepted_objective_value": { + "type": "integer" + }, + "objective_efficiency": { + "type": "number" + }, + "domain_breakdowns": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "attempted": { + "type": "integer" + }, + "accepted": { + "type": "integer" + }, + "rejected": { + "type": "integer" + }, + "pending": { + "type": "integer" + }, + "total_objective_value": { + "type": "integer" + }, + "accepted_objective_value": { + "type": "integer" + }, + "objective_efficiency": { + "type": "number" + }, + "acceptance_rate": { + "type": "number" + } + }, + "required": [ + "label", + "attempted", + "accepted", + "rejected", + "pending", + "total_objective_value", + "accepted_objective_value" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total", + "accepted", + "rejected", + "ignored", + "pending", + "lifecycle", + "errors", + "zero_touch", + "acceptance_rate", + "waste_rate", + "zero_touch_rate" + ], + "additionalProperties": false + }, + "experiments": { + "type": [ + "null", + "object" + ], + "properties": { + "assignments": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "cumulative_counts": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + } + }, + "required": [ + "assignments" + ], + "additionalProperties": false + }, + "graders": { + "type": [ + "null", + "object" + ], + "properties": { + "version": { + "type": "integer" + }, + "results": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "value": { + "type": [ + "null", + "number" + ] + }, + "unit": { + "type": "string" + }, + "passed": { + "type": [ + "null", + "boolean" + ] + }, + "direction": { + "type": "string" + }, + "threshold": { + "type": [ + "null", + "number" + ] + }, + "message": { + "type": "string" + }, + "error": { + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "additionalProperties": false + } + }, + "total": { + "type": "integer" + }, + "passed": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "unavailable_count": { + "type": "integer" + } + }, + "required": [ + "results", + "total", + "passed", + "failed", + "error_count", + "unavailable_count" + ], + "additionalProperties": false + } + }, + "required": [ + "overview", + "metrics", + "downloaded_files" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "run1_id": { + "type": "integer" + }, + "run2_id": { + "type": "integer" + }, + "firewall_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_id": { + "type": "integer" + }, + "run2_id": { + "type": "integer" + }, + "new_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "removed_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "status_changes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "volume_changes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "summary": { + "type": "object", + "properties": { + "new_domain_count": { + "type": "integer" + }, + "removed_domain_count": { + "type": "integer" + }, + "status_change_count": { + "type": "integer" + }, + "volume_change_count": { + "type": "integer" + }, + "has_anomalies": { + "type": "boolean" + }, + "anomaly_count": { + "type": "integer" + } + }, + "required": [ + "new_domain_count", + "removed_domain_count", + "status_change_count", + "volume_change_count", + "has_anomalies", + "anomaly_count" + ], + "additionalProperties": false + } + }, + "required": [ + "run1_id", + "run2_id", + "summary" + ], + "additionalProperties": false + }, + "mcp_tools_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "new_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "run1_error_count": { + "type": "integer" + }, + "run2_error_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_name", + "status" + ], + "additionalProperties": false + } + }, + "removed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "run1_error_count": { + "type": "integer" + }, + "run2_error_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_name", + "status" + ], + "additionalProperties": false + } + }, + "changed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "run1_error_count": { + "type": "integer" + }, + "run2_error_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_name", + "status" + ], + "additionalProperties": false + } + }, + "summary": { + "type": "object", + "properties": { + "new_tool_count": { + "type": "integer" + }, + "removed_tool_count": { + "type": "integer" + }, + "changed_tool_count": { + "type": "integer" + }, + "has_anomalies": { + "type": "boolean" + }, + "anomaly_count": { + "type": "integer" + } + }, + "required": [ + "new_tool_count", + "removed_tool_count", + "changed_tool_count", + "has_anomalies", + "anomaly_count" + ], + "additionalProperties": false + } + }, + "required": [ + "summary" + ], + "additionalProperties": false + }, + "run_metrics_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_token_usage": { + "type": "integer" + }, + "run2_token_usage": { + "type": "integer" + }, + "token_usage_change": { + "type": "string" + }, + "run1_duration": { + "type": "string" + }, + "run2_duration": { + "type": "string" + }, + "duration_change": { + "type": "string" + }, + "run1_turns": { + "type": "integer" + }, + "run2_turns": { + "type": "integer" + }, + "turns_change": { + "type": "integer" + }, + "run1_tokens_per_turn": { + "type": "integer" + }, + "run2_tokens_per_turn": { + "type": "integer" + }, + "tokens_per_turn_change": { + "type": "string" + }, + "run1_working_set_rebuild_factor": { + "type": [ + "null", + "number" + ] + }, + "run2_working_set_rebuild_factor": { + "type": [ + "null", + "number" + ] + }, + "working_set_rebuild_factor_change": { + "type": "string" + }, + "token_usage_details": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_input_tokens": { + "type": "integer" + }, + "run2_input_tokens": { + "type": "integer" + }, + "input_tokens_change": { + "type": "string" + }, + "run1_output_tokens": { + "type": "integer" + }, + "run2_output_tokens": { + "type": "integer" + }, + "output_tokens_change": { + "type": "string" + }, + "run1_cache_read_tokens": { + "type": "integer" + }, + "run2_cache_read_tokens": { + "type": "integer" + }, + "cache_read_tokens_change": { + "type": "string" + }, + "run1_cache_write_tokens": { + "type": "integer" + }, + "run2_cache_write_tokens": { + "type": "integer" + }, + "cache_write_tokens_change": { + "type": "string" + }, + "run1_aic": { + "type": "number" + }, + "run2_aic": { + "type": "number" + }, + "aic_change": { + "type": "string" + }, + "run1_total_requests": { + "type": "integer" + }, + "run2_total_requests": { + "type": "integer" + }, + "requests_delta": { + "type": "string" + }, + "run1_cache_efficiency": { + "type": "number" + }, + "run2_cache_efficiency": { + "type": "number" + }, + "cache_efficiency_change": { + "type": "string" + } + }, + "required": [ + "run1_input_tokens", + "run2_input_tokens", + "run1_output_tokens", + "run2_output_tokens", + "run1_cache_read_tokens", + "run2_cache_read_tokens", + "run1_cache_write_tokens", + "run2_cache_write_tokens", + "run1_total_requests", + "run2_total_requests", + "run1_cache_efficiency", + "run2_cache_efficiency" + ], + "additionalProperties": false + }, + "github_rate_limit_details": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_total_api_calls": { + "type": "integer" + }, + "run2_total_api_calls": { + "type": "integer" + }, + "api_calls_change": { + "type": "string" + }, + "run1_core_consumed": { + "type": "integer" + }, + "run2_core_consumed": { + "type": "integer" + }, + "core_consumed_change": { + "type": "string" + }, + "run1_core_remaining": { + "type": "integer" + }, + "run2_core_remaining": { + "type": "integer" + }, + "run1_core_limit": { + "type": "integer" + }, + "run2_core_limit": { + "type": "integer" + } + }, + "required": [ + "run1_total_api_calls", + "run2_total_api_calls" + ], + "additionalProperties": false + }, + "tool_calls_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "new_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "removed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "changed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "all_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "bash_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_total_calls": { + "type": "integer" + }, + "run2_total_calls": { + "type": "integer" + }, + "total_calls_change": { + "type": "string" + }, + "commands": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + } + }, + "required": [ + "run1_total_calls", + "run2_total_calls" + ], + "additionalProperties": false + }, + "summary": { + "type": "object", + "properties": { + "new_tool_count": { + "type": "integer" + }, + "removed_tool_count": { + "type": "integer" + }, + "changed_tool_count": { + "type": "integer" + }, + "run1_total_calls": { + "type": "integer" + }, + "run2_total_calls": { + "type": "integer" + } + }, + "required": [ + "new_tool_count", + "removed_tool_count", + "changed_tool_count", + "run1_total_calls", + "run2_total_calls" + ], + "additionalProperties": false + } + }, + "required": [ + "summary" + ], + "additionalProperties": false + } + }, + "required": [ + "run1_token_usage", + "run2_token_usage" + ], + "additionalProperties": false + } + }, + "required": [ + "run1_id", + "run2_id" + ], + "additionalProperties": false + }, + { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "run1_id": { + "type": "integer" + }, + "run2_id": { + "type": "integer" + }, + "firewall_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_id": { + "type": "integer" + }, + "run2_id": { + "type": "integer" + }, + "new_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "removed_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "status_changes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "volume_changes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_allowed": { + "type": "integer" + }, + "run1_blocked": { + "type": "integer" + }, + "run2_allowed": { + "type": "integer" + }, + "run2_blocked": { + "type": "integer" + }, + "run1_status": { + "type": "string" + }, + "run2_status": { + "type": "string" + }, + "volume_change": { + "type": "string" + } + }, + "required": [ + "domain", + "status", + "run1_allowed", + "run1_blocked", + "run2_allowed", + "run2_blocked" + ], + "additionalProperties": false + } + }, + "summary": { + "type": "object", + "properties": { + "new_domain_count": { + "type": "integer" + }, + "removed_domain_count": { + "type": "integer" + }, + "status_change_count": { + "type": "integer" + }, + "volume_change_count": { + "type": "integer" + }, + "has_anomalies": { + "type": "boolean" + }, + "anomaly_count": { + "type": "integer" + } + }, + "required": [ + "new_domain_count", + "removed_domain_count", + "status_change_count", + "volume_change_count", + "has_anomalies", + "anomaly_count" + ], + "additionalProperties": false + } + }, + "required": [ + "run1_id", + "run2_id", + "summary" + ], + "additionalProperties": false + }, + "mcp_tools_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "new_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "run1_error_count": { + "type": "integer" + }, + "run2_error_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_name", + "status" + ], + "additionalProperties": false + } + }, + "removed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "run1_error_count": { + "type": "integer" + }, + "run2_error_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_name", + "status" + ], + "additionalProperties": false + } + }, + "changed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "run1_error_count": { + "type": "integer" + }, + "run2_error_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_name", + "status" + ], + "additionalProperties": false + } + }, + "summary": { + "type": "object", + "properties": { + "new_tool_count": { + "type": "integer" + }, + "removed_tool_count": { + "type": "integer" + }, + "changed_tool_count": { + "type": "integer" + }, + "has_anomalies": { + "type": "boolean" + }, + "anomaly_count": { + "type": "integer" + } + }, + "required": [ + "new_tool_count", + "removed_tool_count", + "changed_tool_count", + "has_anomalies", + "anomaly_count" + ], + "additionalProperties": false + } + }, + "required": [ + "summary" + ], + "additionalProperties": false + }, + "run_metrics_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_token_usage": { + "type": "integer" + }, + "run2_token_usage": { + "type": "integer" + }, + "token_usage_change": { + "type": "string" + }, + "run1_duration": { + "type": "string" + }, + "run2_duration": { + "type": "string" + }, + "duration_change": { + "type": "string" + }, + "run1_turns": { + "type": "integer" + }, + "run2_turns": { + "type": "integer" + }, + "turns_change": { + "type": "integer" + }, + "run1_tokens_per_turn": { + "type": "integer" + }, + "run2_tokens_per_turn": { + "type": "integer" + }, + "tokens_per_turn_change": { + "type": "string" + }, + "run1_working_set_rebuild_factor": { + "type": [ + "null", + "number" + ] + }, + "run2_working_set_rebuild_factor": { + "type": [ + "null", + "number" + ] + }, + "working_set_rebuild_factor_change": { + "type": "string" + }, + "token_usage_details": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_input_tokens": { + "type": "integer" + }, + "run2_input_tokens": { + "type": "integer" + }, + "input_tokens_change": { + "type": "string" + }, + "run1_output_tokens": { + "type": "integer" + }, + "run2_output_tokens": { + "type": "integer" + }, + "output_tokens_change": { + "type": "string" + }, + "run1_cache_read_tokens": { + "type": "integer" + }, + "run2_cache_read_tokens": { + "type": "integer" + }, + "cache_read_tokens_change": { + "type": "string" + }, + "run1_cache_write_tokens": { + "type": "integer" + }, + "run2_cache_write_tokens": { + "type": "integer" + }, + "cache_write_tokens_change": { + "type": "string" + }, + "run1_aic": { + "type": "number" + }, + "run2_aic": { + "type": "number" + }, + "aic_change": { + "type": "string" + }, + "run1_total_requests": { + "type": "integer" + }, + "run2_total_requests": { + "type": "integer" + }, + "requests_delta": { + "type": "string" + }, + "run1_cache_efficiency": { + "type": "number" + }, + "run2_cache_efficiency": { + "type": "number" + }, + "cache_efficiency_change": { + "type": "string" + } + }, + "required": [ + "run1_input_tokens", + "run2_input_tokens", + "run1_output_tokens", + "run2_output_tokens", + "run1_cache_read_tokens", + "run2_cache_read_tokens", + "run1_cache_write_tokens", + "run2_cache_write_tokens", + "run1_total_requests", + "run2_total_requests", + "run1_cache_efficiency", + "run2_cache_efficiency" + ], + "additionalProperties": false + }, + "github_rate_limit_details": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_total_api_calls": { + "type": "integer" + }, + "run2_total_api_calls": { + "type": "integer" + }, + "api_calls_change": { + "type": "string" + }, + "run1_core_consumed": { + "type": "integer" + }, + "run2_core_consumed": { + "type": "integer" + }, + "core_consumed_change": { + "type": "string" + }, + "run1_core_remaining": { + "type": "integer" + }, + "run2_core_remaining": { + "type": "integer" + }, + "run1_core_limit": { + "type": "integer" + }, + "run2_core_limit": { + "type": "integer" + } + }, + "required": [ + "run1_total_api_calls", + "run2_total_api_calls" + ], + "additionalProperties": false + }, + "tool_calls_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "new_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "removed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "changed_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "all_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + }, + "bash_diff": { + "type": [ + "null", + "object" + ], + "properties": { + "run1_total_calls": { + "type": "integer" + }, + "run2_total_calls": { + "type": "integer" + }, + "total_calls_change": { + "type": "string" + }, + "commands": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "is_anomaly": { + "type": "boolean" + }, + "anomaly_note": { + "type": "string" + }, + "run1_call_count": { + "type": "integer" + }, + "run2_call_count": { + "type": "integer" + }, + "call_count_change": { + "type": "string" + }, + "run1_max_input_size": { + "type": "integer" + }, + "run2_max_input_size": { + "type": "integer" + }, + "run1_max_output_size": { + "type": "integer" + }, + "run2_max_output_size": { + "type": "integer" + } + }, + "required": [ + "name", + "status", + "run1_call_count", + "run2_call_count" + ], + "additionalProperties": false + } + } + }, + "required": [ + "run1_total_calls", + "run2_total_calls" + ], + "additionalProperties": false + }, + "summary": { + "type": "object", + "properties": { + "new_tool_count": { + "type": "integer" + }, + "removed_tool_count": { + "type": "integer" + }, + "changed_tool_count": { + "type": "integer" + }, + "run1_total_calls": { + "type": "integer" + }, + "run2_total_calls": { + "type": "integer" + } + }, + "required": [ + "new_tool_count", + "removed_tool_count", + "changed_tool_count", + "run1_total_calls", + "run2_total_calls" + ], + "additionalProperties": false + } + }, + "required": [ + "summary" + ], + "additionalProperties": false + } + }, + "required": [ + "run1_token_usage", + "run2_token_usage" + ], + "additionalProperties": false + } + }, + "required": [ + "run1_id", + "run2_id" + ], + "additionalProperties": false + } + } + ] +} diff --git a/schemas/logs.schema.json b/schemas/logs.schema.json new file mode 100644 index 00000000000..436461fad78 --- /dev/null +++ b/schemas/logs.schema.json @@ -0,0 +1,2946 @@ +{ + "oneOf": [ + { + "type": "object", + "properties": { + "summary": { + "type": "object", + "properties": { + "total_runs": { + "type": "integer" + }, + "total_duration": { + "type": "string" + }, + "total_aic": { + "type": "number" + }, + "total_tokens": { + "type": "integer" + }, + "total_action_minutes": { + "type": "number" + }, + "total_turns": { + "type": "integer" + }, + "total_steering_events": { + "type": "integer" + }, + "total_errors": { + "type": "integer" + }, + "total_warnings": { + "type": "integer" + }, + "total_missing_tools": { + "type": "integer" + }, + "total_missing_data": { + "type": "integer" + }, + "total_safe_items": { + "type": "integer" + }, + "total_driver_exit_failures": { + "type": "integer" + }, + "total_agent_logic_failures": { + "type": "integer" + }, + "runs_with_temporary_id_chains": { + "type": "integer" + }, + "runs_with_delegated_temp_targets": { + "type": "integer" + }, + "runs_with_missing_temporary_id_map": { + "type": "integer" + }, + "runs_with_invalid_temporary_id_map": { + "type": "integer" + }, + "total_temporary_id_mappings": { + "type": "integer" + }, + "total_chained_targets": { + "type": "integer" + }, + "total_chained_followup_actions": { + "type": "integer" + }, + "total_closed_temp_targets": { + "type": "integer" + }, + "total_episodes": { + "type": "integer" + }, + "high_confidence_episodes": { + "type": "integer" + }, + "total_github_api_calls": { + "type": "integer" + }, + "engine_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "intentional_failure_runs": { + "type": "integer" + }, + "outcome_accepted": { + "type": "integer" + }, + "outcome_rejected": { + "type": "integer" + }, + "outcome_ignored": { + "type": "integer" + }, + "outcome_pending": { + "type": "integer" + }, + "outcome_acceptance_rate": { + "type": "number" + }, + "outcome_waste_rate": { + "type": "number" + }, + "outcome_zero_touch_rate": { + "type": "number" + } + }, + "required": [ + "total_runs", + "total_duration", + "total_action_minutes", + "total_turns", + "total_errors", + "total_warnings", + "total_missing_tools", + "total_missing_data", + "total_safe_items", + "total_driver_exit_failures", + "total_agent_logic_failures", + "total_episodes", + "high_confidence_episodes" + ], + "additionalProperties": false + }, + "runs": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "run_id": { + "type": "integer" + }, + "number": { + "type": "integer" + }, + "workflow_name": { + "type": "string" + }, + "workflow_path": { + "type": "string" + }, + "intentional_failure": { + "type": "boolean" + }, + "agent": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "engine_id": { + "type": "string" + }, + "status": { + "type": "string" + }, + "conclusion": { + "type": "string" + }, + "classification": { + "type": "string" + }, + "failure_kind": { + "type": "string" + }, + "duration": { + "type": "string" + }, + "action_minutes": { + "type": "number" + }, + "token_usage": { + "type": "integer" + }, + "aic": { + "type": "number" + }, + "ambient_context": { + "type": [ + "null", + "object" + ], + "properties": { + "input_tokens": { + "type": "integer" + }, + "cached_tokens": { + "type": "integer" + }, + "effective_tokens": { + "type": "integer" + } + }, + "required": [ + "input_tokens", + "cached_tokens" + ], + "additionalProperties": false + }, + "working_set": { + "type": [ + "null", + "object" + ], + "properties": { + "measurement_state": { + "type": "string" + }, + "rebuild_factor": { + "type": [ + "null", + "number" + ] + }, + "cumulative_input_tokens": { + "type": "integer" + }, + "peak_input_tokens": { + "type": "integer" + }, + "rebuild_excess_tokens": { + "type": "integer" + }, + "invocations": { + "type": "integer" + } + }, + "required": [ + "measurement_state", + "cumulative_input_tokens", + "peak_input_tokens", + "rebuild_excess_tokens", + "invocations" + ], + "additionalProperties": false + }, + "turns": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "warning_count": { + "type": "integer" + }, + "missing_tool_count": { + "type": "integer" + }, + "missing_data_count": { + "type": "integer" + }, + "safe_items_count": { + "type": "integer" + }, + "manifest_entry_count": { + "type": "integer" + }, + "temporary_id_map_status": { + "type": "string" + }, + "temporary_id_mappings": { + "type": "integer" + }, + "chained_target_count": { + "type": "integer" + }, + "chained_followup_action_count": { + "type": "integer" + }, + "delegated_temp_target_count": { + "type": "integer" + }, + "closed_temp_target_count": { + "type": "integer" + }, + "created_at": { + "type": "string" + }, + "started_at": { + "type": "string" + }, + "updated_at": { + "type": "string" + }, + "url": { + "type": "string" + }, + "logs_path": { + "type": "string" + }, + "audit_path": { + "type": "string" + }, + "event": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "head_sha": { + "type": "string" + }, + "display_title": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "ref": { + "type": "string" + }, + "sha": { + "type": "string" + }, + "actor": { + "type": "string" + }, + "run_attempt": { + "type": "string" + }, + "target_repo": { + "type": "string" + }, + "event_name": { + "type": "string" + }, + "comparison": { + "type": [ + "null", + "object" + ], + "properties": { + "baseline_found": { + "type": "boolean" + }, + "baseline": { + "type": [ + "null", + "object" + ], + "properties": { + "run_id": { + "type": "integer" + }, + "workflow_name": { + "type": "string" + }, + "conclusion": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "selection": { + "type": "string" + }, + "matched_on": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "run_id" + ], + "additionalProperties": false + }, + "delta": { + "type": [ + "null", + "object" + ], + "properties": { + "turns": { + "type": "object", + "properties": { + "before": { + "type": "integer" + }, + "after": { + "type": "integer" + }, + "changed": { + "type": "boolean" + } + }, + "required": [ + "before", + "after", + "changed" + ], + "additionalProperties": false + }, + "posture": { + "type": "object", + "properties": { + "before": { + "type": "string" + }, + "after": { + "type": "string" + }, + "changed": { + "type": "boolean" + } + }, + "required": [ + "before", + "after", + "changed" + ], + "additionalProperties": false + }, + "blocked_requests": { + "type": "object", + "properties": { + "before": { + "type": "integer" + }, + "after": { + "type": "integer" + }, + "changed": { + "type": "boolean" + } + }, + "required": [ + "before", + "after", + "changed" + ], + "additionalProperties": false + }, + "mcp_failure": { + "type": [ + "null", + "object" + ], + "properties": { + "before": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "after": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "newly_present": { + "type": "boolean" + } + }, + "required": [ + "newly_present" + ], + "additionalProperties": false + } + }, + "required": [ + "turns", + "posture", + "blocked_requests" + ], + "additionalProperties": false + }, + "classification": { + "type": [ + "null", + "object" + ], + "properties": { + "label": { + "type": "string" + }, + "reason_codes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "label" + ], + "additionalProperties": false + }, + "recommendation": { + "type": [ + "null", + "object" + ], + "properties": { + "action": { + "type": "string" + } + }, + "required": [ + "action" + ], + "additionalProperties": false + } + }, + "required": [ + "baseline_found" + ], + "additionalProperties": false + }, + "task_domain": { + "type": [ + "null", + "object" + ], + "properties": { + "name": { + "type": "string" + }, + "label": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "name", + "label" + ], + "additionalProperties": false + }, + "behavior_fingerprint": { + "type": [ + "null", + "object" + ], + "properties": { + "execution_style": { + "type": "string" + }, + "tool_breadth": { + "type": "string" + }, + "actuation_style": { + "type": "string" + }, + "resource_profile": { + "type": "string" + }, + "dispatch_mode": { + "type": "string" + }, + "agentic_fraction": { + "type": "number" + } + }, + "required": [ + "execution_style", + "tool_breadth", + "actuation_style", + "resource_profile", + "dispatch_mode", + "agentic_fraction" + ], + "additionalProperties": false + }, + "agentic_assessments": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "evidence": { + "type": "string" + }, + "recommendation": { + "type": "string" + } + }, + "required": [ + "kind", + "severity", + "summary" + ], + "additionalProperties": false + } + }, + "context": { + "type": [ + "null", + "object" + ], + "properties": { + "repo": { + "type": "string" + }, + "run_id": { + "type": "string" + }, + "workflow_id": { + "type": "string" + }, + "workflow_call_id": { + "type": "string" + }, + "time": { + "type": "string" + }, + "actor": { + "type": "string" + }, + "event_type": { + "type": "string" + }, + "item_type": { + "type": "string" + }, + "item_number": { + "type": "string" + }, + "comment_id": { + "type": "string" + } + }, + "required": [ + "repo", + "run_id", + "workflow_id" + ], + "additionalProperties": false + }, + "token_usage_summary": { + "type": [ + "null", + "object" + ], + "properties": { + "total_input_tokens": { + "type": "integer" + }, + "total_output_tokens": { + "type": "integer" + }, + "total_cache_read_tokens": { + "type": "integer" + }, + "total_cache_write_tokens": { + "type": "integer" + }, + "total_requests": { + "type": "integer" + }, + "total_steering_events": { + "type": "integer" + }, + "total_duration_ms": { + "type": "integer" + }, + "total_response_bytes": { + "type": "integer" + }, + "cache_efficiency": { + "type": "number" + }, + "total_effective_tokens": { + "type": "integer" + }, + "total_aic": { + "type": "number" + }, + "ambient_context": { + "type": [ + "null", + "object" + ], + "properties": { + "input_tokens": { + "type": "integer" + }, + "cached_tokens": { + "type": "integer" + }, + "effective_tokens": { + "type": "integer" + } + }, + "required": [ + "input_tokens", + "cached_tokens" + ], + "additionalProperties": false + }, + "by_model": { + "type": "object", + "additionalProperties": { + "type": [ + "null", + "object" + ], + "properties": { + "provider": { + "type": "string" + }, + "input_tokens": { + "type": "integer" + }, + "output_tokens": { + "type": "integer" + }, + "cache_read_tokens": { + "type": "integer" + }, + "cache_write_tokens": { + "type": "integer" + }, + "reasoning_tokens": { + "type": "integer" + }, + "effective_tokens": { + "type": "integer" + }, + "requests": { + "type": "integer" + }, + "duration_ms": { + "type": "integer" + }, + "response_bytes": { + "type": "integer" + }, + "aic": { + "type": "number" + } + }, + "required": [ + "provider", + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cache_write_tokens", + "requests", + "duration_ms", + "response_bytes" + ], + "additionalProperties": false + } + }, + "subagent_model_requests": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "agent_name": { + "type": "string" + }, + "requested_model": { + "type": "string" + }, + "invocation_count": { + "type": "integer" + }, + "effective_model": { + "type": "string" + }, + "reason_code": { + "type": "string" + } + }, + "required": [ + "agent_name", + "requested_model", + "invocation_count" + ], + "additionalProperties": false + } + }, + "subagent_model_actuals": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "model": { + "type": "string" + }, + "provider": { + "type": "string" + }, + "requests": { + "type": "integer" + } + }, + "required": [ + "model", + "requests" + ], + "additionalProperties": false + } + }, + "mismatch_count": { + "type": "integer" + }, + "warnings": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "total_input_tokens", + "total_output_tokens", + "total_cache_read_tokens", + "total_cache_write_tokens", + "total_requests", + "total_duration_ms", + "total_response_bytes", + "cache_efficiency", + "by_model" + ], + "additionalProperties": false + }, + "github_api_calls": { + "type": "integer" + }, + "avg_time_between_turns": { + "type": "string" + }, + "experiments": { + "type": [ + "null", + "object" + ], + "properties": { + "assignments": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "cumulative_counts": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + } + }, + "required": [ + "assignments" + ], + "additionalProperties": false + }, + "graders": { + "type": [ + "null", + "object" + ], + "properties": { + "version": { + "type": "integer" + }, + "results": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "status": { + "type": "string" + }, + "value": { + "type": [ + "null", + "number" + ] + }, + "unit": { + "type": "string" + }, + "passed": { + "type": [ + "null", + "boolean" + ] + }, + "direction": { + "type": "string" + }, + "threshold": { + "type": [ + "null", + "number" + ] + }, + "message": { + "type": "string" + }, + "error": { + "type": "string" + } + }, + "required": [ + "id", + "status" + ], + "additionalProperties": false + } + }, + "total": { + "type": "integer" + }, + "passed": { + "type": "integer" + }, + "failed": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "unavailable_count": { + "type": "integer" + } + }, + "required": [ + "results", + "total", + "passed", + "failed", + "error_count", + "unavailable_count" + ], + "additionalProperties": false + } + }, + "required": [ + "run_id", + "number", + "workflow_name", + "workflow_path", + "status", + "classification", + "created_at", + "url", + "logs_path", + "event", + "branch" + ], + "additionalProperties": false + } + }, + "episodes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "episode_id": { + "type": "string" + }, + "kind": { + "type": "string" + }, + "confidence": { + "type": "string" + }, + "reasons": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "root_run_id": { + "type": "integer" + }, + "run_ids": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + }, + "workflow_names": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "primary_workflow": { + "type": "string" + }, + "total_runs": { + "type": "integer" + }, + "total_tokens": { + "type": "integer" + }, + "total_aic": { + "type": "number" + }, + "total_duration": { + "type": "string" + }, + "risky_node_count": { + "type": "integer" + }, + "changed_node_count": { + "type": "integer" + }, + "write_capable_node_count": { + "type": "integer" + }, + "missing_tool_count": { + "type": "integer" + }, + "mcp_failure_count": { + "type": "integer" + }, + "blocked_request_count": { + "type": "integer" + }, + "latest_success_fallback_count": { + "type": "integer" + }, + "new_mcp_failure_run_count": { + "type": "integer" + }, + "blocked_request_increase_run_count": { + "type": "integer" + }, + "blocked_request_at_cap": { + "type": "boolean" + }, + "resource_heavy_node_count": { + "type": "integer" + }, + "poor_control_node_count": { + "type": "integer" + }, + "risk_distribution": { + "type": "string" + }, + "escalation_eligible": { + "type": "boolean" + }, + "escalation_reason": { + "type": "string" + }, + "suggested_route": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "organization": { + "type": "string" + }, + "manifest_entry_count": { + "type": "integer" + }, + "temporary_id_mappings": { + "type": "integer" + }, + "chained_target_count": { + "type": "integer" + }, + "chained_followup_action_count": { + "type": "integer" + }, + "delegated_temp_target_count": { + "type": "integer" + }, + "closed_temp_target_count": { + "type": "integer" + }, + "tool_calls": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "tool": { + "type": "string" + }, + "server": { + "type": "string" + }, + "tokens": { + "type": "integer" + }, + "duration_ms": { + "type": "integer" + }, + "status": { + "type": "string" + }, + "error": { + "type": "string" + } + }, + "required": [ + "tool", + "server", + "tokens", + "duration_ms", + "status" + ], + "additionalProperties": false + } + } + }, + "required": [ + "episode_id", + "kind", + "confidence", + "run_ids", + "workflow_names", + "total_runs", + "total_tokens", + "total_duration", + "risky_node_count", + "changed_node_count", + "write_capable_node_count", + "missing_tool_count", + "mcp_failure_count", + "blocked_request_count", + "latest_success_fallback_count", + "new_mcp_failure_run_count", + "blocked_request_increase_run_count", + "resource_heavy_node_count", + "poor_control_node_count", + "risk_distribution", + "escalation_eligible" + ], + "additionalProperties": false + } + }, + "edges": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "source_run_id": { + "type": "integer" + }, + "target_run_id": { + "type": "integer" + }, + "edge_type": { + "type": "string" + }, + "confidence": { + "type": "string" + }, + "reasons": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "source_repo": { + "type": "string" + }, + "source_ref": { + "type": "string" + }, + "event_type": { + "type": "string" + }, + "episode_id": { + "type": "string" + } + }, + "required": [ + "source_run_id", + "target_run_id", + "edge_type", + "confidence" + ], + "additionalProperties": false + } + }, + "tool_usage": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "total_calls": { + "type": "integer" + }, + "runs": { + "type": "integer" + }, + "max_output_size": { + "type": "integer" + }, + "max_duration": { + "type": "string" + } + }, + "required": [ + "name", + "total_calls", + "runs" + ], + "additionalProperties": false + } + }, + "mcp_tool_usage": { + "type": [ + "null", + "object" + ], + "properties": { + "summary": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "call_count": { + "type": "integer" + }, + "total_input_size": { + "type": "integer" + }, + "total_output_size": { + "type": "integer" + }, + "max_input_size": { + "type": "integer" + }, + "max_output_size": { + "type": "integer" + }, + "avg_duration": { + "type": "string" + }, + "max_duration": { + "type": "string" + }, + "error_count": { + "type": "integer" + } + }, + "required": [ + "server_name", + "tool_name", + "call_count", + "total_input_size", + "total_output_size", + "max_input_size", + "max_output_size" + ], + "additionalProperties": false + } + }, + "servers": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "tool_call_count": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "request_count": { + "type": "integer" + }, + "total_input_size": { + "type": "integer" + }, + "total_output_size": { + "type": "integer" + }, + "avg_duration": { + "type": "string" + } + }, + "required": [ + "server_name", + "tool_call_count", + "request_count", + "total_input_size", + "total_output_size" + ], + "additionalProperties": false + } + }, + "tool_calls": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "tool_call_id": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "server_name": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "method": { + "type": "string" + }, + "input_size": { + "type": "integer" + }, + "output_size": { + "type": "integer" + }, + "duration": { + "type": "string" + }, + "status": { + "type": "string" + }, + "error": { + "type": "string" + }, + "effective_token_delta": { + "type": "integer" + } + }, + "required": [ + "timestamp", + "server_name", + "tool_name", + "input_size", + "output_size", + "status" + ], + "additionalProperties": false + } + }, + "filtered_events": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "timestamp": { + "type": "string" + }, + "server_id": { + "type": "string" + }, + "tool_name": { + "type": "string" + }, + "description": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "secrecy_tags": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "integrity_tags": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "author_association": { + "type": "string" + }, + "author_login": { + "type": "string" + }, + "html_url": { + "type": "string" + }, + "number": { + "type": "string" + } + }, + "required": [ + "timestamp", + "server_id", + "tool_name", + "reason" + ], + "additionalProperties": false + } + }, + "integrity": { + "type": [ + "null", + "object" + ], + "properties": { + "total_filtered": { + "type": "integer" + }, + "runs_with_filtered_events": { + "type": "integer" + }, + "filtered_server_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "filtered_tool_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "filtered_reason_counts": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + } + }, + "required": [ + "total_filtered" + ], + "additionalProperties": false + } + }, + "required": [ + "summary", + "tool_calls" + ], + "additionalProperties": false + }, + "observability_insights": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "category": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "title": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "evidence": { + "type": "string" + } + }, + "required": [ + "category", + "severity", + "title", + "summary" + ], + "additionalProperties": false + } + }, + "errors_and_warnings": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "message": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "engine": { + "type": "string" + }, + "run_id": { + "type": "integer" + }, + "run_url": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "pattern_id": { + "type": "string" + } + }, + "required": [ + "type", + "message", + "count", + "run_id", + "run_url" + ], + "additionalProperties": false + } + }, + "missing_tools": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "tool": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "workflows": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "first_reason": { + "type": "string" + }, + "run_ids": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + } + }, + "required": [ + "tool", + "count", + "workflows", + "first_reason", + "run_ids" + ], + "additionalProperties": false + } + }, + "missing_data": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "data_type": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "workflows": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "first_reason": { + "type": "string" + }, + "run_ids": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + } + }, + "required": [ + "data_type", + "count", + "workflows", + "first_reason", + "run_ids" + ], + "additionalProperties": false + } + }, + "mcp_failures": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "workflows": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "run_ids": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + } + }, + "required": [ + "server_name", + "count", + "workflows", + "run_ids" + ], + "additionalProperties": false + } + }, + "access_log": { + "type": [ + "null", + "object" + ], + "properties": { + "total_requests": { + "type": "integer" + }, + "allowed_requests": { + "type": "integer" + }, + "blocked_requests": { + "type": "integer" + }, + "allowed_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "blocked_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "by_workflow": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "total_requests": { + "type": "integer" + }, + "allowed_count": { + "type": "integer" + }, + "blocked_count": { + "type": "integer" + }, + "allowed_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "blocked_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "total_requests", + "allowed_count", + "blocked_count" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_requests", + "allowed_requests", + "blocked_requests", + "allowed_domains", + "blocked_domains" + ], + "additionalProperties": false + }, + "firewall_log": { + "type": [ + "null", + "object" + ], + "properties": { + "total_requests": { + "type": "integer" + }, + "allowed_requests": { + "type": "integer" + }, + "blocked_requests": { + "type": "integer" + }, + "allowed_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "blocked_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "requests_by_domain": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "allowed": { + "type": "integer" + }, + "blocked": { + "type": "integer" + } + }, + "required": [ + "allowed", + "blocked" + ], + "additionalProperties": false + } + }, + "by_workflow": { + "type": "object", + "additionalProperties": { + "type": [ + "null", + "object" + ], + "properties": { + "allowed_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "blocked_domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "total_requests": { + "type": "integer" + }, + "allowed_requests": { + "type": "integer" + }, + "blocked_requests": { + "type": "integer" + }, + "requests_by_domain": { + "type": "object", + "additionalProperties": { + "type": "object", + "properties": { + "allowed": { + "type": "integer" + }, + "blocked": { + "type": "integer" + } + }, + "required": [ + "allowed", + "blocked" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_requests", + "allowed_requests", + "blocked_requests" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_requests", + "allowed_requests", + "blocked_requests", + "allowed_domains", + "blocked_domains" + ], + "additionalProperties": false + }, + "redacted_domains": { + "type": [ + "null", + "object" + ], + "properties": { + "total_domains": { + "type": "integer" + }, + "domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "by_workflow": { + "type": "object", + "additionalProperties": { + "type": [ + "null", + "object" + ], + "properties": { + "total_domains": { + "type": "integer" + }, + "domains": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + }, + "required": [ + "total_domains", + "domains" + ], + "additionalProperties": false + } + } + }, + "required": [ + "total_domains", + "domains" + ], + "additionalProperties": false + }, + "continuation": { + "type": [ + "null", + "object" + ], + "properties": { + "message": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "start_date": { + "type": "string" + }, + "end_date": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "after_run_id": { + "type": "integer" + }, + "before_run_id": { + "type": "integer" + }, + "ignore_workflow_runs": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + }, + "timeout": { + "type": "integer" + }, + "max_github_api_rate_limit": { + "type": "integer" + }, + "max_storage": { + "type": "integer" + }, + "prune_older_runs": { + "type": "boolean" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "continuations": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "repository": { + "type": "string" + }, + "message": { + "type": "string" + }, + "workflow_name": { + "type": "string" + }, + "count": { + "type": "integer" + }, + "start_date": { + "type": "string" + }, + "end_date": { + "type": "string" + }, + "engine": { + "type": "string" + }, + "branch": { + "type": "string" + }, + "after_run_id": { + "type": "integer" + }, + "before_run_id": { + "type": "integer" + }, + "ignore_workflow_runs": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + }, + "timeout": { + "type": "integer" + }, + "max_github_api_rate_limit": { + "type": "integer" + }, + "max_storage": { + "type": "integer" + }, + "prune_older_runs": { + "type": "boolean" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + } + }, + "logs_location": { + "type": "string" + }, + "message": { + "type": "string" + }, + "stale_warning": { + "type": "string" + }, + "github_api_rate_limit": { + "type": [ + "null", + "object" + ], + "properties": { + "host": { + "type": "string" + }, + "start": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + }, + "end": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + } + }, + "required": [ + "host" + ], + "additionalProperties": false + }, + "github_api_rate_limits": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "host": { + "type": "string" + }, + "start": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + }, + "end": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + } + }, + "required": [ + "host" + ], + "additionalProperties": false + } + } + }, + "required": [ + "summary", + "runs", + "episodes", + "edges", + "logs_location" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "runs_analyzed": { + "type": "integer" + }, + "runs_with_data": { + "type": "integer" + }, + "runs_without_data": { + "type": "integer" + }, + "summary": { + "type": "object", + "properties": { + "total_requests": { + "type": "integer" + }, + "total_allowed": { + "type": "integer" + }, + "total_blocked": { + "type": "integer" + }, + "overall_deny_rate": { + "type": "number" + }, + "unique_domains": { + "type": "integer" + } + }, + "required": [ + "total_requests", + "total_allowed", + "total_blocked", + "overall_deny_rate", + "unique_domains" + ], + "additionalProperties": false + }, + "metrics_trend": { + "type": "object", + "properties": { + "total_tokens": { + "type": "integer" + }, + "avg_tokens": { + "type": "integer" + }, + "median_tokens": { + "type": "number" + }, + "stddev_tokens": { + "type": "number" + }, + "min_tokens": { + "type": "integer" + }, + "max_tokens": { + "type": "integer" + }, + "total_turns": { + "type": "integer" + }, + "avg_turns": { + "type": "number" + }, + "median_turns": { + "type": "number" + }, + "stddev_turns": { + "type": "number" + }, + "max_turns": { + "type": "integer" + }, + "avg_duration_ns": { + "type": "integer" + }, + "median_duration_ns": { + "type": "integer" + }, + "stddev_duration_ns": { + "type": "integer" + }, + "min_duration_ns": { + "type": "integer" + }, + "max_duration_ns": { + "type": "integer" + }, + "token_spikes": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + } + }, + "required": [ + "total_tokens", + "avg_tokens", + "median_tokens", + "stddev_tokens", + "min_tokens", + "max_tokens", + "total_turns", + "avg_turns", + "median_turns", + "stddev_turns", + "max_turns", + "avg_duration_ns", + "median_duration_ns", + "stddev_duration_ns", + "min_duration_ns", + "max_duration_ns" + ], + "additionalProperties": false + }, + "mcp_health": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "server_name": { + "type": "string" + }, + "runs_connected": { + "type": "integer" + }, + "total_runs": { + "type": "integer" + }, + "total_calls": { + "type": "integer" + }, + "total_errors": { + "type": "integer" + }, + "error_rate": { + "type": "number" + }, + "unreliable": { + "type": "boolean" + } + }, + "required": [ + "server_name", + "runs_connected", + "total_runs", + "total_calls", + "total_errors", + "error_rate", + "unreliable" + ], + "additionalProperties": false + } + }, + "error_trend": { + "type": "object", + "properties": { + "runs_with_errors": { + "type": "integer" + }, + "total_errors": { + "type": "integer" + }, + "avg_errors_per_run": { + "type": "number" + }, + "runs_with_warnings": { + "type": "integer" + }, + "total_warnings": { + "type": "integer" + } + }, + "required": [ + "runs_with_errors", + "total_errors", + "avg_errors_per_run", + "runs_with_warnings", + "total_warnings" + ], + "additionalProperties": false + }, + "domain_inventory": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string" + }, + "seen_in_runs": { + "type": "integer" + }, + "total_allowed": { + "type": "integer" + }, + "total_blocked": { + "type": "integer" + }, + "overall_status": { + "type": "string" + }, + "per_run_status": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "run_id": { + "type": "integer" + }, + "status": { + "type": "string" + }, + "allowed": { + "type": "integer" + }, + "blocked": { + "type": "integer" + } + }, + "required": [ + "run_id", + "status", + "allowed", + "blocked" + ], + "additionalProperties": false + } + } + }, + "required": [ + "domain", + "seen_in_runs", + "total_allowed", + "total_blocked", + "overall_status", + "per_run_status" + ], + "additionalProperties": false + } + }, + "per_run_breakdown": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "run_id": { + "type": "integer" + }, + "workflow_name": { + "type": "string" + }, + "conclusion": { + "type": "string" + }, + "duration_ns": { + "type": "integer" + }, + "total_requests": { + "type": "integer" + }, + "allowed": { + "type": "integer" + }, + "blocked": { + "type": "integer" + }, + "deny_rate": { + "type": "number" + }, + "unique_domains": { + "type": "integer" + }, + "tokens": { + "type": "integer" + }, + "turns": { + "type": "integer" + }, + "mcp_errors": { + "type": "integer" + }, + "error_count": { + "type": "integer" + }, + "has_data": { + "type": "boolean" + }, + "token_spike": { + "type": "boolean" + } + }, + "required": [ + "run_id", + "workflow_name", + "conclusion", + "duration_ns", + "total_requests", + "allowed", + "blocked", + "deny_rate", + "unique_domains", + "tokens", + "turns", + "mcp_errors", + "error_count", + "has_data" + ], + "additionalProperties": false + } + }, + "drain3_insights": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "category": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "title": { + "type": "string" + }, + "summary": { + "type": "string" + }, + "evidence": { + "type": "string" + } + }, + "required": [ + "category", + "severity", + "title", + "summary" + ], + "additionalProperties": false + } + }, + "cluster_analysis": { + "type": [ + "null", + "object" + ], + "properties": { + "clusters": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "dimension": { + "type": "string" + }, + "value": { + "type": "string" + }, + "run_ids": { + "type": [ + "null", + "array" + ], + "items": { + "type": "integer" + } + }, + "count": { + "type": "integer" + }, + "metrics": { + "type": "object", + "properties": { + "avg_tokens": { + "type": "integer" + }, + "median_tokens": { + "type": "number" + }, + "stddev_tokens": { + "type": "number" + }, + "avg_turns": { + "type": "number" + }, + "avg_duration_ns": { + "type": "integer" + }, + "avg_errors_per_run": { + "type": "number" + }, + "success_rate": { + "type": "number" + } + }, + "required": [ + "avg_tokens", + "median_tokens", + "stddev_tokens", + "avg_turns", + "avg_duration_ns", + "avg_errors_per_run", + "success_rate" + ], + "additionalProperties": false + } + }, + "required": [ + "dimension", + "value", + "run_ids", + "count", + "metrics" + ], + "additionalProperties": false + } + }, + "patterns": { + "type": [ + "null", + "array" + ], + "items": { + "type": "object", + "properties": { + "kind": { + "type": "string" + }, + "severity": { + "type": "string" + }, + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "evidence": { + "type": "string" + } + }, + "required": [ + "kind", + "severity", + "title", + "description" + ], + "additionalProperties": false + } + } + }, + "required": [ + "clusters", + "patterns" + ], + "additionalProperties": false + }, + "github_api_rate_limit": { + "type": [ + "null", + "object" + ], + "properties": { + "host": { + "type": "string" + }, + "start": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + }, + "end": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + } + }, + "required": [ + "host" + ], + "additionalProperties": false + }, + "github_api_rate_limits": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "properties": { + "host": { + "type": "string" + }, + "start": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + }, + "end": { + "type": [ + "null", + "object" + ], + "properties": { + "limit": { + "type": "integer" + }, + "remaining": { + "type": "integer" + }, + "reset": { + "type": "integer" + }, + "used": { + "type": "integer" + } + }, + "required": [ + "limit", + "remaining", + "reset", + "used" + ], + "additionalProperties": false + } + }, + "required": [ + "host" + ], + "additionalProperties": false + } + } + }, + "required": [ + "runs_analyzed", + "runs_with_data", + "runs_without_data", + "summary", + "metrics_trend", + "error_trend", + "domain_inventory", + "per_run_breakdown" + ], + "additionalProperties": false + } + ] +}