diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e75d41e85..eaef1606e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1423,7 +1423,9 @@ jobs: runs-on: ubuntu-26.04 needs: [changes, upgrade-matrix] if: ${{ !startsWith(github.ref, 'refs/tags/') && needs.changes.outputs.code == 'true' }} - timeout-minutes: 10 + # A leg takes about 3 minutes. The rest is IMAGE_WAIT below: room to wait + # for the image of a tag pushed moments before this run started. + timeout-minutes: 35 strategy: fail-fast: false matrix: @@ -1446,6 +1448,8 @@ jobs: run: go build -o ../solidping . - name: Upgrade ${{ matrix.tag }} to HEAD + env: + IMAGE_WAIT: "1500" run: | set -euo pipefail command -v sqlite3 >/dev/null || sudo apt-get install -y sqlite3 diff --git a/scripts/upgrade-test.sh b/scripts/upgrade-test.sh index 3ddc60a77..f154dbef3 100755 --- a/scripts/upgrade-test.sh +++ b/scripts/upgrade-test.sh @@ -10,6 +10,8 @@ # HEAD_BIN the HEAD server binary (default: ./solidping, see `make test-upgrade`) # SKIP_TAGS space-separated tags left out of the matrix (default: SKIP_TAGS below) # PG_IMAGE Postgres image for the old and new side (default: docker-compose.yml's) +# IMAGE_WAIT seconds to wait for the released image to be published (default: 0). +# CI sets it: a tag exists minutes before the image its own CI pushes. # # Needs: bash, docker, curl, jq, git (matrix), sqlite3 (sqlite engine). @@ -121,6 +123,22 @@ fail() { exit 1 } +# pull_image : pulls it, retrying every 30 s for IMAGE_WAIT seconds. A +# release tag enters the matrix the moment it is pushed, but its image only +# exists once the tag's own CI has built it (about 20 minutes later): a PR whose +# CI starts in between failed every leg from that tag on "not found". +pull_image() { + local image=$1 deadline=$((SECONDS + ${IMAGE_WAIT:-0})) err + until err=$(docker pull -q "$image" 2>&1 >/dev/null); do + if ((SECONDS >= deadline)); then + echo "$err" >&2 + return 1 + fi + echo " $image is not available yet, retrying in 30 s" >&2 + sleep 30 + done +} + # wait_health : succeeds once /api/mgmt/health answers ok. wait_health() { local port=$1 deadline=$((SECONDS + $2)) @@ -222,6 +240,7 @@ fi # ---- Old side ------------------------------------------------------------- step "starting $IMAGE_REPO:${from_tag#v}" +pull_image "$IMAGE_REPO:${from_tag#v}" || fail "could not pull $IMAGE_REPO:${from_tag#v}" old_env_args=() for kv in "${common_env[@]}"; do old_env_args+=(-e "$kv"); done docker run -d --name "$old_name" -p 127.0.0.1::4000 "${old_args[@]}" "${old_env_args[@]}" \ diff --git a/server/internal/app/incident_insights_route_test.go b/server/internal/app/incident_insights_route_test.go new file mode 100644 index 000000000..bb7e0043d --- /dev/null +++ b/server/internal/app/incident_insights_route_test.go @@ -0,0 +1,32 @@ +package app + +import ( + "net/http" + "testing" + + "github.com/stretchr/testify/require" +) + +// TestIncidentInsightsRoute walks the real router: the insights route needs a +// token, and with one it is served by the insights handler rather than by +// the `/:uid` route (which would answer 404 INCIDENT_NOT_FOUND). +func TestIncidentInsightsRoute(t *testing.T) { + t.Parallel() + r := require.New(t) + + env := newMCPTestEnv(t) + path := "/api/v1/orgs/" + mcpE2EOrg + "/incidents/insights" + + anon := env.get(t, path, nil) + r.Equal(http.StatusUnauthorized, anon.status, string(anon.body)) + + authed := env.get(t, path+"?period=7d", map[string]string{"Authorization": "Bearer " + env.login(t)}) + r.Equal(http.StatusOK, authed.status, string(authed.body)) + r.Contains(string(authed.body), `"period":"7d"`) + r.Contains(string(authed.body), `"summary"`) + r.Contains(string(authed.body), `"checks"`) + + bad := env.get(t, path+"?period=1y", map[string]string{"Authorization": "Bearer " + env.login(t)}) + r.Equal(http.StatusBadRequest, bad.status, string(bad.body)) + r.Contains(string(bad.body), `"VALIDATION_ERROR"`) +} diff --git a/server/internal/app/openapi/openapi.yaml b/server/internal/app/openapi/openapi.yaml index a90fed552..1732f16e5 100644 --- a/server/internal/app/openapi/openapi.yaml +++ b/server/internal/app/openapi/openapi.yaml @@ -4045,6 +4045,105 @@ paths: "404": $ref: "#/components/responses/NotFound" + /api/v1/orgs/{org}/incidents/insights: + get: + tags: [Incidents] + summary: Incident insights + description: >- + Aggregates the check outages active during a trailing period (started before + its end, still open or resolved inside it; downtime is clipped to the + period): the checks that fail + the most, the time to acknowledge, the time to resolve and how incidents + were resolved. Medians and nearest-rank p90, never means. Rolled-up + (paging-suppressed) incidents count in the checks table and the resolve + time but not in the acknowledge sample or rate. SLO burn and degraded + incidents are not counted. At most 5000 incidents are read, then + `truncated` is true. + operationId: getIncidentInsights + security: + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/OrgPath" + - name: period + in: query + description: Trailing window, one of 7d, 30d (default), 90d. Anything else is a 400 VALIDATION_ERROR. + schema: + type: string + enum: [7d, 30d, 90d] + default: 30d + responses: + "200": + description: Incident insights + content: + application/json: + schema: + type: object + required: [period, windowStart, windowEnd, truncated, summary, checks] + properties: + period: + type: string + windowStart: + type: string + format: date-time + windowEnd: + type: string + format: date-time + truncated: + type: boolean + summary: + type: object + properties: + incidentCount: + type: integer + pagedCount: + type: integer + description: Incidents that were not rolled up, the base of ackRate. + acknowledgedCount: + type: integer + ackRate: + type: number + description: 0 to 1 + timeToAcknowledge: + $ref: "#/components/schemas/IncidentInsightsDuration" + timeToResolve: + $ref: "#/components/schemas/IncidentInsightsDuration" + byResolution: + type: object + properties: + auto: + $ref: "#/components/schemas/IncidentInsightsResolution" + manual: + $ref: "#/components/schemas/IncidentInsightsResolution" + other: + $ref: "#/components/schemas/IncidentInsightsResolution" + checks: + type: array + description: Top 20 checks by incident count, ties broken by downtime. + items: + type: object + properties: + checkUid: + type: string + checkName: + type: string + checkSlug: + type: string + incidentCount: + type: integer + downtimeSeconds: + type: integer + format: int64 + relapseCount: + type: integer + lastIncidentAt: + type: string + format: date-time + nullable: true + "400": + $ref: "#/components/responses/ValidationError" + "401": + $ref: "#/components/responses/Unauthorized" + /api/v1/orgs/{org}/incidents/{uid}: get: tags: [Incidents] @@ -10668,6 +10767,32 @@ components: $ref: "#/components/schemas/Error" schemas: + IncidentInsightsDuration: + type: object + properties: + medianSeconds: + type: integer + format: int64 + nullable: true + p90Seconds: + type: integer + format: int64 + nullable: true + sampleCount: + type: integer + IncidentInsightsResolution: + type: object + properties: + count: + type: integer + medianSeconds: + type: integer + format: int64 + nullable: true + p90Seconds: + type: integer + format: int64 + nullable: true SupportReplyWindow: type: object description: > diff --git a/server/internal/app/server.go b/server/internal/app/server.go index b573fe47d..352339313 100644 --- a/server/internal/app/server.go +++ b/server/internal/app/server.go @@ -1620,6 +1620,8 @@ func (s *Server) SetupRoutes(ctx context.Context) { incidentsHandler := incidents.NewHandler(incidentsService, s.config) orgIncidents := orgGroup("/orgs/:org/incidents") orgIncidents.GET("", incidentsHandler.ListIncidents) + // Registered before "/:uid" so "insights" is not read as an incident uid. + orgIncidents.GET("/insights", incidentsHandler.GetInsights) orgIncidents.GET("/:uid", incidentsHandler.GetIncident) orgIncidents.POST("/:uid/ack", incidentsHandler.AcknowledgeIncident) orgIncidents.POST("/:uid/unack", incidentsHandler.UnacknowledgeIncident) diff --git a/server/internal/checkers/checkbrowser/checker.go b/server/internal/checkers/checkbrowser/checker.go index 361438f8b..11c31d485 100644 --- a/server/internal/checkers/checkbrowser/checker.go +++ b/server/internal/checkers/checkbrowser/checker.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/chromedp/cdproto" "github.com/chromedp/chromedp" checkconfig "github.com/fclairamb/solidping/server/internal/checkers/checkbrowser/config" @@ -329,8 +330,19 @@ func (c *BrowserChecker) captureScreenshot( shot, err := capture(shotCtx) if err != nil { - slog.WarnContext(ctx, "browser check: screenshot capture failed", - "url", cfg.URL, "error", err, "budget_remaining_ms", budgetRemaining.Milliseconds()) + // A page that closed or crashed before the shot is the monitored + // page's doing, not a CDP/infra failure: logged at INFO so it does not + // read as a worker fault in the pod logs. + detached := pageDetached(err) + + level := slog.LevelWarn + if detached { + level = slog.LevelInfo + } + + slog.Log(ctx, level, "browser check: screenshot capture failed", + "url", cfg.URL, "error", err, "budget_remaining_ms", budgetRemaining.Milliseconds(), + "page_detached", detached) setScreenshotError(result, CaptureErrorMessage(err, budgetRemaining)) return @@ -387,9 +399,31 @@ func CaptureErrorMessage(err error, budget time.Duration) string { return "the capture timed out" } + if pageDetached(err) { + return "the page was closed or crashed before the capture" + } + return err.Error() } +// cdpServerError is the JSON-RPC "server error" code Chrome answers with when +// a command targets a session whose page is gone. +const cdpServerError = -32000 + +// pageDetached reports whether a CDP command failed because its page was no +// longer attached: the tab closed itself, crashed, or the target went away +// before the command ran. Chrome reports it as -32000 "Not attached to an +// active page" (or "Session with given id not found"). +func pageDetached(err error) bool { + var cdpErr *cdproto.Error + if !errors.As(err, &cdpErr) || cdpErr.Code != cdpServerError { + return false + } + + return strings.Contains(cdpErr.Message, "Not attached to an active page") || + strings.Contains(cdpErr.Message, "Session with given id not found") +} + // OverCapMessage is the ScreenshotError of a capture dropped over // MaxScreenshotBytes. Shared with the JS runtime. func OverCapMessage(size int) string { diff --git a/server/internal/checkers/checkbrowser/page_detached_test.go b/server/internal/checkers/checkbrowser/page_detached_test.go new file mode 100644 index 000000000..8a9a85631 --- /dev/null +++ b/server/internal/checkers/checkbrowser/page_detached_test.go @@ -0,0 +1,54 @@ +package checkbrowser + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/chromedp/cdproto" + "github.com/stretchr/testify/require" +) + +// errPlainDetachedText carries the CDP wording without being a CDP error. +var errPlainDetachedText = errors.New("plain: Not attached to an active page") + +// TestCaptureErrorMessageNamesADetachedPage: a capture whose page closed or +// crashed first fails with CDP -32000 "Not attached to an active page". That +// is the monitored page's doing, so the operator reads a plain sentence rather +// than the raw protocol error, and the log is not raised as a worker fault. +func TestCaptureErrorMessageNamesADetachedPage(t *testing.T) { + t.Parallel() + + detached := &cdproto.Error{Code: cdpServerError, Message: "Not attached to an active page"} + gone := &cdproto.Error{Code: cdpServerError, Message: "Session with given id not found"} + other := &cdproto.Error{Code: cdpServerError, Message: "Internal error"} + + cases := []struct { + name string + err error + detached bool + message string + }{ + {"detached", detached, true, "the page was closed or crashed before the capture"}, + {"wrapped", fmt.Errorf("capture: %w", detached), true, "the page was closed or crashed before the capture"}, + {"session gone", gone, true, "the page was closed or crashed before the capture"}, + {"other cdp error", other, false, other.Error()}, + {"plain error", errRendererGone, false, errRendererGone.Error()}, + {"deadline", context.DeadlineExceeded, false, "the capture timed out"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + r := require.New(t) + + r.Equal(tc.detached, pageDetached(tc.err)) + r.Equal(tc.message, CaptureErrorMessage(tc.err, 0)) + }) + } + + r := require.New(t) + r.False(pageDetached(nil)) + r.False(pageDetached(errPlainDetachedText)) +} diff --git a/server/internal/checkers/checksftp/checker.go b/server/internal/checkers/checksftp/checker.go index a3502623c..442c625eb 100644 --- a/server/internal/checkers/checksftp/checker.go +++ b/server/internal/checkers/checksftp/checker.go @@ -125,6 +125,13 @@ func (c *SFTPChecker) Execute(ctx context.Context, config checkerdef.Config) (*c }, nil } + // ssh.ClientConfig.Timeout only bounds a dial ssh does itself, and the + // handshake, the SFTP session and Stat read this conn with no deadline: a + // server that stalls after accepting would outlive the worker's deadline. + // Closing the conn when the context ends unblocks every one of them. + stopWatch := context.AfterFunc(ctx, func() { _ = netConn.Close() }) + defer stopWatch() + sshConn, chans, reqs, err := ssh.NewClientConn(netConn, target, sshConfig) if err != nil { _ = netConn.Close() @@ -164,9 +171,33 @@ func (c *SFTPChecker) Execute(ctx context.Context, config checkerdef.Config) (*c connectTime := time.Since(connectStart) metrics["connection_time_ms"] = float64(connectTime.Microseconds()) / microsecondsPerMs + if failed := probeSFTP(ctx, sshClient, cfg.Path, start, metrics, output); failed != nil { + return failed, nil + } + + metrics["total_time_ms"] = float64(time.Since(start).Microseconds()) / microsecondsPerMs + + return &checkerdef.Result{ + Status: checkerdef.StatusUp, + Duration: time.Since(start), + Metrics: metrics, + Output: output, + }, nil +} + +// probeSFTP opens the SFTP session and, when path is set, stats it. It returns +// the failing result, or nil when both succeeded. +func probeSFTP( + ctx context.Context, sshClient *ssh.Client, path string, + start time.Time, metrics, output map[string]any, +) *checkerdef.Result { // Create SFTP client sftpClient, err := sftp.NewClient(sshClient) if err != nil { + if ctx.Err() != nil { + return timeoutResult(start, metrics, output, "SFTP session timeout") + } + return &checkerdef.Result{ Status: checkerdef.StatusDown, Duration: time.Since(start), @@ -174,15 +205,19 @@ func (c *SFTPChecker) Execute(ctx context.Context, config checkerdef.Config) (*c Output: mergeOutput(output, map[string]any{ checkerdef.OutputKeyError: fmt.Sprintf("SFTP session failed: %v", err), }), - }, nil + } } defer func() { _ = sftpClient.Close() }() // Verify path if specified - if cfg.Path != "" { - _, err = sftpClient.Stat(cfg.Path) + if path != "" { + _, err = sftpClient.Stat(path) if err != nil { + if ctx.Err() != nil { + return timeoutResult(start, metrics, output, "path verification timeout") + } + return &checkerdef.Result{ Status: checkerdef.StatusDown, Duration: time.Since(start), @@ -190,18 +225,11 @@ func (c *SFTPChecker) Execute(ctx context.Context, config checkerdef.Config) (*c Output: mergeOutput(output, map[string]any{ checkerdef.OutputKeyError: fmt.Sprintf("path verification failed: %v", err), }), - }, nil + } } } - metrics["total_time_ms"] = float64(time.Since(start).Microseconds()) / microsecondsPerMs - - return &checkerdef.Result{ - Status: checkerdef.StatusUp, - Duration: time.Since(start), - Metrics: metrics, - Output: output, - }, nil + return nil } // hostKeyCallback builds the SSH host key callback for an SFTP check. @@ -225,6 +253,15 @@ func hostKeyCallback(pin string, output map[string]any, observed *string) ssh.Ho } } +func timeoutResult(start time.Time, metrics, output map[string]any, msg string) *checkerdef.Result { + return &checkerdef.Result{ + Status: checkerdef.StatusTimeout, + Duration: time.Since(start), + Metrics: metrics, + Output: mergeOutput(output, map[string]any{checkerdef.OutputKeyError: msg}), + } +} + func mergeOutput(base, extra map[string]any) map[string]any { result := make(map[string]any, len(base)+len(extra)) for k, v := range base { diff --git a/server/internal/checkers/checksftp/context_test.go b/server/internal/checkers/checksftp/context_test.go new file mode 100644 index 000000000..b976053ef --- /dev/null +++ b/server/internal/checkers/checksftp/context_test.go @@ -0,0 +1,72 @@ +package checksftp_test + +import ( + "context" + "net" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/fclairamb/solidping/server/internal/checkers/checkerdef" + "github.com/fclairamb/solidping/server/internal/checkers/checksftp" +) + +// TestExecute_HonorsContextOnAStalledServer: a server that accepts the TCP +// connection and then never speaks used to hold the checker in the SSH +// handshake until the OS gave up, past the worker's deadline, so the worker +// abandoned it ("did not honor context"). Canceling the context must now cut +// the connection and return a timeout promptly. +func TestExecute_HonorsContextOnAStalledServer(t *testing.T) { + t.Parallel() + r := require.New(t) + + listener, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + r.NoError(err) + + t.Cleanup(func() { _ = listener.Close() }) + + go func() { + for { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + + t.Cleanup(func() { _ = conn.Close() }) // held open, silent + } + }() + + host, portStr, err := net.SplitHostPort(listener.Addr().String()) + r.NoError(err) + + port, err := strconv.Atoi(portStr) + r.NoError(err) + + cfg := &checksftp.SFTPConfig{ + Host: host, + Port: port, + Timeout: time.Minute, // the context, not the config, must stop it + Username: "tester", + Password: "s3cret", + } + + ctx, cancel := context.WithTimeout(t.Context(), 300*time.Millisecond) + defer cancel() + + done := make(chan *checkerdef.Result, 1) + + go func() { + result, _ := (&checksftp.SFTPChecker{}).Execute(ctx, cfg) + done <- result + }() + + select { + case result := <-done: + r.NotNil(result) + r.Equal(checkerdef.StatusTimeout, result.Status, "output: %#v", result.Output) + case <-time.After(5 * time.Second): + r.Fail("Execute ignored its context on a stalled SSH handshake") + } +} diff --git a/server/internal/db/incident_active_since_test.go b/server/internal/db/incident_active_since_test.go new file mode 100644 index 000000000..e9e51edd9 --- /dev/null +++ b/server/internal/db/incident_active_since_test.go @@ -0,0 +1,98 @@ +package db_test + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/fclairamb/solidping/server/internal/db" + "github.com/fclairamb/solidping/server/internal/db/models" + "github.com/fclairamb/solidping/server/internal/db/postgres" + "github.com/fclairamb/solidping/server/internal/db/sqlite" + "github.com/fclairamb/solidping/server/internal/testsupport" +) + +func TestListIncidentsActiveSince_SQLite(t *testing.T) { + t.Parallel() + + tempDir, err := os.MkdirTemp("", "sqlite-active-since-*") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(tempDir) }) + + svc, err := sqlite.New(t.Context(), sqlite.Config{DataDir: tempDir}) + require.NoError(t, err) + t.Cleanup(func() { _ = svc.Close() }) + require.NoError(t, svc.Initialize(t.Context())) + + testListIncidentsActiveSince(t, svc) +} + +func TestListIncidentsActiveSince_Postgres(t *testing.T) { + t.Parallel() + + if testing.Short() { + t.Skip("skipping embedded-postgres test in -short mode") + } + + ctx := t.Context() + + svc, err := postgres.New(ctx, &postgres.Config{Embedded: true, Port: testsupport.FreePort(t), RunMode: "test"}) + if err != nil { + testsupport.PostgresUnavailable(t, err) + } + + t.Cleanup(func() { _ = svc.Close() }) + + if initErr := svc.Initialize(ctx); initErr != nil { + testsupport.PostgresInitFailed(t, initErr) + } + + testListIncidentsActiveSince(t, svc) +} + +// testListIncidentsActiveSince pins ActiveSince: open incidents and incidents +// resolved at/after the bound are kept, earlier-resolved ones are not. +func testListIncidentsActiveSince(t *testing.T, svc db.Service) { + t.Helper() + + ctx := t.Context() + r := require.New(t) + + org := models.NewOrganization("active-since-org", "") + r.NoError(svc.CreateOrganization(ctx, org)) + check := models.NewCheck(org.UID, "active-since-check", "http") + r.NoError(svc.CreateCheck(ctx, check)) + + now := time.Now().UTC() + bound := now.Add(-24 * time.Hour) + + mk := func(started time.Time, resolved *time.Time) string { + inc := models.NewIncident(org.UID, check.UID, started, "x") + inc.ResolvedAt = resolved + r.NoError(svc.CreateIncident(ctx, inc)) + + return inc.UID + } + + before := bound.Add(-time.Hour) + inside := bound.Add(time.Hour) + stillOpen := mk(bound.Add(-48*time.Hour), nil) + resolvedInside := mk(bound.Add(-48*time.Hour), &inside) + resolvedBefore := mk(bound.Add(-48*time.Hour), &before) + + list, _, err := svc.ListIncidents(ctx, &models.ListIncidentsFilter{ + OrganizationUID: org.UID, ActiveSince: &bound, Limit: 10, + }) + r.NoError(err) + + got := map[string]bool{} + for _, inc := range list { + got[inc.UID] = true + } + + r.True(got[stillOpen], "open incident is active") + r.True(got[resolvedInside], "incident resolved after the bound is active") + r.False(got[resolvedBefore], "incident resolved before the bound is not") +} diff --git a/server/internal/db/models/incident.go b/server/internal/db/models/incident.go index 81d83a9cc..e590345af 100644 --- a/server/internal/db/models/incident.go +++ b/server/internal/db/models/incident.go @@ -224,11 +224,15 @@ type ListIncidentsFilter struct { // check outages. Callers that compute DOWNTIME from incidents (availability, // SLO status, uptime reports) pass IncidentKindCheck, or a burn alert would // count as downtime against the very objective that produced it. - Kinds []string - Since *time.Time // Optional: incidents started after this time - Until *time.Time // Optional: incidents started before this time - HideSuppressed bool // Optional: hide rolled-up (paging-suppressed) incidents - CausedByUID string // Optional: only incidents whose caused_by_incident_uid equals this + Kinds []string + Since *time.Time // Optional: incidents started after this time + Until *time.Time // Optional: incidents started before this time + // ActiveSince keeps incidents still open or resolved at/after this instant + // (resolved_at IS NULL OR resolved_at >= value). With Until it selects the + // incidents overlapping a window. + ActiveSince *time.Time + HideSuppressed bool // Optional: hide rolled-up (paging-suppressed) incidents + CausedByUID string // Optional: only incidents whose caused_by_incident_uid equals this // AckedOnly returns only incidents with non-NULL acknowledged_at (and // expired snoozes, since snoozed-but-expired is just acked again). Set by // the handler when ?state=acked is requested. diff --git a/server/internal/db/postgres/postgres.go b/server/internal/db/postgres/postgres.go index 27662c1a3..8781e46c0 100644 --- a/server/internal/db/postgres/postgres.go +++ b/server/internal/db/postgres/postgres.go @@ -4147,6 +4147,10 @@ func applyIncidentsFilter(query *bun.SelectQuery, filter *models.ListIncidentsFi query = query.Where("started_at < ?", *filter.Until) } + if filter.ActiveSince != nil { + query = query.Where("(resolved_at IS NULL OR resolved_at >= ?)", *filter.ActiveSince) + } + if filter.HideSuppressed { query = query.Where("paging_suppressed = ?", false) } diff --git a/server/internal/db/sqlite/sqlite.go b/server/internal/db/sqlite/sqlite.go index ec49c7bc9..68fe73f7f 100644 --- a/server/internal/db/sqlite/sqlite.go +++ b/server/internal/db/sqlite/sqlite.go @@ -4107,6 +4107,10 @@ func applyIncidentsFilter(query *bun.SelectQuery, filter *models.ListIncidentsFi query = query.Where("started_at < ?", *filter.Until) } + if filter.ActiveSince != nil { + query = query.Where("(resolved_at IS NULL OR resolved_at >= ?)", *filter.ActiveSince) + } + if filter.HideSuppressed { query = query.Where("paging_suppressed = ?", false) } diff --git a/server/internal/email/templates/uptime-report.html b/server/internal/email/templates/uptime-report.html index e68a69a05..ab547ac1a 100644 --- a/server/internal/email/templates/uptime-report.html +++ b/server/internal/email/templates/uptime-report.html @@ -35,6 +35,17 @@

{{t "report.title"}}

{{t "report.labels.incidents"}}{{.IncidentCount}}{{if .LongestIncident}} · {{t "report.incidentStats.longest" "duration" .LongestIncident}}{{end}}{{if .AverageIncident}} · {{t "report.incidentStats.average" "duration" .AverageIncident}}{{end}}{{if .ShowIncidentDelta}}
{{.IncidentDeltaText}} {{t "report.versus" "period" .PreviousPeriodLabel "previous" .PreviousIncidentCount}}{{end}} {{if .TotalDowntime}}{{t "report.labels.totalDowntime"}}{{.TotalDowntime}}{{end}} + {{if .ShowInsights}} +

{{if .MedianTimeToResolve}}{{t "report.insights.resolve" "duration" .MedianTimeToResolve}} · {{end}}{{if .MedianTimeToAcknowledge}}{{t "report.insights.acknowledge" "duration" .MedianTimeToAcknowledge}} · {{end}}{{t "report.insights.acknowledged" "count" .AcknowledgedCount "total" .PagedCount}}

+ {{if .TopFailingChecks}} +

{{t "report.insights.title"}}

+ + {{range .TopFailingChecks}} + + {{end}} + + {{end}} + {{end}} {{if .HasLatency}}

{{t "report.latency.title"}}

@@ -105,7 +116,14 @@

{{t "report.slo.title"}}

{{end}}{{t "common.textField" "label" (t "report.labels.checksMonitored") "value" .CheckCount}} {{t "common.textField" "label" (t "report.labels.incidents") "value" .IncidentCount}}{{if .LongestIncident}} ({{t "report.incidentStats.longest" "duration" .LongestIncident}}{{if .AverageIncident}}, {{t "report.incidentStats.average" "duration" .AverageIncident}}{{end}}){{end}}{{if .ShowIncidentDelta}} ({{.IncidentDeltaText}} {{t "report.text.versus" "period" .PreviousPeriodLabel "previous" .PreviousIncidentCount}}){{end}} {{if .TotalDowntime}}{{t "common.textField" "label" (t "report.labels.totalDowntime") "value" .TotalDowntime}} -{{end}}{{if .HasLatency}} +{{end}}{{if .ShowInsights}}{{if .MedianTimeToResolve}}{{t "report.insights.resolve" "duration" .MedianTimeToResolve}} +{{end}}{{if .MedianTimeToAcknowledge}}{{t "report.insights.acknowledge" "duration" .MedianTimeToAcknowledge}} +{{end}}{{t "report.insights.acknowledged" "count" .AcknowledgedCount "total" .PagedCount}} +{{if .TopFailingChecks}} +{{t "report.insights.title"}} +{{range .TopFailingChecks}} - {{t "common.textField" "label" .Name "value" .Detail}} +{{if .URL}} {{.URL}} +{{end}}{{end}}{{end}}{{end}}{{if .HasLatency}} {{t "report.text.latencyHeading"}} - {{t "common.textField" "label" (t "report.latency.average") "value" .AvgResponseTime}}{{if .ShowResponseDelta}} ({{.ResponseDeltaText}} {{t "report.text.versus" "period" .PreviousPeriodLabel "previous" .PreviousAvgResponseTime}}){{end}} {{if .MinResponseTime}} - {{t "common.textField" "label" (t "report.latency.range") "value" (printf "%s / %s" .MinResponseTime .MaxResponseTime)}} diff --git a/server/internal/handlers/emailpreview/fixtures_report.go b/server/internal/handlers/emailpreview/fixtures_report.go index 0929375ff..a179a7c91 100644 --- a/server/internal/handlers/emailpreview/fixtures_report.go +++ b/server/internal/handlers/emailpreview/fixtures_report.go @@ -44,6 +44,8 @@ func uptimeReportFixture(loc i18n.Locale) map[string]any { objective := report.SLOs[0] + topFailing := topFailingMaps(report.TopFailingChecks) + return map[string]any{ keyOrgName: report.OrgName, "PeriodLabel": report.PeriodLabel, @@ -57,6 +59,13 @@ func uptimeReportFixture(loc i18n.Locale) map[string]any { "AverageIncident": report.AverageIncident, "TotalDowntime": report.TotalDowntime, + "ShowInsights": report.ShowInsights, + "MedianTimeToResolve": report.MedianTimeToResolve, + "MedianTimeToAcknowledge": report.MedianTimeToAcknowledge, + "AcknowledgedCount": report.AcknowledgedCount, + "PagedCount": report.PagedCount, + "TopFailingChecks": topFailing, + // Period-over-period trend. Present in the fixture so the preview // exercises the delta spans rather than only their absence. "PreviousPeriodLabel": report.PreviousPeriodLabel, @@ -110,7 +119,6 @@ func uptimeReportPreview() *uptimereport.Data { year := time.Now().Year() july := time.Date(year, time.July, 1, 0, 0, 0, 0, paris) june := july.AddDate(0, -1, 0) - attainment := 99.95 return &uptimereport.Data{ OrgName: fixtureOrgName, @@ -120,6 +128,11 @@ func uptimeReportPreview() *uptimereport.Data { CheckCount: 2, IncidentCount: 3, + ShowInsights: true, + AcknowledgedCount: 2, + PagedCount: 3, + TopFailingChecks: previewTopFailing(), + HasPreviousData: true, PreviousIncidentCount: 5, ShowAvailabilityDelta: true, @@ -145,14 +158,7 @@ func uptimeReportPreview() *uptimereport.Data { Days: []uptimereport.DayCell{{Color: "#d1d5db", Span: 31, Wide: true}}, }, }, - SLOs: []uptimereport.SLORow{{ - Name: "API availability", - HasData: true, - State: slo.StateHealthy, - Attainment: &attainment, - Target: 99.9, - Budget: 21*time.Minute + 30*time.Second, - }}, + SLOs: previewSLOs(), DashboardURL: fixtureDashboardURL, Figures: uptimereport.Figures{ @@ -167,15 +173,56 @@ func uptimeReportPreview() *uptimereport.Data { LongestIncident: 42 * time.Minute, AverageIncident: 21*time.Minute + 40*time.Second, TotalDowntime: time.Hour + 5*time.Minute, - AvgResponse: 171, - MinResponse: 42, - MaxResponse: 3200, - HasResponseRange: true, - PreviousAvgResponse: 158, - SlowSamples: 4, - SlowPeaks: 2, - StripStart: time.Date(year, time.July, 1, 0, 0, 0, 0, time.UTC), - StripDays: 31, + + MedianTimeToResolve: 18 * time.Minute, + MedianTimeToAcknowledge: 4 * time.Minute, + + AvgResponse: 171, + MinResponse: 42, + MaxResponse: 3200, + HasResponseRange: true, + PreviousAvgResponse: 158, + SlowSamples: 4, + SlowPeaks: 2, + StripStart: time.Date(year, time.July, 1, 0, 0, 0, 0, time.UTC), + StripDays: 31, + }, + } +} + +// previewTopFailing is the "Most incidents" block of the preview. +func previewTopFailing() []uptimereport.FailingCheckRow { + return []uptimereport.FailingCheckRow{ + { + Name: fixtureCheckName, IncidentCount: 2, RelapseCount: 1, + URL: "https://solidping.example/d/orgs/acme/checks/chk-prod-api", }, + { + Name: "Marketing site", IncidentCount: 1, + URL: "https://solidping.example/d/orgs/acme/checks/chk-marketing", + }, + } +} + +func topFailingMaps(rows []uptimereport.FailingCheckRow) []map[string]any { + out := make([]map[string]any, 0, len(rows)) + + for i := range rows { + out = append(out, map[string]any{keyName: rows[i].Name, "URL": rows[i].URL, "Detail": rows[i].Detail}) } + + return out +} + +func previewSLOs() []uptimereport.SLORow { + attainment := 99.95 + + return []uptimereport.SLORow{{ + Name: "API availability", + HasData: true, + State: slo.StateHealthy, + Attainment: &attainment, + Target: 99.9, + Budget: 21*time.Minute + 30*time.Second, + }} } diff --git a/server/internal/handlers/incidents/export_test.go b/server/internal/handlers/incidents/export_test.go index 67e567349..538729291 100644 --- a/server/internal/handlers/incidents/export_test.go +++ b/server/internal/handlers/incidents/export_test.go @@ -75,3 +75,9 @@ func (s *Service) RollUpExistingChildrenForTest( ) { s.rollUpExistingChildren(ctx, parentCheck, parent, onset) } + +// SetInsightsLimitsForTest shrinks the insights page size and load cap so +// pagination and truncation can be exercised without thousands of rows. +func (s *Service) SetInsightsLimitsForTest(pageSize, maxLoaded int) { + s.insightsPageSize, s.insightsMaxLoaded = pageSize, maxLoaded +} diff --git a/server/internal/handlers/incidents/insights.go b/server/internal/handlers/incidents/insights.go new file mode 100644 index 000000000..aeed49b10 --- /dev/null +++ b/server/internal/handlers/incidents/insights.go @@ -0,0 +1,189 @@ +package incidents + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/fclairamb/solidping/server/internal/db/models" + "github.com/fclairamb/solidping/server/internal/handlers/base" + "github.com/fclairamb/solidping/server/internal/httpx" + "github.com/fclairamb/solidping/server/internal/incidentinsights" +) + +const ( + defaultInsightsPageSize = 500 + defaultInsightsMaxLoaded = 5000 + insightsTopChecks = 20 + insightsDefaultDays = 30 +) + +var errInvalidInsightsPeriod = errors.New("invalid period: must be one of 7d, 30d, 90d") + +// InsightsCheckRow is one row of the "top failing checks" table. +type InsightsCheckRow struct { + incidentinsights.CheckRow + CheckName string `json:"checkName"` + CheckSlug string `json:"checkSlug"` +} + +// InsightsResponse is the body of GET /orgs/:org/incidents/insights. +type InsightsResponse struct { + Period string `json:"period"` + WindowStart time.Time `json:"windowStart"` + WindowEnd time.Time `json:"windowEnd"` + Truncated bool `json:"truncated"` + Summary incidentinsights.Summary `json:"summary"` + Checks []InsightsCheckRow `json:"checks"` +} + +func parseInsightsPeriod(raw string) (string, int, error) { + switch raw { + case "", "30d": + return "30d", insightsDefaultDays, nil + case "7d": + return "7d", 7, nil + case "90d": + return "90d", 90, nil + default: + return "", 0, errInvalidInsightsPeriod + } +} + +// GetInsights handles GET /api/v1/orgs/:org/incidents/insights. +func (h *Handler) GetInsights(writer http.ResponseWriter, req *http.Request) error { + period, days, err := parseInsightsPeriod(req.URL.Query().Get("period")) + if err != nil { + return h.WriteError(writer, http.StatusBadRequest, base.ErrorCodeValidationError, err.Error()) + } + + response, err := h.svc.GetInsights(req.Context(), httpx.Param(req, "org"), period, days) + if err != nil { + return h.handleError(writer, req, err) + } + + return h.WriteJSON(writer, http.StatusOK, response) +} + +// GetInsights aggregates the check outages of the trailing window. +func (s *Service) GetInsights(ctx context.Context, orgSlug, period string, days int) (*InsightsResponse, error) { + org, err := s.db.GetOrganizationBySlug(ctx, orgSlug) + if err != nil { + return nil, ErrOrganizationNotFound + } + + now := s.clock.Now() + windowStart := now.AddDate(0, 0, -days) + + incidents, truncated, err := s.loadInsightsIncidents(ctx, org.UID, windowStart, now) + if err != nil { + return nil, err + } + + computed := incidentinsights.Compute(incidents, windowStart, now, now) + + rows := computed.Checks + if len(rows) > insightsTopChecks { + rows = rows[:insightsTopChecks] + } + + response := &InsightsResponse{ + Period: period, + WindowStart: windowStart, + WindowEnd: now, + Truncated: truncated, + Summary: computed.Summary, + Checks: make([]InsightsCheckRow, 0, len(rows)), + } + + checkUIDs := make([]string, 0, len(rows)) + for idx := range rows { + checkUIDs = append(checkUIDs, rows[idx].CheckUID) + } + + checks := map[string]*models.Check{} + if len(checkUIDs) > 0 { + checks, err = s.db.GetChecksByUIDs(ctx, org.UID, checkUIDs) + if err != nil { + return nil, fmt.Errorf("failed to get checks: %w", err) + } + } + + for idx := range rows { + row := &rows[idx] + out := InsightsCheckRow{CheckRow: *row} + + if check := checks[row.CheckUID]; check != nil { + if check.Name != nil { + out.CheckName = *check.Name + } + + if check.Slug != nil { + out.CheckSlug = *check.Slug + } + } + + response.Checks = append(response.Checks, out) + } + + return response, nil +} + +// loadInsightsIncidents pages through the check outages active during the +// window (started before its end, still open or resolved inside it), stopping +// at the load cap (truncated is then true). +func (s *Service) loadInsightsIncidents( + ctx context.Context, orgUID string, since, until time.Time, +) ([]*models.Incident, bool, error) { + pageSize, maxLoaded := s.insightsPageSize, s.insightsMaxLoaded + if pageSize <= 0 { + pageSize = defaultInsightsPageSize + } + + if maxLoaded <= 0 { + maxLoaded = defaultInsightsMaxLoaded + } + + var all []*models.Incident + + var ( + cursorTS *time.Time + cursorUID *string + ) + + for { + page, _, err := s.db.ListIncidents(ctx, &models.ListIncidentsFilter{ + OrganizationUID: orgUID, + Kinds: []string{models.IncidentKindCheck}, + ActiveSince: &since, + Until: &until, + CursorTimestamp: cursorTS, + CursorUID: cursorUID, + Limit: pageSize + 1, + }) + if err != nil { + return nil, false, fmt.Errorf("failed to list incidents: %w", err) + } + + hasMore := len(page) > pageSize + if hasMore { + page = page[:pageSize] + } + + all = append(all, page...) + + if !hasMore { + return all, false, nil + } + + if len(all) >= maxLoaded { + return all[:maxLoaded], true, nil + } + + last := page[len(page)-1] + ts, uid := last.StartedAt, last.UID + cursorTS, cursorUID = &ts, &uid + } +} diff --git a/server/internal/handlers/incidents/insights_test.go b/server/internal/handlers/incidents/insights_test.go new file mode 100644 index 000000000..e13ae92f3 --- /dev/null +++ b/server/internal/handlers/incidents/insights_test.go @@ -0,0 +1,277 @@ +package incidents_test + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/fclairamb/solidping/server/internal/config" + "github.com/fclairamb/solidping/server/internal/db" + "github.com/fclairamb/solidping/server/internal/db/models" + "github.com/fclairamb/solidping/server/internal/db/sqlite" + "github.com/fclairamb/solidping/server/internal/handlers/incidents" + "github.com/fclairamb/solidping/server/internal/httpx" + "github.com/fclairamb/solidping/server/internal/jobs/jobsvc" + "github.com/fclairamb/solidping/server/internal/notifier" + "github.com/fclairamb/solidping/server/internal/utils/clock" +) + +func newInsightsRouter(dbSvc *sqlite.Service) *httpx.Router { + jobs := jobsvc.NewService(dbSvc.DB(), dbSvc, notifier.NewLocalEventNotifier(), nil) + svc := incidents.NewService(dbSvc, jobs, clock.Real{}, nil) + handler := incidents.NewHandler(svc, &config.Config{}) + + router := httpx.New() + router.GET("/orgs/:org/incidents/insights", handler.GetInsights) + + return router +} + +func fetchInsights( + t *testing.T, router *httpx.Router, org, query string, +) (*httptest.ResponseRecorder, incidents.InsightsResponse) { + t.Helper() + + req := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/orgs/"+org+"/incidents/insights"+query, nil) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + + var out incidents.InsightsResponse + if rec.Code == http.StatusOK { + require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &out)) + } + + return rec, out +} + +func TestGetInsights(t *testing.T) { + t.Parallel() + + ctx := t.Context() + r := require.New(t) + + dbSvc, err := sqlite.New(ctx, sqlite.Config{InMemory: true}) + r.NoError(err) + t.Cleanup(func() { _ = dbSvc.Close() }) + r.NoError(dbSvc.Initialize(ctx)) + + org := models.NewOrganization("insights-org", "") + r.NoError(dbSvc.CreateOrganization(ctx, org)) + other := models.NewOrganization("insights-other", "") + r.NoError(dbSvc.CreateOrganization(ctx, other)) + empty := models.NewOrganization("insights-empty", "") + r.NoError(dbSvc.CreateOrganization(ctx, empty)) + + check := models.NewCheck(org.UID, "acme-api", "http") + r.NoError(dbSvc.CreateCheck(ctx, check)) + otherCheck := models.NewCheck(other.UID, "acme-other", "http") + r.NoError(dbSvc.CreateCheck(ctx, otherCheck)) + + now := time.Now() + seed := func(orgUID, checkUID, kind string, mod func(*models.Incident)) { + inc := models.NewIncident(orgUID, checkUID, now.Add(-2*time.Hour), "incident") + inc.Kind = kind + if mod != nil { + mod(inc) + } + r.NoError(dbSvc.CreateIncident(ctx, inc)) + } + + ackedAt := now.Add(-2*time.Hour + 5*time.Minute) + seed(org.UID, check.UID, models.IncidentKindCheck, func(i *models.Incident) { i.AcknowledgedAt = &ackedAt }) + // Rolled up: counted in the checks table, left out of the ack rate. + seed(org.UID, check.UID, models.IncidentKindCheck, func(i *models.Incident) { i.PagingSuppressed = true }) + // Not counted: another kind, and another org. + seed(org.UID, check.UID, models.IncidentKindSLOBurn, nil) + seed(other.UID, otherCheck.UID, models.IncidentKindCheck, nil) + + router := newInsightsRouter(dbSvc) + + t.Run("counts check outages of this org only", func(t *testing.T) { + t.Parallel() + r := require.New(t) + + rec, got := fetchInsights(t, router, org.Slug, "") + r.Equal(http.StatusOK, rec.Code, rec.Body.String()) + r.Equal("30d", got.Period) + r.Equal(2, got.Summary.IncidentCount) + r.Len(got.Checks, 1) + r.Equal(2, got.Checks[0].IncidentCount) + r.Equal("acme-api", got.Checks[0].CheckSlug) + r.False(got.Truncated) + }) + + t.Run("rolled-up incident is not in the ack rate", func(t *testing.T) { + t.Parallel() + r := require.New(t) + + _, got := fetchInsights(t, router, org.Slug, "?period=7d") + r.Equal(1, got.Summary.PagedCount) + r.Equal(1, got.Summary.AcknowledgedCount) + r.InDelta(1.0, got.Summary.AckRate, 0) + r.Equal(int64(300), *got.Summary.TimeToAcknowledge.MedianSeconds) + }) + + t.Run("period validation", func(t *testing.T) { + t.Parallel() + r := require.New(t) + + rec, _ := fetchInsights(t, router, org.Slug, "?period=1y") + r.Equal(http.StatusBadRequest, rec.Code) + + var body struct { + Code string `json:"code"` + } + r.NoError(json.Unmarshal(rec.Body.Bytes(), &body)) + r.Equal("VALIDATION_ERROR", body.Code) + }) + + t.Run("empty org", func(t *testing.T) { + t.Parallel() + r := require.New(t) + + rec, got := fetchInsights(t, router, empty.Slug, "?period=90d") + r.Equal(http.StatusOK, rec.Code) + r.Zero(got.Summary.IncidentCount) + r.Empty(got.Checks) + r.Nil(got.Summary.TimeToAcknowledge.MedianSeconds) + }) +} + +func newInsightsDB(t *testing.T) *sqlite.Service { + t.Helper() + + dbSvc, err := sqlite.New(t.Context(), sqlite.Config{InMemory: true}) + require.NoError(t, err) + t.Cleanup(func() { _ = dbSvc.Close() }) + require.NoError(t, dbSvc.Initialize(t.Context())) + + return dbSvc +} + +func newInsightsRouterFor(dbSvc db.Service, limits ...int) *httpx.Router { + svc := incidents.NewService(dbSvc, nil, clock.Real{}, nil) + if len(limits) == 2 { + svc.SetInsightsLimitsForTest(limits[0], limits[1]) + } + + router := httpx.New() + router.GET("/orgs/:org/incidents/insights", incidents.NewHandler(svc, &config.Config{}).GetInsights) + + return router +} + +// TestGetInsightsWindowOverlap: incidents active during the period count even +// when they started before it; one resolved before the window start does not. +func TestGetInsightsWindowOverlap(t *testing.T) { + t.Parallel() + + ctx := t.Context() + r := require.New(t) + dbSvc := newInsightsDB(t) + + org := models.NewOrganization("overlap-org", "") + r.NoError(dbSvc.CreateOrganization(ctx, org)) + + openCheck := models.NewCheck(org.UID, "still-open", "http") + r.NoError(dbSvc.CreateCheck(ctx, openCheck)) + oldCheck := models.NewCheck(org.UID, "long-resolved", "http") + r.NoError(dbSvc.CreateCheck(ctx, oldCheck)) + crossCheck := models.NewCheck(org.UID, "resolved-inside", "http") + r.NoError(dbSvc.CreateCheck(ctx, crossCheck)) + + now := time.Now() + started := now.Add(-20 * 24 * time.Hour) // before the 7d window + + r.NoError(dbSvc.CreateIncident(ctx, models.NewIncident(org.UID, openCheck.UID, started, "x"))) + + oldResolvedAt := now.Add(-10 * 24 * time.Hour) + old := models.NewIncident(org.UID, oldCheck.UID, started, "x") + old.ResolvedAt = &oldResolvedAt + r.NoError(dbSvc.CreateIncident(ctx, old)) + + crossResolvedAt := now.Add(-3 * 24 * time.Hour) + cross := models.NewIncident(org.UID, crossCheck.UID, started, "x") + cross.ResolvedAt = &crossResolvedAt + r.NoError(dbSvc.CreateIncident(ctx, cross)) + + rec, got := fetchInsights(t, newInsightsRouterFor(dbSvc), org.Slug, "?period=7d") + r.Equal(http.StatusOK, rec.Code, rec.Body.String()) + r.Equal(2, got.Summary.IncidentCount, "open and resolved-inside count, resolved-before does not") + + slugs := map[string]bool{} + for _, row := range got.Checks { + slugs[row.CheckSlug] = true + } + + r.True(slugs["still-open"], "positive control: an open outage from before the window") + r.True(slugs["resolved-inside"]) + r.False(slugs["long-resolved"], "resolved before the window start") +} + +// TestGetInsightsPaginationAndTruncation uses a tiny page size and cap. +func TestGetInsightsPaginationAndTruncation(t *testing.T) { + t.Parallel() + + ctx := t.Context() + r := require.New(t) + dbSvc := newInsightsDB(t) + + org := models.NewOrganization("paging-org", "") + r.NoError(dbSvc.CreateOrganization(ctx, org)) + check := models.NewCheck(org.UID, "paged", "http") + r.NoError(dbSvc.CreateCheck(ctx, check)) + + for idx := range 7 { + inc := models.NewIncident(org.UID, check.UID, time.Now().Add(-time.Duration(idx+1)*time.Hour), "x") + r.NoError(dbSvc.CreateIncident(ctx, inc)) + } + + // 7 incidents over pages of 3: every page is followed, nothing truncated. + rec, got := fetchInsights(t, newInsightsRouterFor(dbSvc, 3, 100), org.Slug, "?period=7d") + r.Equal(http.StatusOK, rec.Code, rec.Body.String()) + r.Equal(7, got.Summary.IncidentCount) + r.False(got.Truncated) + + // A cap of 5 over pages of 3: truncated at exactly 5. + rec, got = fetchInsights(t, newInsightsRouterFor(dbSvc, 3, 5), org.Slug, "?period=7d") + r.Equal(http.StatusOK, rec.Code, rec.Body.String()) + r.Equal(5, got.Summary.IncidentCount) + r.True(got.Truncated) +} + +var errChecksLookup = errors.New("checks lookup failed") + +type failingChecksDB struct { + db.Service +} + +func (failingChecksDB) GetChecksByUIDs(context.Context, string, []string) (map[string]*models.Check, error) { + return nil, errChecksLookup +} + +// TestGetInsightsChecksLookupError: a failed check lookup is a 500, not a +// silently nameless table. +func TestGetInsightsChecksLookupError(t *testing.T) { + t.Parallel() + + ctx := t.Context() + r := require.New(t) + dbSvc := newInsightsDB(t) + + org := models.NewOrganization("lookup-err-org", "") + r.NoError(dbSvc.CreateOrganization(ctx, org)) + check := models.NewCheck(org.UID, "chk-lookup", "http") + r.NoError(dbSvc.CreateCheck(ctx, check)) + r.NoError(dbSvc.CreateIncident(ctx, models.NewIncident(org.UID, check.UID, time.Now().Add(-time.Hour), "x"))) + + rec, _ := fetchInsights(t, newInsightsRouterFor(failingChecksDB{dbSvc}), org.Slug, "?period=7d") + r.Equal(http.StatusInternalServerError, rec.Code, rec.Body.String()) + r.Contains(rec.Body.String(), "INTERNAL_ERROR") +} diff --git a/server/internal/handlers/incidents/service.go b/server/internal/handlers/incidents/service.go index eb415cdab..a3322bfa8 100644 --- a/server/internal/handlers/incidents/service.go +++ b/server/internal/handlers/incidents/service.go @@ -254,6 +254,10 @@ type Service struct { jobsSvc jobsvc.Service clock clock.Clock + // insightsPageSize and insightsMaxLoaded override the insights loader's + // defaults when positive (tests). + insightsPageSize, insightsMaxLoaded int + // publications is the status-page publication hook. Nil-safe: every call // site goes through the publish* helpers below, which no-op when unset // (status pages disabled, tests). diff --git a/server/internal/i18n/locales/de/report.json b/server/internal/i18n/locales/de/report.json index df1baea11..c07edeafb 100644 --- a/server/internal/i18n/locales/de/report.json +++ b/server/internal/i18n/locales/de/report.json @@ -89,5 +89,14 @@ "perCheckTruncated": "Pro Prüfung, schlechteste zuerst (angezeigt werden die {shown} Prüfungen mit der niedrigsten Verfügbarkeit von insgesamt {total}):", "objectivesHeading": "Ziele:", "budgetRemaining": "verbleibendes Budget {budget}" + }, + "insights": { + "title": "Meiste Vorfälle", + "resolve": "Mediane Zeit bis zur Lösung: {duration}", + "acknowledge": "Mediane Zeit bis zur Bestätigung: {duration}", + "acknowledged": "{count} von {total} bestätigt", + "rowDetail": "{incidents}, {relapses}", + "relapses_one": "{count} Rückfall", + "relapses_other": "{count} Rückfälle" } } diff --git a/server/internal/i18n/locales/en/report.json b/server/internal/i18n/locales/en/report.json index 9bf9bb195..e3e7d36fb 100644 --- a/server/internal/i18n/locales/en/report.json +++ b/server/internal/i18n/locales/en/report.json @@ -89,5 +89,14 @@ "perCheckTruncated": "Per check, worst first (showing the {shown} lowest-uptime checks of {total}):", "objectivesHeading": "Objectives:", "budgetRemaining": "budget remaining {budget}" + }, + "insights": { + "title": "Most incidents", + "resolve": "Median time to resolve: {duration}", + "acknowledge": "Median time to acknowledge: {duration}", + "acknowledged": "{count} of {total} acknowledged", + "rowDetail": "{incidents}, {relapses}", + "relapses_one": "{count} relapse", + "relapses_other": "{count} relapses" } } diff --git a/server/internal/i18n/locales/es/report.json b/server/internal/i18n/locales/es/report.json index 59fd31ddd..a8907f5f3 100644 --- a/server/internal/i18n/locales/es/report.json +++ b/server/internal/i18n/locales/es/report.json @@ -89,5 +89,14 @@ "perCheckTruncated": "Por comprobación, las peores primero (se muestran las {shown} comprobaciones con menor disponibilidad de un total de {total}):", "objectivesHeading": "Objetivos:", "budgetRemaining": "presupuesto restante {budget}" + }, + "insights": { + "title": "Más incidentes", + "resolve": "Tiempo mediano de resolución: {duration}", + "acknowledge": "Tiempo mediano de reconocimiento: {duration}", + "acknowledged": "{count} de {total} reconocidos", + "rowDetail": "{incidents}, {relapses}", + "relapses_one": "{count} recaída", + "relapses_other": "{count} recaídas" } } diff --git a/server/internal/i18n/locales/fr/report.json b/server/internal/i18n/locales/fr/report.json index 15c63629f..e6c5dc48b 100644 --- a/server/internal/i18n/locales/fr/report.json +++ b/server/internal/i18n/locales/fr/report.json @@ -89,5 +89,14 @@ "perCheckTruncated": "Par contrôle, les moins bons en premier (les {shown} contrôles sur {total} à la plus faible disponibilité) :", "objectivesHeading": "Objectifs :", "budgetRemaining": "budget restant {budget}" + }, + "insights": { + "title": "Le plus d'incidents", + "resolve": "Délai médian de résolution : {duration}", + "acknowledge": "Délai médian d'acquittement : {duration}", + "acknowledged": "{count} sur {total} acquittés", + "rowDetail": "{incidents}, {relapses}", + "relapses_one": "{count} rechute", + "relapses_other": "{count} rechutes" } } diff --git a/server/internal/incidentinsights/insights.go b/server/internal/incidentinsights/insights.go new file mode 100644 index 000000000..e36b91aa6 --- /dev/null +++ b/server/internal/incidentinsights/insights.go @@ -0,0 +1,260 @@ +// Package incidentinsights aggregates incident rows into the numbers a team +// asks for after a month of on-call: which checks fail the most, how long it +// takes to acknowledge, how long incidents last. Everything here is a pure +// function over already-loaded incidents so that the insights endpoint and the +// uptime report share one definition. +package incidentinsights + +import ( + "math" + "sort" + "time" + + "github.com/fclairamb/solidping/server/internal/db/models" +) + +// DurationStats is the median and p90 of a sample of durations. The durations +// are nil, not zero, when the sample is empty. +type DurationStats struct { + MedianSeconds *int64 `json:"medianSeconds"` + P90Seconds *int64 `json:"p90Seconds"` + SampleCount int `json:"sampleCount"` +} + +// ResolutionStats is the DurationStats of one resolution bucket plus its count. +type ResolutionStats struct { + Count int `json:"count"` + MedianSeconds *int64 `json:"medianSeconds"` + P90Seconds *int64 `json:"p90Seconds"` +} + +// ByResolution splits resolved incidents by how they were closed. +type ByResolution struct { + Auto ResolutionStats `json:"auto"` + Manual ResolutionStats `json:"manual"` + Other ResolutionStats `json:"other"` +} + +// Summary is the window-wide part of the insights. +type Summary struct { + IncidentCount int `json:"incidentCount"` + // PagedCount is the incidents somebody could have acknowledged: every one + // except the rolled-up (paging-suppressed) ones. It is the ackRate base. + PagedCount int `json:"pagedCount"` + AcknowledgedCount int `json:"acknowledgedCount"` + AckRate float64 `json:"ackRate"` + TimeToAcknowledge DurationStats `json:"timeToAcknowledge"` + TimeToResolve DurationStats `json:"timeToResolve"` + ByResolution ByResolution `json:"byResolution"` +} + +// CheckRow aggregates the incidents of one check. +type CheckRow struct { + CheckUID string `json:"checkUid"` + IncidentCount int `json:"incidentCount"` + DowntimeSeconds int64 `json:"downtimeSeconds"` + RelapseCount int `json:"relapseCount"` + LastIncidentAt *time.Time `json:"lastIncidentAt"` +} + +// Insights is the result of Compute. +type Insights struct { + Summary Summary `json:"summary"` + Checks []CheckRow `json:"checks"` +} + +// Compute aggregates incidents over [windowStart, windowEnd). An open incident +// runs to now for downtime. Incidents entirely outside the window are ignored. +func Compute(incidents []*models.Incident, windowStart, windowEnd, now time.Time) Insights { + var ( + ackSample []int64 + resolveSample []int64 + autoSample []int64 + manualSample []int64 + otherSample []int64 + summary Summary + ) + + rows := map[string]*CheckRow{} + + for _, inc := range incidents { + downtime, inWindow := clippedDowntime(inc, windowStart, windowEnd, now) + if !inWindow { + continue + } + + summary.IncidentCount++ + + row := rows[inc.CheckUID] + if row == nil { + row = &CheckRow{CheckUID: inc.CheckUID} + rows[inc.CheckUID] = row + } + + row.IncidentCount++ + row.DowntimeSeconds += downtime + row.RelapseCount += inc.RelapseCount + + if row.LastIncidentAt == nil || inc.StartedAt.After(*row.LastIncidentAt) { + startedAt := inc.StartedAt + row.LastIncidentAt = &startedAt + } + + if !inc.PagingSuppressed { + summary.PagedCount++ + + if inc.AcknowledgedAt != nil { + summary.AcknowledgedCount++ + ackSample = append(ackSample, nonNegative(inc.AcknowledgedAt.Sub(onset(inc)))) + } + } + + if inc.ResolvedAt == nil { + continue + } + + dur := nonNegative(inc.ResolvedAt.Sub(inc.StartedAt)) + + switch resolutionBucket(inc) { + case models.ResolutionTypeAuto: + autoSample = append(autoSample, dur) + resolveSample = append(resolveSample, dur) + case models.ResolutionTypeManual: + manualSample = append(manualSample, dur) + resolveSample = append(resolveSample, dur) + default: + otherSample = append(otherSample, dur) + } + } + + if summary.PagedCount > 0 { + summary.AckRate = float64(summary.AcknowledgedCount) / float64(summary.PagedCount) + } + + summary.TimeToAcknowledge = stats(ackSample) + summary.TimeToResolve = stats(resolveSample) + summary.ByResolution = ByResolution{ + Auto: resolutionStats(autoSample), + Manual: resolutionStats(manualSample), + Other: resolutionStats(otherSample), + } + + return Insights{Summary: summary, Checks: sortedRows(rows)} +} + +// onset is the start of the incident's current episode: reopening clears the +// acknowledgement, so the surviving one belongs to the last episode (same rule +// as incidentClockFloor in the incidents service). +func onset(inc *models.Incident) time.Time { + if inc.LastReopenedAt != nil && inc.LastReopenedAt.After(inc.StartedAt) { + return *inc.LastReopenedAt + } + + return inc.StartedAt +} + +// resolutionBucket maps a resolution type to auto, manual or "other". +func resolutionBucket(inc *models.Incident) string { + if inc.ResolutionType == nil { + return "other" + } + + switch *inc.ResolutionType { + case models.ResolutionTypeAuto, models.ResolutionTypeManual: + return *inc.ResolutionType + default: + return "other" + } +} + +// clippedDowntime is the incident's wall-clock outage clamped to the window, +// an open incident running to now. The bool is false when it does not overlap +// the window at all. (availability.IncidentBlock drops incidents that started +// before the window, which would lose the tail of one that straddles it.) +func clippedDowntime(inc *models.Incident, windowStart, windowEnd, now time.Time) (int64, bool) { + start := inc.StartedAt + end := now + + if inc.ResolvedAt != nil { + end = *inc.ResolvedAt + } + + if !start.Before(windowEnd) || end.Before(windowStart) { + return 0, false + } + + if start.Before(windowStart) { + start = windowStart + } + + if end.After(windowEnd) { + end = windowEnd + } + + return nonNegative(end.Sub(start)), true +} + +func nonNegative(d time.Duration) int64 { + if d < 0 { + return 0 + } + + return int64(d.Seconds()) +} + +func sortedRows(rows map[string]*CheckRow) []CheckRow { + out := make([]CheckRow, 0, len(rows)) + for _, row := range rows { + out = append(out, *row) + } + + sort.Slice(out, func(i, j int) bool { + if out[i].IncidentCount != out[j].IncidentCount { + return out[i].IncidentCount > out[j].IncidentCount + } + + if out[i].DowntimeSeconds != out[j].DowntimeSeconds { + return out[i].DowntimeSeconds > out[j].DowntimeSeconds + } + + return out[i].CheckUID < out[j].CheckUID + }) + + return out +} + +func stats(sample []int64) DurationStats { + median, p90 := percentiles(sample) + + return DurationStats{MedianSeconds: median, P90Seconds: p90, SampleCount: len(sample)} +} + +func resolutionStats(sample []int64) ResolutionStats { + median, p90 := percentiles(sample) + + return ResolutionStats{Count: len(sample), MedianSeconds: median, P90Seconds: p90} +} + +func percentiles(sample []int64) (*int64, *int64) { + if len(sample) == 0 { + return nil, nil + } + + sorted := append([]int64(nil), sample...) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + + median := nearestRank(sorted, 0.5) + p90 := nearestRank(sorted, 0.9) + + return &median, &p90 +} + +// nearestRank returns the nearest-rank percentile of an ascending sample. +func nearestRank(sorted []int64, pct float64) int64 { + rank := int(math.Ceil(pct * float64(len(sorted)))) + if rank < 1 { + rank = 1 + } + + return sorted[rank-1] +} diff --git a/server/internal/incidentinsights/insights_test.go b/server/internal/incidentinsights/insights_test.go new file mode 100644 index 000000000..9cc292263 --- /dev/null +++ b/server/internal/incidentinsights/insights_test.go @@ -0,0 +1,199 @@ +package incidentinsights_test + +import ( + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/fclairamb/solidping/server/internal/db/models" + "github.com/fclairamb/solidping/server/internal/incidentinsights" +) + +func winStart() time.Time { return time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC) } +func winEnd() time.Time { return time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC) } +func now() time.Time { return winEnd() } + +func at(minutes int) time.Time { return winStart().Add(time.Duration(minutes) * time.Minute) } + +func tp(v time.Time) *time.Time { return &v } + +func sp(v string) *string { return &v } + +func inc(check string, startMin int) *models.Incident { + return &models.Incident{UID: fmt.Sprintf("%s-%d", check, startMin), CheckUID: check, StartedAt: at(startMin)} +} + +func resolved(i *models.Incident, endMin int, kind string) *models.Incident { + i.ResolvedAt = tp(at(endMin)) + i.ResolutionType = sp(kind) + + return i +} + +func acked(i *models.Incident, minute int) *models.Incident { + i.AcknowledgedAt = tp(at(minute)) + + return i +} + +func TestPercentiles(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + minutes []int + median, p90 int64 + wantNoSample bool + }{ + {name: "single", minutes: []int{5}, median: 300, p90: 300}, + {name: "odd", minutes: []int{1, 2, 3, 4, 5}, median: 180, p90: 300}, + {name: "even", minutes: []int{4, 1, 3, 2}, median: 120, p90: 240}, + {name: "empty", wantNoSample: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + r := require.New(t) + + var list []*models.Incident + for i, m := range tc.minutes { + list = append(list, acked(inc("c", i*100), i*100+m)) + } + + got := incidentinsights.Compute(list, winStart(), winEnd(), now()).Summary.TimeToAcknowledge + r.Equal(len(tc.minutes), got.SampleCount) + + if tc.wantNoSample { + r.Nil(got.MedianSeconds) + r.Nil(got.P90Seconds) + + return + } + + r.Equal(tc.median, *got.MedianSeconds) + r.Equal(tc.p90, *got.P90Seconds) + }) + } +} + +func TestCompute_EmptyInput(t *testing.T) { + t.Parallel() + r := require.New(t) + + got := incidentinsights.Compute(nil, winStart(), winEnd(), now()) + r.Zero(got.Summary.IncidentCount) + r.Zero(got.Summary.AckRate) + r.Nil(got.Summary.TimeToAcknowledge.MedianSeconds) + r.Nil(got.Summary.TimeToResolve.MedianSeconds) + r.Empty(got.Checks) +} + +func TestCompute_NoAcknowledged(t *testing.T) { + t.Parallel() + r := require.New(t) + + got := incidentinsights.Compute( + []*models.Incident{resolved(inc("c", 0), 10, models.ResolutionTypeAuto)}, winStart(), winEnd(), now()) + r.Zero(got.Summary.AckRate) + r.Zero(got.Summary.TimeToAcknowledge.SampleCount) + r.Nil(got.Summary.TimeToAcknowledge.MedianSeconds) + r.Equal(1, got.Summary.IncidentCount) +} + +func TestCompute_ReopenedAckMeasuredFromReopen(t *testing.T) { + t.Parallel() + r := require.New(t) + + i := acked(inc("c", 0), 130) + i.LastReopenedAt = tp(at(120)) + + got := incidentinsights.Compute([]*models.Incident{i}, winStart(), winEnd(), now()) + r.Equal(int64(600), *got.Summary.TimeToAcknowledge.MedianSeconds) + r.InDelta(1.0, got.Summary.AckRate, 0) +} + +func TestCompute_OpenIncident(t *testing.T) { + t.Parallel() + r := require.New(t) + + i := inc("c", 0) + i.StartedAt = winEnd().Add(-time.Hour) + + got := incidentinsights.Compute([]*models.Incident{i}, winStart(), winEnd(), now()) + r.Equal(1, got.Summary.IncidentCount) + r.Zero(got.Summary.TimeToResolve.SampleCount) + r.Equal(int64(3600), got.Checks[0].DowntimeSeconds) +} + +func TestCompute_ClippedToWindow(t *testing.T) { + t.Parallel() + r := require.New(t) + + i := inc("c", 0) + i.StartedAt = winStart().Add(-time.Hour) + resolved(i, 30, models.ResolutionTypeAuto) + + got := incidentinsights.Compute([]*models.Incident{i}, winStart(), winEnd(), now()) + r.Equal(int64(1800), got.Checks[0].DowntimeSeconds) +} + +func TestCompute_ResolutionSplit(t *testing.T) { + t.Parallel() + r := require.New(t) + + list := []*models.Incident{ + resolved(inc("a", 0), 10, models.ResolutionTypeAuto), + resolved(inc("a", 100), 130, models.ResolutionTypeManual), + resolved(inc("a", 200), 1200, models.ResolutionTypeExpired), + resolved(inc("a", 300), 1300, models.ResolutionTypeCheckDeleted), + } + + got := incidentinsights.Compute(list, winStart(), winEnd(), now()).Summary + r.Equal(1, got.ByResolution.Auto.Count) + r.Equal(1, got.ByResolution.Manual.Count) + r.Equal(2, got.ByResolution.Other.Count) + r.Equal(2, got.TimeToResolve.SampleCount) + r.Equal(int64(600), *got.TimeToResolve.MedianSeconds) +} + +func TestCompute_SuppressedLeftOutOfAck(t *testing.T) { + t.Parallel() + r := require.New(t) + + rolled := resolved(inc("c", 0), 10, models.ResolutionTypeAuto) + rolled.PagingSuppressed = true + paged := acked(inc("c", 100), 105) + + got := incidentinsights.Compute([]*models.Incident{rolled, paged}, winStart(), winEnd(), now()).Summary + r.Equal(2, got.IncidentCount) + r.Equal(1, got.PagedCount) + r.InDelta(1.0, got.AckRate, 0) + r.Equal(1, got.TimeToResolve.SampleCount) +} + +func TestCompute_CheckRows(t *testing.T) { + t.Parallel() + r := require.New(t) + + a1 := resolved(inc("a", 0), 10, models.ResolutionTypeAuto) + a1.RelapseCount = 2 + a2 := resolved(inc("a", 100), 120, models.ResolutionTypeAuto) + a2.RelapseCount = 1 + b1 := resolved(inc("b", 50), 250, models.ResolutionTypeAuto) + b2 := resolved(inc("b", 300), 301, models.ResolutionTypeAuto) + c1 := resolved(inc("c", 10), 11, models.ResolutionTypeAuto) + + got := incidentinsights.Compute([]*models.Incident{c1, a1, b1, a2, b2}, winStart(), winEnd(), now()).Checks + r.Len(got, 3) + // a and b tie on 2 incidents: b has more downtime and ranks first. + r.Equal("b", got[0].CheckUID) + r.Equal(int64(200*60+60), got[0].DowntimeSeconds) + r.Equal("a", got[1].CheckUID) + r.Equal(3, got[1].RelapseCount) + r.Equal(int64(30*60), got[1].DowntimeSeconds) + r.Equal(at(100), *got[1].LastIncidentAt) + r.Equal("c", got[2].CheckUID) +} diff --git a/server/internal/uptimereport/builder_test.go b/server/internal/uptimereport/builder_test.go index 7460fe692..cc4089265 100644 --- a/server/internal/uptimereport/builder_test.go +++ b/server/internal/uptimereport/builder_test.go @@ -2,6 +2,7 @@ package uptimereport import ( "context" + "errors" "fmt" "testing" "time" @@ -428,3 +429,250 @@ func TestBuilderReportsAverageIncidentDuration(t *testing.T) { r.Equal("20m", data.AverageIncident) r.Equal("40m", data.TotalDowntime) } + +func (e *builderEnv) addIncident( + t *testing.T, check *models.Check, startedAt time.Time, resolveAfter, ackAfter time.Duration, relapses int, +) { + t.Helper() + + incident := models.NewIncident(e.org.UID, check.UID, startedAt, "down") + incident.RelapseCount = relapses + + if resolveAfter > 0 { + resolvedAt := startedAt.Add(resolveAfter) + resolutionType := models.ResolutionTypeAuto + incident.State = models.IncidentStateResolved + incident.ResolvedAt = &resolvedAt + incident.ResolutionType = &resolutionType + } + + if ackAfter > 0 { + ackedAt := startedAt.Add(ackAfter) + incident.AcknowledgedAt = &ackedAt + } + + require.NoError(t, e.db.CreateIncident(e.ctx, incident)) +} + +func TestBuilderFillsIncidentInsights(t *testing.T) { + t.Parallel() + + r := require.New(t) + + env := newBuilderEnv(t) + now := time.Now().UTC() + window := currentMonth(now) + day := 24 * time.Hour + + api := env.createCheck(t, "api") + web := env.createCheck(t, "web") + outside := env.createCheck(t, "outside") + + for _, check := range []*models.Check{api, web, outside} { + env.addRaw(t, check, now.Add(-time.Hour), 90, 100, 150) + } + + env.addIncident(t, api, window.Start.Add(day), 10*time.Minute, 2*time.Minute, 1) + env.addIncident(t, api, window.Start.Add(2*day), 30*time.Minute, 0, 2) + env.addIncident(t, web, window.Start.Add(3*day), 20*time.Minute, 4*time.Minute, 0) + // Not in the schedule's scope: must not be counted anywhere. + env.addIncident(t, outside, window.Start.Add(4*day), 5*time.Hour, 0, 9) + env.addIncident(t, outside, window.Start.Add(5*day), 5*time.Hour, 0, 9) + env.addIncident(t, outside, window.Start.Add(6*day), 5*time.Hour, 0, 9) + + schedule := monthlySchedule(env.org.UID) + schedule.CheckUIDs = []string{api.UID, web.UID} + + data, err := env.builder.Build(env.ctx, env.org, schedule, window, now) + r.NoError(err) + + r.True(data.ShowInsights) + r.Equal(3, data.IncidentCount) + r.Equal("20m", data.MedianTimeToResolve) + r.Equal("2m", data.MedianTimeToAcknowledge) // nearest-rank median of 2m and 4m + r.Equal(2, data.AcknowledgedCount) + r.Equal(3, data.PagedCount) + + r.Len(data.TopFailingChecks, 2) + r.Equal("api", data.TopFailingChecks[0].Name) + r.Equal(2, data.TopFailingChecks[0].IncidentCount) + r.Equal(3, data.TopFailingChecks[0].RelapseCount) + r.Equal("web", data.TopFailingChecks[1].Name) + r.Equal(1, data.TopFailingChecks[1].IncidentCount) +} + +func TestBuilderLeavesInsightsEmptyWithoutIncidents(t *testing.T) { + t.Parallel() + + r := require.New(t) + + env := newBuilderEnv(t) + now := time.Now().UTC() + + check := env.createCheck(t, "api") + env.addRaw(t, check, now.Add(-time.Hour), 100, 100, 150) + + data, err := env.builder.Build(env.ctx, env.org, monthlySchedule(env.org.UID), currentMonth(now), now) + r.NoError(err) + + r.False(data.ShowInsights) + r.Empty(data.MedianTimeToResolve) + r.Empty(data.MedianTimeToAcknowledge) + r.Zero(data.AcknowledgedCount) + r.Empty(data.TopFailingChecks) +} + +func TestBuilderOmitsTopFailingForASingleCheckScope(t *testing.T) { + t.Parallel() + + r := require.New(t) + + env := newBuilderEnv(t) + now := time.Now().UTC() + window := currentMonth(now) + + check := env.createCheck(t, "api") + env.addRaw(t, check, now.Add(-time.Hour), 90, 100, 150) + env.addIncident(t, check, window.Start.Add(24*time.Hour), 10*time.Minute, time.Minute, 0) + + data, err := env.builder.Build(env.ctx, env.org, monthlySchedule(env.org.UID), window, now) + r.NoError(err) + + r.True(data.ShowInsights) + r.Empty(data.TopFailingChecks) +} + +// TestBuilderInsightsCountOutagesFromBeforeThePeriod: an outage that began +// before the window and is still open is in the insights (positive control), +// one resolved before the window start is not, and neither moves the +// incident-count figure, which counts by start date. +func TestBuilderInsightsCountOutagesFromBeforeThePeriod(t *testing.T) { + t.Parallel() + + r := require.New(t) + + env := newBuilderEnv(t) + now := time.Now().UTC() + window := currentMonth(now) + day := 24 * time.Hour + + stuck := env.createCheck(t, "stuck") + old := env.createCheck(t, "old") + + for _, check := range []*models.Check{stuck, old} { + env.addRaw(t, check, now.Add(-time.Hour), 90, 100, 150) + } + + env.addIncident(t, stuck, window.Start.Add(-10*day), 0, 0, 3) // still open + env.addIncident(t, old, window.Start.Add(-20*day), 2*day, 0, 0) + + schedule := monthlySchedule(env.org.UID) + schedule.CheckUIDs = []string{stuck.UID, old.UID} + + data, err := env.builder.Build(env.ctx, env.org, schedule, window, now) + r.NoError(err) + + r.True(data.ShowInsights) + r.Len(data.TopFailingChecks, 1) + r.Equal("stuck", data.TopFailingChecks[0].Name) + r.Equal(3, data.TopFailingChecks[0].RelapseCount) + r.Zero(data.IncidentCount, "the incident count is by start date and is unchanged") +} + +// TestBuilderCountsAGroupIncidentOnceAcrossMembers: a group incident is listed +// for each in-scope member and counts once. +func TestBuilderCountsAGroupIncidentOnceAcrossMembers(t *testing.T) { + t.Parallel() + + r := require.New(t) + + env := newBuilderEnv(t) + now := time.Now().UTC() + window := currentMonth(now) + + first := env.createCheck(t, "grp-a") + second := env.createCheck(t, "grp-b") + + for _, check := range []*models.Check{first, second} { + env.addRaw(t, check, now.Add(-time.Hour), 90, 100, 150) + } + + group := models.NewCheckGroup(env.org.UID, "Group", "grp") + r.NoError(env.db.CreateCheckGroup(env.ctx, group)) + + startedAt := window.Start.Add(24 * time.Hour) + incident := models.NewIncident(env.org.UID, first.UID, startedAt, "group down") + incident.CheckGroupUID = &group.UID + r.NoError(env.db.CreateIncident(env.ctx, incident)) + + for _, check := range []*models.Check{first, second} { + r.NoError(env.db.UpsertIncidentMemberCheck(env.ctx, &models.IncidentMemberCheck{ + IncidentUID: incident.UID, CheckUID: check.UID, + JoinedAt: startedAt, FirstFailureAt: startedAt, LastFailureAt: startedAt, FailureCount: 1, + })) + } + + // Positive control: each member's own load returns the incident. + for _, check := range []*models.Check{first, second} { + listed, _, err := env.db.ListIncidents(env.ctx, &models.ListIncidentsFilter{ + OrganizationUID: env.org.UID, MemberCheckUID: check.UID, + Kinds: []string{models.IncidentKindCheck}, ActiveSince: &window.Start, Until: &window.End, Limit: 10, + }) + r.NoError(err) + r.Len(listed, 1) + } + + schedule := monthlySchedule(env.org.UID) + schedule.CheckUIDs = []string{first.UID, second.UID} + + data, err := env.builder.Build(env.ctx, env.org, schedule, window, now) + r.NoError(err) + + r.True(data.ShowInsights) + r.Equal(1, data.PagedCount) + + total := 0 + for _, row := range data.TopFailingChecks { + total += row.IncidentCount + } + + r.Equal(1, total) +} + +var errListIncidents = errors.New("list incidents failed") + +// failingInsightsDB fails only the insights load (the one filtering by +// ActiveSince), so the incident block still succeeds. +type failingInsightsDB struct { + db.Service +} + +func (f failingInsightsDB) ListIncidents( + ctx context.Context, filter *models.ListIncidentsFilter, +) ([]*models.Incident, int64, error) { + if filter.ActiveSince != nil { + return nil, 0, errListIncidents + } + + return f.Service.ListIncidents(ctx, filter) +} + +// TestBuilderFailsWhenTheInsightsIncidentLoadFails: the report errors with a +// wrapped cause instead of going out without its insights. +func TestBuilderFailsWhenTheInsightsIncidentLoadFails(t *testing.T) { + t.Parallel() + + r := require.New(t) + + env := newBuilderEnv(t) + now := time.Now().UTC() + + check := env.createCheck(t, "api") + env.addRaw(t, check, now.Add(-time.Hour), 90, 100, 150) + + builder := NewBuilder(failingInsightsDB{env.db}, &config.Config{}, nil) + + _, err := builder.Build(env.ctx, env.org, monthlySchedule(env.org.UID), currentMonth(now), now) + r.ErrorIs(err, errListIncidents) + r.Contains(err.Error(), "list incidents for insights") +} diff --git a/server/internal/uptimereport/locale_report_test.go b/server/internal/uptimereport/locale_report_test.go index 768610836..1d3b1865b 100644 --- a/server/internal/uptimereport/locale_report_test.go +++ b/server/internal/uptimereport/locale_report_test.go @@ -47,6 +47,7 @@ func englishLeftovers() []string { "Uptime report", "Availability", "Response time", "Objectives", "Open dashboard", "worst first", "No data", "Checks monitored", "Total downtime", "Budget remaining", " vs ", "uptime", "longest", "average", "samples", "Gray means", + "Most incidents", "Median time", "acknowledged", "relapse", } } @@ -281,3 +282,35 @@ func TestUptimeReportPluralsSurviveTheJob(t *testing.T) { r.Contains(preheader[1], tc.want, "%s, %d incidents", tc.lang, tc.count) } } + +// TestUptimeReportInsightsAreWrittenInEveryLanguage pins the new strings: each +// locale renders its own figures and block title, with no raw key left. +func TestUptimeReportInsightsAreWrittenInEveryLanguage(t *testing.T) { + t.Parallel() + + want := map[string][]string{ + i18n.LangEnglish: {"Median time to resolve: 18m", "2 of 3 acknowledged", "Most incidents", "2 incidents, 1 relapse"}, + i18n.LangFrench: { + "Délai médian de résolution", "2 sur 3 acquittés", "Le plus d'incidents", "2 incidents, 1 rechute", + }, + i18n.LangGerman: {"Mediane Zeit bis zur Lösung", "2 von 3 bestätigt", "Meiste Vorfälle", "2 Vorfälle, 1 Rückfall"}, + i18n.LangSpanish: { + "Tiempo mediano de resolución", "2 de 3 reconocidos", "Más incidentes", //nolint:misspell // Spanish + "2 incidentes, 1 recaída", //nolint:misspell // Spanish + }, + } + + formatter, err := email.NewFormatter() + require.NoError(t, err) + + for lang, phrases := range want { + _, body, text := renderReport(t, formatter, i18n.New(lang, "Europe/Paris"), sampleData()) + + for _, phrase := range phrases { + require.Contains(t, visibleText(body), phrase, lang) + require.Contains(t, text, phrase, lang) + } + + require.NotContains(t, text, "report.insights", lang) + } +} diff --git a/server/internal/uptimereport/render_test.go b/server/internal/uptimereport/render_test.go index 1846f839a..b41970800 100644 --- a/server/internal/uptimereport/render_test.go +++ b/server/internal/uptimereport/render_test.go @@ -42,6 +42,17 @@ func sampleData() *uptimereport.Data { CheckCount: 2, IncidentCount: 3, + ShowInsights: true, + AcknowledgedCount: 2, + PagedCount: 3, + TopFailingChecks: []uptimereport.FailingCheckRow{ + { + Name: "Production API", IncidentCount: 2, RelapseCount: 1, + URL: "https://solidping.example/d/orgs/acme/checks/chk-prod-api", + }, + {Name: "Marketing site", IncidentCount: 1}, + }, + HasPreviousData: true, PreviousIncidentCount: 5, @@ -98,15 +109,18 @@ func sampleData() *uptimereport.Data { LongestIncident: 42 * time.Minute, AverageIncident: 21*time.Minute + 40*time.Second, TotalDowntime: time.Hour + 5*time.Minute, - AvgResponse: 171, - MinResponse: 42, - MaxResponse: 3200, - HasResponseRange: true, - PreviousAvgResponse: 158, - SlowSamples: 4, - SlowPeaks: 2, - StripStart: time.Date(2026, time.July, 1, 0, 0, 0, 0, time.UTC), - StripDays: 31, + + MedianTimeToResolve: 18 * time.Minute, + MedianTimeToAcknowledge: 4 * time.Minute, + AvgResponse: 171, + MinResponse: 42, + MaxResponse: 3200, + HasResponseRange: true, + PreviousAvgResponse: 158, + SlowSamples: 4, + SlowPeaks: 2, + StripStart: time.Date(2026, time.July, 1, 0, 0, 0, 0, time.UTC), + StripDays: 31, }, } @@ -299,6 +313,7 @@ func TestUptimeReportRendersNoDataHonestly(t *testing.T) { data.AvailabilityPct = "" data.Checks = nil data.SLOs = nil + data.TopFailingChecks = nil _, html, text, err := formatter.Format(uptimereport.TemplateName, i18n.English, roundTrip(t, data)) r.NoError(err) @@ -457,6 +472,7 @@ func TestUptimeReportEveryViewModelFieldSurvivesTheRoundTrip(t *testing.T) { requireEveryFieldIsKeyed(t, data.Checks[0]) requireEveryFieldIsKeyed(t, data.Checks[0].Days[0]) requireEveryFieldIsKeyed(t, data.SLOs[0]) + requireEveryFieldIsKeyed(t, data.TopFailingChecks[0]) // And the whole thing still renders with no unresolved key. formatter, err := email.NewFormatter() @@ -628,3 +644,88 @@ func TestUptimeReportDownAllPeriodStatesTheFactAndDropsLatency(t *testing.T) { r.NotContains(html, "color:#15803d") r.NotContains(html, `bgcolor="#15803d"`) } + +// insightsCopy is what the three figures and the block render as in English +// for sampleData. +const ( + insightsResolve = "Median time to resolve: 18m" + insightsAcknowledge = "Median time to acknowledge: 4m" + insightsAcknowledged = "2 of 3 acknowledged" + insightsTitle = "Most incidents" +) + +func renderInsights(t *testing.T, data *uptimereport.Data) (string, string) { + t.Helper() + + formatter, err := email.NewFormatter() + require.NoError(t, err) + + _, html, text, err := formatter.Format(uptimereport.TemplateName, i18n.English, roundTrip(t, data.In(i18n.English))) + require.NoError(t, err) + + return visibleText(html), text +} + +func TestUptimeReportRendersInsightsLineAndBlock(t *testing.T) { + t.Parallel() + + html, text := renderInsights(t, sampleData()) + + for _, body := range []string{html, text} { + require.Contains(t, body, insightsResolve) + require.Contains(t, body, insightsAcknowledge) + require.Contains(t, body, insightsAcknowledged) + require.Contains(t, body, insightsTitle) + require.Contains(t, body, "2 incidents, 1 relapse") + require.Contains(t, body, "1 incident, 0 relapses") + } +} + +func TestUptimeReportOmitsInsightsWithoutIncidents(t *testing.T) { + t.Parallel() + + data := sampleData() + data.IncidentCount = 0 + data.ShowInsights = false + data.AcknowledgedCount, data.PagedCount = 0, 0 + data.TopFailingChecks = nil + data.Figures.MedianTimeToResolve, data.Figures.MedianTimeToAcknowledge = 0, 0 + + html, text := renderInsights(t, data) + + for _, body := range []string{html, text} { + require.NotContains(t, body, "Median time") + require.NotContains(t, body, "acknowledged") + require.NotContains(t, body, insightsTitle) + } +} + +func TestUptimeReportInsightsWithNothingAcknowledged(t *testing.T) { + t.Parallel() + + data := sampleData() + data.AcknowledgedCount = 0 + data.Figures.MedianTimeToAcknowledge = 0 + + html, text := renderInsights(t, data) + + for _, body := range []string{html, text} { + require.NotContains(t, body, "Median time to acknowledge") + require.Contains(t, body, "0 of 3 acknowledged") + require.Contains(t, body, insightsResolve) + } +} + +func TestUptimeReportOmitsMostIncidentsForASingleCheck(t *testing.T) { + t.Parallel() + + data := sampleData() + data.TopFailingChecks = nil + + html, text := renderInsights(t, data) + + for _, body := range []string{html, text} { + require.NotContains(t, body, insightsTitle) + require.Contains(t, body, insightsAcknowledged) + } +} diff --git a/server/internal/uptimereport/report.go b/server/internal/uptimereport/report.go index 3157ff3a5..830893be0 100644 --- a/server/internal/uptimereport/report.go +++ b/server/internal/uptimereport/report.go @@ -22,6 +22,7 @@ import ( "github.com/fclairamb/solidping/server/internal/handlers/slos" "github.com/fclairamb/solidping/server/internal/i18n" "github.com/fclairamb/solidping/server/internal/i18n/recipient" + "github.com/fclairamb/solidping/server/internal/incidentinsights" "github.com/fclairamb/solidping/server/internal/slo" "github.com/fclairamb/solidping/server/internal/systemconfig" "github.com/fclairamb/solidping/server/internal/uptimebar" @@ -164,6 +165,22 @@ type Data struct { AverageIncident string `json:"AverageIncident,omitempty"` TotalDowntime string `json:"TotalDowntime,omitempty"` + // --- Incident insights (package incidentinsights) -------------------- + // + // ShowInsights is false when the period had no incident: neither the + // medians line nor the block is rendered. The medians are empty when + // nothing was acknowledged / resolved. AcknowledgedCount and PagedCount + // carry no omitempty: "0 of 3 acknowledged" must survive the round trip. + ShowInsights bool `json:"ShowInsights"` + MedianTimeToResolve string `json:"MedianTimeToResolve,omitempty"` + MedianTimeToAcknowledge string `json:"MedianTimeToAcknowledge,omitempty"` + AcknowledgedCount int `json:"AcknowledgedCount"` + // PagedCount is the incidents somebody could have acknowledged. + PagedCount int `json:"PagedCount"` + // TopFailingChecks is up to maxTopFailing checks, most incidents first. + // Empty (block omitted) for a scope of a single check. + TopFailingChecks []FailingCheckRow `json:"TopFailingChecks,omitempty"` + // --- Period-over-period trend ------------------------------------- // // Every Show* flag is false when there is no previous-period data, and the @@ -275,6 +292,10 @@ type Figures struct { AverageIncident time.Duration TotalDowntime time.Duration + // Incident insight medians (zero when there is no sample). + MedianTimeToResolve time.Duration + MedianTimeToAcknowledge time.Duration + // Response times in milliseconds (HasLatency, the range when // HasResponseRange, the previous average when ShowResponseDelta). AvgResponse float64 @@ -391,6 +412,10 @@ func (b *Builder) measure( applyIncidents(data, incidents) + if insightsErr := b.loadInsights(ctx, data, org, checks, checkUIDs, window, now, baseURL); insightsErr != nil { + return nil, insightsErr + } + if trendErr := b.applyPreviousPeriod( ctx, org.UID, schedule, checkUIDs, window, now, hints, data, overall, incidents.Count, ); trendErr != nil { @@ -476,6 +501,79 @@ func applyIncidents(data *Data, incidents availability.PeriodIncidents) { data.Figures.TotalDowntime = time.Duration(incidents.TotalDowntimeSeconds) * time.Second } +// maxTopFailing caps the "Most incidents" block. +const maxTopFailing = 5 + +// FailingCheckRow is one line of the "Most incidents" block. +// +//nolint:tagliatelle // template view-model keys are PascalCase; see Data. +type FailingCheckRow struct { + Name string `json:"Name"` + URL string `json:"URL,omitempty"` + IncidentCount int `json:"IncidentCount"` + RelapseCount int `json:"RelapseCount"` + // Detail is "4 incidents, 2 relapses", written in the reader's language. + Detail string `json:"Detail,omitempty"` +} + +// applyInsights fills the incident insights from the incidents of the scope's +// checks, using the same computation as the dashboard page. +func applyInsights( + data *Data, checks []*models.Check, incidents []*models.Incident, + window slo.Window, now time.Time, baseURL, orgSlug string, +) { + byUID := make(map[string]*models.Check, len(checks)) + for _, check := range checks { + byUID[check.UID] = check + } + + scoped := make([]*models.Incident, 0, len(incidents)) + + for _, inc := range incidents { + if _, ok := byUID[inc.CheckUID]; ok { + scoped = append(scoped, inc) + } + } + + result := incidentinsights.Compute(scoped, window.Start.UTC(), window.End.UTC(), now.UTC()) + if result.Summary.IncidentCount == 0 { + return + } + + data.ShowInsights = true + data.AcknowledgedCount = result.Summary.AcknowledgedCount + data.PagedCount = result.Summary.PagedCount + + if m := result.Summary.TimeToResolve.MedianSeconds; m != nil { + data.Figures.MedianTimeToResolve = time.Duration(*m) * time.Second + } + + if m := result.Summary.TimeToAcknowledge.MedianSeconds; m != nil { + data.Figures.MedianTimeToAcknowledge = time.Duration(*m) * time.Second + } + + // The table already names the one check of a single-check scope. + if len(checks) < 2 { + return + } + + for i := range result.Checks { + if len(data.TopFailingChecks) == maxTopFailing { + break + } + + row := &result.Checks[i] + + check := byUID[row.CheckUID] + data.TopFailingChecks = append(data.TopFailingChecks, FailingCheckRow{ + Name: checkDisplayName(check), + URL: checkReportURL(baseURL, orgSlug, check), + IncidentCount: row.IncidentCount, + RelapseCount: row.RelapseCount, + }) + } +} + // buildCheckRows folds the per-check window stats into rows and, at the same // time, into the scope-level total. Returning both from one pass is what keeps // "the headline" and "the sum of the table" from ever disagreeing. @@ -659,24 +757,75 @@ func (b *Builder) scopeChecks( func (b *Builder) incidentBlock( ctx context.Context, orgUID string, checkUIDs []string, window slo.Window, now time.Time, ) (availability.PeriodIncidents, error) { + // Started inside the window: the downtime and count figures are about the + // incidents that began in the period. + all, err := b.listCheckIncidents(ctx, orgUID, checkUIDs, &models.ListIncidentsFilter{ + Since: &window.Start, + Until: &window.End, + Kinds: []string{models.IncidentKindCheck}, + }) + if err != nil { + return availability.PeriodIncidents{}, fmt.Errorf("list incidents: %w", err) + } + + return availability.IncidentBlock(all, window.Start.UTC(), window.End.UTC(), now.UTC()), nil +} + +// loadInsights fills the incident insights; a failed load fails the report +// rather than sending it without them. +func (b *Builder) loadInsights( + ctx context.Context, data *Data, org *models.Organization, checks []*models.Check, + checkUIDs []string, window slo.Window, now time.Time, baseURL string, +) error { + incidents, err := b.insightsIncidents(ctx, org.UID, checkUIDs, window) + if err != nil { + return err + } + + applyInsights(data, checks, incidents, window, now, baseURL, org.Slug) + + return nil +} + +// insightsIncidents loads the incidents active during the window (started +// before its end, still open or resolved inside it), the population of the +// incident insights page. incidentBlock counts by start date, so an outage that +// began before the window would be missing here. +func (b *Builder) insightsIncidents( + ctx context.Context, orgUID string, checkUIDs []string, window slo.Window, +) ([]*models.Incident, error) { + all, err := b.listCheckIncidents(ctx, orgUID, checkUIDs, &models.ListIncidentsFilter{ + ActiveSince: &window.Start, + Until: &window.End, + Kinds: []string{models.IncidentKindCheck}, + }) + if err != nil { + return nil, fmt.Errorf("list incidents for insights: %w", err) + } + + return all, nil +} + +// listCheckIncidents returns the check incidents of every check (a group +// incident listed by several members once), narrowed by the time fields of base. +func (b *Builder) listCheckIncidents( + ctx context.Context, orgUID string, checkUIDs []string, base *models.ListIncidentsFilter, +) ([]*models.Incident, error) { seen := make(map[string]struct{}) all := make([]*models.Incident, 0) for _, checkUID := range checkUIDs { - filter := &models.ListIncidentsFilter{ - OrganizationUID: orgUID, - MemberCheckUID: checkUID, - // Check incidents only — the digest reports downtime, not budget - // alerts (spec 2026-08-21-08). - Kinds: []string{models.IncidentKindCheck}, - Since: &window.Start, - Until: &window.End, - Limit: incidentFetchLimit, - } - - incidents, _, err := b.db.ListIncidents(ctx, filter) + filter := *base + filter.OrganizationUID = orgUID + filter.MemberCheckUID = checkUID + // Check incidents only: the digest reports downtime, not budget + // alerts (spec 2026-08-21-08). + filter.Kinds = []string{models.IncidentKindCheck} + filter.Limit = incidentFetchLimit + + incidents, _, err := b.db.ListIncidents(ctx, &filter) if err != nil { - return availability.PeriodIncidents{}, fmt.Errorf("list incidents: %w", err) + return nil, err //nolint:wrapcheck // wrapped by the callers } for _, inc := range incidents { @@ -690,7 +839,7 @@ func (b *Builder) incidentBlock( } } - return availability.IncidentBlock(all, window.Start.UTC(), window.End.UTC(), now.UTC()), nil + return all, nil } // sloRows evaluates every enabled objective whose scope intersects the report's. diff --git a/server/internal/uptimereport/size_test.go b/server/internal/uptimereport/size_test.go index 7cc2f562c..c4a550e86 100644 --- a/server/internal/uptimereport/size_test.go +++ b/server/internal/uptimereport/size_test.go @@ -50,6 +50,9 @@ func worstCaseMonthlyReport(t *testing.T) *Data { AvailabilityColor: "#c2410c", CheckCount: maxCheckRows + 87, IncidentCount: 42, + ShowInsights: true, + AcknowledgedCount: 30, + PagedCount: 42, HasPreviousData: true, ShowAvailabilityDelta: true, AvailabilityDeltaColor: deltaBadColor, @@ -77,15 +80,18 @@ func worstCaseMonthlyReport(t *testing.T) *Data { LongestIncident: 3*time.Hour + 12*time.Minute, AverageIncident: 18 * time.Minute, TotalDowntime: 12*time.Hour + 36*time.Minute, - AvgResponse: 412, - MinResponse: 37, - MaxResponse: 9800, - HasResponseRange: true, - PreviousAvgResponse: 292, - SlowSamples: 318, - SlowPeaks: 24, - StripStart: time.Date(2026, time.July, 1, 0, 0, 0, 0, time.UTC), - StripDays: days, + + MedianTimeToResolve: 2*time.Hour + 41*time.Minute, + MedianTimeToAcknowledge: 13*time.Minute + 20*time.Second, + AvgResponse: 412, + MinResponse: 37, + MaxResponse: 9800, + HasResponseRange: true, + PreviousAvgResponse: 292, + SlowSamples: 318, + SlowPeaks: 24, + StripStart: time.Date(2026, time.July, 1, 0, 0, 0, 0, time.UTC), + StripDays: days, }, } @@ -108,6 +114,15 @@ func worstCaseMonthlyReport(t *testing.T) *Data { }) } + for i := range maxTopFailing { + data.TopFailingChecks = append(data.TopFailingChecks, FailingCheckRow{ + Name: fmt.Sprintf("top-failing-check-with-a-fairly-long-name-%02d.example", i), + URL: fmt.Sprintf("https://solidping.example/d/orgs/acme/checks/chk-%02d", i), + IncidentCount: 40 - i, + RelapseCount: 12 - i, + }) + } + // Exactly what Build does before handing the view model to the mailer, // including the note it raises when the budget drops a strip — the test // must never hand-write that text, or it would pass while the builder diff --git a/server/internal/uptimereport/write.go b/server/internal/uptimereport/write.go index b39e1ffc1..b64ae4c28 100644 --- a/server/internal/uptimereport/write.go +++ b/server/internal/uptimereport/write.go @@ -56,6 +56,7 @@ func (d *Data) In(loc i18n.Locale) *Data { out := *d out.Checks = slices.Clone(d.Checks) out.SLOs = slices.Clone(d.SLOs) + out.TopFailingChecks = slices.Clone(d.TopFailingChecks) out.readers = nil out.written = nil out.write(loc) @@ -84,6 +85,9 @@ func (d *Data) write(loc i18n.Locale) { d.AverageIncident = spanIfAny(loc, fig.AverageIncident) d.TotalDowntime = spanIfAny(loc, fig.TotalDowntime) + d.MedianTimeToResolve = spanIfAny(loc, fig.MedianTimeToResolve) + d.MedianTimeToAcknowledge = spanIfAny(loc, fig.MedianTimeToAcknowledge) + d.writeTrend(loc) d.writeLatency(loc) @@ -92,6 +96,13 @@ func (d *Data) write(loc i18n.Locale) { row.AvailabilityPct = pctIf(loc, row.HasData, row.Availability) } + for i := range d.TopFailingChecks { + row := &d.TopFailingChecks[i] + row.Detail = loc.T("report.insights.rowDetail", + "incidents", loc.Tn("report.incidents", row.IncidentCount), + "relapses", loc.Tn("report.insights.relapses", row.RelapseCount)) + } + for i := range d.SLOs { d.SLOs[i].write(loc) } diff --git a/specs/done/2026/11/2026-11-13-01-incident-insights-page.md b/specs/done/2026/11/2026-11-13-01-incident-insights-page.md new file mode 100644 index 000000000..00bec618f --- /dev/null +++ b/specs/done/2026/11/2026-11-13-01-incident-insights-page.md @@ -0,0 +1,108 @@ +--- +model: sonnet +effort: medium +pr: batch +automerge: false +--- + +# Incident insights page: top failing checks, time to acknowledge, time to resolve + +## Problem + +The dashboard lists incidents one by one but answers none of the questions a team asks after a month of on-call: + +- Which checks fail the most ? +- How long does it take us to acknowledge an incident ? +- How long do incidents last, and how many recover on their own ? + +The data is already on the `incidents` row (`server/internal/db/models/incident.go:87-105`): `started_at`, `acknowledged_at`, `resolved_at`, `resolution_type`, `relapse_count`, `last_reopened_at`. Nothing aggregates it. No migration is needed. + +## Proposal + +### 1. Pure computation package + +New package `server/internal/incidentinsights`, pure functions over `[]*models.Incident` (no DB access), so the uptime report can reuse it (spec `2026-11-13-02`). + +`Compute(incidents, windowStart, windowEnd, now) Insights` returns: + +- **`summary`** + - `incidentCount` + - `acknowledgedCount` and `ackRate` (0 to 1) + - `timeToAcknowledge`: `{ medianSeconds, p90Seconds, sampleCount }` + - `timeToResolve`: same shape, over resolved incidents + - `byResolution`: one entry for `auto`, one for `manual`, one for `other`, each with `count`, `medianSeconds`, `p90Seconds` +- **`checks`**: one row per check, with `checkUid`, `incidentCount`, `downtimeSeconds`, `relapseCount` (sum), `lastIncidentAt` + +Rules: + +- **Time to acknowledge** = `acknowledged_at` minus the incident's current onset. The onset is `last_reopened_at` when it is after `started_at`, else `started_at`. Reopening clears `acknowledged_at` (`server/internal/handlers/incidents/service.go:1769-1773`), so the surviving ack belongs to the last episode. Same rule as `incidentClockFloor` (`service.go:1173`). Only incidents with a non-null `acknowledged_at` are in the sample. +- **Ack rate** = acknowledged incidents / all incidents in the window. It is always returned next to the ack time, because most incidents recover before anyone acks and the ack time alone flatters. +- **Time to resolve** = `resolved_at` minus `started_at`, resolved incidents only. +- **Resolution split**: `auto` = `ResolutionTypeAuto`, `manual` = `ResolutionTypeManual`, `other` = every other value (`incident.go:23-44`: expired, escalated, disabled, check_deleted, check_disabled). `other` is counted but left out of the overall `timeToResolve` median, since those closures say nothing about recovery. +- **Median and p90**, never the mean (one 6 hour outage wrecks a mean). Nearest-rank on the sorted sample. An empty sample returns `sampleCount: 0` and null durations, not zero. +- **Downtime per check**: reuse `availability.IncidentBlock` (`server/internal/handlers/availability/service.go:332`), which clips an incident to the window and counts an open one up to `now`. +- No per-person breakdown. `acknowledged_by` is not read. + +### 2. API endpoint + +`GET /api/v1/orgs/:org/incidents/insights?period=7d|30d|90d` (default `30d`), registered next to the other incident routes in `server/internal/app/server.go:1594`, before `/:uid`. + +- Same read permission as `ListIncidents`. +- Loads incidents with `ListIncidentsFilter` (`incident.go:210`): `OrganizationUID`, `Since` = window start, `Kinds` = check outages only (no SLO burn alerts, no degraded incidents). +- Rolled-up (paging-suppressed) incidents count in the checks table and in time to resolve. They are left out of the acknowledge sample and of the ack rate, since nobody was paged. +- Page through the list. Stop at 5,000 incidents and return `truncated: true`. +- Checks table: top 20 by `incidentCount`, ties broken by `downtimeSeconds`. Each row carries the check name and slug so the page needs no second request. +- Unknown `period` answers 400 `VALIDATION_ERROR`. +- Add the endpoint to `server/internal/app/openapi/openapi.yaml`. + +### 3. Dashboard page + +New route `web/dash0/src/routes/orgs/$org/incidents.insights.tsx`, reached from an "Insights" button in the header of the incidents list (`web/dash0/src/routes/orgs/$org/incidents.index.tsx`). + +- Period selector (7, 30, 90 days), stored in the URL search params. +- **Top failing checks** first: a table with check (link to the check page), incidents, downtime, relapses, last incident. Sortable client-side on the three numeric columns. Each incident count links to the incidents list filtered on that check. +- **Response times** below: three cards. + - Time to acknowledge: median, p90, and "12 of 40 acknowledged". + - Time to resolve: median, p90. + - Resolution: count and median for auto-recovered and for manually resolved, count only for other. +- A card with an empty sample shows a dash and the reason ("No acknowledged incident in this period"), never `0s`. +- Empty state when the period has no incident. +- Truncation notice when `truncated` is true. +- Reuse the primitives of the design reference (`web/dash0/src/routes/orgs/$org/design-reference.tsx`). Usable on mobile: the table scrolls or stacks, no fixed widths. +- `data-testid` on the period selector, each card, each table row and the empty state. +- Durations are formatted with the helper the incident pages already use. + +### 4. Docs and changelog + +- A short section in the incidents page of `web/docs/docs/` describing the three numbers and how each is computed. +- A `CHANGELOG.md` entry (conventions: `wiki/conventions/changelog.md`). + +## Tests + +- `server/internal/incidentinsights/insights_test.go`, table-driven: + - median and p90 on odd, even and single-element samples + - empty input: zero counts, null durations + - no acknowledged incident: `ackRate` 0, `timeToAcknowledge.sampleCount` 0 + - reopened incident: ack time is measured from `last_reopened_at`, not `started_at` + - open incident: counted, absent from the resolve sample, downtime runs to `now` + - incident started before the window: downtime clipped to the window + - resolution split: `expired` and `check_deleted` land in `other` and stay out of the overall median + - checks rows: sums of incidents, downtime and relapses per check, order and tie-break +- `server/internal/handlers/incidents/`: handler test for the endpoint + - 401 without a token + - 400 `VALIDATION_ERROR` on `period=1y` + - incidents of another org are not counted + - an SLO burn incident is not counted (positive control: the same test counts a check outage) + - a rolled-up incident counts in the checks table and not in the ack rate +- `web/dash0/e2e/incident-insights.spec.ts`: + - the page shows the failing check and its incident count after an incident is created + - switching the period updates the URL and the numbers + - an org with no incident shows the empty state + - a mobile viewport has no horizontal page scroll + +## To verify + +- The exact name of the kind constant for check outages (`IncidentKindCheck` ?) near `incident.go:71`. +- How a group incident (`check_group_uid` set, `incident.go:108`) should be attributed: its `check_uid`, or each member check. Default to `check_uid`. +- Whether the incidents list already accepts a check filter in its URL search params, for the count links. +- Whether dash0 strings are translated (see `locale_pages_test.go`), and add the keys if so. diff --git a/specs/done/2026/11/2026-11-13-02-uptime-report-incident-insights.md b/specs/done/2026/11/2026-11-13-02-uptime-report-incident-insights.md new file mode 100644 index 000000000..4c8ee26b4 --- /dev/null +++ b/specs/done/2026/11/2026-11-13-02-uptime-report-incident-insights.md @@ -0,0 +1,53 @@ +--- +model: sonnet +effort: medium +pr: batch +automerge: false +--- + +# Add response times and most-failing checks to the scheduled uptime report + +## Problem + +The scheduled uptime report (`server/internal/uptimereport/report.go`) gives the incident count and the total downtime (`report.go:159-165`), and sorts its check table by availability (`report.go:520`). It does not say how fast incidents were acknowledged or resolved, nor which checks opened the most incidents. A check that flaps ten times for 30 seconds each keeps a good availability and stays at the bottom of the table. + +Spec `2026-11-13-01` adds these numbers to the dashboard. The report should carry the same ones, computed by the same code. + +Depends on `2026-11-13-01-incident-insights-page` (package `server/internal/incidentinsights`). + +## Proposal + +1. In `Builder.measure` (`report.go:334`), call `incidentinsights.Compute` on the incidents of the report window, restricted to the checks in scope (`scopeChecks`, `report.go:602`). Reuse the incidents the builder already loads for `incidentBlock` (`report.go:659`) if it has them, no second query. +2. Extend `Data` (`report.go:140`) with: + - `MedianTimeToResolve` and `MedianTimeToAcknowledge`, formatted like `TotalDowntime` + - `AcknowledgedCount` (shown as "12 of 40 acknowledged") + - `TopFailingChecks`: up to 5 rows with name, link (`checkReportURL`, `report.go:791`), incident count and relapse count, ordered by incident count +3. Render them in the email (`write.go` and the report template): + - one line under the existing incident figures: median time to resolve, median time to acknowledge, acknowledged count + - a "Most incidents" block listing the 5 rows +4. Hide rules: + - no incident in the period: neither the line nor the block is rendered + - no acknowledged incident: the acknowledge figure is omitted, the acknowledged count still shows "0 of N" + - a single check in scope: the "Most incidents" block is omitted (the table already says it) +5. Add the strings to every report locale, and the new fields to the preview fixtures (`server/internal/handlers/emailpreview/fixtures_report.go`). +6. Keep the email inside the size budget checked by `size_test.go`. +7. `CHANGELOG.md` entry, and a line in the report docs page of `web/docs/docs/`. + +## Tests + +- `server/internal/uptimereport/builder_test.go`: + - a window with acknowledged and resolved incidents fills the three figures and orders `TopFailingChecks` by incident count + - a window with no incident leaves them empty + - an incident on a check outside the schedule's scope is not counted +- `server/internal/uptimereport/render_test.go`: + - the line and the block are rendered when there are incidents + - neither is rendered with zero incidents (positive control: the same test renders them with one incident) + - the acknowledge figure is absent and "0 of N acknowledged" present when nothing was acknowledged + - the block is absent with a single check in scope +- `locale_report_test.go`: the new strings exist in every locale. +- `size_test.go`: still passes with 5 top rows and the longest locale. + +## To verify + +- Whether `incidentBlock` (`report.go:659`) keeps the incident slice or only the aggregate. If only the aggregate, have it return the slice. +- The template file the report is rendered from, and the locale files it reads. diff --git a/web/dash0/e2e/account-notifications-discord.spec.ts b/web/dash0/e2e/account-notifications-discord.spec.ts index cfcb57fa2..760cb8f06 100644 --- a/web/dash0/e2e/account-notifications-discord.spec.ts +++ b/web/dash0/e2e/account-notifications-discord.spec.ts @@ -82,9 +82,18 @@ async function stubAccount(page: Page, opts: StubOptions = {}) { return; } - const response = await route.fetch(); - const body = await response.json(); - await route.fulfill({ response, json: { ...body, hasPassword } }); + // A test that asserts an absence can end while this fetch is still in + // flight; the page is then gone and the fetch is disposed. That is teardown, + // not a failure, so it must not be reported against the test. + try { + const response = await route.fetch(); + const body = await response.json(); + await route.fulfill({ response, json: { ...body, hasPassword } }); + } catch (error) { + if (!/disposed|closed/i.test(String(error))) { + throw error; + } + } }); await page.route("**/api/v1/config", async (route) => { diff --git a/web/dash0/e2e/incident-insights.spec.ts b/web/dash0/e2e/incident-insights.spec.ts new file mode 100644 index 000000000..ae384e25b --- /dev/null +++ b/web/dash0/e2e/incident-insights.spec.ts @@ -0,0 +1,138 @@ +import type { Page } from "@playwright/test"; + +import { test, expect, DASH_BASE } from "./fixtures"; + +/** + * Spec 2026-11-13-01: the incident insights page (top failing checks, time to + * acknowledge, time to resolve), reached from the incidents list. + * + * The first test runs against the real seeded org. The others mock the + * insights endpoint so they can assert exact numbers and the empty state. + */ + +const emptyDuration = { medianSeconds: null, p90Seconds: null, sampleCount: 0 }; +const emptyResolution = { count: 0, medianSeconds: null, p90Seconds: null }; + +function insightsBody(period: string, incidentCount: number) { + return { + period, + windowStart: new Date(Date.now() - 86_400_000).toISOString(), + windowEnd: new Date().toISOString(), + truncated: false, + summary: { + incidentCount, + pagedCount: incidentCount, + acknowledgedCount: 0, + ackRate: 0, + timeToAcknowledge: emptyDuration, + timeToResolve: + incidentCount > 0 + ? { medianSeconds: 600, p90Seconds: 900, sampleCount: incidentCount } + : emptyDuration, + byResolution: { + auto: incidentCount > 0 ? { count: incidentCount, medianSeconds: 600, p90Seconds: 900 } : emptyResolution, + manual: emptyResolution, + other: emptyResolution, + }, + }, + checks: + incidentCount > 0 + ? [ + { + checkUid: "99999999-9999-9999-9999-999999999991", + checkName: "Acme API", + checkSlug: "acme-api", + incidentCount, + downtimeSeconds: 1200, + relapseCount: 1, + lastIncidentAt: new Date().toISOString(), + }, + ] + : [], + }; +} + +async function mockInsights(page: Page, countFor: (period: string) => number) { + await page.route("**/api/v1/orgs/*/incidents/insights*", async (route) => { + const period = new URL(route.request().url()).searchParams.get("period") ?? "30d"; + await route.fulfill({ json: insightsBody(period, countFor(period)) }); + }); +} + +test.describe("Incident insights", () => { + test("lists a failing check with its incident count, reached from the incidents list", async ({ + authenticatedPage, + }) => { + const page = authenticatedPage; + + await page.goto(`${DASH_BASE}/orgs/test/incidents`); + await page.getByTestId("incidents-insights-button").click(); + await expect(page).toHaveURL(/\/incidents\/insights/); + + await page.getByTestId("insights-period-90d").click(); + await expect(page).toHaveURL(/period=90d/); + + const row = page.getByTestId("insights-check-row").first(); + await expect(row).toBeVisible({ timeout: 15000 }); + const count = Number(await row.getByTestId("insights-incident-count").innerText()); + expect(count).toBeGreaterThanOrEqual(1); + await expect(page.getByTestId("insights-card-ack")).toBeVisible(); + await expect(page.getByTestId("insights-card-resolve")).toBeVisible(); + await expect(page.getByTestId("insights-card-resolution")).toBeVisible(); + }); + + test("switching the period updates the URL and the numbers", async ({ authenticatedPage }) => { + const page = authenticatedPage; + await mockInsights(page, (period) => (period === "7d" ? 2 : 5)); + + await page.goto(`${DASH_BASE}/orgs/test/incidents/insights`); + await expect(page.getByTestId("insights-incident-count")).toHaveText("5"); + // No acknowledged incident: a dash and the reason, never "0s". + await expect(page.getByTestId("insights-card-ack")).toContainText("No acknowledged incident"); + + await page.getByTestId("insights-period-7d").click(); + await expect(page).toHaveURL(/period=7d/); + await expect(page.getByTestId("insights-incident-count")).toHaveText("2"); + }); + + test("an org with no incident shows the empty state", async ({ authenticatedPage }) => { + const page = authenticatedPage; + await mockInsights(page, () => 0); + + await page.goto(`${DASH_BASE}/orgs/test/incidents/insights`); + await expect(page.getByTestId("insights-empty")).toBeVisible(); + await expect(page.getByTestId("insights-check-row")).toHaveCount(0); + }); + + test("a phone viewport has no horizontal page scroll", async ({ authenticatedPage }) => { + const page = authenticatedPage; + await page.setViewportSize({ width: 375, height: 812 }); + await mockInsights(page, () => 5); + + await page.goto(`${DASH_BASE}/orgs/test/incidents/insights`); + await expect(page.getByTestId("insights-check-row")).toBeVisible(); + + const overflow = await page.evaluate( + () => document.documentElement.scrollWidth > document.documentElement.clientWidth + 1, + ); + expect(overflow).toBe(false); + }); + + // No page.route: the page talks to the real insights endpoint. The test-mode + // seed (server/test/testdata) has an active check incident on "Captured + // Check" that stays open for the whole run. + test("against the real endpoint, a seeded incident shows its check and count", async ({ + authenticatedPage, + }) => { + const page = authenticatedPage; + + await page.goto(`${DASH_BASE}/orgs/test/incidents/insights?period=30d`); + + const row = page.getByTestId("insights-check-row").filter({ + has: page.getByRole("link", { name: "Captured Check" }), + }); + await expect(row).toHaveCount(1); + await expect(row.getByTestId("insights-incident-count")).toHaveText(/^[1-9]\d*$/); + await expect(page.getByTestId("insights-empty")).toHaveCount(0); + }); +}); diff --git a/web/dash0/src/api/hooks.ts b/web/dash0/src/api/hooks.ts index 1eb146171..a3ba9b7e0 100644 --- a/web/dash0/src/api/hooks.ts +++ b/web/dash0/src/api/hooks.ts @@ -2667,6 +2667,58 @@ export function useIncidents( }); } +export type InsightsPeriod = "7d" | "30d" | "90d"; + +export interface InsightsDuration { + medianSeconds: number | null; + p90Seconds: number | null; + sampleCount: number; +} + +export interface InsightsResolution { + count: number; + medianSeconds: number | null; + p90Seconds: number | null; +} + +export interface IncidentInsights { + period: InsightsPeriod; + truncated: boolean; + summary: { + incidentCount: number; + pagedCount: number; + acknowledgedCount: number; + ackRate: number; + timeToAcknowledge: InsightsDuration; + timeToResolve: InsightsDuration; + byResolution: { + auto: InsightsResolution; + manual: InsightsResolution; + other: InsightsResolution; + }; + }; + checks: { + checkUid: string; + checkName: string; + checkSlug: string; + incidentCount: number; + downtimeSeconds: number; + relapseCount: number; + lastIncidentAt: string | null; + }[]; +} + +export function useIncidentInsights(org: string, period: InsightsPeriod) { + return useQuery({ + queryKey: ["incident-insights", org, period], + queryFn: () => + apiFetch( + `/api/v1/orgs/${org}/incidents/insights?period=${period}`, + ), + enabled: !!org, + }); +} + export function useIncident(org: string, uid: string) { return useQuery({ queryKey: ["incident", org, uid], diff --git a/web/dash0/src/i18n-html-lang.test.ts b/web/dash0/src/i18n-html-lang.test.ts new file mode 100644 index 000000000..87a43eae9 --- /dev/null +++ b/web/dash0/src/i18n-html-lang.test.ts @@ -0,0 +1,34 @@ +/** + * @vitest-environment jsdom + */ +import { afterEach, describe, expect, it } from "vitest"; +import i18n, { i18nReady } from "./i18n"; + +// The static index.html says lang="en". When the app renders another +// language, a stale lang makes the browser offer to auto-translate a page +// that is already in the visitor's language, and the translator's DOM edits +// then crash React reconciliation (removeChild NotFoundError). +describe(" follows the rendered language", () => { + afterEach(async () => { + await i18n.changeLanguage("en"); + }); + + it("is the rendered language after a change", async () => { + await i18nReady; + await i18n.changeLanguage("es"); + expect(document.documentElement.lang).toBe("es"); + + await i18n.changeLanguage("en"); + expect(document.documentElement.lang).toBe("en"); + }); + + it("uses the base language for a regional variant", async () => { + await i18n.changeLanguage("fr-CA"); + expect(document.documentElement.lang).toBe("fr"); + }); + + it("stays English for a language the app does not ship", async () => { + await i18n.changeLanguage("pt-BR"); + expect(document.documentElement.lang).toBe("en"); + }); +}); diff --git a/web/dash0/src/i18n.ts b/web/dash0/src/i18n.ts index a7c240e2c..50bb9410f 100644 --- a/web/dash0/src/i18n.ts +++ b/web/dash0/src/i18n.ts @@ -89,6 +89,16 @@ const enResources = { support: en_support, }; +// Keep on the language actually rendered (index.html ships "en"). +// A stale lang makes the browser offer to auto-translate a page already in the +// visitor's language, and the translator's DOM edits crash React's +// reconciliation (removeChild NotFoundError). resolvedLanguage is the shipped +// language in use: "fr" for fr-CA, "en" for a language the app lacks. +i18n.on("languageChanged", () => { + if (typeof document === "undefined") return; + document.documentElement.lang = i18n.resolvedLanguage ?? "en"; +}); + const initPromise = i18n .use(LanguageDetector) .use(initReactI18next) diff --git a/web/dash0/src/locales/de/incidents.json b/web/dash0/src/locales/de/incidents.json index 518329cae..7693588b9 100644 --- a/web/dash0/src/locales/de/incidents.json +++ b/web/dash0/src/locales/de/incidents.json @@ -336,5 +336,38 @@ "noAffected": "Für diese Seite wurden keine betroffenen Komponenten ermittelt.", "noUpdates": "Noch keine Updates.", "automated": "automatisch" + }, + "insights": { + "title": "Incident-Auswertung", + "subtitle": "Welche Checks am häufigsten ausfallen und wie schnell Sie bestätigen und beheben.", + "button": "Auswertung", + "back": "Zurück zu den Incidents", + "period": "Zeitraum", + "days7": "7 Tage", + "days30": "30 Tage", + "days90": "90 Tage", + "topChecks": "Häufigste Ausfälle", + "colCheck": "Check", + "colIncidents": "Incidents", + "colDowntime": "Ausfallzeit", + "colRelapses": "Rückfälle", + "colLast": "Letzter Incident", + "responseTimes": "Reaktionszeiten", + "tta": "Zeit bis zur Bestätigung", + "ttr": "Zeit bis zur Behebung", + "resolution": "Behebung", + "median": "Median", + "p90": "p90", + "ackedOf": "{{acked}} von {{total}} bestätigt", + "noAck": "Kein bestätigter Incident in diesem Zeitraum", + "noResolve": "Kein behobener Incident in diesem Zeitraum", + "auto": "Selbst erholt", + "manual": "Manuell behoben", + "other": "Sonstige Abschlüsse", + "n": "{{n}} Incidents", + "medianValue": "Median {{value}}", + "empty": "Kein Incident in diesem Zeitraum", + "truncated": "Es werden nur die 5.000 neuesten Incidents berücksichtigt, die Zahlen sind daher unvollständig.", + "noData": "Keine Daten" } } diff --git a/web/dash0/src/locales/en/incidents.json b/web/dash0/src/locales/en/incidents.json index ca35c6dc1..454258581 100644 --- a/web/dash0/src/locales/en/incidents.json +++ b/web/dash0/src/locales/en/incidents.json @@ -336,5 +336,38 @@ "noAffected": "No affected components resolved for this page.", "noUpdates": "No updates yet.", "automated": "automated" + }, + "insights": { + "title": "Incident insights", + "subtitle": "Which checks fail the most, and how fast you acknowledge and resolve.", + "button": "Insights", + "back": "Back to incidents", + "period": "Period", + "days7": "7 days", + "days30": "30 days", + "days90": "90 days", + "topChecks": "Top failing checks", + "colCheck": "Check", + "colIncidents": "Incidents", + "colDowntime": "Downtime", + "colRelapses": "Relapses", + "colLast": "Last incident", + "responseTimes": "Response times", + "tta": "Time to acknowledge", + "ttr": "Time to resolve", + "resolution": "Resolution", + "median": "Median", + "p90": "p90", + "ackedOf": "{{acked}} of {{total}} acknowledged", + "noAck": "No acknowledged incident in this period", + "noResolve": "No resolved incident in this period", + "auto": "Auto-recovered", + "manual": "Manually resolved", + "other": "Other closures", + "n": "{{n}} incidents", + "medianValue": "median {{value}}", + "empty": "No incident in this period", + "truncated": "Only the 5,000 most recent incidents are included, so these numbers are partial.", + "noData": "No data" } } diff --git a/web/dash0/src/locales/es/incidents.json b/web/dash0/src/locales/es/incidents.json index 4c336bf8d..e70230d84 100644 --- a/web/dash0/src/locales/es/incidents.json +++ b/web/dash0/src/locales/es/incidents.json @@ -336,5 +336,38 @@ "noAffected": "No se han identificado componentes afectados para esta página.", "noUpdates": "Todavía no hay actualizaciones.", "automated": "automática" + }, + "insights": { + "title": "Estadísticas de incidentes", + "subtitle": "Qué checks fallan más y con qué rapidez se reconocen y resuelven.", + "button": "Estadísticas", + "back": "Volver a los incidentes", + "period": "Período", + "days7": "7 días", + "days30": "30 días", + "days90": "90 días", + "topChecks": "Checks que más fallan", + "colCheck": "Check", + "colIncidents": "Incidentes", + "colDowntime": "Inactividad", + "colRelapses": "Recaídas", + "colLast": "Último incidente", + "responseTimes": "Tiempos de respuesta", + "tta": "Tiempo de reconocimiento", + "ttr": "Tiempo de resolución", + "resolution": "Resolución", + "median": "Mediana", + "p90": "p90", + "ackedOf": "{{acked}} de {{total}} reconocidos", + "noAck": "Ningún incidente reconocido en este período", + "noResolve": "Ningún incidente resuelto en este período", + "auto": "Recuperados solos", + "manual": "Resueltos manualmente", + "other": "Otros cierres", + "n": "{{n}} incidentes", + "medianValue": "mediana {{value}}", + "empty": "Ningún incidente en este período", + "truncated": "Solo se incluyen los 5.000 incidentes más recientes, por lo que estas cifras son parciales.", + "noData": "Sin datos" } } diff --git a/web/dash0/src/locales/fr/incidents.json b/web/dash0/src/locales/fr/incidents.json index 8a8d1a630..1e50759b6 100644 --- a/web/dash0/src/locales/fr/incidents.json +++ b/web/dash0/src/locales/fr/incidents.json @@ -336,5 +336,38 @@ "noAffected": "Aucun composant affecté n'a été identifié pour cette page.", "noUpdates": "Aucune mise à jour pour le moment.", "automated": "automatique" + }, + "insights": { + "title": "Statistiques des incidents", + "subtitle": "Quels checks tombent le plus, et à quelle vitesse vous acquittez et résolvez.", + "button": "Statistiques", + "back": "Retour aux incidents", + "period": "Période", + "days7": "7 jours", + "days30": "30 jours", + "days90": "90 jours", + "topChecks": "Checks les plus en échec", + "colCheck": "Check", + "colIncidents": "Incidents", + "colDowntime": "Indisponibilité", + "colRelapses": "Rechutes", + "colLast": "Dernier incident", + "responseTimes": "Temps de réaction", + "tta": "Délai d'acquittement", + "ttr": "Délai de résolution", + "resolution": "Résolution", + "median": "Médiane", + "p90": "p90", + "ackedOf": "{{acked}} sur {{total}} acquittés", + "noAck": "Aucun incident acquitté sur cette période", + "noResolve": "Aucun incident résolu sur cette période", + "auto": "Rétablis seuls", + "manual": "Résolus manuellement", + "other": "Autres clôtures", + "n": "{{n}} incident(s)", + "medianValue": "médiane {{value}}", + "empty": "Aucun incident sur cette période", + "truncated": "Seuls les 5 000 incidents les plus récents sont pris en compte : ces chiffres sont partiels.", + "noData": "Aucune donnée" } } diff --git a/web/dash0/src/locales/identical-allowlist.ts b/web/dash0/src/locales/identical-allowlist.ts index e55fb3e42..7fcec02f7 100644 --- a/web/dash0/src/locales/identical-allowlist.ts +++ b/web/dash0/src/locales/identical-allowlist.ts @@ -348,6 +348,10 @@ export const DASH0_IDENTICAL_ALLOWLIST: IdenticalAllowlist = { "events:audit.targetTypes.integration": ["de"], // "Integration" "feedback:tool_text": ["de"], // "Text" "incidents:title": ["fr"], // "Incidents" + "incidents:insights.colCheck": ["fr", "de", "es"], // "Check" + "incidents:insights.colIncidents": ["fr", "de"], // "Incidents" + "incidents:insights.p90": ["fr", "de", "es"], // "p90" + "incidents:insights.median": ["de"], // "Median" "incidents:table.incident": ["fr"], // "Incident" "incidents:fallbackTitle": ["fr"], // "Incident" "incidents:snoozeDialog.until15m": ["fr"], // "15 minutes" diff --git a/web/dash0/src/routeTree.gen.ts b/web/dash0/src/routeTree.gen.ts index b3904c632..06cc9715e 100644 --- a/web/dash0/src/routeTree.gen.ts +++ b/web/dash0/src/routeTree.gen.ts @@ -109,6 +109,7 @@ import { Route as OrgsOrgMaintenanceWindowsMaintenanceWindowUidRouteImport } fro import { Route as OrgsOrgJobsJobUidRouteImport } from './routes/orgs/$org/jobs.$jobUid' import { Route as OrgsOrgIntegrationsNewRouteImport } from './routes/orgs/$org/integrations.new' import { Route as OrgsOrgIntegrationsIntegrationUidRouteImport } from './routes/orgs/$org/integrations.$integrationUid' +import { Route as OrgsOrgIncidentsInsightsRouteImport } from './routes/orgs/$org/incidents.insights' import { Route as OrgsOrgIncidentsIncidentUidRouteImport } from './routes/orgs/$org/incidents.$incidentUid' import { Route as OrgsOrgEscalationPoliciesNewRouteImport } from './routes/orgs/$org/escalation-policies.new' import { Route as OrgsOrgEscalationPoliciesUidRouteImport } from './routes/orgs/$org/escalation-policies.$uid' @@ -692,6 +693,12 @@ const OrgsOrgIntegrationsIntegrationUidRoute = path: '/$integrationUid', getParentRoute: () => OrgsOrgIntegrationsRoute, } as any) +const OrgsOrgIncidentsInsightsRoute = + OrgsOrgIncidentsInsightsRouteImport.update({ + id: '/insights', + path: '/insights', + getParentRoute: () => OrgsOrgIncidentsRoute, + } as any) const OrgsOrgIncidentsIncidentUidRoute = OrgsOrgIncidentsIncidentUidRouteImport.update({ id: '/$incidentUid', @@ -1063,6 +1070,7 @@ export interface FileRoutesByFullPath { '/orgs/$org/escalation-policies/$uid': typeof OrgsOrgEscalationPoliciesUidRoute '/orgs/$org/escalation-policies/new': typeof OrgsOrgEscalationPoliciesNewRoute '/orgs/$org/incidents/$incidentUid': typeof OrgsOrgIncidentsIncidentUidRoute + '/orgs/$org/incidents/insights': typeof OrgsOrgIncidentsInsightsRoute '/orgs/$org/integrations/$integrationUid': typeof OrgsOrgIntegrationsIntegrationUidRoute '/orgs/$org/integrations/new': typeof OrgsOrgIntegrationsNewRoute '/orgs/$org/jobs/$jobUid': typeof OrgsOrgJobsJobUidRoute @@ -1201,6 +1209,7 @@ export interface FileRoutesByTo { '/orgs/$org/escalation-policies/$uid': typeof OrgsOrgEscalationPoliciesUidRoute '/orgs/$org/escalation-policies/new': typeof OrgsOrgEscalationPoliciesNewRoute '/orgs/$org/incidents/$incidentUid': typeof OrgsOrgIncidentsIncidentUidRoute + '/orgs/$org/incidents/insights': typeof OrgsOrgIncidentsInsightsRoute '/orgs/$org/integrations/$integrationUid': typeof OrgsOrgIntegrationsIntegrationUidRoute '/orgs/$org/integrations/new': typeof OrgsOrgIntegrationsNewRoute '/orgs/$org/jobs/$jobUid': typeof OrgsOrgJobsJobUidRoute @@ -1351,6 +1360,7 @@ export interface FileRoutesById { '/orgs/$org/escalation-policies/$uid': typeof OrgsOrgEscalationPoliciesUidRoute '/orgs/$org/escalation-policies/new': typeof OrgsOrgEscalationPoliciesNewRoute '/orgs/$org/incidents/$incidentUid': typeof OrgsOrgIncidentsIncidentUidRoute + '/orgs/$org/incidents/insights': typeof OrgsOrgIncidentsInsightsRoute '/orgs/$org/integrations/$integrationUid': typeof OrgsOrgIntegrationsIntegrationUidRoute '/orgs/$org/integrations/new': typeof OrgsOrgIntegrationsNewRoute '/orgs/$org/jobs/$jobUid': typeof OrgsOrgJobsJobUidRoute @@ -1510,6 +1520,7 @@ export interface FileRouteTypes { | '/orgs/$org/escalation-policies/$uid' | '/orgs/$org/escalation-policies/new' | '/orgs/$org/incidents/$incidentUid' + | '/orgs/$org/incidents/insights' | '/orgs/$org/integrations/$integrationUid' | '/orgs/$org/integrations/new' | '/orgs/$org/jobs/$jobUid' @@ -1648,6 +1659,7 @@ export interface FileRouteTypes { | '/orgs/$org/escalation-policies/$uid' | '/orgs/$org/escalation-policies/new' | '/orgs/$org/incidents/$incidentUid' + | '/orgs/$org/incidents/insights' | '/orgs/$org/integrations/$integrationUid' | '/orgs/$org/integrations/new' | '/orgs/$org/jobs/$jobUid' @@ -1797,6 +1809,7 @@ export interface FileRouteTypes { | '/orgs/$org/escalation-policies/$uid' | '/orgs/$org/escalation-policies/new' | '/orgs/$org/incidents/$incidentUid' + | '/orgs/$org/incidents/insights' | '/orgs/$org/integrations/$integrationUid' | '/orgs/$org/integrations/new' | '/orgs/$org/jobs/$jobUid' @@ -2618,6 +2631,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof OrgsOrgIntegrationsIntegrationUidRouteImport parentRoute: typeof OrgsOrgIntegrationsRoute } + '/orgs/$org/incidents/insights': { + id: '/orgs/$org/incidents/insights' + path: '/insights' + fullPath: '/orgs/$org/incidents/insights' + preLoaderRoute: typeof OrgsOrgIncidentsInsightsRouteImport + parentRoute: typeof OrgsOrgIncidentsRoute + } '/orgs/$org/incidents/$incidentUid': { id: '/orgs/$org/incidents/$incidentUid' path: '/$incidentUid' @@ -3143,12 +3163,14 @@ const OrgsOrgEscalationPoliciesRouteWithChildren = interface OrgsOrgIncidentsRouteChildren { OrgsOrgIncidentsIncidentUidRoute: typeof OrgsOrgIncidentsIncidentUidRoute + OrgsOrgIncidentsInsightsRoute: typeof OrgsOrgIncidentsInsightsRoute OrgsOrgIncidentsIndexRoute: typeof OrgsOrgIncidentsIndexRoute OrgsOrgIncidentsIncidentUidNotificationsNotificationUidRoute: typeof OrgsOrgIncidentsIncidentUidNotificationsNotificationUidRoute } const OrgsOrgIncidentsRouteChildren: OrgsOrgIncidentsRouteChildren = { OrgsOrgIncidentsIncidentUidRoute: OrgsOrgIncidentsIncidentUidRoute, + OrgsOrgIncidentsInsightsRoute: OrgsOrgIncidentsInsightsRoute, OrgsOrgIncidentsIndexRoute: OrgsOrgIncidentsIndexRoute, OrgsOrgIncidentsIncidentUidNotificationsNotificationUidRoute: OrgsOrgIncidentsIncidentUidNotificationsNotificationUidRoute, diff --git a/web/dash0/src/routes/orgs/$org/incidents.index.tsx b/web/dash0/src/routes/orgs/$org/incidents.index.tsx index b1ecf9ac5..f7ee21206 100644 --- a/web/dash0/src/routes/orgs/$org/incidents.index.tsx +++ b/web/dash0/src/routes/orgs/$org/incidents.index.tsx @@ -3,6 +3,7 @@ import { useTranslation } from "react-i18next"; import { createFileRoute, Link, useNavigate } from "@tanstack/react-router"; import { AlertTriangle, + BarChart3, Check, CheckCircle, Layers, @@ -320,6 +321,23 @@ function IncidentsIndexPage() { description={t("subtitle")} docsHref="/docs/features/incidents" className="flex-wrap" + actions={ + + } />
@@ -418,6 +436,7 @@ function IncidentsIndexPage() {
+ ))} + + } + /> + + {isLoading || !data || !summary ? ( + + ) : summary.incidentCount === 0 ? ( + + + {t("insights.empty")} + + + ) : ( + <> + {data.truncated && ( +
+ {t("insights.truncated")} +
+ )} + + + + {t("insights.topChecks")} + + + + + + {t("insights.colCheck")} + setSortKey("incidentCount")} + testId="insights-sort-incidents" + /> + setSortKey("downtimeSeconds")} + testId="insights-sort-downtime" + /> + setSortKey("relapseCount")} + testId="insights-sort-relapses" + /> + {t("insights.colLast")} + + + + {rows.map((row) => ( + + + + {row.checkName || row.checkSlug || row.checkUid} + + + + + {row.incidentCount} + + + + {formatDuration(row.downtimeSeconds * 1000)} + + {row.relapseCount} + + {row.lastIncidentAt ? ( + + ) : ( + "-" + )} + + + ))} + +
+ + + +

+ {t("insights.responseTimes")} +

+
+ + + + + + {t("insights.resolution")} + + + + {(["auto", "manual", "other"] as const).map((kind) => { + const stat = summary.byResolution[kind]; + const median = seconds(stat.medianSeconds); + return ( +
+ {t(`insights.${kind}`)} + + {t("insights.n", { n: stat.count })} + {kind !== "other" && median + ? ` · ${t("insights.medianValue", { value: median })}` + : ""} + +
+ ); + })} + + +
+ + )} + + ); +} + +function SortHead({ + label, + active, + onClick, + testId, +}: { + label: string; + active: boolean; + onClick: () => void; + testId: string; +}) { + return ( + + + + ); +} + +function StatCard({ + testId, + title, + median, + p90, + empty, + footer, +}: { + testId: string; + title: string; + median: string | null; + p90: string | null; + empty: string; + footer?: string; +}) { + const { t } = useTranslation("incidents"); + + return ( + + + {title} + + + {median === null ? ( + <> +
-
+

{empty}

+ + ) : ( + <> +
{median}
+

+ {t("insights.median")} · {t("insights.p90")} {p90} +

+ + )} + {footer &&

{footer}

} + + + ); +} diff --git a/web/docs/docs/features/incidents.md b/web/docs/docs/features/incidents.md index 0fb0b5ee3..89d876a35 100644 --- a/web/docs/docs/features/incidents.md +++ b/web/docs/docs/features/incidents.md @@ -304,6 +304,16 @@ the next evaluation. Degraded incidents do not count in the check's availability: the **Outages** column of the availability table counts outages only. +## Incident insights {#insights} + +The **Insights** button in the header of the incidents list opens a page that answers the questions a team asks after a few weeks on call. Pick a window of 7, 30 or 90 days. It covers the incidents active during the period: those that started inside it, and those that started earlier and were still open or resolved inside it. Only check outages count: SLO burn alerts and degraded incidents are left out. + +- **Top failing checks**: the 20 checks with the most incidents active in the window (ties broken by downtime), with their downtime, relapses and last incident. Downtime is the wall-clock time of the incidents, clipped to the window; an open incident counts up to now. +- **Time to acknowledge**: the time between the start of the incident's current episode and its acknowledgement. When an incident reopens, the acknowledgement is cleared, so the time is measured from the reopening. It is always shown with "N of M acknowledged", because most incidents recover before anybody acknowledges them and the time alone would flatter you. Incidents rolled up under a root cause (nobody was paged) are not part of it. +- **Time to resolve**: the time between the start and the resolution of incidents that were auto-recovered or resolved manually. Closures that say nothing about recovery (expired, escalated, disabled check, deleted check) are counted under "Other" and left out of the median. + +Every duration is a **median** and a **p90** (nearest rank), never a mean, so one six-hour outage does not distort the picture. With no data the card shows a dash, not zero. The same numbers are available from `GET /api/v1/orgs/{org}/incidents/insights?period=30d`. At most 5,000 incidents are read; the response then carries `truncated: true`. + ## Notification Events | Event | When | Description | diff --git a/web/docs/docs/features/slos.md b/web/docs/docs/features/slos.md index c9e57dc34..b553f8e55 100644 --- a/web/docs/docs/features/slos.md +++ b/web/docs/docs/features/slos.md @@ -119,6 +119,13 @@ scope. - **Incidents**: how many, the longest, the average, and the total downtime, each against the previous period. There is deliberately no MTBF: with the handful of incidents a typical month produces it is not a meaningful number. +- **Incident response**: the median time to resolve, the median time to + acknowledge, and "12 of 40 acknowledged", computed by the same code as the + incident insights page of the dashboard. A **Most incidents** block lists the + five checks that opened the most incidents, with their relapses, so a check + that flaps all month is not hidden behind a good availability. Nothing is + shown for a period without incident, and the block is left out when the + report covers a single check. - **Response time**: average, fastest and slowest, and how many probes were slower than one second. Recent periods count individual *samples*; older data is stored as rollups, which can only report *peaks* — a rolled-up period whose diff --git a/wiki/testing/test-layers.md b/wiki/testing/test-layers.md index d154e7a86..ed0b88c19 100644 --- a/wiki/testing/test-layers.md +++ b/wiki/testing/test-layers.md @@ -148,6 +148,11 @@ recent `v*` tags plus the newest tag of the previous minor, each on both engines in `ci.yml`, both gating the `ci` job). Locally, `make test-upgrade` runs the whole matrix and needs docker, jq and sqlite3. A single leg: `HEAD_BIN=./solidping scripts/upgrade-test.sh sqlite v0.38.6`. +A tag enters the matrix the moment it is pushed, about 20 minutes before its own CI publishes +its image. CI sets `IMAGE_WAIT=1500` so a leg retries the pull that long instead of failing on +"not found" (the job timeout is 35 minutes for that reason). The default is 0: locally a +missing image fails at once. + It complements `scripts/check-frozen-migrations.sh` (static) and does not replace it. CI builds its own Go-only binary for the `upgrade` job (no frontends), so the layer also runs on