feat: incident insights page and uptime report insights

.github/workflows/ci.yml

@@ -1423,7 +1423,9 @@ jobs:
     runs-on: ubuntu-26.04
     needs: [changes, upgrade-matrix]
     if: ${{ !startsWith(github.ref, 'refs/tags/') && needs.changes.outputs.code == 'true' }}
-    timeout-minutes: 10
+    # A leg takes about 3 minutes. The rest is IMAGE_WAIT below: room to wait
+    # for the image of a tag pushed moments before this run started.
+    timeout-minutes: 35
     strategy:
       fail-fast: false
       matrix:
@@ -1446,6 +1448,8 @@ jobs:
         run: go build -o ../solidping .
 
       - name: Upgrade ${{ matrix.tag }} to HEAD
+        env:
+          IMAGE_WAIT: "1500"
         run: |
           set -euo pipefail
           command -v sqlite3 >/dev/null || sudo apt-get install -y sqlite3

scripts/upgrade-test.sh

@@ -10,6 +10,8 @@
 #   HEAD_BIN   the HEAD server binary (default: ./solidping, see `make test-upgrade`)
 #   SKIP_TAGS  space-separated tags left out of the matrix (default: SKIP_TAGS below)
 #   PG_IMAGE   Postgres image for the old and new side (default: docker-compose.yml's)
+#   IMAGE_WAIT seconds to wait for the released image to be published (default: 0).
+#              CI sets it: a tag exists minutes before the image its own CI pushes.
 #
 # Needs: bash, docker, curl, jq, git (matrix), sqlite3 (sqlite engine).
 
@@ -121,6 +123,22 @@ fail() {
     exit 1
 }
 
+# pull_image <image>: pulls it, retrying every 30 s for IMAGE_WAIT seconds. A
+# release tag enters the matrix the moment it is pushed, but its image only
+# exists once the tag's own CI has built it (about 20 minutes later): a PR whose
+# CI starts in between failed every leg from that tag on "not found".
+pull_image() {
+    local image=$1 deadline=$((SECONDS + ${IMAGE_WAIT:-0})) err
+    until err=$(docker pull -q "$image" 2>&1 >/dev/null); do
+        if ((SECONDS >= deadline)); then
+            echo "$err" >&2
+            return 1
+        fi
+        echo "    $image is not available yet, retrying in 30 s" >&2
+        sleep 30
+    done
+}
+
 # wait_health <port> <seconds>: succeeds once /api/mgmt/health answers ok.
 wait_health() {
     local port=$1 deadline=$((SECONDS + $2))
@@ -222,6 +240,7 @@ fi
 
 # ---- Old side -------------------------------------------------------------
 step "starting $IMAGE_REPO:${from_tag#v}"
+pull_image "$IMAGE_REPO:${from_tag#v}" || fail "could not pull $IMAGE_REPO:${from_tag#v}"
 old_env_args=()
 for kv in "${common_env[@]}"; do old_env_args+=(-e "$kv"); done
 docker run -d --name "$old_name" -p 127.0.0.1::4000 "${old_args[@]}" "${old_env_args[@]}" \

server/internal/app/incident_insights_route_test.go

@@ -0,0 +1,32 @@
+package app
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// TestIncidentInsightsRoute walks the real router: the insights route needs a
+// token, and with one it is served by the insights handler rather than by
+// the `/:uid` route (which would answer 404 INCIDENT_NOT_FOUND).
+func TestIncidentInsightsRoute(t *testing.T) {
+	t.Parallel()
+	r := require.New(t)
+
+	env := newMCPTestEnv(t)
+	path := "/api/v1/orgs/" + mcpE2EOrg + "/incidents/insights"
+
+	anon := env.get(t, path, nil)
+	r.Equal(http.StatusUnauthorized, anon.status, string(anon.body))
+
+	authed := env.get(t, path+"?period=7d", map[string]string{"Authorization": "Bearer " + env.login(t)})
+	r.Equal(http.StatusOK, authed.status, string(authed.body))
+	r.Contains(string(authed.body), `"period":"7d"`)
+	r.Contains(string(authed.body), `"summary"`)
+	r.Contains(string(authed.body), `"checks"`)
+
+	bad := env.get(t, path+"?period=1y", map[string]string{"Authorization": "Bearer " + env.login(t)})
+	r.Equal(http.StatusBadRequest, bad.status, string(bad.body))
+	r.Contains(string(bad.body), `"VALIDATION_ERROR"`)
+}

server/internal/app/openapi/openapi.yaml

@@ -4045,6 +4045,105 @@ paths:
         "404":
           $ref: "#/components/responses/NotFound"
 
+  /api/v1/orgs/{org}/incidents/insights:
+    get:
+      tags: [Incidents]
+      summary: Incident insights
+      description: >-
+        Aggregates the check outages active during a trailing period (started before
+        its end, still open or resolved inside it; downtime is clipped to the
+        period): the checks that fail
+        the most, the time to acknowledge, the time to resolve and how incidents
+        were resolved. Medians and nearest-rank p90, never means. Rolled-up
+        (paging-suppressed) incidents count in the checks table and the resolve
+        time but not in the acknowledge sample or rate. SLO burn and degraded
+        incidents are not counted. At most 5000 incidents are read, then
+        `truncated` is true.
+      operationId: getIncidentInsights
+      security:
+        - bearerAuth: []
+      parameters:
+        - $ref: "#/components/parameters/OrgPath"
+        - name: period
+          in: query
+          description: Trailing window, one of 7d, 30d (default), 90d. Anything else is a 400 VALIDATION_ERROR.
+          schema:
+            type: string
+            enum: [7d, 30d, 90d]
+            default: 30d
+      responses:
+        "200":
+          description: Incident insights
+          content:
+            application/json:
+              schema:
+                type: object
+                required: [period, windowStart, windowEnd, truncated, summary, checks]
+                properties:
+                  period:
+                    type: string
+                  windowStart:
+                    type: string
+                    format: date-time
+                  windowEnd:
+                    type: string
+                    format: date-time
+                  truncated:
+                    type: boolean
+                  summary:
+                    type: object
+                    properties:
+                      incidentCount:
+                        type: integer
+                      pagedCount:
+                        type: integer
+                        description: Incidents that were not rolled up, the base of ackRate.
+                      acknowledgedCount:
+                        type: integer
+                      ackRate:
+                        type: number
+                        description: 0 to 1
+                      timeToAcknowledge:
+                        $ref: "#/components/schemas/IncidentInsightsDuration"
+                      timeToResolve:
+                        $ref: "#/components/schemas/IncidentInsightsDuration"
+                      byResolution:
+                        type: object
+                        properties:
+                          auto:
+                            $ref: "#/components/schemas/IncidentInsightsResolution"
+                          manual:
+                            $ref: "#/components/schemas/IncidentInsightsResolution"
+                          other:
+                            $ref: "#/components/schemas/IncidentInsightsResolution"
+                  checks:
+                    type: array
+                    description: Top 20 checks by incident count, ties broken by downtime.
+                    items:
+                      type: object
+                      properties:
+                        checkUid:
+                          type: string
+                        checkName:
+                          type: string
+                        checkSlug:
+                          type: string
+                        incidentCount:
+                          type: integer
+                        downtimeSeconds:
+                          type: integer
+                          format: int64
+                        relapseCount:
+                          type: integer
+                        lastIncidentAt:
+                          type: string
+                          format: date-time
+                          nullable: true
+        "400":
+          $ref: "#/components/responses/ValidationError"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+
   /api/v1/orgs/{org}/incidents/{uid}:
     get:
       tags: [Incidents]
@@ -10668,6 +10767,32 @@ components:
             $ref: "#/components/schemas/Error"
 
   schemas:
+    IncidentInsightsDuration:
+      type: object
+      properties:
+        medianSeconds:
+          type: integer
+          format: int64
+          nullable: true
+        p90Seconds:
+          type: integer
+          format: int64
+          nullable: true
+        sampleCount:
+          type: integer
+    IncidentInsightsResolution:
+      type: object
+      properties:
+        count:
+          type: integer
+        medianSeconds:
+          type: integer
+          format: int64
+          nullable: true
+        p90Seconds:
+          type: integer
+          format: int64
+          nullable: true
     SupportReplyWindow:
       type: object
       description: >

server/internal/app/server.go

@@ -1620,6 +1620,8 @@ func (s *Server) SetupRoutes(ctx context.Context) {
 	incidentsHandler := incidents.NewHandler(incidentsService, s.config)
 	orgIncidents := orgGroup("/orgs/:org/incidents")
 	orgIncidents.GET("", incidentsHandler.ListIncidents)
+	// Registered before "/:uid" so "insights" is not read as an incident uid.
+	orgIncidents.GET("/insights", incidentsHandler.GetInsights)
 	orgIncidents.GET("/:uid", incidentsHandler.GetIncident)
 	orgIncidents.POST("/:uid/ack", incidentsHandler.AcknowledgeIncident)
 	orgIncidents.POST("/:uid/unack", incidentsHandler.UnacknowledgeIncident)

server/internal/checkers/checkbrowser/checker.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chromedp/cdproto"
 	"github.com/chromedp/chromedp"
 
 	checkconfig "github.com/fclairamb/solidping/server/internal/checkers/checkbrowser/config"
@@ -329,8 +330,19 @@ func (c *BrowserChecker) captureScreenshot(
 
 	shot, err := capture(shotCtx)
 	if err != nil {
-		slog.WarnContext(ctx, "browser check: screenshot capture failed",
-			"url", cfg.URL, "error", err, "budget_remaining_ms", budgetRemaining.Milliseconds())
+		// A page that closed or crashed before the shot is the monitored
+		// page's doing, not a CDP/infra failure: logged at INFO so it does not
+		// read as a worker fault in the pod logs.
+		detached := pageDetached(err)
+
+		level := slog.LevelWarn
+		if detached {
+			level = slog.LevelInfo
+		}
+
+		slog.Log(ctx, level, "browser check: screenshot capture failed",
+			"url", cfg.URL, "error", err, "budget_remaining_ms", budgetRemaining.Milliseconds(),
+			"page_detached", detached)
 		setScreenshotError(result, CaptureErrorMessage(err, budgetRemaining))
 
 		return
@@ -387,9 +399,31 @@ func CaptureErrorMessage(err error, budget time.Duration) string {
 		return "the capture timed out"
 	}
 
+	if pageDetached(err) {
+		return "the page was closed or crashed before the capture"
+	}
+
 	return err.Error()
 }
 
+// cdpServerError is the JSON-RPC "server error" code Chrome answers with when
+// a command targets a session whose page is gone.
+const cdpServerError = -32000
+
+// pageDetached reports whether a CDP command failed because its page was no
+// longer attached: the tab closed itself, crashed, or the target went away
+// before the command ran. Chrome reports it as -32000 "Not attached to an
+// active page" (or "Session with given id not found").
+func pageDetached(err error) bool {
+	var cdpErr *cdproto.Error
+	if !errors.As(err, &cdpErr) || cdpErr.Code != cdpServerError {
+		return false
+	}
+
+	return strings.Contains(cdpErr.Message, "Not attached to an active page") ||
+		strings.Contains(cdpErr.Message, "Session with given id not found")
+}
+
 // OverCapMessage is the ScreenshotError of a capture dropped over
 // MaxScreenshotBytes. Shared with the JS runtime.
 func OverCapMessage(size int) string {

server/internal/checkers/checkbrowser/page_detached_test.go

@@ -0,0 +1,54 @@
+package checkbrowser
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/chromedp/cdproto"
+	"github.com/stretchr/testify/require"
+)
+
+// errPlainDetachedText carries the CDP wording without being a CDP error.
+var errPlainDetachedText = errors.New("plain: Not attached to an active page")
+
+// TestCaptureErrorMessageNamesADetachedPage: a capture whose page closed or
+// crashed first fails with CDP -32000 "Not attached to an active page". That
+// is the monitored page's doing, so the operator reads a plain sentence rather
+// than the raw protocol error, and the log is not raised as a worker fault.
+func TestCaptureErrorMessageNamesADetachedPage(t *testing.T) {
+	t.Parallel()
+
+	detached := &cdproto.Error{Code: cdpServerError, Message: "Not attached to an active page"}
+	gone := &cdproto.Error{Code: cdpServerError, Message: "Session with given id not found"}
+	other := &cdproto.Error{Code: cdpServerError, Message: "Internal error"}
+
+	cases := []struct {
+		name     string
+		err      error
+		detached bool
+		message  string
+	}{
+		{"detached", detached, true, "the page was closed or crashed before the capture"},
+		{"wrapped", fmt.Errorf("capture: %w", detached), true, "the page was closed or crashed before the capture"},
+		{"session gone", gone, true, "the page was closed or crashed before the capture"},
+		{"other cdp error", other, false, other.Error()},
+		{"plain error", errRendererGone, false, errRendererGone.Error()},
+		{"deadline", context.DeadlineExceeded, false, "the capture timed out"},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			r := require.New(t)
+
+			r.Equal(tc.detached, pageDetached(tc.err))
+			r.Equal(tc.message, CaptureErrorMessage(tc.err, 0))
+		})
+	}
+
+	r := require.New(t)
+	r.False(pageDetached(nil))
+	r.False(pageDetached(errPlainDetachedText))
+}