Obervability with OpenTelemetry

.docker/grafana-datasources.yaml

@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    isDefault: true
+    url: http://victoria-metrics:8428
+  - name: VictoriaLogs
+    type: victoriametrics-logs-datasource
+    access: proxy
+    url: http://victoria-logs:9428
+  - name: Jaeger
+    type: jaeger
+    access: proxy
+    url: http://victoria-traces:10428/select/jaeger

.docker/otel-collector.yaml

@@ -0,0 +1,35 @@
+# Same pipelines as the cluster collector, each signal sent to its own Victoria backend.
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+processors:
+  batch:
+
+exporters:
+  otlphttp/metrics:
+    endpoint: http://victoria-metrics:8428/opentelemetry
+  otlphttp/logs:
+    endpoint: http://victoria-logs:9428/insert/opentelemetry
+  otlphttp/traces:
+    endpoint: http://victoria-traces:10428/insert/opentelemetry
+
+service:
+  pipelines:
+    metrics:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlphttp/metrics]
+    logs:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlphttp/logs]
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlphttp/traces]

.gitignore

@@ -13,6 +13,7 @@ appsettings.Development.json
 *.csproj.user
 TestResults/*
 Local.runsettings
+.env
 
 # NPM
 firebase.config.js

CONTRIBUTING.md

@@ -21,10 +21,10 @@ If you do not agree with these terms, do not submit a contribution.
 
 1. [.NET 10.0 SDK](https://dotnet.microsoft.com/download).
 2. Node.js, for the scripts in `scripts/`.
-3. A MongoDB database, for example:
+3. A MongoDB database, for example the one of [Containers](#containers):
 
    ```bash
-   docker run --name mongodb -d -p 27017:27017 mongo:9.0
+   docker compose up -d mongodb
    ```
 
    A local `mongod` or a [MongoDB Atlas](https://cloud.mongodb.com/) cluster works too.
@@ -276,7 +276,53 @@ Variable                       | Default                             | Purpose
 A failed test leaves a full-page screenshot and a Playwright trace in `test/BlazorApp.PlaywrightTests/bin/<config>/net10.0/e2e-diagnostics`, and prints their paths in its output.
 A trace opens at [trace.playwright.dev](https://trace.playwright.dev).
 
-## Container images
+## Containers
+
+`compose.yaml` builds and runs both applications, with two optional profiles:
+
+- `mongodb`: a MongoDB whose indexes are created on its first start, from `scripts/mongodb-create-index.js`.
+- `observability`: an OpenTelemetry Collector sending to VictoriaMetrics, VictoriaLogs and VictoriaTraces, read in Grafana, the same pipelines as the cluster.
+
+Every port is bound to `127.0.0.1`.
+
+Service          | URL
+-----------------|-------------------------------------------------------------------------------
+`blazorapp`      | [localhost:9001](http://localhost:9001)
+`webapi`         | [localhost:9002/scalar](http://localhost:9002/scalar)
+`mongodb`        | `mongodb://localhost:27017`
+`otel-collector` | `http://localhost:4317` (OTLP gRPC), `http://localhost:4318` (OTLP HTTP)
+`grafana`        | [localhost:3000](http://localhost:3000), signed in as an anonymous administrator
+
+Values come from a `.env` file at the repository root, gitignored, read by compose itself:
+
+```ini
+FIREBASE_PROJECT_ID=xxxx
+FIREBASE_AUTH_DOMAIN=xxxx.firebaseapp.com
+FIREBASE_APIKEY=xxxx
+FIREBASE_SERVICE_ACCOUNT='{"type":"service_account",...}'
+TMDB_APIKEY=xxxx
+# Only when MongoDB runs outside compose, host.docker.internal being the host.
+# MONGODB_CONNECTION_STRING=mongodb://host.docker.internal:27017
+# MONGODB_DATABASE=keeptrack_dev
+# Only with the observability profile.
+# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317
+```
+
+`IGDB_CLIENTID`, `IGDB_CLIENTSECRET`, `DISCOGS_TOKEN` and `GOOGLEBOOKS_APIKEY` set the other [providers](#reference-data-providers).
+A single-quoted value is taken as is, so the service account JSON keeps its `$` and quotes.
+
+```bash
+docker compose --profile mongodb --profile observability up -d --build   # everything
+docker compose --profile mongodb up -d --build                           # the applications and MongoDB
+docker compose up -d grafana otel-collector                              # the observability stack alone
+docker compose --profile mongodb --profile observability down            # stops, keeping the volumes
+```
+
+A service named on the command line starts whatever its profile, so `docker compose up -d grafana otel-collector` also serves the applications run from an IDE.
+They send to it with `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317`, in `launchSettings.json` or the run configuration, and appear in Grafana under **Explore > Jaeger** as `keeptrack-webapi` and `keeptrack-blazorapp`.
+[Telemetry](docs/operations.md#telemetry) describes what is exported.
+
+The images alone:
 
 ```bash
 docker build . -t devprofr/keeptrack-blazorapp:local -f src/BlazorApp/Dockerfile

Directory.Build.props

@@ -5,7 +5,7 @@
 
   <PropertyGroup>
     <!-- Tip: Edit this value to change the current MAJOR.MINOR.PATCH version -->
-    <VersionPrefix>2.4.5</VersionPrefix>
+    <VersionPrefix>2.4.6</VersionPrefix>
   </PropertyGroup>
 
   <Choose>

Directory.Packages.props

@@ -25,6 +25,11 @@
     <PackageVersion Include="Microsoft.Testing.Extensions.CodeCoverage" Version="18.12.0" />
     <PackageVersion Include="Microsoft.Testing.Extensions.TrxReport" Version="2.5.1" />
     <PackageVersion Include="MongoDB.Bson" Version="3.13.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.20.1" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.20.1" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.20.1" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.20.1" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.20.1" />
     <PackageVersion Include="MongoDB.Driver" Version="3.13.0" />
     <PackageVersion Include="Moq" Version="4.21.0" />
     <PackageVersion Include="Riok.Mapperly" Version="4.3.1" />

Keeptrack.slnx

@@ -57,6 +57,7 @@
     <File Path="scripts/unset-tvshow-want-to-watch.js" />
   </Folder>
   <Project Path="src/BlazorApp/BlazorApp.csproj" />
+  <Project Path="src/Common.Hosting/Common.Hosting.csproj" />
   <Project Path="src/Common.System/Common.System.csproj" />
   <Project Path="src/Domain/Domain.csproj" />
   <Project Path="src/Infrastructure.MongoDb/Infrastructure.MongoDb.csproj" />

compose.yaml

@@ -0,0 +1,138 @@
+# Both applications, with MongoDB and the observability stack as opt-in profiles.
+# Values come from a gitignored .env file next to this one, see CONTRIBUTING.md.
+# Every port is bound to 127.0.0.1, since nothing here asks for a sign-in on its API.
+
+services:
+  webapi:
+    image: devprofr/keeptrack-webapi:local
+    build:
+      context: .
+      dockerfile: src/WebApi/Dockerfile
+    environment:
+      ASPNETCORE_ENVIRONMENT: Development
+      Features__IsHttpsRedirectionEnabled: "false"
+      Features__IsScalarEnabled: "true"
+      AllowedOrigins__0: http://localhost:9001
+      Authentication__JwtBearer__Authority: https://securetoken.google.com/${FIREBASE_PROJECT_ID:-}
+      Authentication__JwtBearer__TokenValidation__Issuer: https://securetoken.google.com/${FIREBASE_PROJECT_ID:-}
+      Authentication__JwtBearer__TokenValidation__Audience: ${FIREBASE_PROJECT_ID:-}
+      Infrastructure__MongoDB__ConnectionString: ${MONGODB_CONNECTION_STRING:-mongodb://mongodb:27017}
+      Infrastructure__MongoDB__DatabaseName: ${MONGODB_DATABASE:-keeptrack_dev}
+      Tmdb__ApiKey: ${TMDB_APIKEY:-}
+      Igdb__ClientId: ${IGDB_CLIENTID:-}
+      Igdb__ClientSecret: ${IGDB_CLIENTSECRET:-}
+      Discogs__Token: ${DISCOGS_TOKEN:-}
+      GoogleBooks__ApiKey: ${GOOGLEBOOKS_APIKEY:-}
+      Logging__LogLevel__Keeptrack: Debug
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
+      OTEL_SERVICE_NAME: keeptrack-webapi
+      OTEL_RESOURCE_ATTRIBUTES: service.namespace=keeptrack,deployment.environment.name=compose
+    ports:
+      - 127.0.0.1:9002:8080
+    depends_on:
+      # Not required, so the applications also start against a MongoDB outside compose.
+      mongodb:
+        condition: service_started
+        required: false
+
+  blazorapp:
+    image: devprofr/keeptrack-blazorapp:local
+    build:
+      context: .
+      dockerfile: src/BlazorApp/Dockerfile
+    environment:
+      ASPNETCORE_ENVIRONMENT: Development
+      Features__IsHttpsRedirectionEnabled: "false"
+      WebApi__BaseUrl: http://webapi:8080
+      Firebase__WebAppConfiguration__ApiKey: ${FIREBASE_APIKEY:-}
+      Firebase__WebAppConfiguration__AuthDomain: ${FIREBASE_AUTH_DOMAIN:-}
+      Firebase__WebAppConfiguration__ProjectId: ${FIREBASE_PROJECT_ID:-}
+      Firebase__ServiceAccount: ${FIREBASE_SERVICE_ACCOUNT:-}
+      Logging__LogLevel__Keeptrack: Debug
+      OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-}
+      OTEL_SERVICE_NAME: keeptrack-blazorapp
+      OTEL_RESOURCE_ATTRIBUTES: service.namespace=keeptrack,deployment.environment.name=compose
+    ports:
+      - 127.0.0.1:9001:8080
+    depends_on:
+      - webapi
+
+  mongodb:
+    image: mongo:9.0
+    profiles: [mongodb]
+    environment:
+      MONGO_INITDB_DATABASE: ${MONGODB_DATABASE:-keeptrack_dev}
+    volumes:
+      - mongodb:/data/db
+      # Runs once, on an empty volume, so a new database starts with its indexes.
+      - ./scripts/mongodb-create-index.js:/docker-entrypoint-initdb.d/mongodb-create-index.js:ro
+    ports:
+      - 127.0.0.1:27017:27017
+
+  otel-collector:
+    image: otel/opentelemetry-collector:0.163.2
+    profiles: [observability]
+    command: [--config=/etc/otelcol/config.yaml]
+    volumes:
+      - ./.docker/otel-collector.yaml:/etc/otelcol/config.yaml:ro
+    ports:
+      # Published, so the applications started from an IDE send to it too.
+      - 127.0.0.1:4317:4317
+      - 127.0.0.1:4318:4318
+    depends_on:
+      - victoria-metrics
+      - victoria-logs
+      - victoria-traces
+
+  victoria-metrics:
+    image: victoriametrics/victoria-metrics:v1.155.0
+    profiles: [observability]
+    command: [-storageDataPath=/storage, -retentionPeriod=7d]
+    volumes:
+      - victoria-metrics:/storage
+    ports:
+      - 127.0.0.1:8428:8428
+
+  victoria-logs:
+    image: victoriametrics/victoria-logs:v1.53.0
+    profiles: [observability]
+    command: [-storageDataPath=/storage, -retentionPeriod=7d]
+    volumes:
+      - victoria-logs:/storage
+    ports:
+      - 127.0.0.1:9428:9428
+
+  victoria-traces:
+    image: victoriametrics/victoria-traces:v0.12.1
+    profiles: [observability]
+    command: [-storageDataPath=/storage, -retentionPeriod=7d]
+    volumes:
+      - victoria-traces:/storage
+    ports:
+      - 127.0.0.1:10428:10428
+
+  grafana:
+    image: grafana/grafana:13.3.0
+    profiles: [observability]
+    environment:
+      GF_PLUGINS_PREINSTALL: victoriametrics-logs-datasource
+      # An anonymous administrator is enough for a stack only reachable from this machine.
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Admin
+      GF_AUTH_DISABLE_LOGIN_FORM: "true"
+    volumes:
+      - ./.docker/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro
+      - grafana:/var/lib/grafana
+    ports:
+      - 127.0.0.1:3000:3000
+    depends_on:
+      - victoria-metrics
+      - victoria-logs
+      - victoria-traces
+
+volumes:
+  mongodb:
+  victoria-metrics:
+  victoria-logs:
+  victoria-traces:
+  grafana:

docs/README.md

@@ -27,5 +27,5 @@ Repository rules and conventions are in [AGENTS.md](../AGENTS.md), local setup i
 
 - [Testing](testing.md): suites, databases, cleanup, end-to-end design.
 - [CI and code quality](ci-and-quality.md): pipelines, IstarCI, settled Sonar findings.
-- [Operations](operations.md): deployment, indexes, migrations, backups.
+- [Operations](operations.md): deployment, telemetry, indexes, migrations, backups.
 - [Backlog](backlog.md): known debt and gaps.

docs/architecture.md

@@ -8,11 +8,12 @@ The UI only talks to the API over HTTP, and the API alone reads and writes the d
 Layered architecture across small single-purpose projects in `src/`, with `Domain` at the center and nothing referencing outward.
 
 - **`Common.System`**, depending on nothing: cross-cutting primitives (`IHasId`, `IHasIdAndOwnerId`, `PagedRequest`, `PagedResult<T>`, `TitleNormalizer`, `ListSort`, `RefuelCostCheck`).
+- **`Common.Hosting`**, depending on nothing of the solution: the OpenTelemetry setup that `WebApi` and `BlazorApp` both call.
 - **`Domain`**, depending on `Common.System`: business models (`Models/*Model`), repository interfaces (`Repositories/I*Repository`) and pure services (`Services/`), with no persistence or web concerns.
 - **`Infrastructure.MongoDb`**, depending on `Domain`: BSON `Entities/`, `Repositories/`, `Mappers/`.
 - **`WebApi.Contracts`**, depending on `Common.System`: the public REST DTOs (`Dto/`), shared with `BlazorApp` so the client deserializes without duplicate classes.
-- **`WebApi`**, depending on `Infrastructure.MongoDb`, `Domain` and `WebApi.Contracts`: controllers, DTO mappers, DI, JWT auth, OpenAPI/Scalar.
-- **`BlazorApp`**, depending on `Common.System` and `WebApi.Contracts`: the Blazor Server UI over HTTP, never referencing `Domain` or `Infrastructure.MongoDb`.
+- **`WebApi`**, depending on `Common.Hosting`, `Infrastructure.MongoDb`, `Domain` and `WebApi.Contracts`: controllers, DTO mappers, DI, JWT auth, OpenAPI/Scalar.
+- **`BlazorApp`**, depending on `Common.Hosting`, `Common.System` and `WebApi.Contracts`: the Blazor Server UI over HTTP, never referencing `Domain` or `Infrastructure.MongoDb`.
 
 Controllers live in `WebApi/Controllers/`, with any real computation in `Domain/Services/`.
 `WebApi/Import/` and `WebApi/ReferenceData/` use an older feature-folder shape that new code does not extend (see [backlog](backlog.md)).

docs/operations.md

@@ -12,6 +12,13 @@ Configuration keys are in [CONTRIBUTING.md](../CONTRIBUTING.md#configuration).
 
 Several replicas are supported (see [Architecture](architecture.md#scaling)).
 
+## Telemetry
+
+Both applications export traces, metrics and logs over OTLP once `OTEL_EXPORTER_OTLP_ENDPOINT` is set, and export nothing otherwise.
+The other standard variables configure the rest: `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`, `OTEL_EXPORTER_OTLP_PROTOCOL`.
+Traces cover the incoming requests, the provider calls and every MongoDB command, the driver emitting its spans from the `MongoDB.Driver` source.
+The `/healthz` probes are left out of the traces, since the probes of every replica would outnumber the real requests.
+
 ## Database indexes
 
 `scripts/mongodb-create-index.js` is idempotent, and is re-run after every change to it and after every migration script:

src/BlazorApp/BlazorApp.csproj

@@ -18,6 +18,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <ProjectReference Include="..\Common.Hosting\Common.Hosting.csproj" />
     <ProjectReference Include="..\Common.System\Common.System.csproj" />
     <ProjectReference Include="..\WebApi.Contracts\WebApi.Contracts.csproj" />
   </ItemGroup>

src/BlazorApp/GlobalUsings.cs

@@ -4,6 +4,7 @@
 global using Keeptrack.BlazorApp.Components.Inventory.Clients;
 global using Keeptrack.BlazorApp.DataProtection;
 global using Keeptrack.BlazorApp.DependencyInjection;
+global using Keeptrack.Common.Hosting;
 global using Microsoft.AspNetCore.Authentication.Cookies;
 global using Microsoft.AspNetCore.DataProtection;
 global using Microsoft.AspNetCore.Http.HttpResults;

src/BlazorApp/Program.cs

@@ -1,4 +1,5 @@
 var builder = WebApplication.CreateBuilder(args);
+builder.AddKeeptrackTelemetry();
 
 // adds services to the container
 builder.Services.AddRazorComponents()

src/Common.Hosting/Common.Hosting.csproj

@@ -0,0 +1,23 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <AssemblyName>Keeptrack.Common.Hosting</AssemblyName>
+    <RootNamespace>Keeptrack.Common.Hosting</RootNamespace>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <FrameworkReference Include="Microsoft.AspNetCore.App" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
+  </ItemGroup>
+
+</Project>