diff --git a/.docker/grafana-datasources.yaml b/.docker/grafana-datasources.yaml new file mode 100644 index 00000000..3afd7389 --- /dev/null +++ b/.docker/grafana-datasources.yaml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + isDefault: true + url: http://victoria-metrics:8428 + - name: VictoriaLogs + type: victoriametrics-logs-datasource + access: proxy + url: http://victoria-logs:9428 + - name: Jaeger + type: jaeger + access: proxy + url: http://victoria-traces:10428/select/jaeger diff --git a/.docker/otel-collector.yaml b/.docker/otel-collector.yaml new file mode 100644 index 00000000..06693b52 --- /dev/null +++ b/.docker/otel-collector.yaml @@ -0,0 +1,35 @@ +# Same pipelines as the cluster collector, each signal sent to its own Victoria backend. + +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + +exporters: + otlphttp/metrics: + endpoint: http://victoria-metrics:8428/opentelemetry + otlphttp/logs: + endpoint: http://victoria-logs:9428/insert/opentelemetry + otlphttp/traces: + endpoint: http://victoria-traces:10428/insert/opentelemetry + +service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp/metrics] + logs: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp/logs] + traces: + receivers: [otlp] + processors: [batch] + exporters: [otlphttp/traces] diff --git a/.gitignore b/.gitignore index 11e8acdc..9a9d4a45 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ appsettings.Development.json *.csproj.user TestResults/* Local.runsettings +.env # NPM firebase.config.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9b12dccf..e381a0ca 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,10 +21,10 @@ If you do not agree with these terms, do not submit a contribution. 1. [.NET 10.0 SDK](https://dotnet.microsoft.com/download). 2. Node.js, for the scripts in `scripts/`. -3. A MongoDB database, for example: +3. A MongoDB database, for example the one of [Containers](#containers): ```bash - docker run --name mongodb -d -p 27017:27017 mongo:9.0 + docker compose up -d mongodb ``` A local `mongod` or a [MongoDB Atlas](https://cloud.mongodb.com/) cluster works too. @@ -276,7 +276,53 @@ Variable | Default | Purpose A failed test leaves a full-page screenshot and a Playwright trace in `test/BlazorApp.PlaywrightTests/bin//net10.0/e2e-diagnostics`, and prints their paths in its output. A trace opens at [trace.playwright.dev](https://trace.playwright.dev). -## Container images +## Containers + +`compose.yaml` builds and runs both applications, with two optional profiles: + +- `mongodb`: a MongoDB whose indexes are created on its first start, from `scripts/mongodb-create-index.js`. +- `observability`: an OpenTelemetry Collector sending to VictoriaMetrics, VictoriaLogs and VictoriaTraces, read in Grafana, the same pipelines as the cluster. + +Every port is bound to `127.0.0.1`. + +Service | URL +-----------------|------------------------------------------------------------------------------- +`blazorapp` | [localhost:9001](http://localhost:9001) +`webapi` | [localhost:9002/scalar](http://localhost:9002/scalar) +`mongodb` | `mongodb://localhost:27017` +`otel-collector` | `http://localhost:4317` (OTLP gRPC), `http://localhost:4318` (OTLP HTTP) +`grafana` | [localhost:3000](http://localhost:3000), signed in as an anonymous administrator + +Values come from a `.env` file at the repository root, gitignored, read by compose itself: + +```ini +FIREBASE_PROJECT_ID=xxxx +FIREBASE_AUTH_DOMAIN=xxxx.firebaseapp.com +FIREBASE_APIKEY=xxxx +FIREBASE_SERVICE_ACCOUNT='{"type":"service_account",...}' +TMDB_APIKEY=xxxx +# Only when MongoDB runs outside compose, host.docker.internal being the host. +# MONGODB_CONNECTION_STRING=mongodb://host.docker.internal:27017 +# MONGODB_DATABASE=keeptrack_dev +# Only with the observability profile. +# OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4317 +``` + +`IGDB_CLIENTID`, `IGDB_CLIENTSECRET`, `DISCOGS_TOKEN` and `GOOGLEBOOKS_APIKEY` set the other [providers](#reference-data-providers). +A single-quoted value is taken as is, so the service account JSON keeps its `$` and quotes. + +```bash +docker compose --profile mongodb --profile observability up -d --build # everything +docker compose --profile mongodb up -d --build # the applications and MongoDB +docker compose up -d grafana otel-collector # the observability stack alone +docker compose --profile mongodb --profile observability down # stops, keeping the volumes +``` + +A service named on the command line starts whatever its profile, so `docker compose up -d grafana otel-collector` also serves the applications run from an IDE. +They send to it with `OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317`, in `launchSettings.json` or the run configuration, and appear in Grafana under **Explore > Jaeger** as `keeptrack-webapi` and `keeptrack-blazorapp`. +[Telemetry](docs/operations.md#telemetry) describes what is exported. + +The images alone: ```bash docker build . -t devprofr/keeptrack-blazorapp:local -f src/BlazorApp/Dockerfile diff --git a/Directory.Build.props b/Directory.Build.props index a07d1fa8..38f136bc 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -5,7 +5,7 @@ - 2.4.5 + 2.4.6 diff --git a/Directory.Packages.props b/Directory.Packages.props index 3575f1d1..47e1dfb6 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -25,6 +25,11 @@ + + + + + diff --git a/Keeptrack.slnx b/Keeptrack.slnx index 394896cf..1ef209e0 100644 --- a/Keeptrack.slnx +++ b/Keeptrack.slnx @@ -57,6 +57,7 @@ + diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..4c0b5b45 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,138 @@ +# Both applications, with MongoDB and the observability stack as opt-in profiles. +# Values come from a gitignored .env file next to this one, see CONTRIBUTING.md. +# Every port is bound to 127.0.0.1, since nothing here asks for a sign-in on its API. + +services: + webapi: + image: devprofr/keeptrack-webapi:local + build: + context: . + dockerfile: src/WebApi/Dockerfile + environment: + ASPNETCORE_ENVIRONMENT: Development + Features__IsHttpsRedirectionEnabled: "false" + Features__IsScalarEnabled: "true" + AllowedOrigins__0: http://localhost:9001 + Authentication__JwtBearer__Authority: https://securetoken.google.com/${FIREBASE_PROJECT_ID:-} + Authentication__JwtBearer__TokenValidation__Issuer: https://securetoken.google.com/${FIREBASE_PROJECT_ID:-} + Authentication__JwtBearer__TokenValidation__Audience: ${FIREBASE_PROJECT_ID:-} + Infrastructure__MongoDB__ConnectionString: ${MONGODB_CONNECTION_STRING:-mongodb://mongodb:27017} + Infrastructure__MongoDB__DatabaseName: ${MONGODB_DATABASE:-keeptrack_dev} + Tmdb__ApiKey: ${TMDB_APIKEY:-} + Igdb__ClientId: ${IGDB_CLIENTID:-} + Igdb__ClientSecret: ${IGDB_CLIENTSECRET:-} + Discogs__Token: ${DISCOGS_TOKEN:-} + GoogleBooks__ApiKey: ${GOOGLEBOOKS_APIKEY:-} + Logging__LogLevel__Keeptrack: Debug + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_SERVICE_NAME: keeptrack-webapi + OTEL_RESOURCE_ATTRIBUTES: service.namespace=keeptrack,deployment.environment.name=compose + ports: + - 127.0.0.1:9002:8080 + depends_on: + # Not required, so the applications also start against a MongoDB outside compose. + mongodb: + condition: service_started + required: false + + blazorapp: + image: devprofr/keeptrack-blazorapp:local + build: + context: . + dockerfile: src/BlazorApp/Dockerfile + environment: + ASPNETCORE_ENVIRONMENT: Development + Features__IsHttpsRedirectionEnabled: "false" + WebApi__BaseUrl: http://webapi:8080 + Firebase__WebAppConfiguration__ApiKey: ${FIREBASE_APIKEY:-} + Firebase__WebAppConfiguration__AuthDomain: ${FIREBASE_AUTH_DOMAIN:-} + Firebase__WebAppConfiguration__ProjectId: ${FIREBASE_PROJECT_ID:-} + Firebase__ServiceAccount: ${FIREBASE_SERVICE_ACCOUNT:-} + Logging__LogLevel__Keeptrack: Debug + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} + OTEL_SERVICE_NAME: keeptrack-blazorapp + OTEL_RESOURCE_ATTRIBUTES: service.namespace=keeptrack,deployment.environment.name=compose + ports: + - 127.0.0.1:9001:8080 + depends_on: + - webapi + + mongodb: + image: mongo:9.0 + profiles: [mongodb] + environment: + MONGO_INITDB_DATABASE: ${MONGODB_DATABASE:-keeptrack_dev} + volumes: + - mongodb:/data/db + # Runs once, on an empty volume, so a new database starts with its indexes. + - ./scripts/mongodb-create-index.js:/docker-entrypoint-initdb.d/mongodb-create-index.js:ro + ports: + - 127.0.0.1:27017:27017 + + otel-collector: + image: otel/opentelemetry-collector:0.163.2 + profiles: [observability] + command: [--config=/etc/otelcol/config.yaml] + volumes: + - ./.docker/otel-collector.yaml:/etc/otelcol/config.yaml:ro + ports: + # Published, so the applications started from an IDE send to it too. + - 127.0.0.1:4317:4317 + - 127.0.0.1:4318:4318 + depends_on: + - victoria-metrics + - victoria-logs + - victoria-traces + + victoria-metrics: + image: victoriametrics/victoria-metrics:v1.155.0 + profiles: [observability] + command: [-storageDataPath=/storage, -retentionPeriod=7d] + volumes: + - victoria-metrics:/storage + ports: + - 127.0.0.1:8428:8428 + + victoria-logs: + image: victoriametrics/victoria-logs:v1.53.0 + profiles: [observability] + command: [-storageDataPath=/storage, -retentionPeriod=7d] + volumes: + - victoria-logs:/storage + ports: + - 127.0.0.1:9428:9428 + + victoria-traces: + image: victoriametrics/victoria-traces:v0.12.1 + profiles: [observability] + command: [-storageDataPath=/storage, -retentionPeriod=7d] + volumes: + - victoria-traces:/storage + ports: + - 127.0.0.1:10428:10428 + + grafana: + image: grafana/grafana:13.3.0 + profiles: [observability] + environment: + GF_PLUGINS_PREINSTALL: victoriametrics-logs-datasource + # An anonymous administrator is enough for a stack only reachable from this machine. + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + GF_AUTH_DISABLE_LOGIN_FORM: "true" + volumes: + - ./.docker/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml:ro + - grafana:/var/lib/grafana + ports: + - 127.0.0.1:3000:3000 + depends_on: + - victoria-metrics + - victoria-logs + - victoria-traces + +volumes: + mongodb: + victoria-metrics: + victoria-logs: + victoria-traces: + grafana: diff --git a/docs/README.md b/docs/README.md index 168e5a16..e100ffcb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -27,5 +27,5 @@ Repository rules and conventions are in [AGENTS.md](../AGENTS.md), local setup i - [Testing](testing.md): suites, databases, cleanup, end-to-end design. - [CI and code quality](ci-and-quality.md): pipelines, IstarCI, settled Sonar findings. -- [Operations](operations.md): deployment, indexes, migrations, backups. +- [Operations](operations.md): deployment, telemetry, indexes, migrations, backups. - [Backlog](backlog.md): known debt and gaps. diff --git a/docs/architecture.md b/docs/architecture.md index 36794b9f..593de3d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -8,11 +8,12 @@ The UI only talks to the API over HTTP, and the API alone reads and writes the d Layered architecture across small single-purpose projects in `src/`, with `Domain` at the center and nothing referencing outward. - **`Common.System`**, depending on nothing: cross-cutting primitives (`IHasId`, `IHasIdAndOwnerId`, `PagedRequest`, `PagedResult`, `TitleNormalizer`, `ListSort`, `RefuelCostCheck`). +- **`Common.Hosting`**, depending on nothing of the solution: the OpenTelemetry setup that `WebApi` and `BlazorApp` both call. - **`Domain`**, depending on `Common.System`: business models (`Models/*Model`), repository interfaces (`Repositories/I*Repository`) and pure services (`Services/`), with no persistence or web concerns. - **`Infrastructure.MongoDb`**, depending on `Domain`: BSON `Entities/`, `Repositories/`, `Mappers/`. - **`WebApi.Contracts`**, depending on `Common.System`: the public REST DTOs (`Dto/`), shared with `BlazorApp` so the client deserializes without duplicate classes. -- **`WebApi`**, depending on `Infrastructure.MongoDb`, `Domain` and `WebApi.Contracts`: controllers, DTO mappers, DI, JWT auth, OpenAPI/Scalar. -- **`BlazorApp`**, depending on `Common.System` and `WebApi.Contracts`: the Blazor Server UI over HTTP, never referencing `Domain` or `Infrastructure.MongoDb`. +- **`WebApi`**, depending on `Common.Hosting`, `Infrastructure.MongoDb`, `Domain` and `WebApi.Contracts`: controllers, DTO mappers, DI, JWT auth, OpenAPI/Scalar. +- **`BlazorApp`**, depending on `Common.Hosting`, `Common.System` and `WebApi.Contracts`: the Blazor Server UI over HTTP, never referencing `Domain` or `Infrastructure.MongoDb`. Controllers live in `WebApi/Controllers/`, with any real computation in `Domain/Services/`. `WebApi/Import/` and `WebApi/ReferenceData/` use an older feature-folder shape that new code does not extend (see [backlog](backlog.md)). diff --git a/docs/operations.md b/docs/operations.md index d540e766..e52a6b19 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -12,6 +12,13 @@ Configuration keys are in [CONTRIBUTING.md](../CONTRIBUTING.md#configuration). Several replicas are supported (see [Architecture](architecture.md#scaling)). +## Telemetry + +Both applications export traces, metrics and logs over OTLP once `OTEL_EXPORTER_OTLP_ENDPOINT` is set, and export nothing otherwise. +The other standard variables configure the rest: `OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`, `OTEL_EXPORTER_OTLP_PROTOCOL`. +Traces cover the incoming requests, the provider calls and every MongoDB command, the driver emitting its spans from the `MongoDB.Driver` source. +The `/healthz` probes are left out of the traces, since the probes of every replica would outnumber the real requests. + ## Database indexes `scripts/mongodb-create-index.js` is idempotent, and is re-run after every change to it and after every migration script: diff --git a/src/BlazorApp/BlazorApp.csproj b/src/BlazorApp/BlazorApp.csproj index 03c8e147..f7fa3099 100644 --- a/src/BlazorApp/BlazorApp.csproj +++ b/src/BlazorApp/BlazorApp.csproj @@ -18,6 +18,7 @@ + diff --git a/src/BlazorApp/GlobalUsings.cs b/src/BlazorApp/GlobalUsings.cs index a0037488..f0b46247 100644 --- a/src/BlazorApp/GlobalUsings.cs +++ b/src/BlazorApp/GlobalUsings.cs @@ -4,6 +4,7 @@ global using Keeptrack.BlazorApp.Components.Inventory.Clients; global using Keeptrack.BlazorApp.DataProtection; global using Keeptrack.BlazorApp.DependencyInjection; +global using Keeptrack.Common.Hosting; global using Microsoft.AspNetCore.Authentication.Cookies; global using Microsoft.AspNetCore.DataProtection; global using Microsoft.AspNetCore.Http.HttpResults; diff --git a/src/BlazorApp/Program.cs b/src/BlazorApp/Program.cs index 45211763..f2da59f5 100644 --- a/src/BlazorApp/Program.cs +++ b/src/BlazorApp/Program.cs @@ -1,4 +1,5 @@ var builder = WebApplication.CreateBuilder(args); +builder.AddKeeptrackTelemetry(); // adds services to the container builder.Services.AddRazorComponents() diff --git a/src/Common.Hosting/Common.Hosting.csproj b/src/Common.Hosting/Common.Hosting.csproj new file mode 100644 index 00000000..25a5c611 --- /dev/null +++ b/src/Common.Hosting/Common.Hosting.csproj @@ -0,0 +1,23 @@ + + + + net10.0 + Keeptrack.Common.Hosting + Keeptrack.Common.Hosting + enable + enable + + + + + + + + + + + + + + + diff --git a/src/Common.Hosting/TelemetryExtensions.cs b/src/Common.Hosting/TelemetryExtensions.cs new file mode 100644 index 00000000..2a5111ac --- /dev/null +++ b/src/Common.Hosting/TelemetryExtensions.cs @@ -0,0 +1,44 @@ +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using OpenTelemetry; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; + +namespace Keeptrack.Common.Hosting; + +/// +/// OpenTelemetry of both applications, configured by the standard OTEL_* environment variables (service name, resource attributes, protocol). +/// +public static class TelemetryExtensions +{ + private static readonly PathString s_healthCheckPath = new("/healthz"); + + public static IHostApplicationBuilder AddKeeptrackTelemetry(this IHostApplicationBuilder builder) + { + // Off without a collector, so a local run or a test host never retries exports to an endpoint nobody listens on. + if (string.IsNullOrWhiteSpace(builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"])) + { + return builder; + } + + builder.Services.AddOpenTelemetry() + .WithTracing(tracing => tracing + // The probes of every replica would otherwise outnumber the real requests. + .AddAspNetCoreInstrumentation(options => options.Filter = context => !context.Request.Path.StartsWithSegments(s_healthCheckPath)) + .AddHttpClientInstrumentation() + // The driver emits a span per command from its own ActivitySource, with no extra package. + .AddSource("MongoDB.Driver")) + .WithMetrics(metrics => metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddRuntimeInstrumentation()) + .WithLogging(_ => { }, options => + { + options.IncludeFormattedMessage = true; + options.IncludeScopes = true; + }) + .UseOtlpExporter(); + return builder; + } +} diff --git a/src/WebApi/GlobalUsings.cs b/src/WebApi/GlobalUsings.cs index f5bd6aaf..80ae83ff 100644 --- a/src/WebApi/GlobalUsings.cs +++ b/src/WebApi/GlobalUsings.cs @@ -1,4 +1,5 @@ global using System; +global using Keeptrack.Common.Hosting; global using Keeptrack.WebApi; global using Keeptrack.WebApi.Contracts.Dto; global using Keeptrack.WebApi.DependencyInjection; diff --git a/src/WebApi/Program.cs b/src/WebApi/Program.cs index f41e0d7b..d0caa548 100644 --- a/src/WebApi/Program.cs +++ b/src/WebApi/Program.cs @@ -1,4 +1,5 @@ var builder = WebApplication.CreateBuilder(args); +builder.AddKeeptrackTelemetry(); var configuration = new AppConfiguration(builder.Configuration); diff --git a/src/WebApi/WebApi.csproj b/src/WebApi/WebApi.csproj index c260d6e2..7e285399 100644 --- a/src/WebApi/WebApi.csproj +++ b/src/WebApi/WebApi.csproj @@ -24,6 +24,7 @@ + diff --git a/test/WebApi.UnitTests/Hosting/TelemetryExtensionsTest.cs b/test/WebApi.UnitTests/Hosting/TelemetryExtensionsTest.cs new file mode 100644 index 00000000..c83d0bb9 --- /dev/null +++ b/test/WebApi.UnitTests/Hosting/TelemetryExtensionsTest.cs @@ -0,0 +1,47 @@ +using System.Diagnostics; +using System.Linq; +using AwesomeAssertions; +using Keeptrack.Common.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Logs; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using Xunit; + +namespace Keeptrack.WebApi.UnitTests.Hosting; + +[Trait("Category", "UnitTests")] +public class TelemetryExtensionsTest +{ + [Fact] + public void AddKeeptrackTelemetry_RegistersNothing_WhenNoOtlpEndpointIsSet() + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings()); + + builder.AddKeeptrackTelemetry(); + + using var app = builder.Build(); + app.Services.GetService().Should().BeNull(); + app.Services.GetService().Should().BeNull(); + app.Services.GetServices().Should().NotContain(x => x is OpenTelemetryLoggerProvider); + } + + [Fact] + public void AddKeeptrackTelemetry_ExportsTracesMetricsAndLogs_WhenAnOtlpEndpointIsSet() + { + var builder = Host.CreateEmptyApplicationBuilder(new HostApplicationBuilderSettings()); + builder.Configuration["OTEL_EXPORTER_OTLP_ENDPOINT"] = "http://localhost:4317"; + + builder.AddKeeptrackTelemetry(); + + using var app = builder.Build(); + app.Services.GetService().Should().NotBeNull(); + app.Services.GetService().Should().NotBeNull(); + app.Services.GetServices().Should().Contain(x => x is OpenTelemetryLoggerProvider); + new[] { "Microsoft.AspNetCore", "System.Net.Http", "MongoDB.Driver" } + .Where(name => !new ActivitySource(name).HasListeners()) + .Should().BeEmpty("incoming requests, provider calls and MongoDB commands are all traced"); + } +}