From 478966132e97ff09c9a46a93b4df3fca56194dff Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Sat, 31 Oct 2026 19:09:10 -0400 Subject: [PATCH 1/2] feat(observability): add logs config and Rust log export - Add observability.logs (bool or {enabled}) to cog.yaml parsing, schema, and validation, reserve COG_LOGS_*, and set COG_LOGS_CONFIGURED/ENABLED in generated and custom-Dockerfile images. - Turn PythonObservabilityCheck into a function that checks the coglet capability flag for each enabled signal. - Export coglet's own tracing events from the parent through opentelemetry-appender-tracing. The layer drops re-emitted model output (coglet::setup, coglet::user, and coglet::prediction events with a source field) and non-coglet targets. The worker does not export, because its events already reach the parent. --- crates/Cargo.lock | 13 ++ crates/coglet-python/src/lib.rs | 25 +++ crates/coglet/Cargo.toml | 9 +- crates/coglet/src/lib.rs | 2 + crates/coglet/src/logs.rs | 252 ++++++++++++++++++++++ crates/coglet/src/worker.rs | 2 + pkg/config/config.go | 8 +- pkg/config/config_file.go | 39 ++++ pkg/config/config_test.go | 3 +- pkg/config/data/config_schema_v1.0.json | 20 +- pkg/config/env.go | 1 + pkg/config/parse.go | 7 + pkg/config/validate.go | 8 +- pkg/config/validate_test.go | 28 ++- pkg/dockerfile/standard_generator.go | 29 ++- pkg/dockerfile/standard_generator_test.go | 27 ++- pkg/image/build.go | 9 +- pkg/image/build_test.go | 22 +- 18 files changed, 483 insertions(+), 21 deletions(-) create mode 100644 crates/coglet/src/logs.rs diff --git a/crates/Cargo.lock b/crates/Cargo.lock index dc593214d5..427e94eb93 100644 --- a/crates/Cargo.lock +++ b/crates/Cargo.lock @@ -274,6 +274,7 @@ dependencies = [ "mime_guess", "nix 0.31.3", "opentelemetry", + "opentelemetry-appender-tracing", "opentelemetry-jaeger-propagator", "opentelemetry-otlp", "opentelemetry_sdk", @@ -1702,6 +1703,18 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0080f0dc1d7c786f467cd85a4e395fcab11ee852004f39a29a18ab7c25d837" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "opentelemetry-http" version = "0.32.0" diff --git a/crates/coglet-python/src/lib.rs b/crates/coglet-python/src/lib.rs index e1c91d70ca..260c4160d2 100644 --- a/crates/coglet-python/src/lib.rs +++ b/crates/coglet-python/src/lib.rs @@ -102,6 +102,7 @@ fn init_tracing( _to_stderr: bool, setup_log_tx: Option>, #[cfg(feature = "tracing")] tracer: Option, + #[cfg(feature = "tracing")] logs: Option<&coglet_core::logs::LoggingRuntime>, ) -> Result>, String> { let filter = if std::env::var("RUST_LOG").is_ok() { EnvFilter::from_default_env().add_directive("coglet::trace=trace".parse().unwrap()) @@ -132,6 +133,11 @@ fn init_tracing( #[cfg(not(feature = "tracing"))] let otel_layer = tracing_subscriber::layer::Identity::new(); + #[cfg(feature = "tracing")] + let logs_layer = logs.map(|runtime| runtime.layer()); + #[cfg(not(feature = "tracing"))] + let logs_layer = tracing_subscriber::layer::Identity::new(); + if let Some(tx) = setup_log_tx { let accumulator = coglet_core::SetupLogAccumulator::new(tx); @@ -140,6 +146,7 @@ fn init_tracing( .with(filter) .with(sentry_layer) .with(otel_layer) + .with(logs_layer) .with(accumulator) .with(fmt::layer().json().with_writer(std::io::stderr)); subscriber.try_init().map_err(|error| error.to_string())?; @@ -148,6 +155,7 @@ fn init_tracing( .with(filter) .with(sentry_layer) .with(otel_layer) + .with(logs_layer) .with(accumulator) .with(fmt::layer().with_writer(std::io::stderr)); subscriber.try_init().map_err(|error| error.to_string())?; @@ -159,6 +167,7 @@ fn init_tracing( .with(filter) .with(sentry_layer) .with(otel_layer) + .with(logs_layer) .with(fmt::layer().json().with_writer(std::io::stderr)); subscriber.try_init().map_err(|error| error.to_string())?; } else { @@ -166,6 +175,7 @@ fn init_tracing( .with(filter) .with(sentry_layer) .with(otel_layer) + .with(logs_layer) .with(fmt::layer().with_writer(std::io::stderr)); subscriber.try_init().map_err(|error| error.to_string())?; } @@ -349,6 +359,11 @@ fn serve_impl( let _entered = rt.enter(); coglet_core::trace::TracingRuntime::from_env(coglet_core::trace::ProcessRole::Parent) }; + #[cfg(feature = "tracing")] + let logs_runtime = { + let _entered = rt.enter(); + coglet_core::logs::LoggingRuntime::from_env() + }; let (setup_log_tx, setup_log_rx) = tokio::sync::mpsc::unbounded_channel(); #[cfg(feature = "tracing")] @@ -356,6 +371,7 @@ fn serve_impl( false, Some(setup_log_tx), trace_runtime.as_ref().map(|runtime| runtime.tracer()), + logs_runtime.as_ref(), ) .map_err(PyErr::new::)?; #[cfg(not(feature = "tracing"))] @@ -421,6 +437,10 @@ fn serve_impl( } #[cfg(feature = "tracing")] coglet_core::runtime_metrics::shutdown(); + #[cfg(feature = "tracing")] + if let Some(runtime) = logs_runtime.as_ref() { + runtime.shutdown(); + } return result; }; @@ -441,6 +461,10 @@ fn serve_impl( } #[cfg(feature = "tracing")] coglet_core::runtime_metrics::shutdown(); + #[cfg(feature = "tracing")] + if let Some(runtime) = logs_runtime.as_ref() { + runtime.shutdown(); + } result } @@ -700,6 +724,7 @@ fn coglet(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add("__version__", env!("COGLET_PEP440_VERSION"))?; m.add("__build__", BuildInfo::new())?; m.add("_supports_observability_metrics", cfg!(feature = "tracing"))?; + m.add("_supports_observability_logs", cfg!(feature = "tracing"))?; // Frozen server object m.add("server", CogletServer {})?; diff --git a/crates/coglet/Cargo.toml b/crates/coglet/Cargo.toml index e348dcda15..70187e1905 100644 --- a/crates/coglet/Cargo.toml +++ b/crates/coglet/Cargo.toml @@ -14,6 +14,7 @@ categories.workspace = true default = ["tracing", "tracing-grpc"] tracing = [ "dep:opentelemetry", + "dep:opentelemetry-appender-tracing", "dep:opentelemetry-jaeger-propagator", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", @@ -22,6 +23,7 @@ tracing = [ "opentelemetry-otlp/reqwest-blocking-client", "opentelemetry-otlp/trace", "opentelemetry-otlp/metrics", + "opentelemetry-otlp/logs", ] tracing-grpc = [ "tracing", @@ -74,10 +76,11 @@ rustls.workspace = true # Observability tracing.workspace = true tracing-subscriber.workspace = true -opentelemetry = { version = "=0.32.0", default-features = false, features = ["trace", "metrics"], optional = true } +opentelemetry = { version = "=0.32.0", default-features = false, features = ["trace", "metrics", "logs"], optional = true } +opentelemetry-appender-tracing = { version = "=0.32.0", default-features = false, optional = true } opentelemetry-jaeger-propagator = { version = "=0.32.0", default-features = false, optional = true } opentelemetry-otlp = { version = "=0.32.0", default-features = false, optional = true } -opentelemetry_sdk = { version = "=0.32.0", default-features = false, features = ["trace", "metrics"], optional = true } +opentelemetry_sdk = { version = "=0.32.0", default-features = false, features = ["trace", "metrics", "logs"], optional = true } tracing-opentelemetry = { version = "=0.33.0", default-features = false, optional = true } [target.'cfg(unix)'.dependencies] @@ -89,4 +92,4 @@ tempfile = "3" wiremock = "0.6" tower = { version = "0.5", features = ["util"] } http-body-util = "0.1" -opentelemetry_sdk = { version = "=0.32.0", default-features = false, features = ["testing", "trace", "metrics"] } +opentelemetry_sdk = { version = "=0.32.0", default-features = false, features = ["testing", "trace", "metrics", "logs"] } diff --git a/crates/coglet/src/lib.rs b/crates/coglet/src/lib.rs index 4fdc60bcbf..7f09b789a0 100644 --- a/crates/coglet/src/lib.rs +++ b/crates/coglet/src/lib.rs @@ -8,6 +8,8 @@ mod version; pub mod bridge; mod fd_redirect; +#[cfg(feature = "tracing")] +pub mod logs; pub mod orchestrator; pub mod permit; #[cfg(feature = "tracing")] diff --git a/crates/coglet/src/logs.rs b/crates/coglet/src/logs.rs new file mode 100644 index 0000000000..4dd97ea45a --- /dev/null +++ b/crates/coglet/src/logs.rs @@ -0,0 +1,252 @@ +//! OpenTelemetry log export for coglet's own runtime events. +//! +//! Only the parent process exports. Worker events reach the parent through +//! `ControlResponse::WorkerLog` and are re-emitted there, so a worker-side +//! exporter would send them twice. + +use std::time::Duration; + +use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; +use opentelemetry_otlp::{LogExporter, Protocol, WithExportConfig as _}; +use opentelemetry_sdk::logs::SdkLoggerProvider; +use tracing::Metadata; +use tracing_subscriber::Layer; +use tracing_subscriber::filter::filter_fn; +use tracing_subscriber::registry::LookupSpan; + +use crate::trace::{ProcessRole, base_resource, env_bool}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum OtlpProtocol { + HttpProtobuf, + Grpc, +} + +#[derive(Debug, Eq, PartialEq)] +struct LogsConfig { + endpoint: String, + append_logs_path: bool, + protocol: OtlpProtocol, +} + +impl LogsConfig { + fn from_env() -> Result, String> { + if !env_bool("COG_LOGS_CONFIGURED", false)? + || !env_bool("COG_LOGS_ENABLED", true)? + || env_bool("OTEL_SDK_DISABLED", false)? + || std::env::var("OTEL_LOGS_EXPORTER").as_deref() == Ok("none") + { + return Ok(None); + } + + let (endpoint, append_logs_path) = match std::env::var("OTEL_EXPORTER_OTLP_LOGS_ENDPOINT") { + Ok(value) if !value.trim().is_empty() => (value, false), + Ok(_) => { + eprintln!("Logs enabled without an OTLP endpoint; log export disabled"); + return Ok(None); + } + Err(_) => match std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT") { + Ok(value) if !value.trim().is_empty() => (value, true), + _ => { + eprintln!("Logs enabled without an OTLP endpoint; log export disabled"); + return Ok(None); + } + }, + }; + + let protocol = match std::env::var("OTEL_EXPORTER_OTLP_LOGS_PROTOCOL") + .or_else(|_| std::env::var("OTEL_EXPORTER_OTLP_PROTOCOL")) + .unwrap_or_else(|_| "http/protobuf".to_string()) + .as_str() + { + "http" | "http/protobuf" => OtlpProtocol::HttpProtobuf, + "grpc" => OtlpProtocol::Grpc, + value => return Err(format!("unsupported OTLP protocol {value:?}")), + }; + + Ok(Some(Self { + endpoint, + append_logs_path, + protocol, + })) + } +} + +pub struct LoggingRuntime { + provider: SdkLoggerProvider, +} + +impl LoggingRuntime { + /// Builds the parent's log exporter, or returns `None` when logs are + /// disabled or misconfigured. Configuration errors never stop serving. + pub fn from_env() -> Option { + match LogsConfig::from_env().and_then(|config| config.map(build_exporter).transpose()) { + Ok(Some(exporter)) => Some(Self::new( + SdkLoggerProvider::builder() + .with_resource(base_resource(ProcessRole::Parent)) + .with_batch_exporter(exporter) + .build(), + )), + Ok(None) => None, + Err(error) => { + eprintln!("Invalid OpenTelemetry logs configuration; log export disabled: {error}"); + None + } + } + } + + fn new(provider: SdkLoggerProvider) -> Self { + Self { provider } + } + + /// A tracing layer that exports coglet's own events. Model output that the + /// parent re-emits as tracing events is excluded. + pub fn layer(&self) -> impl Layer + use + where + S: tracing::Subscriber + for<'a> LookupSpan<'a>, + { + OpenTelemetryTracingBridge::new(&self.provider).with_filter(filter_fn(exports_event)) + } + + /// Shutdown drains the batch processor, so no separate flush is needed. + pub fn shutdown(&self) { + if let Err(error) = self.provider.shutdown_with_timeout(Duration::from_secs(5)) { + eprintln!("Failed to shut down OpenTelemetry logs provider: {error}"); + } + } +} + +fn exports_event(metadata: &Metadata<'_>) -> bool { + if !metadata.is_event() || !metadata.target().starts_with("coglet") { + return false; + } + match metadata.target() { + // Model stdout/stderr re-emitted during setup and outside predictions. + "coglet::setup" | "coglet::user" => false, + // Model output lines carry a `source` field; Cog's own prediction + // events on this target do not. + "coglet::prediction" => metadata.fields().field("source").is_none(), + _ => true, + } +} + +fn build_exporter(config: LogsConfig) -> Result { + let builder = LogExporter::builder(); + match config.protocol { + OtlpProtocol::HttpProtobuf => builder + .with_http() + .with_protocol(Protocol::HttpBinary) + .with_endpoint(http_logs_endpoint( + &config.endpoint, + config.append_logs_path, + )) + .build() + .map_err(|error| error.to_string()), + OtlpProtocol::Grpc => { + #[cfg(feature = "tracing-grpc")] + { + builder + .with_tonic() + .with_endpoint(config.endpoint) + .build() + .map_err(|error| error.to_string()) + } + #[cfg(not(feature = "tracing-grpc"))] + { + Err("gRPC logs support is not compiled in".to_string()) + } + } + } +} + +fn http_logs_endpoint(endpoint: &str, append_logs_path: bool) -> String { + if !append_logs_path { + return endpoint.to_string(); + } + + let suffix_start = endpoint.find(['?', '#']).unwrap_or(endpoint.len()); + let (path, suffix) = endpoint.split_at(suffix_start); + let path = path.trim_end_matches('/'); + if path.ends_with("/v1/logs") { + return format!("{path}{suffix}"); + } + format!("{path}/v1/logs{suffix}") +} + +#[cfg(test)] +mod tests { + use opentelemetry_sdk::logs::{InMemoryLogExporter, SimpleLogProcessor}; + use tracing_subscriber::layer::SubscriberExt as _; + + use super::*; + + fn exported_bodies(emit: impl FnOnce()) -> Vec<(String, String)> { + let exporter = InMemoryLogExporter::default(); + let runtime = LoggingRuntime::new( + SdkLoggerProvider::builder() + .with_log_processor(SimpleLogProcessor::new(exporter.clone())) + .build(), + ); + let subscriber = tracing_subscriber::registry().with(runtime.layer()); + tracing::subscriber::with_default(subscriber, emit); + + let logs = exporter + .get_emitted_logs() + .unwrap() + .into_iter() + .map(|log| { + let body = log + .record + .body() + .map(|body| format!("{body:?}")) + .unwrap_or_default(); + ( + log.record + .target() + .map(ToString::to_string) + .unwrap_or_default(), + body, + ) + }) + .collect(); + runtime.shutdown(); + logs + } + + #[test] + fn http_logs_endpoint_appends_signal_path_once() { + assert_eq!( + http_logs_endpoint("http://collector:4318", true), + "http://collector:4318/v1/logs" + ); + assert_eq!( + http_logs_endpoint("http://collector:4318/v1/logs/", true), + "http://collector:4318/v1/logs" + ); + assert_eq!( + http_logs_endpoint("http://collector:4318/?token=secret", true), + "http://collector:4318/v1/logs?token=secret" + ); + assert_eq!( + http_logs_endpoint("http://collector:4318/custom", false), + "http://collector:4318/custom" + ); + } + + #[test] + fn exports_cog_events_but_not_model_output() { + let logs = exported_bodies(|| { + tracing::info!(target: "coglet::prediction", prediction_id = "p1", "Starting prediction"); + tracing::info!(target: "coglet::prediction", prediction_id = "p1", source = "stdout", "model print"); + tracing::info!(target: "coglet::setup", source = "stderr", "setup print"); + tracing::info!(target: "coglet::user", "user print"); + tracing::warn!(target: "coglet::orchestrator", "Worker exited"); + tracing::info!(target: "opentelemetry_sdk", "internal"); + tracing::info!(target: "hyper::client", "transport"); + }); + + let targets: Vec<_> = logs.iter().map(|(target, _)| target.as_str()).collect(); + assert_eq!(targets, ["coglet::prediction", "coglet::orchestrator"]); + assert!(logs[0].1.contains("Starting prediction"), "{logs:?}"); + } +} diff --git a/crates/coglet/src/worker.rs b/crates/coglet/src/worker.rs index 5eaba0a79a..22abd0efd6 100644 --- a/crates/coglet/src/worker.rs +++ b/crates/coglet/src/worker.rs @@ -125,6 +125,8 @@ fn init_worker_tracing( EnvFilter::new(filter_str) }; + // No OTel logs layer here: the parent exports these events after + // WorkerTracingLayer relays them, so exporting here too would duplicate them. let worker_layer = WorkerTracingLayer::new(tx); #[cfg(feature = "tracing")] diff --git a/pkg/config/config.go b/pkg/config/config.go index 2cb9c4bc5c..a669200f8a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -79,13 +79,15 @@ type Observability struct { Config string `json:"config,omitempty" yaml:"config,omitempty"` Traces *Tracing `json:"traces,omitempty" yaml:"traces,omitempty"` Metrics *Metrics `json:"metrics,omitempty" yaml:"metrics,omitempty"` + Logs *Logs `json:"logs,omitempty" yaml:"logs,omitempty"` } // AnyTelemetryEnabled reports whether this image opts into an OpenTelemetry signal. func (o *Observability) AnyTelemetryEnabled() bool { return o != nil && ((o.Traces != nil && o.Traces.Enabled) || - (o.Metrics != nil && o.Metrics.Enabled)) + (o.Metrics != nil && o.Metrics.Enabled) || + (o.Logs != nil && o.Logs.Enabled)) } type Tracing struct { @@ -100,6 +102,10 @@ type Metrics struct { Enabled bool `json:"enabled" yaml:"enabled"` } +type Logs struct { + Enabled bool `json:"enabled" yaml:"enabled"` +} + // WeightSourceConfig describes where to import weights from. // This is the "source" sub-object inside a weights entry. type WeightSourceConfig struct { diff --git a/pkg/config/config_file.go b/pkg/config/config_file.go index 5a2d70a563..d679a1b327 100644 --- a/pkg/config/config_file.go +++ b/pkg/config/config_file.go @@ -69,6 +69,7 @@ type observabilityFile struct { Config *string `json:"config,omitempty" yaml:"config,omitempty"` Traces *tracingFile `json:"traces,omitempty" yaml:"traces,omitempty"` Metrics *metricsFile `json:"metrics,omitempty" yaml:"metrics,omitempty"` + Logs *logsFile `json:"logs,omitempty" yaml:"logs,omitempty"` } type tracingFile struct { @@ -83,6 +84,10 @@ type metricsFile struct { Enabled *bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` } +type logsFile struct { + Enabled *bool `json:"enabled,omitempty" yaml:"enabled,omitempty"` +} + // UnmarshalYAML accepts a boolean shorthand or the full tracing object. func (t *tracingFile) UnmarshalYAML(unmarshal func(any) error) error { var enabled bool @@ -151,6 +156,40 @@ func (m *metricsFile) UnmarshalJSON(data []byte) error { return nil } +// UnmarshalYAML accepts a boolean shorthand or the full logs object. +func (l *logsFile) UnmarshalYAML(unmarshal func(any) error) error { + var enabled bool + if err := unmarshal(&enabled); err == nil { + l.Enabled = &enabled + return nil + } + + type rawLogsFile logsFile + var raw rawLogsFile + if err := unmarshal(&raw); err != nil { + return fmt.Errorf("logs must be a boolean or mapping: %w", err) + } + *l = logsFile(raw) + return nil +} + +// UnmarshalJSON accepts a boolean shorthand or the full logs object. +func (l *logsFile) UnmarshalJSON(data []byte) error { + var enabled bool + if err := json.Unmarshal(data, &enabled); err == nil { + l.Enabled = &enabled + return nil + } + + type rawLogsFile logsFile + var raw rawLogsFile + if err := json.Unmarshal(data, &raw); err != nil { + return fmt.Errorf("logs must be a boolean or object: %w", err) + } + *l = logsFile(raw) + return nil +} + // UnmarshalYAML implements custom YAML unmarshaling for runItemFile // to support both string and object forms. func (r *runItemFile) UnmarshalYAML(unmarshal func(any) error) error { diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 138419e677..bcd8671612 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -861,12 +861,13 @@ func TestObservabilityConfigParsing(t *testing.T) { } func TestObservabilityBooleanSignalParsing(t *testing.T) { - cfgFile, err := parseBytes([]byte("observability:\n traces: true\n metrics: true\n")) + cfgFile, err := parseBytes([]byte("observability:\n traces: true\n metrics: true\n logs: true\n")) require.NoError(t, err) cfg, err := configFileToConfig(cfgFile) require.NoError(t, err) require.True(t, cfg.Observability.Traces.Enabled) require.True(t, cfg.Observability.Metrics.Enabled) + require.True(t, cfg.Observability.Logs.Enabled) require.True(t, cfg.Observability.AnyTelemetryEnabled()) } diff --git a/pkg/config/data/config_schema_v1.0.json b/pkg/config/data/config_schema_v1.0.json index 7ff0fb9c05..62b4fe5970 100644 --- a/pkg/config/data/config_schema_v1.0.json +++ b/pkg/config/data/config_schema_v1.0.json @@ -213,7 +213,7 @@ "type": "string", "minLength": 1, "pattern": "\\.py$", - "description": "An optional Python module that customizes OpenTelemetry providers and runtime metrics." + "description": "An optional Python module that customizes OpenTelemetry providers, runtime metrics, and instrumentation." }, "traces": { "$id": "#/properties/observability/properties/traces", @@ -268,6 +268,24 @@ } } ] + }, + "logs": { + "$id": "#/properties/observability/properties/logs", + "description": "Enables OpenTelemetry log export for model Python logging and Cog runtime events. Use true for defaults or an object to control enablement.", + "oneOf": [ + { "type": "boolean" }, + { + "type": "object", + "required": ["enabled"], + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean", + "description": "Whether to enable log export in this image." + } + } + } + ] } } }, diff --git a/pkg/config/env.go b/pkg/config/env.go index 69f532bc7a..1575ea025a 100644 --- a/pkg/config/env.go +++ b/pkg/config/env.go @@ -26,6 +26,7 @@ var environmentVariableDenyList = []string{ "COG_OBSERVABILITY_*", "COG_TRACE_*", "COG_METRICS_*", + "COG_LOGS_*", "OTEL_*", // Nvidia "LIBRARY_PATH", diff --git a/pkg/config/parse.go b/pkg/config/parse.go index 6c49f4b1df..f23f176b8d 100644 --- a/pkg/config/parse.go +++ b/pkg/config/parse.go @@ -165,6 +165,13 @@ func configFileToConfig(cfg *configFile) (*Config, error) { config.Observability.Metrics.Enabled = *metrics.Enabled } } + if cfg.Observability.Logs != nil { + logs := cfg.Observability.Logs + config.Observability.Logs = &Logs{} + if logs.Enabled != nil { + config.Observability.Logs.Enabled = *logs.Enabled + } + } } config.Environment = cfg.Environment diff --git a/pkg/config/validate.go b/pkg/config/validate.go index 5d60cc09eb..a911792442 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -86,7 +86,7 @@ func validateObservability(cfg *configFile, opts *validateOptions, result *Valid if cfg.Observability.Config != nil { validateObservabilityConfig(*cfg.Observability.Config, opts, result) if !observabilityFileAnyTelemetryEnabled(cfg.Observability) { - result.AddError(&ValidationError{Field: "observability.config", Value: *cfg.Observability.Config, Message: "requires observability.traces.enabled or observability.metrics.enabled to be true"}) + result.AddError(&ValidationError{Field: "observability.config", Value: *cfg.Observability.Config, Message: "requires observability.traces.enabled, observability.metrics.enabled, or observability.logs.enabled to be true"}) } } if cfg.Observability.Traces != nil { @@ -95,12 +95,16 @@ func validateObservability(cfg *configFile, opts *validateOptions, result *Valid if cfg.Observability.Metrics != nil && cfg.Observability.Metrics.Enabled == nil { result.AddError(&ValidationError{Field: "observability.metrics.enabled", Message: "is required"}) } + if cfg.Observability.Logs != nil && cfg.Observability.Logs.Enabled == nil { + result.AddError(&ValidationError{Field: "observability.logs.enabled", Message: "is required"}) + } } func observabilityFileAnyTelemetryEnabled(observability *observabilityFile) bool { return observability != nil && ((observability.Traces != nil && observability.Traces.Enabled != nil && *observability.Traces.Enabled) || - (observability.Metrics != nil && observability.Metrics.Enabled != nil && *observability.Metrics.Enabled)) + (observability.Metrics != nil && observability.Metrics.Enabled != nil && *observability.Metrics.Enabled) || + (observability.Logs != nil && observability.Logs.Enabled != nil && *observability.Logs.Enabled)) } func validateTracing(traces *tracingFile, result *ValidationResult) { diff --git a/pkg/config/validate_test.go b/pkg/config/validate_test.go index 42f8a657b9..c81d193993 100644 --- a/pkg/config/validate_test.go +++ b/pkg/config/validate_test.go @@ -242,6 +242,26 @@ func TestValidateObservabilityMetrics(t *testing.T) { } } +func TestValidateObservabilityLogs(t *testing.T) { + tests := []struct { + name string + logs *logsFile + wantErrors bool + }{ + {name: "enabled", logs: &logsFile{Enabled: new(true)}}, + {name: "disabled", logs: &logsFile{Enabled: new(false)}}, + {name: "missing enabled", logs: &logsFile{}, wantErrors: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := &configFile{Observability: &observabilityFile{Logs: test.logs}} + result := ValidateConfigFile(cfg) + require.Equal(t, test.wantErrors, result.HasErrors(), "errors: %v", result.Errors) + }) + } +} + func TestValidateObservabilityConfig(t *testing.T) { projectDir := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(projectDir, "telemetry.py"), []byte("# telemetry"), 0o644)) @@ -254,12 +274,14 @@ func TestValidateObservabilityConfig(t *testing.T) { configPath string traces *tracingFile metrics *metricsFile + logs *logsFile wantError string }{ {name: "valid", configPath: "telemetry.py", traces: &tracingFile{Enabled: new(true)}}, {name: "metrics only", configPath: "telemetry.py", metrics: &metricsFile{Enabled: new(true)}}, - {name: "no signal enabled", configPath: "telemetry.py", wantError: "requires observability.traces.enabled or observability.metrics.enabled to be true"}, - {name: "all signals disabled", configPath: "telemetry.py", traces: &tracingFile{Enabled: new(false)}, wantError: "requires observability.traces.enabled or observability.metrics.enabled to be true"}, + {name: "logs only", configPath: "telemetry.py", logs: &logsFile{Enabled: new(true)}}, + {name: "no signal enabled", configPath: "telemetry.py", wantError: "requires observability.traces.enabled, observability.metrics.enabled, or observability.logs.enabled to be true"}, + {name: "all signals disabled", configPath: "telemetry.py", traces: &tracingFile{Enabled: new(false)}, wantError: "requires observability.traces.enabled, observability.metrics.enabled, or observability.logs.enabled to be true"}, {name: "absolute", configPath: filepath.Join(projectDir, "telemetry.py"), traces: &tracingFile{Enabled: new(true)}, wantError: "project-relative"}, {name: "parent component", configPath: "nested/../telemetry.py", traces: &tracingFile{Enabled: new(true)}, wantError: "project-relative"}, {name: "wrong extension", configPath: "telemetry.txt", traces: &tracingFile{Enabled: new(true)}, wantError: "ending in .py"}, @@ -269,7 +291,7 @@ func TestValidateObservabilityConfig(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - cfg := &configFile{Observability: &observabilityFile{Config: new(test.configPath), Traces: test.traces, Metrics: test.metrics}} + cfg := &configFile{Observability: &observabilityFile{Config: new(test.configPath), Traces: test.traces, Metrics: test.metrics, Logs: test.logs}} result := ValidateConfigFile(cfg, WithProjectDir(projectDir)) if test.wantError == "" { require.False(t, result.HasErrors(), "errors: %v", result.Errors) diff --git a/pkg/dockerfile/standard_generator.go b/pkg/dockerfile/standard_generator.go index 1ceb048bd1..e0cc5e0ee5 100644 --- a/pkg/dockerfile/standard_generator.go +++ b/pkg/dockerfile/standard_generator.go @@ -31,8 +31,7 @@ const uvPip = "uv pip" const observabilityConfigBuildPath = "telemetry.py" const observabilityConfigRuntimePath = "/.cog/telemetry.py" const PythonObservabilityRequirements = "opentelemetry-api==1.44.0 opentelemetry-sdk==1.44.0 opentelemetry-exporter-otlp-proto-http==1.44.0 opentelemetry-exporter-otlp-proto-grpc==1.44.0" -const PythonObservabilityCheck = `python -c "import cog._telemetry; from coglet import _impl; raise SystemExit(0 if getattr(_impl, '_supports_observability_metrics', False) else 1)"` -const PythonObservabilityCheckError = "OpenTelemetry metrics require matching cog and coglet builds with metrics support" +const PythonObservabilityCheckError = "OpenTelemetry metrics and logs require matching cog and coglet builds that support them" const uvBreakSystemPackages = "--break-system-packages" const PrecompilePythonCommand = "RUN find / -type f -name \"*.py[co]\" -delete && find / -type f -name \"*.py\" -exec touch -t 197001010000 {} \\; && find / -type f -name \"*.py\" -printf \"%h\\n\" | sort -u | /usr/bin/python3 -m compileall --invalidation-mode timestamp -o 2 -j 0" const STANDARD_GENERATOR_NAME = "STANDARD_GENERATOR" @@ -416,6 +415,12 @@ func (g *StandardGenerator) cogEnvVars() []string { `ENV COG_METRICS_ENABLED=true`, ) } + if g.Config.Observability != nil && g.Config.Observability.Logs != nil && g.Config.Observability.Logs.Enabled { + envs = append(envs, + `ENV COG_LOGS_CONFIGURED=true`, + `ENV COG_LOGS_ENABLED=true`, + ) + } return envs } @@ -771,12 +776,28 @@ func (g *StandardGenerator) installPythonObservabilityDependencies() string { if g.strip { install += " && " + StripDebugSymbolsCommand } - if g.Config.Observability.Metrics != nil && g.Config.Observability.Metrics.Enabled { - install += " && (" + PythonObservabilityCheck + " || (echo \"" + PythonObservabilityCheckError + "\" >&2; exit 1))" + if check := PythonObservabilityCheck(g.Config.Observability); check != "" { + install += " && (" + check + " || (echo \"" + PythonObservabilityCheckError + "\" >&2; exit 1))" } return install } +// PythonObservabilityCheck returns a command that fails when the installed +// coglet lacks support for an enabled signal, or "" when no check is needed. +func PythonObservabilityCheck(observability *config.Observability) string { + var checks []string + if observability.Metrics != nil && observability.Metrics.Enabled { + checks = append(checks, "getattr(_impl, '_supports_observability_metrics', False)") + } + if observability.Logs != nil && observability.Logs.Enabled { + checks = append(checks, "getattr(_impl, '_supports_observability_logs', False)") + } + if len(checks) == 0 { + return "" + } + return `python -c "import cog._telemetry; from coglet import _impl; raise SystemExit(0 if ` + strings.Join(checks, " and ") + ` else 1)"` +} + // installCogFromPyPI installs the cog SDK from PyPI. // preRelease adds --pre to allow pip to resolve pre-release packages. func (g *StandardGenerator) installCogFromPyPI(config *wheels.WheelConfig, preRelease bool) (string, error) { diff --git a/pkg/dockerfile/standard_generator_test.go b/pkg/dockerfile/standard_generator_test.go index 1375a54159..eec65b5cad 100644 --- a/pkg/dockerfile/standard_generator_test.go +++ b/pkg/dockerfile/standard_generator_test.go @@ -1541,7 +1541,7 @@ func TestPythonObservabilityDependenciesAreOptIn(t *testing.T) { }, }} require.Contains(t, enabled.installPythonObservabilityDependencies(), PythonObservabilityRequirements) - assert.NotContains(t, enabled.installPythonObservabilityDependencies(), PythonObservabilityCheck) + assert.NotContains(t, enabled.installPythonObservabilityDependencies(), "_supports_observability") enabled.strip = true require.Contains(t, enabled.installPythonObservabilityDependencies(), StripDebugSymbolsCommand) @@ -1553,6 +1553,29 @@ func TestPythonObservabilityDependenciesAreOptIn(t *testing.T) { }, }} require.Contains(t, metricsOnly.installPythonObservabilityDependencies(), PythonObservabilityRequirements) - assert.Contains(t, metricsOnly.installPythonObservabilityDependencies(), PythonObservabilityCheck) + assert.Contains(t, metricsOnly.installPythonObservabilityDependencies(), PythonObservabilityCheck(metricsOnly.Config.Observability)) assert.Contains(t, metricsOnly.installPythonObservabilityDependencies(), PythonObservabilityCheckError) + assert.NotContains(t, metricsOnly.installPythonObservabilityDependencies(), "_supports_observability_logs") + + logsOnly := &StandardGenerator{Config: &config.Config{ + Build: &config.Build{}, + Observability: &config.Observability{ + Logs: &config.Logs{Enabled: true}, + }, + }} + require.Contains(t, logsOnly.installPythonObservabilityDependencies(), PythonObservabilityRequirements) + assert.Contains(t, logsOnly.installPythonObservabilityDependencies(), "_supports_observability_logs") + assert.NotContains(t, logsOnly.installPythonObservabilityDependencies(), "_supports_observability_metrics") + assert.Contains(t, logsOnly.cogEnvVars(), `ENV COG_LOGS_CONFIGURED=true`) + assert.Contains(t, logsOnly.cogEnvVars(), `ENV COG_LOGS_ENABLED=true`) +} + +func TestPythonObservabilityCheckRequiresEachEnabledSignal(t *testing.T) { + require.Empty(t, PythonObservabilityCheck(&config.Observability{Traces: &config.Tracing{Enabled: true}})) + + check := PythonObservabilityCheck(&config.Observability{ + Metrics: &config.Metrics{Enabled: true}, + Logs: &config.Logs{Enabled: true}, + }) + assert.Contains(t, check, "getattr(_impl, '_supports_observability_metrics', False) and getattr(_impl, '_supports_observability_logs', False)") } diff --git a/pkg/image/build.go b/pkg/image/build.go index 8ebc34da0c..59bdb3e9b7 100644 --- a/pkg/image/build.go +++ b/pkg/image/build.go @@ -663,8 +663,8 @@ func observabilityDockerfile(baseImage string, observability *config.Observabili fmt.Fprintln(&b, "USER root") } fmt.Fprintf(&b, "RUN python -m pip install --no-cache-dir --break-system-packages %s\n", dockerfile.PythonObservabilityRequirements) - if observability.Metrics != nil && observability.Metrics.Enabled { - fmt.Fprintf(&b, "RUN (%s || (echo \"%s\" >&2; exit 1))\n", dockerfile.PythonObservabilityCheck, dockerfile.PythonObservabilityCheckError) + if check := dockerfile.PythonObservabilityCheck(observability); check != "" { + fmt.Fprintf(&b, "RUN (%s || (echo \"%s\" >&2; exit 1))\n", check, dockerfile.PythonObservabilityCheckError) } if imageUser != "" { fmt.Fprintf(&b, "USER %s\n", strconv.Quote(imageUser)) @@ -687,6 +687,11 @@ func observabilityDockerfile(baseImage string, observability *config.Observabili } else { fmt.Fprint(&b, "ENV COG_METRICS_CONFIGURED=false\nENV COG_METRICS_ENABLED=false\n") } + if observability.Logs != nil && observability.Logs.Enabled { + fmt.Fprint(&b, "ENV COG_LOGS_CONFIGURED=true\nENV COG_LOGS_ENABLED=true\n") + } else { + fmt.Fprint(&b, "ENV COG_LOGS_CONFIGURED=false\nENV COG_LOGS_ENABLED=false\n") + } if observability.Config != "" { fmt.Fprintf(&b, "COPY --from=%s telemetry.py /.cog/telemetry.py\nENV COG_OBSERVABILITY_CONFIG=\"/.cog/telemetry.py\"\n", cogBuildContextName) } else { diff --git a/pkg/image/build_test.go b/pkg/image/build_test.go index 7764e8814c..199338b78e 100644 --- a/pkg/image/build_test.go +++ b/pkg/image/build_test.go @@ -169,8 +169,9 @@ func TestAddObservabilityToCustomDockerfileImageUsesStagedConfig(t *testing.T) { require.Contains(t, build.DockerfileContents, "USER root") require.Contains(t, build.DockerfileContents, "USER \"1000:1000\"") require.Equal(t, map[string]string{cogBuildContextName: "/tmp/build-cache"}, build.BuildContexts) - assert.NotContains(t, build.DockerfileContents, dockerfilepkg.PythonObservabilityCheck) + assert.NotContains(t, build.DockerfileContents, "_supports_observability") assert.Contains(t, build.DockerfileContents, "ENV COG_METRICS_CONFIGURED=false") + assert.Contains(t, build.DockerfileContents, "ENV COG_LOGS_CONFIGURED=false") } func TestAddObservabilityToCustomDockerfileImageSupportsMetricsOnly(t *testing.T) { @@ -188,13 +189,30 @@ func TestAddObservabilityToCustomDockerfileImageSupportsMetricsOnly(t *testing.T assert.Contains(t, build.DockerfileContents, "COPY --from=cog_build telemetry.py /.cog/telemetry.py") assert.Contains(t, build.DockerfileContents, "ENV COG_METRICS_CONFIGURED=true") assert.Contains(t, build.DockerfileContents, "ENV COG_METRICS_ENABLED=true") - assert.Contains(t, build.DockerfileContents, dockerfilepkg.PythonObservabilityCheck) + assert.Contains(t, build.DockerfileContents, dockerfilepkg.PythonObservabilityCheck(observability)) assert.Contains(t, build.DockerfileContents, dockerfilepkg.PythonObservabilityCheckError) assert.Contains(t, build.DockerfileContents, "ENV COG_TRACE_CONFIGURED=false") assert.Contains(t, build.DockerfileContents, "USER root") assert.Contains(t, build.DockerfileContents, "USER \"1000:1000\"") } +func TestAddObservabilityToCustomDockerfileImageSupportsLogsOnly(t *testing.T) { + dockerCommand := &recordingCommand{MockCommand: dockertest.NewMockCommand()} + observability := &config.Observability{Logs: &config.Logs{Enabled: true}} + + err := addObservabilityToCustomDockerfileImage(t.Context(), dockerCommand, "my-image", observability, "plain", "/tmp/build-cache") + + require.NoError(t, err) + require.Len(t, dockerCommand.builds, 1) + build := dockerCommand.builds[0] + assert.Contains(t, build.DockerfileContents, dockerfilepkg.PythonObservabilityRequirements) + assert.Contains(t, build.DockerfileContents, "_supports_observability_logs") + assert.Contains(t, build.DockerfileContents, "ENV COG_LOGS_CONFIGURED=true") + assert.Contains(t, build.DockerfileContents, "ENV COG_LOGS_ENABLED=true") + assert.Contains(t, build.DockerfileContents, "ENV COG_TRACE_CONFIGURED=false") + assert.Contains(t, build.DockerfileContents, "ENV COG_METRICS_CONFIGURED=false") +} + func TestAddObservabilityToCustomDockerfileImageDisablesInheritedTelemetry(t *testing.T) { dockerCommand := &recordingCommand{MockCommand: dockertest.NewMockCommand()} observability := &config.Observability{Metrics: &config.Metrics{Enabled: false}} From 01fd5b9cfb1693a2c613a5ef9bcd70009db9a618 Mon Sep 17 00:00:00 2001 From: Anish Sahoo Date: Sat, 31 Oct 2026 19:09:10 -0400 Subject: [PATCH 2/2] feat(observability): export Python logging records over OTLP - Install a Python LoggerProvider when logs are enabled, with a create_logger_provider(resource) factory in telemetry.py. - Export records by wrapping logging.Logger.callHandlers instead of adding a root handler, so logging.lastResort, basicConfig(), stderr, and the prediction logs field behave as before. Records follow the model's own levels, filters, and disabled loggers, and keep extra fields as attributes. - Reuse the SDK's attribute mapping and severity conversion without the deprecated LoggingHandler. Non-string messages stay structured, matching the SDK handler. - Skip opentelemetry loggers, re-entrant calls, and OTel SDK export threads so a failed export cannot queue more records. - Add a logs integration test, a custom logger provider in the hello-concurrency example, and logs docs. --- README.md | 2 +- architecture/04-container-runtime.md | 5 +- crates/coglet-python/src/worker_bridge.rs | 1 + docs/environment.md | 10 +- docs/llms.txt | 85 ++++++++-- docs/observability.md | 59 ++++++- docs/python.md | 4 + docs/yaml.md | 10 +- examples/hello-concurrency/README.md | 10 +- examples/hello-concurrency/cog.yaml | 2 + examples/hello-concurrency/run.py | 5 +- examples/hello-concurrency/telemetry.py | 21 +++ .../tests/observability_logs.txtar | 59 +++++++ python/cog/_telemetry.py | 156 +++++++++++++++++- python/tests/test_telemetry.py | 144 +++++++++++++++- 15 files changed, 523 insertions(+), 50 deletions(-) create mode 100644 integration-tests/tests/observability_logs.txtar diff --git a/README.md b/README.md index 863e1b2c26..98020f518b 100644 --- a/README.md +++ b/README.md @@ -229,7 +229,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development environme - [Using Cog with Windows 11](docs/wsl2/wsl2.md) - [Browse the example models in this repo](docs/examples.md) - [Deploy models with Cog](docs/deploy.md) -- [Configure OpenTelemetry tracing and metrics](docs/observability.md) +- [Configure OpenTelemetry tracing, metrics, and logs](docs/observability.md) - [`cog.yaml` reference](docs/yaml.md) to learn how to define your model's environment - [Run interface reference](docs/python.md) to learn how the `Runner` interface works - [Training interface reference](docs/training.md) to learn how to add a fine-tuning API to your model diff --git a/architecture/04-container-runtime.md b/architecture/04-container-runtime.md index ae4934c4bc..089da88998 100644 --- a/architecture/04-container-runtime.md +++ b/architecture/04-container-runtime.md @@ -391,7 +391,9 @@ Opt-in OpenTelemetry tracing uses a provider in the parent process, a provider i Metrics use separate ownership. The parent creates the runtime MeterProvider after worker setup and exports fixed prediction, rejection, setup, and slot-state measurements. The Python worker creates the model MeterProvider for arbitrary model-owned instruments. It returns a runtime metric configuration in its `Ready` control message so the parent can omit selected fixed instruments before recording terminal setup status. -Models may provide `observability.config`, which Cog validates and stages at a fixed image path. During worker setup, Cog loads that module, creates selected Python tracer and meter providers, validates them, installs them globally, and then calls optional `configure_instrumentation()` before predictor import. Cog owns provider flush and shutdown. Configuration failures stop model setup because the user explicitly selected that module. +Logs follow the same split. The parent attaches an OpenTelemetry appender to its `tracing` subscriber and exports Cog's own events, including worker events relayed over the control channel; it skips the events that re-emit model stdout and stderr. The worker exports Python `logging` records by wrapping `logging.Logger.callHandlers`, which runs after logger level, disabled, and filter checks but adds no handler, so `lastResort` and `basicConfig()` behave as before. + +Models may provide `observability.config`, which Cog validates and stages at a fixed image path. During worker setup, Cog loads that module, creates selected Python tracer, meter, and logger providers, validates them, installs them globally, and then calls optional `configure_instrumentation()` before predictor import. Cog owns provider flush and shutdown. Configuration failures stop model setup because the user explicitly selected that module. The framework trace covers HTTP handling, validation, the logical prediction lifetime, worker execution, input preparation, predictor invocation, output upload, and setup. Model code creates ordinary child spans through `opentelemetry.trace`. @@ -415,6 +417,7 @@ If the healthcheck fails, the HTTP `/health-check` endpoint returns `UNHEALTHY` | `COG_SETUP_TIMEOUT` | none | Setup timeout in seconds (0 is ignored) | | `COG_TRACE_ENABLED` | image | `true` in tracing-enabled images; absent otherwise | | `COG_METRICS_ENABLED` | image | `true` in metrics-enabled images; absent otherwise | +| `COG_LOGS_ENABLED` | image | `true` in logs-enabled images; absent otherwise | | `COG_OBSERVABILITY_CONFIG` | none | Internal path to staged Python telemetry config | | `COG_THROTTLE_RESPONSE_INTERVAL` | 0.5s | Webhook response throttling interval | | `LOG_FORMAT` | json | Set to `console` for human-readable log output | diff --git a/crates/coglet-python/src/worker_bridge.rs b/crates/coglet-python/src/worker_bridge.rs index f496552826..031f277391 100644 --- a/crates/coglet-python/src/worker_bridge.rs +++ b/crates/coglet-python/src/worker_bridge.rs @@ -64,6 +64,7 @@ fn python_telemetry_enabled() -> bool { } (env_true("COG_TRACE_CONFIGURED", false) && env_true("COG_TRACE_ENABLED", true)) || (env_true("COG_METRICS_CONFIGURED", false) && env_true("COG_METRICS_ENABLED", true)) + || (env_true("COG_LOGS_CONFIGURED", false) && env_true("COG_LOGS_ENABLED", true)) } fn runtime_metrics_config_from_python( diff --git a/docs/environment.md b/docs/environment.md index 4682d77b97..be05300106 100644 --- a/docs/environment.md +++ b/docs/environment.md @@ -224,22 +224,26 @@ $ COG_LOG_LEVEL=debug docker run -p 5000:5000 my-model ### OpenTelemetry -Tracing and metrics must first be enabled under `observability` in `cog.yaml`. Runtime settings may disable an enabled image but cannot enable a signal that did not opt in. +Tracing, metrics, and logs must first be enabled under `observability` in `cog.yaml`. Runtime settings may disable an enabled image but cannot enable a signal that did not opt in. | Variable | Purpose | | ------------------------------------- | ---------------------------------------------- | | `COG_TRACE_ENABLED` | Set to `false` to disable tracing at runtime. | | `COG_METRICS_ENABLED` | Set to `false` to disable metrics at runtime. | +| `COG_LOGS_ENABLED` | Set to `false` to disable logs at runtime. | | `OTEL_SDK_DISABLED` | Hard-disable OpenTelemetry SDK initialization. | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | Default collector endpoint for both signals. | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Default collector endpoint for all signals. | | `OTEL_EXPORTER_OTLP_PROTOCOL` | Default protocol: `http/protobuf` or `grpc`. | | `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Trace-specific collector endpoint. | | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | Metric-specific collector endpoint. | +| `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | Log-specific collector endpoint. | | `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` | Trace-specific protocol override. | | `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL` | Metric-specific protocol override. | +| `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` | Log-specific protocol override. | | `OTEL_EXPORTER_OTLP_HEADERS` | Collector authentication headers. | | `OTEL_TRACES_EXPORTER` | Set to `none` to disable trace export. | | `OTEL_METRICS_EXPORTER` | Set to `none` to disable metric export. | +| `OTEL_LOGS_EXPORTER` | Set to `none` to disable log export. | | `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes as `key=value` pairs. | | `OTEL_SERVICE_NAME` | Service name, default `cog`. | | `OTEL_TRACES_SAMPLER` | Runtime sampler override. | @@ -248,7 +252,7 @@ Tracing and metrics must first be enabled under `observability` in `cog.yaml`. R When an enabled default signal has no endpoint, Cog logs one warning and continues without that provider. A custom Python provider configured by `observability.config` may still use another exporter or no exporter. Delivery failures from Cog's built-in exporters never change prediction results; custom processors and exporters are model-owned code. -`COG_OBSERVABILITY_*`, `COG_TRACE_*`, `COG_METRICS_*`, and `OTEL_*` are reserved from the general `cog.yaml` `environment` list. `COG_OBSERVABILITY_CONFIG` is internal and points to the validated file staged in the image. Supply supported `COG_TRACE_*`, `COG_METRICS_*`, and `OTEL_*` settings to the running container instead. +`COG_OBSERVABILITY_*`, `COG_TRACE_*`, `COG_METRICS_*`, `COG_LOGS_*`, and `OTEL_*` are reserved from the general `cog.yaml` `environment` list. `COG_OBSERVABILITY_CONFIG` is internal and points to the validated file staged in the image. Supply supported `COG_TRACE_*`, `COG_METRICS_*`, `COG_LOGS_*`, and `OTEL_*` settings to the running container instead. ### `COG_THROTTLE_RESPONSE_INTERVAL` diff --git a/docs/llms.txt b/docs/llms.txt index 4b6bd94871..2e25ffa7fe 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -229,7 +229,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for how to set up a development environme - [Using Cog with Windows 11](docs/wsl2/wsl2.md) - [Browse the example models in this repo](docs/examples.md) - [Deploy models with Cog](docs/deploy.md) -- [Configure OpenTelemetry tracing and metrics](docs/observability.md) +- [Configure OpenTelemetry tracing, metrics, and logs](docs/observability.md) - [`cog.yaml` reference](docs/yaml.md) to learn how to define your model's environment - [Run interface reference](docs/python.md) to learn how the `Runner` interface works - [Training interface reference](docs/training.md) to learn how to add a fine-tuning API to your model @@ -1253,22 +1253,26 @@ $ COG_LOG_LEVEL=debug docker run -p 5000:5000 my-model ### OpenTelemetry -Tracing and metrics must first be enabled under `observability` in `cog.yaml`. Runtime settings may disable an enabled image but cannot enable a signal that did not opt in. +Tracing, metrics, and logs must first be enabled under `observability` in `cog.yaml`. Runtime settings may disable an enabled image but cannot enable a signal that did not opt in. | Variable | Purpose | | ------------------------------------- | ---------------------------------------------- | | `COG_TRACE_ENABLED` | Set to `false` to disable tracing at runtime. | | `COG_METRICS_ENABLED` | Set to `false` to disable metrics at runtime. | +| `COG_LOGS_ENABLED` | Set to `false` to disable logs at runtime. | | `OTEL_SDK_DISABLED` | Hard-disable OpenTelemetry SDK initialization. | -| `OTEL_EXPORTER_OTLP_ENDPOINT` | Default collector endpoint for both signals. | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | Default collector endpoint for all signals. | | `OTEL_EXPORTER_OTLP_PROTOCOL` | Default protocol: `http/protobuf` or `grpc`. | | `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Trace-specific collector endpoint. | | `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | Metric-specific collector endpoint. | +| `OTEL_EXPORTER_OTLP_LOGS_ENDPOINT` | Log-specific collector endpoint. | | `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` | Trace-specific protocol override. | | `OTEL_EXPORTER_OTLP_METRICS_PROTOCOL` | Metric-specific protocol override. | +| `OTEL_EXPORTER_OTLP_LOGS_PROTOCOL` | Log-specific protocol override. | | `OTEL_EXPORTER_OTLP_HEADERS` | Collector authentication headers. | | `OTEL_TRACES_EXPORTER` | Set to `none` to disable trace export. | | `OTEL_METRICS_EXPORTER` | Set to `none` to disable metric export. | +| `OTEL_LOGS_EXPORTER` | Set to `none` to disable log export. | | `OTEL_RESOURCE_ATTRIBUTES` | Resource attributes as `key=value` pairs. | | `OTEL_SERVICE_NAME` | Service name, default `cog`. | | `OTEL_TRACES_SAMPLER` | Runtime sampler override. | @@ -1277,7 +1281,7 @@ Tracing and metrics must first be enabled under `observability` in `cog.yaml`. R When an enabled default signal has no endpoint, Cog logs one warning and continues without that provider. A custom Python provider configured by `observability.config` may still use another exporter or no exporter. Delivery failures from Cog's built-in exporters never change prediction results; custom processors and exporters are model-owned code. -`COG_OBSERVABILITY_*`, `COG_TRACE_*`, `COG_METRICS_*`, and `OTEL_*` are reserved from the general `cog.yaml` `environment` list. `COG_OBSERVABILITY_CONFIG` is internal and points to the validated file staged in the image. Supply supported `COG_TRACE_*`, `COG_METRICS_*`, and `OTEL_*` settings to the running container instead. +`COG_OBSERVABILITY_*`, `COG_TRACE_*`, `COG_METRICS_*`, `COG_LOGS_*`, and `OTEL_*` are reserved from the general `cog.yaml` `environment` list. `COG_OBSERVABILITY_CONFIG` is internal and points to the validated file staged in the image. Supply supported `COG_TRACE_*`, `COG_METRICS_*`, `COG_LOGS_*`, and `OTEL_*` settings to the running container instead. ### `COG_THROTTLE_RESPONSE_INTERVAL` @@ -2479,15 +2483,13 @@ class Runner(BaseRunner): # Observability -Cog can join incoming distributed traces, export fixed runtime metrics, and make standard OpenTelemetry APIs available to model code. Signals are opt-in and use OTLP, so they work with collectors and backends that support OpenTelemetry. +Cog can join incoming distributed traces, export fixed runtime metrics, export logs, and make standard OpenTelemetry APIs available to model code. Signals are opt-in and use OTLP, so they work with collectors and backends that support OpenTelemetry. -Cog has two telemetry ownership domains. The Rust parent owns fixed runtime metrics. The Python worker owns model-authored spans and metrics. Both can export to the same collector, but a Python provider never replaces the parent runtime provider. - -OpenTelemetry log export is not supported. +Cog has two telemetry ownership domains. The Rust parent owns fixed runtime metrics and Cog's own log events. The Python worker owns model-authored spans, metrics, and logs. Both can export to the same collector, but a Python provider never replaces a parent provider. ## Enable telemetry -Enable either signal with an object: +Enable each signal with an object: ```yaml observability: @@ -2496,11 +2498,13 @@ observability: sampler: parentbased_always_off metrics: enabled: true + logs: + enabled: true ``` -The boolean forms `traces: true` and `metrics: true` are accepted as shorthand. +The boolean forms `traces: true`, `metrics: true`, and `logs: true` are accepted as shorthand. -An image can enable either signal independently. Runtime configuration can disable an enabled signal, but cannot enable a signal omitted from the image. +An image can enable each signal independently. Runtime configuration can disable an enabled signal, but cannot enable a signal omitted from the image. Configure the collector when running the image: @@ -2512,7 +2516,7 @@ OTEL_SERVICE_NAME=cog Cog supports OTLP gRPC and HTTP/protobuf. Collector endpoints, authentication headers, certificates, and other `OTEL_*` values are runtime configuration and cannot be set through the general `cog.yaml` `environment` list. -Framework telemetry starts only when the image opts in and a collector endpoint is present. `COG_TRACE_ENABLED=false` and `COG_METRICS_ENABLED=false` disable the entire matching signal, including custom Python providers. `OTEL_SDK_DISABLED=true` disables all signals. `OTEL_TRACES_EXPORTER=none` and `OTEL_METRICS_EXPORTER=none` disable only the matching built-in provider. An explicitly configured Python factory can still use a different exporter or no exporter. +Framework telemetry starts only when the image opts in and a collector endpoint is present. `COG_TRACE_ENABLED=false`, `COG_METRICS_ENABLED=false`, and `COG_LOGS_ENABLED=false` disable the entire matching signal, including custom Python providers. `OTEL_SDK_DISABLED=true` disables all signals. `OTEL_TRACES_EXPORTER=none`, `OTEL_METRICS_EXPORTER=none`, and `OTEL_LOGS_EXPORTER=none` disable only the matching built-in provider. An explicitly configured Python factory can still use a different exporter or no exporter. ## Getting started with tracing @@ -2604,16 +2608,21 @@ observability: enabled: true metrics: enabled: true + logs: + enabled: true ``` Cog validates the file during configuration, copies it to a fixed path in the image, and loads it before importing the model. Provider factories are optional. A missing factory uses Cog's default provider for that signal. Factories receive Cog's base `Resource` and may merge or replace its attributes. Existing zero-argument `create_tracer_provider()` factories remain supported. ```python from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.trace import SpanLimits, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased @@ -2638,6 +2647,12 @@ def create_meter_provider(resource: Resource) -> MeterProvider: ) +def create_logger_provider(resource: Resource) -> LoggerProvider: + provider = LoggerProvider(resource=resource, shutdown_on_exit=False) + provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + return provider + + def configure_instrumentation() -> None: from opentelemetry.instrumentation.requests import RequestsInstrumentor @@ -2705,6 +2720,36 @@ def configure_runtime_metrics() -> RuntimeMetricsConfig: Set `enabled=False` to disable all current and future built-in Cog metrics. This does not disable the Python `MeterProvider`, so model metrics can still export. +## Logs + +`logs.enabled: true` exports two kinds of log records: + +- Model records from Python's `logging` module, exported by the worker. Each record keeps its severity, logger name (as the instrumentation scope), message, source location, exception details, and `extra` fields as attributes. When tracing is enabled, records written during a prediction carry its trace and span IDs. +- Cog's own runtime events, such as prediction start and completion, setup progress, and errors, exported by the parent. Worker runtime events are relayed through the parent and keep only their message. + +Model code needs no OpenTelemetry imports: + +```python +import logging + +log = logging.getLogger(__name__) + + +class Runner(BaseRunner): + def run(self, prompt: str) -> str: + result = self.model(prompt) + log.info("generated", extra={"prompt_tokens": result.prompt_tokens}) + return result.text +``` + +Cog does not change log levels. A record is exported only if the model's logging configuration lets it through, so with Python's defaults only `WARNING` and above are exported. Call `logging.basicConfig(level=logging.INFO)` or set a logger's level to export more. Records from disabled loggers or removed by a logger's filters are not exported. + +Export does not add a logging handler. Output that reaches stderr today, including the prediction's `logs` field, is unchanged, and `logging.basicConfig()` behaves as before. Text written with `print()` or directly to stdout or stderr is not exported. + +Put values you want to filter or group on in `extra`. Attributes are queryable in every backend. A message that is a dict or list with no format arguments, such as `log.info({"cache": "miss"})`, is exported as a structured body, matching the OpenTelemetry SDK handler, but backends differ in whether they index body fields. + +`extra` fields and exception tracebacks are exported verbatim. Avoid logging inputs, credentials, or other sensitive data you do not want in your collector. + ## Streaming predictions Normal JSON and Server-Sent Events requests share the same prediction, worker, and model spans. The HTTP span may end after the SSE response starts, while `cog.prediction`, `cog.prediction.invoke`, and model spans continue until generation finishes or the request is canceled. @@ -2821,7 +2866,7 @@ Cog provides defaults for `service.version` and the process-local `service.insta - Unreachable collector: Cog's built-in exporters retry or drop without failing predictions. - Malformed parent context: ignore it and continue. - Worker shutdown: flush and shut down parent and worker providers with bounded best effort; call custom Python provider cleanup synchronously. -- Forced termination, crashes, and OOM: final spans may be lost. +- Forced termination, crashes, and OOM: final spans and log records may be lost. If metrics telemetry configuration fails before the worker sends `Ready`, and the runtime metric exporter is configured, Cog uses the default runtime metric selection only long enough to record the failed setup result. The failed configuration cannot supply its own selections. @@ -3358,6 +3403,10 @@ class Runner(BaseRunner): Cog installs the `MeterProvider` before importing the model. Do not call `set_meter_provider()` in model code. Use `observability.config` to provide a custom provider or to disable selected [runtime metrics](observability.md#runtime-metric-selection). +### OpenTelemetry logs + +When `observability.logs` is enabled, records from Python's `logging` module are also exported to the OpenTelemetry collector. Cog does not add a handler or change log levels, so stderr output and the prediction's `logs` field are unchanged. `extra` fields become log attributes. See [Logs](observability.md#logs) for what is exported. + ## OpenTelemetry spans When `observability.traces` is enabled, Cog installs an OpenTelemetry tracer provider before importing the model. Model spans use the standard API and automatically join the active prediction trace: @@ -4320,7 +4369,7 @@ concurrency: ## `observability` -OpenTelemetry tracing and metrics are disabled by default. Enable either signal with an object: +OpenTelemetry tracing, metrics, and logs are disabled by default. Enable each signal with an object: ```yaml observability: @@ -4329,13 +4378,15 @@ observability: sampler: parentbased_always_off metrics: enabled: true + logs: + enabled: true ``` -The boolean forms `traces: true` and `metrics: true` are accepted as shorthand. +The boolean forms `traces: true`, `metrics: true`, and `logs: true` are accepted as shorthand. -`config` is an optional project-relative Python file for customizing Python telemetry providers and selecting Cog runtime metrics. It requires at least one enabled signal. The file may define `create_tracer_provider(resource)`, `create_meter_provider(resource)`, `configure_runtime_metrics()`, and `configure_instrumentation()`. Cog installs selected providers before importing the model and flushes and shuts them down with the worker. Note that `configure_runtime_metrics()` is only read when `observability.metrics.enabled` is true; it is ignored silently for traces-only images. +`config` is an optional project-relative Python file for customizing Python telemetry providers and selecting Cog runtime metrics. It requires at least one enabled signal. The file may define `create_tracer_provider(resource)`, `create_meter_provider(resource)`, `create_logger_provider(resource)`, `configure_runtime_metrics()`, and `configure_instrumentation()`. Cog installs selected providers before importing the model and flushes and shuts them down with the worker. Note that `configure_runtime_metrics()` is only read when `observability.metrics.enabled` is true; it is ignored silently for traces-only images. -This hook affects model-authored Python spans and metrics only. Cog's Rust parent continues to own fixed runtime metrics. See [Observability](observability.md#custom-python-telemetry) for examples and lifecycle details. +This hook affects model-authored Python spans, metrics, and logs only. Cog's Rust parent continues to own fixed runtime metrics. See [Observability](observability.md#custom-python-telemetry) for examples and lifecycle details. The default sampler continues sampled caller traces but does not start new traces. Supported sampler names are `always_on`, `always_off`, `traceidratio`, `parentbased_always_on`, `parentbased_always_off`, and `parentbased_traceidratio`. Ratio samplers require `sampler_arg` as a string between `"0"` and `"1"`. diff --git a/docs/observability.md b/docs/observability.md index 4aa26c42ef..c5de4f5fcb 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -1,14 +1,12 @@ # Observability -Cog can join incoming distributed traces, export fixed runtime metrics, and make standard OpenTelemetry APIs available to model code. Signals are opt-in and use OTLP, so they work with collectors and backends that support OpenTelemetry. +Cog can join incoming distributed traces, export fixed runtime metrics, export logs, and make standard OpenTelemetry APIs available to model code. Signals are opt-in and use OTLP, so they work with collectors and backends that support OpenTelemetry. -Cog has two telemetry ownership domains. The Rust parent owns fixed runtime metrics. The Python worker owns model-authored spans and metrics. Both can export to the same collector, but a Python provider never replaces the parent runtime provider. - -OpenTelemetry log export is not supported. +Cog has two telemetry ownership domains. The Rust parent owns fixed runtime metrics and Cog's own log events. The Python worker owns model-authored spans, metrics, and logs. Both can export to the same collector, but a Python provider never replaces a parent provider. ## Enable telemetry -Enable either signal with an object: +Enable each signal with an object: ```yaml observability: @@ -17,11 +15,13 @@ observability: sampler: parentbased_always_off metrics: enabled: true + logs: + enabled: true ``` -The boolean forms `traces: true` and `metrics: true` are accepted as shorthand. +The boolean forms `traces: true`, `metrics: true`, and `logs: true` are accepted as shorthand. -An image can enable either signal independently. Runtime configuration can disable an enabled signal, but cannot enable a signal omitted from the image. +An image can enable each signal independently. Runtime configuration can disable an enabled signal, but cannot enable a signal omitted from the image. Configure the collector when running the image: @@ -33,7 +33,7 @@ OTEL_SERVICE_NAME=cog Cog supports OTLP gRPC and HTTP/protobuf. Collector endpoints, authentication headers, certificates, and other `OTEL_*` values are runtime configuration and cannot be set through the general `cog.yaml` `environment` list. -Framework telemetry starts only when the image opts in and a collector endpoint is present. `COG_TRACE_ENABLED=false` and `COG_METRICS_ENABLED=false` disable the entire matching signal, including custom Python providers. `OTEL_SDK_DISABLED=true` disables all signals. `OTEL_TRACES_EXPORTER=none` and `OTEL_METRICS_EXPORTER=none` disable only the matching built-in provider. An explicitly configured Python factory can still use a different exporter or no exporter. +Framework telemetry starts only when the image opts in and a collector endpoint is present. `COG_TRACE_ENABLED=false`, `COG_METRICS_ENABLED=false`, and `COG_LOGS_ENABLED=false` disable the entire matching signal, including custom Python providers. `OTEL_SDK_DISABLED=true` disables all signals. `OTEL_TRACES_EXPORTER=none`, `OTEL_METRICS_EXPORTER=none`, and `OTEL_LOGS_EXPORTER=none` disable only the matching built-in provider. An explicitly configured Python factory can still use a different exporter or no exporter. ## Getting started with tracing @@ -125,16 +125,21 @@ observability: enabled: true metrics: enabled: true + logs: + enabled: true ``` Cog validates the file during configuration, copies it to a fixed path in the image, and loads it before importing the model. Provider factories are optional. A missing factory uses Cog's default provider for that signal. Factories receive Cog's base `Resource` and may merge or replace its attributes. Existing zero-argument `create_tracer_provider()` factories remain supported. ```python from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.trace import SpanLimits, TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace.sampling import ParentBased, TraceIdRatioBased @@ -159,6 +164,12 @@ def create_meter_provider(resource: Resource) -> MeterProvider: ) +def create_logger_provider(resource: Resource) -> LoggerProvider: + provider = LoggerProvider(resource=resource, shutdown_on_exit=False) + provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + return provider + + def configure_instrumentation() -> None: from opentelemetry.instrumentation.requests import RequestsInstrumentor @@ -226,6 +237,36 @@ def configure_runtime_metrics() -> RuntimeMetricsConfig: Set `enabled=False` to disable all current and future built-in Cog metrics. This does not disable the Python `MeterProvider`, so model metrics can still export. +## Logs + +`logs.enabled: true` exports two kinds of log records: + +- Model records from Python's `logging` module, exported by the worker. Each record keeps its severity, logger name (as the instrumentation scope), message, source location, exception details, and `extra` fields as attributes. When tracing is enabled, records written during a prediction carry its trace and span IDs. +- Cog's own runtime events, such as prediction start and completion, setup progress, and errors, exported by the parent. Worker runtime events are relayed through the parent and keep only their message. + +Model code needs no OpenTelemetry imports: + +```python +import logging + +log = logging.getLogger(__name__) + + +class Runner(BaseRunner): + def run(self, prompt: str) -> str: + result = self.model(prompt) + log.info("generated", extra={"prompt_tokens": result.prompt_tokens}) + return result.text +``` + +Cog does not change log levels. A record is exported only if the model's logging configuration lets it through, so with Python's defaults only `WARNING` and above are exported. Call `logging.basicConfig(level=logging.INFO)` or set a logger's level to export more. Records from disabled loggers or removed by a logger's filters are not exported. + +Export does not add a logging handler. Output that reaches stderr today, including the prediction's `logs` field, is unchanged, and `logging.basicConfig()` behaves as before. Text written with `print()` or directly to stdout or stderr is not exported. + +Put values you want to filter or group on in `extra`. Attributes are queryable in every backend. A message that is a dict or list with no format arguments, such as `log.info({"cache": "miss"})`, is exported as a structured body, matching the OpenTelemetry SDK handler, but backends differ in whether they index body fields. + +`extra` fields and exception tracebacks are exported verbatim. Avoid logging inputs, credentials, or other sensitive data you do not want in your collector. + ## Streaming predictions Normal JSON and Server-Sent Events requests share the same prediction, worker, and model spans. The HTTP span may end after the SSE response starts, while `cog.prediction`, `cog.prediction.invoke`, and model spans continue until generation finishes or the request is canceled. @@ -342,7 +383,7 @@ Cog provides defaults for `service.version` and the process-local `service.insta - Unreachable collector: Cog's built-in exporters retry or drop without failing predictions. - Malformed parent context: ignore it and continue. - Worker shutdown: flush and shut down parent and worker providers with bounded best effort; call custom Python provider cleanup synchronously. -- Forced termination, crashes, and OOM: final spans may be lost. +- Forced termination, crashes, and OOM: final spans and log records may be lost. If metrics telemetry configuration fails before the worker sends `Ready`, and the runtime metric exporter is configured, Cog uses the default runtime metric selection only long enough to record the failed setup result. The failed configuration cannot supply its own selections. diff --git a/docs/python.md b/docs/python.md index f3a9243923..5565c3f284 100644 --- a/docs/python.md +++ b/docs/python.md @@ -480,6 +480,10 @@ class Runner(BaseRunner): Cog installs the `MeterProvider` before importing the model. Do not call `set_meter_provider()` in model code. Use `observability.config` to provide a custom provider or to disable selected [runtime metrics](observability.md#runtime-metric-selection). +### OpenTelemetry logs + +When `observability.logs` is enabled, records from Python's `logging` module are also exported to the OpenTelemetry collector. Cog does not add a handler or change log levels, so stderr output and the prediction's `logs` field are unchanged. `extra` fields become log attributes. See [Logs](observability.md#logs) for what is exported. + ## OpenTelemetry spans When `observability.traces` is enabled, Cog installs an OpenTelemetry tracer provider before importing the model. Model spans use the standard API and automatically join the active prediction trace: diff --git a/docs/yaml.md b/docs/yaml.md index 4b04328417..7481e7687d 100644 --- a/docs/yaml.md +++ b/docs/yaml.md @@ -222,7 +222,7 @@ concurrency: ## `observability` -OpenTelemetry tracing and metrics are disabled by default. Enable either signal with an object: +OpenTelemetry tracing, metrics, and logs are disabled by default. Enable each signal with an object: ```yaml observability: @@ -231,13 +231,15 @@ observability: sampler: parentbased_always_off metrics: enabled: true + logs: + enabled: true ``` -The boolean forms `traces: true` and `metrics: true` are accepted as shorthand. +The boolean forms `traces: true`, `metrics: true`, and `logs: true` are accepted as shorthand. -`config` is an optional project-relative Python file for customizing Python telemetry providers and selecting Cog runtime metrics. It requires at least one enabled signal. The file may define `create_tracer_provider(resource)`, `create_meter_provider(resource)`, `configure_runtime_metrics()`, and `configure_instrumentation()`. Cog installs selected providers before importing the model and flushes and shuts them down with the worker. Note that `configure_runtime_metrics()` is only read when `observability.metrics.enabled` is true; it is ignored silently for traces-only images. +`config` is an optional project-relative Python file for customizing Python telemetry providers and selecting Cog runtime metrics. It requires at least one enabled signal. The file may define `create_tracer_provider(resource)`, `create_meter_provider(resource)`, `create_logger_provider(resource)`, `configure_runtime_metrics()`, and `configure_instrumentation()`. Cog installs selected providers before importing the model and flushes and shuts them down with the worker. Note that `configure_runtime_metrics()` is only read when `observability.metrics.enabled` is true; it is ignored silently for traces-only images. -This hook affects model-authored Python spans and metrics only. Cog's Rust parent continues to own fixed runtime metrics. See [Observability](observability.md#custom-python-telemetry) for examples and lifecycle details. +This hook affects model-authored Python spans, metrics, and logs only. Cog's Rust parent continues to own fixed runtime metrics. See [Observability](observability.md#custom-python-telemetry) for examples and lifecycle details. The default sampler continues sampled caller traces but does not start new traces. Supported sampler names are `always_on`, `always_off`, `traceidratio`, `parentbased_always_on`, `parentbased_always_off`, and `parentbased_traceidratio`. Ratio samplers require `sampler_arg` as a string between `"0"` and `"1"`. diff --git a/examples/hello-concurrency/README.md b/examples/hello-concurrency/README.md index 47806c6693..3674cb2f33 100644 --- a/examples/hello-concurrency/README.md +++ b/examples/hello-concurrency/README.md @@ -17,9 +17,9 @@ This combined with the async setup and run methods in `run.py` allows Cog to run 4 concurrent predictions. If Cog reaches the max concurrency threshold it will reject subsequent predictions with a `409 Conflict` response. -### Tracing and metrics with Honeycomb +### Tracing, metrics, and logs with Honeycomb -Cog loads `telemetry.py` before importing the model. Its provider factories configure model spans and metrics, while `configure_runtime_metrics()` selects fixed Cog runtime instruments. The model uses the standard `opentelemetry.trace` and `opentelemetry.metrics` APIs. +Cog loads `telemetry.py` before importing the model. Its provider factories configure model spans, metrics, and logs, while `configure_runtime_metrics()` selects fixed Cog runtime instruments. The model uses the standard `opentelemetry.trace` and `opentelemetry.metrics` APIs and Python's `logging` module. Set a Honeycomb API key in your shell, then pass its OTLP configuration at runtime: @@ -37,10 +37,12 @@ cog run \ The `parentbased_always_on` sampler preserves an upstream trace's sampling decision and samples predictions that start a new trace locally. `model.output_tokens` is a model-owned OpenTelemetry counter; `current_scope().record_metric()` continues to populate the prediction response separately. -To print Python spans locally without an OTLP endpoint, run: +The model's `logging` calls are exported as OpenTelemetry logs with the active trace and span IDs, and `extra` fields become log attributes. They still print to stderr and appear in the prediction's `logs` as before. + +To print Python spans or logs locally without an OTLP endpoint, run: ```shell -cog run -e OTEL_DEBUG_TRACES=true -i total=5 -i interval=1 +cog run -e OTEL_DEBUG_TRACES=true -e OTEL_DEBUG_LOGS=true -i total=5 -i interval=1 ``` See [Honeycomb's OpenTelemetry endpoint documentation](https://docs.honeycomb.io/send-data/opentelemetry/#using-the-honeycomb-opentelemetry-endpoint) for regional endpoints and Honeycomb Classic dataset headers. diff --git a/examples/hello-concurrency/cog.yaml b/examples/hello-concurrency/cog.yaml index 516ad9d815..ea4ab68a33 100644 --- a/examples/hello-concurrency/cog.yaml +++ b/examples/hello-concurrency/cog.yaml @@ -11,3 +11,5 @@ observability: sampler: parentbased_always_on metrics: enabled: true + logs: + enabled: true diff --git a/examples/hello-concurrency/run.py b/examples/hello-concurrency/run.py index b8719eb1ad..932540f40f 100644 --- a/examples/hello-concurrency/run.py +++ b/examples/hello-concurrency/run.py @@ -82,5 +82,8 @@ async def run( # pyright: ignore span.set_attribute("metrics.output_tokens", total) duration = time.time() - start_time - logging.info(f"completed prediction in {duration} seconds") + logging.info( + f"completed prediction in {duration} seconds", + extra={"model.output_tokens": total}, + ) span.set_attribute("model.predict_time_seconds", duration) diff --git a/examples/hello-concurrency/telemetry.py b/examples/hello-concurrency/telemetry.py index ac31b6fdca..f560db2400 100644 --- a/examples/hello-concurrency/telemetry.py +++ b/examples/hello-concurrency/telemetry.py @@ -1,8 +1,15 @@ import os from opentelemetry.context import Context +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + BatchLogRecordProcessor, + ConsoleLogRecordExporter, + SimpleLogRecordProcessor, +) from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import Resource @@ -76,5 +83,19 @@ def create_meter_provider(resource: Resource) -> MeterProvider: ) +def create_logger_provider(resource: Resource) -> LoggerProvider: + provider = LoggerProvider( + resource=resource.merge(Resource({"model.name": "hello-concurrency"})), + shutdown_on_exit=False, + ) + if _has_export_endpoint("logs"): + provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + if os.getenv("OTEL_DEBUG_LOGS", "false").lower() == "true": + provider.add_log_record_processor( + SimpleLogRecordProcessor(ConsoleLogRecordExporter()) + ) + return provider + + def configure_runtime_metrics() -> RuntimeMetricsConfig: return RuntimeMetricsConfig(disabled={RuntimeMetric.SETUP_DURATION}) diff --git a/integration-tests/tests/observability_logs.txtar b/integration-tests/tests/observability_logs.txtar new file mode 100644 index 0000000000..aa34574861 --- /dev/null +++ b/integration-tests/tests/observability_logs.txtar @@ -0,0 +1,59 @@ +# A logs-only image loads a custom LoggerProvider before model import. Model +# logging is exported once and still appears in the prediction's logs. +cog build -t $TEST_IMAGE +cog serve + +curl POST /predictions '{"input":{"value":"hello"}}' +stdout '"output":"hello from CustomLoggerProvider, exported 1"' +stdout 'model warning' + +-- cog.yaml -- +build: + python_version: "3.12" +run: "predict.py:Runner" +observability: + config: telemetry.py + logs: + enabled: true + +-- telemetry.py -- +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import ( + InMemoryLogRecordExporter, + SimpleLogRecordProcessor, +) + +exporter = InMemoryLogRecordExporter() + + +class CustomLoggerProvider(LoggerProvider): + pass + + +def create_logger_provider(resource) -> LoggerProvider: + provider = CustomLoggerProvider(resource=resource, shutdown_on_exit=False) + provider.add_log_record_processor(SimpleLogRecordProcessor(exporter)) + return provider + + +-- predict.py -- +import logging +import sys + +from opentelemetry import _logs + +from cog import BaseRunner + +assert type(_logs.get_logger_provider()).__name__ == "CustomLoggerProvider" + +class Runner(BaseRunner): + def run(self, value: str) -> str: + logging.getLogger("model").warning("model warning") + exporter = sys.modules["_cog_telemetry"].exporter + exported = [ + log + for log in exporter.get_finished_logs() + if log.log_record.body == "model warning" + ] + provider_name = type(_logs.get_logger_provider()).__name__ + return f"{value} from {provider_name}, exported {len(exported)}" diff --git a/python/cog/_telemetry.py b/python/cog/_telemetry.py index 48dd7cc9fb..ab69e9541f 100644 --- a/python/cog/_telemetry.py +++ b/python/cog/_telemetry.py @@ -1,17 +1,22 @@ +import contextlib import importlib.util import inspect import logging import os import sys +import threading +import time import uuid from collections.abc import Callable from contextvars import Token from types import ModuleType from typing import Mapping -from opentelemetry import metrics, trace +from opentelemetry import _logs, metrics, trace +from opentelemetry.attributes import _VALID_ANY_VALUE_TYPES from opentelemetry.context import ( Context, + get_current, ) from opentelemetry.context import ( attach as attach_context, @@ -19,6 +24,9 @@ from opentelemetry.context import ( detach as detach_context, ) +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs._internal import std_to_otel +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader from opentelemetry.sdk.resources import OTELResourceDetector, Resource @@ -35,6 +43,7 @@ ) from opentelemetry.trace import ProxyTracerProvider from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.util.types import AnyValue from ._version import __version__ from .telemetry import RuntimeMetricsConfig @@ -43,24 +52,35 @@ _logger = logging.getLogger(__name__) _tracer_provider: TracerProvider | None = None _meter_provider: MeterProvider | None = None +_logger_provider: LoggerProvider | None = None _runtime_metrics_config = RuntimeMetricsConfig() +_original_call_handlers = logging.Logger.callHandlers +_exporting_log = threading.local() +# Python level names that differ from OTel severity text. +_SEVERITY_TEXT = {"WARNING": "WARN", "CRITICAL": "FATAL"} def install_providers() -> RuntimeMetricsConfig: - global _meter_provider, _runtime_metrics_config, _tracer_provider + global _logger_provider, _meter_provider, _runtime_metrics_config, _tracer_provider - if _tracer_provider is not None or _meter_provider is not None: + if ( + _tracer_provider is not None + or _meter_provider is not None + or _logger_provider is not None + ): return _runtime_metrics_config traces_enabled = _traces_enabled() metrics_enabled = _metrics_enabled() - if not traces_enabled and not metrics_enabled: + logs_enabled = _logs_enabled() + if not traces_enabled and not metrics_enabled and not logs_enabled: return _runtime_metrics_config module: ModuleType | None = None _runtime_metrics_config = RuntimeMetricsConfig() tracer_provider: TracerProvider | None = None meter_provider: MeterProvider | None = None + logger_provider: LoggerProvider | None = None try: module = _load_config_from_env() if _has_custom_config() else None if module is not None and metrics_enabled: @@ -72,6 +92,9 @@ def install_providers() -> RuntimeMetricsConfig: meter_provider = ( _build_meter_provider(module, resource) if metrics_enabled else None ) + logger_provider = ( + _build_logger_provider(module, resource) if logs_enabled else None + ) _validate_tracer_provider_collision(tracer_provider) if tracer_provider is not None: @@ -90,12 +113,23 @@ def install_providers() -> RuntimeMetricsConfig: ) _meter_provider = meter_provider + if logger_provider is not None: + _logs.set_logger_provider(logger_provider) + if _logs.get_logger_provider() is not logger_provider: + raise RuntimeError( + "Failed to install Cog's OpenTelemetry LoggerProvider" + ) + _logger_provider = logger_provider + logging.Logger.callHandlers = _call_handlers_and_export # type: ignore[method-assign] + _configure_instrumentation(module) except Exception: if tracer_provider is not _tracer_provider: _shutdown_provider(tracer_provider, "tracing") if meter_provider is not _meter_provider: _shutdown_provider(meter_provider, "metrics") + if logger_provider is not _logger_provider: + _shutdown_provider(logger_provider, "logs") shutdown() raise @@ -119,19 +153,76 @@ def detach(token: Token[Context] | None) -> None: def shutdown() -> None: - global _meter_provider, _tracer_provider + global _logger_provider, _meter_provider, _tracer_provider tracer_provider = _tracer_provider meter_provider = _meter_provider + logger_provider = _logger_provider _tracer_provider = None _meter_provider = None + _logger_provider = None + if logging.Logger.callHandlers is _call_handlers_and_export: + logging.Logger.callHandlers = _original_call_handlers # type: ignore[method-assign] _shutdown_provider(tracer_provider, "tracing") _shutdown_provider(meter_provider, "metrics") + _shutdown_provider(logger_provider, "logs") + + +def _call_handlers_and_export(self: logging.Logger, record: logging.LogRecord) -> None: + # Runs after the logger's disabled and filter checks and before handlers, so + # export follows the model's logging config without adding a handler. A + # root handler would stop logging.lastResort and make basicConfig() a no-op. + _export_log_record(record) + _original_call_handlers(self, record) + + +def _export_log_record(record: logging.LogRecord) -> None: + provider = _logger_provider + if ( + provider is None + or record.name.startswith("opentelemetry") + or getattr(_exporting_log, "active", False) + # OTel SDK export threads: an exporter's HTTP or gRPC client logging a + # failed export would otherwise queue another record for export. + or threading.current_thread().name.startswith("Otel") + ): + return + _exporting_log.active = True + try: + # Logging an export failure here would re-enter this hook. + with contextlib.suppress(Exception): + provider.get_logger(record.name).emit( + _logs.LogRecord( + timestamp=int(record.created * 1e9), + observed_timestamp=time.time_ns(), + context=get_current(), + severity_text=_SEVERITY_TEXT.get( + record.levelname, record.levelname + ), + severity_number=std_to_otel(record.levelno), + body=_log_body(record), + # A staticmethod, so this reuses the SDK's attribute mapping + # without constructing the deprecated handler. + attributes=LoggingHandler._get_attributes(record), + ) + ) + finally: + _exporting_log.active = False + + +def _log_body(record: logging.LogRecord) -> AnyValue: + # Matches the SDK handler: a non-string msg without format args, such as a + # dict, stays structured instead of being converted to a string. + if record.args or isinstance(record.msg, str): + return record.getMessage() + if isinstance(record.msg, _VALID_ANY_VALUE_TYPES): + return record.msg + return str(record.msg) def _shutdown_provider( - provider: TracerProvider | MeterProvider | None, signal: str + provider: TracerProvider | MeterProvider | LoggerProvider | None, signal: str ) -> None: if provider is None: return @@ -153,6 +244,10 @@ def _metrics_enabled() -> bool: return _enabled("COG_METRICS_CONFIGURED", "COG_METRICS_ENABLED") +def _logs_enabled() -> bool: + return _enabled("COG_LOGS_CONFIGURED", "COG_LOGS_ENABLED") + + def _enabled(configured_name: str, enabled_name: str) -> bool: try: return ( @@ -247,6 +342,26 @@ def _build_meter_provider( return provider +def _build_logger_provider( + module: ModuleType | None, resource: Resource +) -> LoggerProvider | None: + factory = getattr(module, "create_logger_provider", None) if module else None + if factory is None: + try: + return _create_default_logger_provider(resource) + except Exception: + _logger.exception("Invalid OpenTelemetry logs configuration; logs disabled") + return None + if not callable(factory): + raise RuntimeError("telemetry.py create_logger_provider must be callable") + provider = _call_provider_factory(factory, resource) + if not isinstance(provider, LoggerProvider): + raise RuntimeError( + "telemetry.py create_logger_provider must return LoggerProvider" + ) + return provider + + def _call_provider_factory( factory: Callable[..., object], resource: Resource ) -> object: @@ -330,6 +445,35 @@ def _create_default_meter_provider(resource: Resource) -> MeterProvider | None: raise +def _create_default_logger_provider(resource: Resource) -> LoggerProvider | None: + if os.environ.get("OTEL_LOGS_EXPORTER", "otlp") == "none": + return None + endpoint, append_path = _endpoint("logs") + if endpoint is None: + return None + protocol = _protocol("logs") + if protocol in {"http", "http/protobuf"}: + from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter as HttpOTLPLogExporter, + ) + + exporter = HttpOTLPLogExporter( + endpoint=_http_endpoint(endpoint, append_path, "logs") + ) + elif protocol == "grpc": + from opentelemetry.exporter.otlp.proto.grpc._log_exporter import ( + OTLPLogExporter as GrpcOTLPLogExporter, + ) + + exporter = GrpcOTLPLogExporter(endpoint=endpoint) + else: + raise RuntimeError(f"Unsupported OTLP protocol: {protocol}") + + provider = LoggerProvider(resource=resource, shutdown_on_exit=False) + provider.add_log_record_processor(BatchLogRecordProcessor(exporter)) + return provider + + def _endpoint(signal: str) -> tuple[str | None, bool]: signal_endpoint = os.environ.get(f"OTEL_EXPORTER_OTLP_{signal.upper()}_ENDPOINT") if signal_endpoint is not None: diff --git a/python/tests/test_telemetry.py b/python/tests/test_telemetry.py index ed9a9b3435..bc10e3acb5 100644 --- a/python/tests/test_telemetry.py +++ b/python/tests/test_telemetry.py @@ -10,7 +10,7 @@ def _run_script(script: str) -> subprocess.CompletedProcess[str]: env = os.environ.copy() for name in list(env): if name.startswith( - ("COG_OBSERVABILITY_", "COG_TRACE_", "COG_METRICS_", "OTEL_") + ("COG_OBSERVABILITY_", "COG_TRACE_", "COG_METRICS_", "COG_LOGS_", "OTEL_") ): del env[name] return subprocess.run( @@ -334,7 +334,7 @@ def configure_instrumentation(): assert result.returncode == 0, result.stderr -@pytest.mark.parametrize("signal", ["TRACE", "METRICS"]) +@pytest.mark.parametrize("signal", ["TRACE", "METRICS", "LOGS"]) def test_signal_boolean_environment_contract(signal: str) -> None: result = _run_script( f""" @@ -342,7 +342,7 @@ def test_signal_boolean_environment_contract(signal: str) -> None: from cog import _telemetry configured = "COG_{signal}_CONFIGURED" enabled = "COG_{signal}_ENABLED" -check = _telemetry._{"traces" if signal == "TRACE" else "metrics"}_enabled +check = _telemetry._{ {"TRACE": "traces", "METRICS": "metrics", "LOGS": "logs"}[signal] }_enabled assert not check() os.environ[configured] = "true" assert check() @@ -367,15 +367,20 @@ def test_signal_boolean_environment_contract(signal: str) -> None: assert result.returncode == 0, result.stderr -@pytest.mark.parametrize("signal", ["TRACE", "METRICS"]) +@pytest.mark.parametrize("signal", ["TRACE", "METRICS", "LOGS"]) def test_only_enabled_signal_creates_provider(tmp_path: Path, signal: str) -> None: config = tmp_path / "telemetry.py" config.write_text( f""" from cog.telemetry import RuntimeMetricsConfig +from opentelemetry.sdk._logs import LoggerProvider from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.trace import TracerProvider +def create_logger_provider(resource): + assert {signal!r} == "LOGS" + return LoggerProvider(shutdown_on_exit=False) + def create_tracer_provider(resource): assert {signal!r} == "TRACE" return TracerProvider(shutdown_on_exit=False) @@ -395,8 +400,10 @@ def configure_runtime_metrics(): os.environ.update({{ "COG_TRACE_CONFIGURED": "true", "COG_METRICS_CONFIGURED": "true", + "COG_LOGS_CONFIGURED": "true", "COG_TRACE_ENABLED": {str(signal == "TRACE").lower()!r}, "COG_METRICS_ENABLED": {str(signal == "METRICS").lower()!r}, + "COG_LOGS_ENABLED": {str(signal == "LOGS").lower()!r}, "COG_OBSERVABILITY_CONFIG": {str(config)!r}, }}) from cog import _telemetry @@ -404,9 +411,138 @@ def configure_runtime_metrics(): config = _telemetry.install_providers() assert (_telemetry._tracer_provider is not None) is {signal == "TRACE"!r} assert (_telemetry._meter_provider is not None) is {signal == "METRICS"!r} +assert (_telemetry._logger_provider is not None) is {signal == "LOGS"!r} if {signal!r} == "METRICS": assert not config.enabled _telemetry.shutdown() """ ) assert result.returncode == 0, result.stderr + + +_LOGS_CONFIG = """ +from opentelemetry.sdk._logs import LoggerProvider +from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter, SimpleLogRecordProcessor + +exporter = InMemoryLogRecordExporter() + +def create_logger_provider(resource): + provider = LoggerProvider(resource=resource, shutdown_on_exit=False) + provider.add_log_record_processor(SimpleLogRecordProcessor(exporter)) + return provider +""" + + +def _logs_script(config: Path, body: str, traces: bool = False) -> str: + return f""" +import logging +import os +import sys +os.environ.update({{ + "COG_LOGS_CONFIGURED": "true", + "COG_TRACE_CONFIGURED": {str(traces).lower()!r}, + "OTEL_TRACES_EXPORTER": "none", + "COG_OBSERVABILITY_CONFIG": {str(config)!r}, +}}) +from cog import _telemetry +_telemetry._CUSTOM_CONFIG_PATH = {str(config)!r} +_telemetry.install_providers() +exporter = sys.modules["_cog_telemetry"].exporter +def exported(): + return [(log.log_record.severity_text, log.log_record.body, dict(log.log_record.attributes or {{}})) for log in exporter.get_finished_logs()] +{body} +""" + + +def test_logs_export_records_without_changing_stderr(tmp_path: Path) -> None: + config = tmp_path / "telemetry.py" + config.write_text(_LOGS_CONFIG) + result = _run_script( + _logs_script( + config, + """ +log = logging.getLogger("model") +log.info("below default level") +try: + raise ValueError("cache miss") +except ValueError: + log.warning("falling back", extra={"prompt_tokens": 42}, exc_info=True) +logging.getLogger("opentelemetry.sdk").warning("sdk internal") +log.warning({"cache": "miss", "attempt": 2}) +log.warning("%s attempts", 3) +import threading +exporter_thread = threading.Thread( + target=lambda: logging.getLogger("urllib3").warning("export failed"), + name="OtelBatchLogRecordProcessor", +) +exporter_thread.start() +exporter_thread.join() +muted = logging.getLogger("muted") +muted.disabled = True +muted.warning("disabled logger") +logging.basicConfig(level=logging.INFO, format="BC %(message)s") +log.info("after basicConfig") + +logs = exported() +assert [(severity, body) for severity, body, _ in logs] == [ + ("WARN", "falling back"), + ("WARN", {"cache": "miss", "attempt": 2}), + ("WARN", "3 attempts"), + ("INFO", "after basicConfig"), +], logs +attributes = logs[0][2] +assert attributes["prompt_tokens"] == 42, attributes +assert attributes["exception.type"] == "ValueError", attributes +assert "code.file.path" in attributes, attributes +assert len(logging.getLogger().handlers) == 1 + +_telemetry.shutdown() +assert logging.Logger.callHandlers is _telemetry._original_call_handlers +""", + ) + ) + assert result.returncode == 0, result.stderr + # Unhandled WARNING still reaches stderr through logging.lastResort, and + # basicConfig() still installs its handler. + assert "falling back" in result.stderr + assert "ValueError: cache miss" in result.stderr + assert "BC after basicConfig" in result.stderr + assert "below default level" not in result.stderr + + +def test_logs_carry_the_active_trace_context(tmp_path: Path) -> None: + config = tmp_path / "telemetry.py" + config.write_text(_LOGS_CONFIG) + result = _run_script( + _logs_script( + config, + """ +from opentelemetry import trace +with trace.get_tracer("test").start_as_current_span("predict") as span: + logging.getLogger("model").warning("inside span") +record = exporter.get_finished_logs()[0].log_record +assert record.trace_id == span.get_span_context().trace_id +assert record.span_id == span.get_span_context().span_id +""", + traces=True, + ) + ) + assert result.returncode == 0, result.stderr + + +def test_default_logger_provider_installs() -> None: + result = _run_script( + """ +import os +os.environ.update({ + "COG_LOGS_CONFIGURED": "true", + "OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:4318", +}) +from cog import _telemetry +from opentelemetry import _logs +from opentelemetry.sdk._logs import LoggerProvider +_telemetry.install_providers() +assert isinstance(_logs.get_logger_provider(), LoggerProvider) +""" + ) + assert result.returncode == 0, result.stderr