From b7e115073b665d8fbfe52a3b6df19510a65114de Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:07:56 +0800 Subject: [PATCH 01/26] fix(sdk): flush concurrent processors inline when executor shut down at exit --- .changelog/5568.fixed | 1 + .../sdk/_logs/_internal/__init__.py | 2197 ++++++------ .../src/opentelemetry/sdk/trace/__init__.py | 2969 +++++++++-------- .../tests/logs/test_multi_log_processor.py | 532 +-- .../tests/trace/test_span_processor.py | 1553 ++++----- 5 files changed, 3672 insertions(+), 3580 deletions(-) create mode 100644 .changelog/5568.fixed diff --git a/.changelog/5568.fixed b/.changelog/5568.fixed new file mode 100644 index 00000000000..e91f3dae308 --- /dev/null +++ b/.changelog/5568.fixed @@ -0,0 +1 @@ +opentelemetry-sdk: run concurrent span/log record processor callbacks inline when the executor is already shut down at interpreter exit, so buffered telemetry is flushed instead of being dropped \ No newline at end of file diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 5b543d01e44..42e329e7c32 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -1,1092 +1,1105 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -import abc -import atexit -import base64 -import concurrent.futures -import json -import logging -import os -import threading -import traceback -import warnings -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from os import environ -from threading import Lock -from time import time_ns -from types import NoneType -from typing import ( # noqa - Any, - Tuple, - Union, - cast, - overload, -) -from weakref import WeakMethod, WeakSet - -from typing_extensions import deprecated - -from opentelemetry._logs import Logger as APILogger -from opentelemetry._logs import LoggerProvider as APILoggerProvider -from opentelemetry._logs import ( - LogRecord, - NoOpLogger, - SeverityNumber, - get_logger, - get_logger_provider, -) -from opentelemetry.attributes import BoundedAttributes -from opentelemetry.context import get_current -from opentelemetry.context.context import Context -from opentelemetry.metrics import MeterProvider, get_meter_provider -from opentelemetry.sdk._logs._internal._exceptions import ( - _copy_log_record_with_exception, - _create_log_record_with_exception, - _set_log_record_exception_attributes, -) -from opentelemetry.sdk._logs._internal._logger_metrics import ( - LoggerMetricsT, - create_logger_metrics, -) -from opentelemetry.sdk.environment_variables import ( - OTEL_ATTRIBUTE_COUNT_LIMIT, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, - OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, - OTEL_SDK_DISABLED, -) -from opentelemetry.sdk.environment_variables._internal import ( - parse_boolean_environment_variable, -) -from opentelemetry.sdk.resources import ( - Resource, - _get_process_dependent_resource, -) -from opentelemetry.sdk.util import ns_to_iso_str -from opentelemetry.sdk.util._configurator import RuleBasedConfigurator -from opentelemetry.sdk.util.instrumentation import ( - InstrumentationScope, -) -from opentelemetry.semconv.attributes import ( - code_attributes, - exception_attributes, -) -from opentelemetry.trace import ( - format_span_id, - format_trace_id, -) -from opentelemetry.util.types import AnyValue, Attributes - -# pylint: disable=too-many-lines - -_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 -_ENV_VALUE_UNSET = "" - -_logger = logging.getLogger(__name__) - - -class BytesEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, bytes): - return base64.b64encode(o).decode() - return super().default(o) - - -class LogRecordDroppedAttributesWarning(UserWarning): - """Custom warning to indicate dropped log attributes due to limits. - - This class is used to filter and handle these specific warnings separately - from other warnings, ensuring that they are only shown once without - interfering with default user warnings. - """ - - -warnings.simplefilter("once", LogRecordDroppedAttributesWarning) - - -@deprecated( - "Use LogRecordDroppedAttributesWarning. Since logs are not stable yet this WILL be removed in future releases." -) -class LogDroppedAttributesWarning(LogRecordDroppedAttributesWarning): - pass - - -class LogRecordLimits: - """This class is based on a SpanLimits class in the Tracing module. - - This class represents the limits that should be enforced on recorded data such as events, links, attributes etc. - - This class does not enforce any limits itself. It only provides a way to read limits from env, - default values and from user provided arguments. - - All limit arguments must be either a non-negative integer, ``None`` or ``LogRecordLimits.UNSET``. - - - All limit arguments are optional. - - If a limit argument is not set, the class will try to read its value from the corresponding - environment variable. - - If the environment variable is not set, the default value, if any, will be used. - - Limit precedence: - - - If a model specific limit is set, it will be used. - - Else if the corresponding global limit is set, it will be used. - - Else if the model specific limit has a default value, the default value will be used. - - Else if the global limit has a default value, the default value will be used. - - Args: - max_attributes: Maximum number of attributes that can be added to a log record (global fallback). - Environment variable: ``OTEL_ATTRIBUTE_COUNT_LIMIT`` - Default: {_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT} - max_attribute_length: Maximum length an attribute value can have (global fallback). Values - longer than the specified length will be truncated. - max_log_record_attributes: Maximum number of attributes that can be added to a log record. - Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT`` - Falls back to ``max_attributes`` when unset. - max_log_record_attribute_length: Maximum length a log record attribute value can have. - Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT`` - Falls back to ``max_attribute_length`` when unset. - """ - - UNSET = -1 - - def __init__( - self, - max_attributes: int | None = None, - max_attribute_length: int | None = None, - max_log_record_attributes: int | None = None, - max_log_record_attribute_length: int | None = None, - ): - # global attribute count - global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) - self.max_attributes = ( - global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT - ) - - # global attribute length - self.max_attribute_length = self._from_env_if_absent( - max_attribute_length, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - ) - - self.max_log_record_attributes = self._from_env_if_absent( - max_log_record_attributes, - OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, - default=self.max_attributes, - ) - - self.max_log_record_attribute_length = self._from_env_if_absent( - max_log_record_attribute_length, - OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, - default=self.max_attribute_length, - ) - - def __repr__(self): - return ( - f"{type(self).__name__}(" - f"max_attributes={self.max_attributes}, " - f"max_attribute_length={self.max_attribute_length}, " - f"max_log_record_attributes={self.max_log_record_attributes}, " - f"max_log_record_attribute_length={self.max_log_record_attribute_length})" - ) - - @classmethod - def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: - if value == cls.UNSET: - return None - - err_msg = "{} must be a non-negative integer but got {}" - - # if no value is provided for the limit, try to load it from env - if value is None: - # return default value if env var is not set - if env_var not in environ: - return default - - str_value = environ.get(env_var, "").strip().lower() - if str_value == _ENV_VALUE_UNSET: - return None - - try: - value = int(str_value) - except ValueError: - raise ValueError(err_msg.format(env_var, str_value)) - - if value < 0: - raise ValueError(err_msg.format(env_var, value)) - return value - - -@deprecated("Use LogRecordLimits. Since logs are not stable yet this WILL be removed in future releases.") -class LogLimits(LogRecordLimits): - pass - - -@dataclass(frozen=True) -class ReadableLogRecord: - """Readable LogRecord should be kept exactly in-sync with ReadWriteLogRecord, only difference is the frozen=True param.""" - - log_record: LogRecord - resource: Resource - instrumentation_scope: InstrumentationScope | None = None - limits: LogRecordLimits | None = None - - @property - def dropped_attributes(self) -> int: - if isinstance(self.log_record.attributes, BoundedAttributes): - return self.log_record.attributes.dropped - return 0 - - def to_json(self, indent: int | None = 4) -> str: - return json.dumps( - { - "body": self.log_record.body, - "severity_number": self.log_record.severity_number.value - if self.log_record.severity_number is not None - else None, - "severity_text": self.log_record.severity_text, - "attributes": (dict(self.log_record.attributes) if bool(self.log_record.attributes) else None), - "dropped_attributes": self.dropped_attributes, - "timestamp": ns_to_iso_str(self.log_record.timestamp) - if self.log_record.timestamp is not None - else None, - "observed_timestamp": ns_to_iso_str(self.log_record.observed_timestamp), - "trace_id": ( - f"0x{format_trace_id(self.log_record.trace_id)}" if self.log_record.trace_id is not None else "" - ), - "span_id": ( - f"0x{format_span_id(self.log_record.span_id)}" if self.log_record.span_id is not None else "" - ), - "trace_flags": self.log_record.trace_flags, - "resource": json.loads(self.resource.to_json()), - "event_name": self.log_record.event_name if self.log_record.event_name else "", - }, - indent=indent, - cls=BytesEncoder, - ) - - -@dataclass -class ReadWriteLogRecord: - """A ReadWriteLogRecord instance represents an event being logged. - ReadWriteLogRecord instances are created and emitted via `Logger` - every time something is logged. They contain all the information - pertinent to the event being logged. - """ - - log_record: LogRecord - resource: Resource | None = Resource.create({}) - instrumentation_scope: InstrumentationScope | None = None - limits: LogRecordLimits = field(default_factory=LogRecordLimits) - - def __post_init__(self): - self.log_record.attributes = BoundedAttributes( - maxlen=self.limits.max_log_record_attributes, - attributes=self.log_record.attributes if self.log_record.attributes else None, - immutable=False, - max_value_len=self.limits.max_log_record_attribute_length, - extended_attributes=True, - ) - if self.dropped_attributes > 0: - warnings.warn( - "Log record attributes were dropped due to limits", - LogRecordDroppedAttributesWarning, - stacklevel=2, - ) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, ReadWriteLogRecord): - return NotImplemented - return self.__dict__ == other.__dict__ - - @property - def dropped_attributes(self) -> int: - if isinstance(self.log_record.attributes, BoundedAttributes): - return self.log_record.attributes.dropped - return 0 - - @classmethod - def _from_api_log_record( - cls, - *, - record: LogRecord, - resource: Resource, - instrumentation_scope: InstrumentationScope | None = None, - limits: LogRecordLimits | None = None, - ) -> ReadWriteLogRecord: - return cls( - log_record=record, - resource=resource, - instrumentation_scope=instrumentation_scope, - **({} if limits is None else {"limits": limits}), - ) - - -class LogRecordProcessor(abc.ABC): - """Interface to hook the log record emitting action. - - Log processors can be registered directly using - :func:`LoggerProvider.add_log_record_processor` and they are invoked - in the same order as they were registered. - - Implementers of custom log processors should be aware of the following: - - Error Handling - -------------- - According to the OpenTelemetry error handling principles, the SDK should - not throw unhandled exceptions at runtime. When implementing a custom - ``LogRecordProcessor``, it is the **processor's responsibility** to handle - any exceptions that may be raised by the exporter's ``export()`` method. - - The ``LogRecordExporter.export()`` method may raise exceptions (e.g., - network errors, timeouts). If these exceptions are not caught, they will - propagate up and potentially crash the application. - - Custom processor implementations should wrap exporter calls in a - try/except block. See ``SimpleLogRecordProcessor`` for a reference - implementation:: - - def on_emit(self, log_record: ReadWriteLogRecord): - try: - self._exporter.export((log_record,)) - except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while exporting logs.") - - The ``BatchLogRecordProcessor`` handles this implicitly since export - operations occur in a background thread where exceptions cannot bubble - up to the caller. - """ - - @abc.abstractmethod - def on_emit(self, log_record: ReadWriteLogRecord) -> None: - """Emits the ``ReadWriteLogRecord``. - - Implementers should handle any exceptions raised during log processing - to prevent application crashes. See the class docstring for details - on error handling expectations. - """ - - def enabled( # pylint: disable=no-self-use,unused-argument - self, - *, - context: Context | None = None, - instrumentation_scope: InstrumentationScope | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - """Returns whether this processor would emit a record for the given arguments. - - Processors that support filtering MAY override this method. The default - implementation returns ``True``. - """ - return True - - @abc.abstractmethod - def shutdown(self) -> None: - """Called when a :class:`opentelemetry.sdk._logs.Logger` is shutdown""" - - @abc.abstractmethod - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Export all the received logs to the configured Exporter that have not yet - been exported. - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. - - Returns: - False if the timeout is exceeded, True otherwise. - """ - - -# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved -# pylint:disable=no-member -class SynchronousMultiLogRecordProcessor(LogRecordProcessor): - """Implementation of class:`LogRecordProcessor` that forwards all received - events to a list of log processors sequentially. - - The underlying log processors are called in sequential order as they were - added. - """ - - def __init__(self) -> None: - # use a tuple to avoid race conditions when adding a new log and - # iterating through it on "emit". - self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] - self._lock = threading.Lock() - - def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: - """Adds a Logprocessor to the list of log processors handled by this instance""" - with self._lock: - self._log_record_processors += (log_record_processor,) - - def on_emit(self, log_record: ReadWriteLogRecord) -> None: - for lp in self._log_record_processors: - lp.on_emit(log_record) - - def enabled( - self, - *, - context: Context | None = None, - instrumentation_scope: InstrumentationScope | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - processors = self._log_record_processors - return any( - lp.enabled( - context=context, - instrumentation_scope=instrumentation_scope, - severity_number=severity_number, - event_name=event_name, - ) - for lp in processors - ) - - def shutdown(self) -> None: - """Shutdown the log processors one by one""" - for lp in self._log_record_processors: - lp.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Force flush the log processors one by one - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. If the first n log processors exceeded the timeout - then remaining log processors will not be flushed. - - Returns: - True if all the log processors flushes the logs within timeout, - False otherwise. - """ - deadline_ns = time_ns() + timeout_millis * 1000000 - all_flushed = True - for lp in self._log_record_processors: - current_ts = time_ns() - if current_ts >= deadline_ns: - return False - - if lp.force_flush((deadline_ns - current_ts) // 1000000) is False: - all_flushed = False - - return all_flushed - - -class ConcurrentMultiLogRecordProcessor(LogRecordProcessor): - """Implementation of :class:`LogRecordProcessor` that forwards all received - events to a list of log processors in parallel. - - Calls to the underlying log processors are forwarded in parallel by - submitting them to a thread pool executor and waiting until each log - processor finished its work. - - Args: - max_workers: The number of threads managed by the thread pool executor - and thus defining how many log processors can work in parallel. - """ - - def __init__(self, max_workers: int = 2) -> None: - # use a tuple to avoid race conditions when adding a new log and - # iterating through it on "emit". - self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] - self._lock = threading.Lock() - self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) - - def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: - with self._lock: - self._log_record_processors += (log_record_processor,) - - def _submit_and_wait( - self, - func: Callable[[LogRecordProcessor], Callable[..., None]], - *args: Any, - **kwargs: Any, - ): - futures = [] - for lp in self._log_record_processors: - future = self._executor.submit(func(lp), *args, **kwargs) - futures.append(future) - for future in futures: - future.result() - - def on_emit(self, log_record: ReadWriteLogRecord) -> None: - self._submit_and_wait(lambda lp: lp.on_emit, log_record) - - def enabled( - self, - *, - context: Context | None = None, - instrumentation_scope: InstrumentationScope | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - processors = self._log_record_processors - return any( - lp.enabled( - context=context, - instrumentation_scope=instrumentation_scope, - severity_number=severity_number, - event_name=event_name, - ) - for lp in processors - ) - - def shutdown(self) -> None: - self._submit_and_wait(lambda lp: lp.shutdown) - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Force flush the log processors in parallel. - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. - - Returns: - True if all the log processors flushes the logs within timeout, - False otherwise. - """ - futures = [] - for lp in self._log_record_processors: - future = self._executor.submit(lp.force_flush, timeout_millis) - futures.append(future) - - done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_millis / 1e3) - - if not_done_futures: - return False - - for future in done_futures: - if future.result() is False: - return False - - return True - - -# skip natural LogRecord attributes -# http://docs.python.org/library/logging.html#logrecord-attributes -_RESERVED_ATTRS = frozenset( - ( - "asctime", - "args", - "created", - "exc_info", - "exc_text", - "filename", - "funcName", - "getMessage", - "message", - "levelname", - "levelno", - "lineno", - "module", - "msecs", - "msg", - "name", - "pathname", - "process", - "processName", - "relativeCreated", - "stack_info", - "thread", - "threadName", - "taskName", - ) -) - - -class LoggingHandler(logging.Handler): - """A handler class which writes logging records, in OTLP format, to - a network destination or file. Supports signals from the `logging` module. - https://docs.python.org/3/library/logging.html - """ - - def __init__( - self, - level: int = logging.NOTSET, - logger_provider: APILoggerProvider | None = None, - ) -> None: - super().__init__(level=level) - self._logger_provider = logger_provider or get_logger_provider() - - warnings.warn( - "`LoggingHandler` in `opentelemetry-sdk` is deprecated. Use the " - "handler from `opentelemetry-instrumentation-logging` instead.", - DeprecationWarning, - ) - - @staticmethod - def _get_attributes(record: logging.LogRecord) -> Attributes: - attributes = {k: v for k, v in vars(record).items() if k not in _RESERVED_ATTRS} - - # Add standard code attributes for logs. - attributes[code_attributes.CODE_FILE_PATH] = record.pathname - attributes[code_attributes.CODE_FUNCTION_NAME] = record.funcName - attributes[code_attributes.CODE_LINE_NUMBER] = record.lineno - - if record.exc_info: - exctype, value, tb = record.exc_info - if exctype is not None: - attributes[exception_attributes.EXCEPTION_TYPE] = exctype.__name__ - if value is not None and value.args: - attributes[exception_attributes.EXCEPTION_MESSAGE] = str(value.args[0]) - if tb is not None: - # https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans/#stacktrace-representation - attributes[exception_attributes.EXCEPTION_STACKTRACE] = "".join( - traceback.format_exception(*record.exc_info) - ) - return attributes - - def _translate(self, record: logging.LogRecord) -> LogRecord: - timestamp = int(record.created * 1e9) - observered_timestamp = time_ns() - attributes = self._get_attributes(record) - severity_number = std_to_otel(record.levelno) - if self.formatter: - body = self.format(record) - else: - # `record.getMessage()` uses `record.msg` as a template to format - # `record.args` into. There is a special case in `record.getMessage()` - # where it will only attempt formatting if args are provided, - # otherwise, it just stringifies `record.msg`. - # - # Since the OTLP body field has a type of 'any' and the logging module - # is sometimes used in such a way that objects incorrectly end up - # set as record.msg, in those cases we would like to bypass - # `record.getMessage()` completely and set the body to the object - # itself instead of its string representation. - # For more background, see: https://github.com/open-telemetry/opentelemetry-python/pull/4216 - if not record.args and not isinstance(record.msg, str): - # if record.msg is not a value we can export, cast it to string - # TODO: https://github.com/open-telemetry/opentelemetry-python/issues/5304 - do something better - # than just casting to a string. - if not isinstance( - record.msg, - ( - NoneType, - bool, - bytes, - int, - float, - str, - Sequence, - Mapping, - ), - ): - body = str(record.msg) - else: - body = record.msg - else: - body = record.getMessage() - - # Map Python log level names to OTel severity text as defined in - # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity - # Python "WARNING" -> OTel "WARN" (see #3548) - # Python "CRITICAL" -> OTel "FATAL" (see #4984) - _python_to_otel_severity_text = { - "WARNING": "WARN", - "CRITICAL": "FATAL", - } - level_name = _python_to_otel_severity_text.get(record.levelname, record.levelname) - - return LogRecord( - timestamp=timestamp, - observed_timestamp=observered_timestamp, - context=get_current() or None, - severity_text=level_name, - severity_number=severity_number, - body=body, - attributes=attributes, - ) - - def emit(self, record: logging.LogRecord) -> None: - """ - Emit a record. Skip emitting if logger is NoOp. - - The record is translated to OTel format, and then sent across the pipeline. - """ - logger = get_logger(record.name, logger_provider=self._logger_provider) - if not isinstance(logger, NoOpLogger): - logger.emit(self._translate(record)) - - def flush(self) -> None: - """ - Flushes the logging output. Skip flushing if logging_provider has no force_flush method. - """ - if hasattr(self._logger_provider, "force_flush") and callable( - self._logger_provider.force_flush # type: ignore[reportAttributeAccessIssue] - ): - # This is done in a separate thread to avoid a potential deadlock, for - # details see https://github.com/open-telemetry/opentelemetry-python/pull/4636. - thread = threading.Thread(target=self._logger_provider.force_flush) # type: ignore[reportAttributeAccessIssue] - thread.start() - - -@dataclass -class _LoggerConfig: - is_enabled: bool = True - - @classmethod - def default(cls) -> _LoggerConfig: - return _LoggerConfig() - - -class Logger(APILogger): - def __init__( - self, - resource: Resource, - multi_log_record_processor: SynchronousMultiLogRecordProcessor | ConcurrentMultiLogRecordProcessor, - instrumentation_scope: InstrumentationScope, - *, - logger_metrics: LoggerMetricsT, - log_record_limits: LogRecordLimits | None = None, - _logger_config: _LoggerConfig, - ): - super().__init__( - instrumentation_scope.name, - instrumentation_scope.version, - instrumentation_scope.schema_url, - instrumentation_scope.attributes, - ) - self._resource = resource - self._multi_log_record_processor = multi_log_record_processor - self._instrumentation_scope = instrumentation_scope - self._logger_metrics = logger_metrics - self._logger_config = _logger_config - self._log_record_limits = log_record_limits or LogRecordLimits() - - def _is_enabled(self) -> bool: - return self._logger_config.is_enabled - - def enabled( - self, - *, - context: Context | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - if not self._is_enabled(): - return False - return self._multi_log_record_processor.enabled( - context=context, - instrumentation_scope=self._instrumentation_scope, - severity_number=severity_number, - event_name=event_name, - ) - - def _set_logger_config(self, logger_config: _LoggerConfig) -> None: - self._logger_config = logger_config - - def _set_resource(self, resource: Resource) -> None: - self._resource = resource - - @property - def instrumentation_scope(self): - return self._instrumentation_scope - - @property - def resource(self): - return self._resource - - # pylint: disable=arguments-differ - def emit( - self, - record: LogRecord | None = None, - *, - timestamp: int | None = None, - observed_timestamp: int | None = None, - context: Context | None = None, - severity_number: SeverityNumber | None = None, - severity_text: str | None = None, - body: AnyValue | None = None, - attributes: Attributes = None, - event_name: str | None = None, - exception: BaseException | None = None, - ) -> None: - """Emits the :class:`ReadWriteLogRecord` by setting instrumentation scope - and forwarding to the processor. - """ - if not self._is_enabled(): - return - # If a record is provided, use it directly - if record is not None: - if not isinstance(record, ReadWriteLogRecord): - if record.exception is not None: - record = _copy_log_record_with_exception(record) - # pylint:disable=protected-access - writable_record = ReadWriteLogRecord._from_api_log_record( - record=record, - resource=self._resource, - instrumentation_scope=self._instrumentation_scope, - limits=self._log_record_limits, - ) - else: - _set_log_record_exception_attributes(record.log_record) - writable_record = record - else: - # Create a record from individual parameters - log_record = _create_log_record_with_exception( - timestamp=timestamp, - observed_timestamp=observed_timestamp, - context=context, - severity_number=severity_number, - severity_text=severity_text, - body=body, - attributes=attributes, - event_name=event_name, - exception=exception, - ) - # pylint:disable=protected-access - writable_record = ReadWriteLogRecord._from_api_log_record( - record=log_record, - resource=self._resource, - instrumentation_scope=self._instrumentation_scope, - limits=self._log_record_limits, - ) - - self._logger_metrics.emit_log() - self._multi_log_record_processor.on_emit(writable_record) - - -_LoggerConfiguratorT = Callable[[InstrumentationScope], _LoggerConfig] -_RuleBasedLoggerConfigurator = RuleBasedConfigurator[_LoggerConfig] - - -def _default_logger_configurator( - _logger_scope: InstrumentationScope, -) -> _LoggerConfig: - return _LoggerConfig.default() - - -def _disable_logger_configurator( - _logger_scope: InstrumentationScope, -) -> _LoggerConfig: - return _LoggerConfig(is_enabled=False) - - -class LoggerProvider(APILoggerProvider): - def __init__( - self, - resource: Resource | None = None, - shutdown_on_exit: bool = True, - multi_log_record_processor: SynchronousMultiLogRecordProcessor - | ConcurrentMultiLogRecordProcessor - | None = None, - *, - meter_provider: MeterProvider | None = None, - log_record_limits: LogRecordLimits | None = None, - _logger_configurator: _LoggerConfiguratorT | None = None, - ): - if resource is None: - self._resource = Resource.create({}) - else: - self._resource = resource - self._multi_log_record_processor = multi_log_record_processor or SynchronousMultiLogRecordProcessor() - self._logger_metrics = create_logger_metrics( - meter_provider or get_meter_provider(), - parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), - ) - disabled = environ.get(OTEL_SDK_DISABLED, "") - self._disabled = disabled.lower().strip() == "true" - self._logger_configurator = _logger_configurator or _default_logger_configurator - self._log_record_limits = log_record_limits or LogRecordLimits() - self._at_exit_handler = None - if shutdown_on_exit: - self._at_exit_handler = atexit.register(self.shutdown) - self._logger_cache = {} - self._logger_cache_lock = Lock() - self._active_loggers: WeakSet[Logger] = WeakSet() - self._active_loggers_lock = Lock() - if hasattr(os, "register_at_fork"): - weak_at_fork = WeakMethod(self._handle_fork) - - def _after_in_child() -> None: - if at_fork := weak_at_fork(): - at_fork() - - os.register_at_fork(after_in_child=_after_in_child) - - def _handle_fork(self) -> None: - self._logger_cache_lock = Lock() - self._active_loggers_lock = Lock() - self._update_resource(_get_process_dependent_resource()) - - @property - def resource(self): - return self._resource - - def _update_resource(self, resource: Resource) -> None: - with self._active_loggers_lock: - self._resource = self._resource.merge(resource) - for logger in list(self._active_loggers): - # pylint: disable-next=protected-access - logger._set_resource(self._resource) - - def _get_logger_no_cache( - self, - name: str, - version: str | None = None, - schema_url: str | None = None, - attributes: Attributes = None, - ) -> Logger: - scope = InstrumentationScope(name, version, schema_url, attributes) - - return Logger( - self._resource, - self._multi_log_record_processor, - scope, - logger_metrics=self._logger_metrics, - _logger_config=self._apply_logger_configurator(scope), - log_record_limits=self._log_record_limits, - ) - - def _get_logger_cached( - self, - name: str, - version: str | None = None, - schema_url: str | None = None, - ) -> Logger: - with self._logger_cache_lock: - key = (name, version, schema_url) - if key in self._logger_cache: - return self._logger_cache[key] - - self._logger_cache[key] = self._get_logger_no_cache(name, version, schema_url) - return self._logger_cache[key] - - def get_logger( - self, - name: str, - version: str | None = None, - schema_url: str | None = None, - attributes: Attributes = None, - ) -> APILogger: - if self._disabled: - return NoOpLogger( - name, - version=version, - schema_url=schema_url, - attributes=attributes, - ) - logger = ( - self._get_logger_cached(name, version, schema_url) - if attributes is None - else self._get_logger_no_cache(name, version, schema_url, attributes) - ) - with self._active_loggers_lock: - self._active_loggers.add(logger) - return logger - - def add_log_record_processor(self, log_record_processor: LogRecordProcessor): - """Registers a new :class:`LogRecordProcessor` for this `LoggerProvider` instance. - - The log processors are invoked in the same order they are registered. - """ - self._multi_log_record_processor.add_log_record_processor(log_record_processor) - - def _set_logger_configurator(self, *, logger_configurator: _LoggerConfiguratorT): - """Set a new LoggerConfigurator for this LoggerProvider. - - Setting a new LoggerConfigurator will result in the configurator being called - for each outstanding Logger and for any newly created loggers thereafter. - Therefore, it is important that the provided function returns quickly. - """ - self._logger_configurator = logger_configurator - with self._active_loggers_lock: - for logger in list(self._active_loggers): - # pylint: disable-next=protected-access - logger._set_logger_config(self._apply_logger_configurator(logger.instrumentation_scope)) - - def _apply_logger_configurator(self, instrumentation_scope: InstrumentationScope) -> _LoggerConfig: - try: - return self._logger_configurator(instrumentation_scope) - # pylint: disable-next=broad-exception-caught - except Exception: - _logger.exception( - "logger configurator failed for scope '%s', using default config", - instrumentation_scope.name, - ) - return _LoggerConfig.default() - - def shutdown(self) -> None: - """Shuts down the log processors.""" - self._multi_log_record_processor.shutdown() - if self._at_exit_handler is not None: - atexit.unregister(self._at_exit_handler) - self._at_exit_handler = None - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Force flush the log processors. - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. - - Returns: - True if all the log processors flushes the logs within timeout, - False otherwise. - """ - return self._multi_log_record_processor.force_flush(timeout_millis) - - -_STD_TO_OTEL = { - 10: SeverityNumber.DEBUG, - 11: SeverityNumber.DEBUG2, - 12: SeverityNumber.DEBUG3, - 13: SeverityNumber.DEBUG4, - 14: SeverityNumber.DEBUG4, - 15: SeverityNumber.DEBUG4, - 16: SeverityNumber.DEBUG4, - 17: SeverityNumber.DEBUG4, - 18: SeverityNumber.DEBUG4, - 19: SeverityNumber.DEBUG4, - 20: SeverityNumber.INFO, - 21: SeverityNumber.INFO2, - 22: SeverityNumber.INFO3, - 23: SeverityNumber.INFO4, - 24: SeverityNumber.INFO4, - 25: SeverityNumber.INFO4, - 26: SeverityNumber.INFO4, - 27: SeverityNumber.INFO4, - 28: SeverityNumber.INFO4, - 29: SeverityNumber.INFO4, - 30: SeverityNumber.WARN, - 31: SeverityNumber.WARN2, - 32: SeverityNumber.WARN3, - 33: SeverityNumber.WARN4, - 34: SeverityNumber.WARN4, - 35: SeverityNumber.WARN4, - 36: SeverityNumber.WARN4, - 37: SeverityNumber.WARN4, - 38: SeverityNumber.WARN4, - 39: SeverityNumber.WARN4, - 40: SeverityNumber.ERROR, - 41: SeverityNumber.ERROR2, - 42: SeverityNumber.ERROR3, - 43: SeverityNumber.ERROR4, - 44: SeverityNumber.ERROR4, - 45: SeverityNumber.ERROR4, - 46: SeverityNumber.ERROR4, - 47: SeverityNumber.ERROR4, - 48: SeverityNumber.ERROR4, - 49: SeverityNumber.ERROR4, - 50: SeverityNumber.FATAL, - 51: SeverityNumber.FATAL2, - 52: SeverityNumber.FATAL3, - 53: SeverityNumber.FATAL4, -} - - -def std_to_otel(levelno: int) -> SeverityNumber: - """ - Map python log levelno as defined in https://docs.python.org/3/library/logging.html#logging-levels - to OTel log severity number as defined here: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#field-severitynumber - """ - if levelno < 10: - return SeverityNumber.UNSPECIFIED - if levelno > 53: - return SeverityNumber.FATAL4 - return _STD_TO_OTEL[levelno] +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import abc +import atexit +import base64 +import concurrent.futures +import json +import logging +import os +import threading +import traceback +import warnings +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from os import environ +from threading import Lock +from time import time_ns +from types import NoneType +from typing import ( # noqa + Any, + Tuple, + Union, + cast, + overload, +) +from weakref import WeakMethod, WeakSet + +from typing_extensions import deprecated + +from opentelemetry._logs import Logger as APILogger +from opentelemetry._logs import LoggerProvider as APILoggerProvider +from opentelemetry._logs import ( + LogRecord, + NoOpLogger, + SeverityNumber, + get_logger, + get_logger_provider, +) +from opentelemetry.attributes import BoundedAttributes +from opentelemetry.context import get_current +from opentelemetry.context.context import Context +from opentelemetry.metrics import MeterProvider, get_meter_provider +from opentelemetry.sdk._logs._internal._exceptions import ( + _copy_log_record_with_exception, + _create_log_record_with_exception, + _set_log_record_exception_attributes, +) +from opentelemetry.sdk._logs._internal._logger_metrics import ( + LoggerMetricsT, + create_logger_metrics, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_ATTRIBUTE_COUNT_LIMIT, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, + OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, + OTEL_SDK_DISABLED, +) +from opentelemetry.sdk.environment_variables._internal import ( + parse_boolean_environment_variable, +) +from opentelemetry.sdk.resources import ( + Resource, + _get_process_dependent_resource, +) +from opentelemetry.sdk.util import ns_to_iso_str +from opentelemetry.sdk.util._configurator import RuleBasedConfigurator +from opentelemetry.sdk.util.instrumentation import ( + InstrumentationScope, +) +from opentelemetry.semconv.attributes import ( + code_attributes, + exception_attributes, +) +from opentelemetry.trace import ( + format_span_id, + format_trace_id, +) +from opentelemetry.util.types import AnyValue, Attributes + +# pylint: disable=too-many-lines + +_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 +_ENV_VALUE_UNSET = "" + +_logger = logging.getLogger(__name__) + + +class BytesEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, bytes): + return base64.b64encode(o).decode() + return super().default(o) + + +class LogRecordDroppedAttributesWarning(UserWarning): + """Custom warning to indicate dropped log attributes due to limits. + + This class is used to filter and handle these specific warnings separately + from other warnings, ensuring that they are only shown once without + interfering with default user warnings. + """ + + +warnings.simplefilter("once", LogRecordDroppedAttributesWarning) + + +@deprecated( + "Use LogRecordDroppedAttributesWarning. Since logs are not stable yet this WILL be removed in future releases." +) +class LogDroppedAttributesWarning(LogRecordDroppedAttributesWarning): + pass + + +class LogRecordLimits: + """This class is based on a SpanLimits class in the Tracing module. + + This class represents the limits that should be enforced on recorded data such as events, links, attributes etc. + + This class does not enforce any limits itself. It only provides a way to read limits from env, + default values and from user provided arguments. + + All limit arguments must be either a non-negative integer, ``None`` or ``LogRecordLimits.UNSET``. + + - All limit arguments are optional. + - If a limit argument is not set, the class will try to read its value from the corresponding + environment variable. + - If the environment variable is not set, the default value, if any, will be used. + + Limit precedence: + + - If a model specific limit is set, it will be used. + - Else if the corresponding global limit is set, it will be used. + - Else if the model specific limit has a default value, the default value will be used. + - Else if the global limit has a default value, the default value will be used. + + Args: + max_attributes: Maximum number of attributes that can be added to a log record (global fallback). + Environment variable: ``OTEL_ATTRIBUTE_COUNT_LIMIT`` + Default: {_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT} + max_attribute_length: Maximum length an attribute value can have (global fallback). Values + longer than the specified length will be truncated. + max_log_record_attributes: Maximum number of attributes that can be added to a log record. + Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT`` + Falls back to ``max_attributes`` when unset. + max_log_record_attribute_length: Maximum length a log record attribute value can have. + Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT`` + Falls back to ``max_attribute_length`` when unset. + """ + + UNSET = -1 + + def __init__( + self, + max_attributes: int | None = None, + max_attribute_length: int | None = None, + max_log_record_attributes: int | None = None, + max_log_record_attribute_length: int | None = None, + ): + # global attribute count + global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) + self.max_attributes = ( + global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT + ) + + # global attribute length + self.max_attribute_length = self._from_env_if_absent( + max_attribute_length, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + ) + + self.max_log_record_attributes = self._from_env_if_absent( + max_log_record_attributes, + OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, + default=self.max_attributes, + ) + + self.max_log_record_attribute_length = self._from_env_if_absent( + max_log_record_attribute_length, + OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, + default=self.max_attribute_length, + ) + + def __repr__(self): + return ( + f"{type(self).__name__}(" + f"max_attributes={self.max_attributes}, " + f"max_attribute_length={self.max_attribute_length}, " + f"max_log_record_attributes={self.max_log_record_attributes}, " + f"max_log_record_attribute_length={self.max_log_record_attribute_length})" + ) + + @classmethod + def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: + if value == cls.UNSET: + return None + + err_msg = "{} must be a non-negative integer but got {}" + + # if no value is provided for the limit, try to load it from env + if value is None: + # return default value if env var is not set + if env_var not in environ: + return default + + str_value = environ.get(env_var, "").strip().lower() + if str_value == _ENV_VALUE_UNSET: + return None + + try: + value = int(str_value) + except ValueError: + raise ValueError(err_msg.format(env_var, str_value)) + + if value < 0: + raise ValueError(err_msg.format(env_var, value)) + return value + + +@deprecated("Use LogRecordLimits. Since logs are not stable yet this WILL be removed in future releases.") +class LogLimits(LogRecordLimits): + pass + + +@dataclass(frozen=True) +class ReadableLogRecord: + """Readable LogRecord should be kept exactly in-sync with ReadWriteLogRecord, only difference is the frozen=True param.""" + + log_record: LogRecord + resource: Resource + instrumentation_scope: InstrumentationScope | None = None + limits: LogRecordLimits | None = None + + @property + def dropped_attributes(self) -> int: + if isinstance(self.log_record.attributes, BoundedAttributes): + return self.log_record.attributes.dropped + return 0 + + def to_json(self, indent: int | None = 4) -> str: + return json.dumps( + { + "body": self.log_record.body, + "severity_number": self.log_record.severity_number.value + if self.log_record.severity_number is not None + else None, + "severity_text": self.log_record.severity_text, + "attributes": (dict(self.log_record.attributes) if bool(self.log_record.attributes) else None), + "dropped_attributes": self.dropped_attributes, + "timestamp": ns_to_iso_str(self.log_record.timestamp) + if self.log_record.timestamp is not None + else None, + "observed_timestamp": ns_to_iso_str(self.log_record.observed_timestamp), + "trace_id": ( + f"0x{format_trace_id(self.log_record.trace_id)}" if self.log_record.trace_id is not None else "" + ), + "span_id": ( + f"0x{format_span_id(self.log_record.span_id)}" if self.log_record.span_id is not None else "" + ), + "trace_flags": self.log_record.trace_flags, + "resource": json.loads(self.resource.to_json()), + "event_name": self.log_record.event_name if self.log_record.event_name else "", + }, + indent=indent, + cls=BytesEncoder, + ) + + +@dataclass +class ReadWriteLogRecord: + """A ReadWriteLogRecord instance represents an event being logged. + ReadWriteLogRecord instances are created and emitted via `Logger` + every time something is logged. They contain all the information + pertinent to the event being logged. + """ + + log_record: LogRecord + resource: Resource | None = Resource.create({}) + instrumentation_scope: InstrumentationScope | None = None + limits: LogRecordLimits = field(default_factory=LogRecordLimits) + + def __post_init__(self): + self.log_record.attributes = BoundedAttributes( + maxlen=self.limits.max_log_record_attributes, + attributes=self.log_record.attributes if self.log_record.attributes else None, + immutable=False, + max_value_len=self.limits.max_log_record_attribute_length, + extended_attributes=True, + ) + if self.dropped_attributes > 0: + warnings.warn( + "Log record attributes were dropped due to limits", + LogRecordDroppedAttributesWarning, + stacklevel=2, + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ReadWriteLogRecord): + return NotImplemented + return self.__dict__ == other.__dict__ + + @property + def dropped_attributes(self) -> int: + if isinstance(self.log_record.attributes, BoundedAttributes): + return self.log_record.attributes.dropped + return 0 + + @classmethod + def _from_api_log_record( + cls, + *, + record: LogRecord, + resource: Resource, + instrumentation_scope: InstrumentationScope | None = None, + limits: LogRecordLimits | None = None, + ) -> ReadWriteLogRecord: + return cls( + log_record=record, + resource=resource, + instrumentation_scope=instrumentation_scope, + **({} if limits is None else {"limits": limits}), + ) + + +class LogRecordProcessor(abc.ABC): + """Interface to hook the log record emitting action. + + Log processors can be registered directly using + :func:`LoggerProvider.add_log_record_processor` and they are invoked + in the same order as they were registered. + + Implementers of custom log processors should be aware of the following: + + Error Handling + -------------- + According to the OpenTelemetry error handling principles, the SDK should + not throw unhandled exceptions at runtime. When implementing a custom + ``LogRecordProcessor``, it is the **processor's responsibility** to handle + any exceptions that may be raised by the exporter's ``export()`` method. + + The ``LogRecordExporter.export()`` method may raise exceptions (e.g., + network errors, timeouts). If these exceptions are not caught, they will + propagate up and potentially crash the application. + + Custom processor implementations should wrap exporter calls in a + try/except block. See ``SimpleLogRecordProcessor`` for a reference + implementation:: + + def on_emit(self, log_record: ReadWriteLogRecord): + try: + self._exporter.export((log_record,)) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while exporting logs.") + + The ``BatchLogRecordProcessor`` handles this implicitly since export + operations occur in a background thread where exceptions cannot bubble + up to the caller. + """ + + @abc.abstractmethod + def on_emit(self, log_record: ReadWriteLogRecord) -> None: + """Emits the ``ReadWriteLogRecord``. + + Implementers should handle any exceptions raised during log processing + to prevent application crashes. See the class docstring for details + on error handling expectations. + """ + + def enabled( # pylint: disable=no-self-use,unused-argument + self, + *, + context: Context | None = None, + instrumentation_scope: InstrumentationScope | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + """Returns whether this processor would emit a record for the given arguments. + + Processors that support filtering MAY override this method. The default + implementation returns ``True``. + """ + return True + + @abc.abstractmethod + def shutdown(self) -> None: + """Called when a :class:`opentelemetry.sdk._logs.Logger` is shutdown""" + + @abc.abstractmethod + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Export all the received logs to the configured Exporter that have not yet + been exported. + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. + + Returns: + False if the timeout is exceeded, True otherwise. + """ + + +# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved +# pylint:disable=no-member +class SynchronousMultiLogRecordProcessor(LogRecordProcessor): + """Implementation of class:`LogRecordProcessor` that forwards all received + events to a list of log processors sequentially. + + The underlying log processors are called in sequential order as they were + added. + """ + + def __init__(self) -> None: + # use a tuple to avoid race conditions when adding a new log and + # iterating through it on "emit". + self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] + self._lock = threading.Lock() + + def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: + """Adds a Logprocessor to the list of log processors handled by this instance""" + with self._lock: + self._log_record_processors += (log_record_processor,) + + def on_emit(self, log_record: ReadWriteLogRecord) -> None: + for lp in self._log_record_processors: + lp.on_emit(log_record) + + def enabled( + self, + *, + context: Context | None = None, + instrumentation_scope: InstrumentationScope | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + processors = self._log_record_processors + return any( + lp.enabled( + context=context, + instrumentation_scope=instrumentation_scope, + severity_number=severity_number, + event_name=event_name, + ) + for lp in processors + ) + + def shutdown(self) -> None: + """Shutdown the log processors one by one""" + for lp in self._log_record_processors: + lp.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Force flush the log processors one by one + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. If the first n log processors exceeded the timeout + then remaining log processors will not be flushed. + + Returns: + True if all the log processors flushes the logs within timeout, + False otherwise. + """ + deadline_ns = time_ns() + timeout_millis * 1000000 + all_flushed = True + for lp in self._log_record_processors: + current_ts = time_ns() + if current_ts >= deadline_ns: + return False + + if lp.force_flush((deadline_ns - current_ts) // 1000000) is False: + all_flushed = False + + return all_flushed + + +class ConcurrentMultiLogRecordProcessor(LogRecordProcessor): + """Implementation of :class:`LogRecordProcessor` that forwards all received + events to a list of log processors in parallel. + + Calls to the underlying log processors are forwarded in parallel by + submitting them to a thread pool executor and waiting until each log + processor finished its work. + + Args: + max_workers: The number of threads managed by the thread pool executor + and thus defining how many log processors can work in parallel. + """ + + def __init__(self, max_workers: int = 2) -> None: + # use a tuple to avoid race conditions when adding a new log and + # iterating through it on "emit". + self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] + self._lock = threading.Lock() + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) + + def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: + with self._lock: + self._log_record_processors += (log_record_processor,) + + def _submit_and_wait( + self, + func: Callable[[LogRecordProcessor], Callable[..., None]], + *args: Any, + **kwargs: Any, + ): + futures = [] + for lp in self._log_record_processors: + try: + future = self._executor.submit(func(lp), *args, **kwargs) + futures.append(future) + except RuntimeError: + # The executor may already be shut down when this runs from an + # atexit handler: CPython runs threading._shutdown() before the + # atexit queue, so the ThreadPoolExecutor is closed by the time + # LoggerProvider.shutdown() runs. Run inline so buffered work is + # still flushed instead of being silently dropped. A + # RuntimeError raised by the callback itself is not caught here + # and still propagates via future.result() below. + _logger.debug( + "Executor is shut down; running LogRecordProcessor callback inline.", + ) + func(lp)(*args, **kwargs) + for future in futures: + future.result() + + def on_emit(self, log_record: ReadWriteLogRecord) -> None: + self._submit_and_wait(lambda lp: lp.on_emit, log_record) + + def enabled( + self, + *, + context: Context | None = None, + instrumentation_scope: InstrumentationScope | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + processors = self._log_record_processors + return any( + lp.enabled( + context=context, + instrumentation_scope=instrumentation_scope, + severity_number=severity_number, + event_name=event_name, + ) + for lp in processors + ) + + def shutdown(self) -> None: + self._submit_and_wait(lambda lp: lp.shutdown) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Force flush the log processors in parallel. + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. + + Returns: + True if all the log processors flushes the logs within timeout, + False otherwise. + """ + futures = [] + for lp in self._log_record_processors: + future = self._executor.submit(lp.force_flush, timeout_millis) + futures.append(future) + + done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_millis / 1e3) + + if not_done_futures: + return False + + for future in done_futures: + if future.result() is False: + return False + + return True + + +# skip natural LogRecord attributes +# http://docs.python.org/library/logging.html#logrecord-attributes +_RESERVED_ATTRS = frozenset( + ( + "asctime", + "args", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "getMessage", + "message", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack_info", + "thread", + "threadName", + "taskName", + ) +) + + +class LoggingHandler(logging.Handler): + """A handler class which writes logging records, in OTLP format, to + a network destination or file. Supports signals from the `logging` module. + https://docs.python.org/3/library/logging.html + """ + + def __init__( + self, + level: int = logging.NOTSET, + logger_provider: APILoggerProvider | None = None, + ) -> None: + super().__init__(level=level) + self._logger_provider = logger_provider or get_logger_provider() + + warnings.warn( + "`LoggingHandler` in `opentelemetry-sdk` is deprecated. Use the " + "handler from `opentelemetry-instrumentation-logging` instead.", + DeprecationWarning, + ) + + @staticmethod + def _get_attributes(record: logging.LogRecord) -> Attributes: + attributes = {k: v for k, v in vars(record).items() if k not in _RESERVED_ATTRS} + + # Add standard code attributes for logs. + attributes[code_attributes.CODE_FILE_PATH] = record.pathname + attributes[code_attributes.CODE_FUNCTION_NAME] = record.funcName + attributes[code_attributes.CODE_LINE_NUMBER] = record.lineno + + if record.exc_info: + exctype, value, tb = record.exc_info + if exctype is not None: + attributes[exception_attributes.EXCEPTION_TYPE] = exctype.__name__ + if value is not None and value.args: + attributes[exception_attributes.EXCEPTION_MESSAGE] = str(value.args[0]) + if tb is not None: + # https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans/#stacktrace-representation + attributes[exception_attributes.EXCEPTION_STACKTRACE] = "".join( + traceback.format_exception(*record.exc_info) + ) + return attributes + + def _translate(self, record: logging.LogRecord) -> LogRecord: + timestamp = int(record.created * 1e9) + observered_timestamp = time_ns() + attributes = self._get_attributes(record) + severity_number = std_to_otel(record.levelno) + if self.formatter: + body = self.format(record) + else: + # `record.getMessage()` uses `record.msg` as a template to format + # `record.args` into. There is a special case in `record.getMessage()` + # where it will only attempt formatting if args are provided, + # otherwise, it just stringifies `record.msg`. + # + # Since the OTLP body field has a type of 'any' and the logging module + # is sometimes used in such a way that objects incorrectly end up + # set as record.msg, in those cases we would like to bypass + # `record.getMessage()` completely and set the body to the object + # itself instead of its string representation. + # For more background, see: https://github.com/open-telemetry/opentelemetry-python/pull/4216 + if not record.args and not isinstance(record.msg, str): + # if record.msg is not a value we can export, cast it to string + # TODO: https://github.com/open-telemetry/opentelemetry-python/issues/5304 - do something better + # than just casting to a string. + if not isinstance( + record.msg, + ( + NoneType, + bool, + bytes, + int, + float, + str, + Sequence, + Mapping, + ), + ): + body = str(record.msg) + else: + body = record.msg + else: + body = record.getMessage() + + # Map Python log level names to OTel severity text as defined in + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity + # Python "WARNING" -> OTel "WARN" (see #3548) + # Python "CRITICAL" -> OTel "FATAL" (see #4984) + _python_to_otel_severity_text = { + "WARNING": "WARN", + "CRITICAL": "FATAL", + } + level_name = _python_to_otel_severity_text.get(record.levelname, record.levelname) + + return LogRecord( + timestamp=timestamp, + observed_timestamp=observered_timestamp, + context=get_current() or None, + severity_text=level_name, + severity_number=severity_number, + body=body, + attributes=attributes, + ) + + def emit(self, record: logging.LogRecord) -> None: + """ + Emit a record. Skip emitting if logger is NoOp. + + The record is translated to OTel format, and then sent across the pipeline. + """ + logger = get_logger(record.name, logger_provider=self._logger_provider) + if not isinstance(logger, NoOpLogger): + logger.emit(self._translate(record)) + + def flush(self) -> None: + """ + Flushes the logging output. Skip flushing if logging_provider has no force_flush method. + """ + if hasattr(self._logger_provider, "force_flush") and callable( + self._logger_provider.force_flush # type: ignore[reportAttributeAccessIssue] + ): + # This is done in a separate thread to avoid a potential deadlock, for + # details see https://github.com/open-telemetry/opentelemetry-python/pull/4636. + thread = threading.Thread(target=self._logger_provider.force_flush) # type: ignore[reportAttributeAccessIssue] + thread.start() + + +@dataclass +class _LoggerConfig: + is_enabled: bool = True + + @classmethod + def default(cls) -> _LoggerConfig: + return _LoggerConfig() + + +class Logger(APILogger): + def __init__( + self, + resource: Resource, + multi_log_record_processor: SynchronousMultiLogRecordProcessor | ConcurrentMultiLogRecordProcessor, + instrumentation_scope: InstrumentationScope, + *, + logger_metrics: LoggerMetricsT, + log_record_limits: LogRecordLimits | None = None, + _logger_config: _LoggerConfig, + ): + super().__init__( + instrumentation_scope.name, + instrumentation_scope.version, + instrumentation_scope.schema_url, + instrumentation_scope.attributes, + ) + self._resource = resource + self._multi_log_record_processor = multi_log_record_processor + self._instrumentation_scope = instrumentation_scope + self._logger_metrics = logger_metrics + self._logger_config = _logger_config + self._log_record_limits = log_record_limits or LogRecordLimits() + + def _is_enabled(self) -> bool: + return self._logger_config.is_enabled + + def enabled( + self, + *, + context: Context | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + if not self._is_enabled(): + return False + return self._multi_log_record_processor.enabled( + context=context, + instrumentation_scope=self._instrumentation_scope, + severity_number=severity_number, + event_name=event_name, + ) + + def _set_logger_config(self, logger_config: _LoggerConfig) -> None: + self._logger_config = logger_config + + def _set_resource(self, resource: Resource) -> None: + self._resource = resource + + @property + def instrumentation_scope(self): + return self._instrumentation_scope + + @property + def resource(self): + return self._resource + + # pylint: disable=arguments-differ + def emit( + self, + record: LogRecord | None = None, + *, + timestamp: int | None = None, + observed_timestamp: int | None = None, + context: Context | None = None, + severity_number: SeverityNumber | None = None, + severity_text: str | None = None, + body: AnyValue | None = None, + attributes: Attributes = None, + event_name: str | None = None, + exception: BaseException | None = None, + ) -> None: + """Emits the :class:`ReadWriteLogRecord` by setting instrumentation scope + and forwarding to the processor. + """ + if not self._is_enabled(): + return + # If a record is provided, use it directly + if record is not None: + if not isinstance(record, ReadWriteLogRecord): + if record.exception is not None: + record = _copy_log_record_with_exception(record) + # pylint:disable=protected-access + writable_record = ReadWriteLogRecord._from_api_log_record( + record=record, + resource=self._resource, + instrumentation_scope=self._instrumentation_scope, + limits=self._log_record_limits, + ) + else: + _set_log_record_exception_attributes(record.log_record) + writable_record = record + else: + # Create a record from individual parameters + log_record = _create_log_record_with_exception( + timestamp=timestamp, + observed_timestamp=observed_timestamp, + context=context, + severity_number=severity_number, + severity_text=severity_text, + body=body, + attributes=attributes, + event_name=event_name, + exception=exception, + ) + # pylint:disable=protected-access + writable_record = ReadWriteLogRecord._from_api_log_record( + record=log_record, + resource=self._resource, + instrumentation_scope=self._instrumentation_scope, + limits=self._log_record_limits, + ) + + self._logger_metrics.emit_log() + self._multi_log_record_processor.on_emit(writable_record) + + +_LoggerConfiguratorT = Callable[[InstrumentationScope], _LoggerConfig] +_RuleBasedLoggerConfigurator = RuleBasedConfigurator[_LoggerConfig] + + +def _default_logger_configurator( + _logger_scope: InstrumentationScope, +) -> _LoggerConfig: + return _LoggerConfig.default() + + +def _disable_logger_configurator( + _logger_scope: InstrumentationScope, +) -> _LoggerConfig: + return _LoggerConfig(is_enabled=False) + + +class LoggerProvider(APILoggerProvider): + def __init__( + self, + resource: Resource | None = None, + shutdown_on_exit: bool = True, + multi_log_record_processor: SynchronousMultiLogRecordProcessor + | ConcurrentMultiLogRecordProcessor + | None = None, + *, + meter_provider: MeterProvider | None = None, + log_record_limits: LogRecordLimits | None = None, + _logger_configurator: _LoggerConfiguratorT | None = None, + ): + if resource is None: + self._resource = Resource.create({}) + else: + self._resource = resource + self._multi_log_record_processor = multi_log_record_processor or SynchronousMultiLogRecordProcessor() + self._logger_metrics = create_logger_metrics( + meter_provider or get_meter_provider(), + parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), + ) + disabled = environ.get(OTEL_SDK_DISABLED, "") + self._disabled = disabled.lower().strip() == "true" + self._logger_configurator = _logger_configurator or _default_logger_configurator + self._log_record_limits = log_record_limits or LogRecordLimits() + self._at_exit_handler = None + if shutdown_on_exit: + self._at_exit_handler = atexit.register(self.shutdown) + self._logger_cache = {} + self._logger_cache_lock = Lock() + self._active_loggers: WeakSet[Logger] = WeakSet() + self._active_loggers_lock = Lock() + if hasattr(os, "register_at_fork"): + weak_at_fork = WeakMethod(self._handle_fork) + + def _after_in_child() -> None: + if at_fork := weak_at_fork(): + at_fork() + + os.register_at_fork(after_in_child=_after_in_child) + + def _handle_fork(self) -> None: + self._logger_cache_lock = Lock() + self._active_loggers_lock = Lock() + self._update_resource(_get_process_dependent_resource()) + + @property + def resource(self): + return self._resource + + def _update_resource(self, resource: Resource) -> None: + with self._active_loggers_lock: + self._resource = self._resource.merge(resource) + for logger in list(self._active_loggers): + # pylint: disable-next=protected-access + logger._set_resource(self._resource) + + def _get_logger_no_cache( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes = None, + ) -> Logger: + scope = InstrumentationScope(name, version, schema_url, attributes) + + return Logger( + self._resource, + self._multi_log_record_processor, + scope, + logger_metrics=self._logger_metrics, + _logger_config=self._apply_logger_configurator(scope), + log_record_limits=self._log_record_limits, + ) + + def _get_logger_cached( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + ) -> Logger: + with self._logger_cache_lock: + key = (name, version, schema_url) + if key in self._logger_cache: + return self._logger_cache[key] + + self._logger_cache[key] = self._get_logger_no_cache(name, version, schema_url) + return self._logger_cache[key] + + def get_logger( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes = None, + ) -> APILogger: + if self._disabled: + return NoOpLogger( + name, + version=version, + schema_url=schema_url, + attributes=attributes, + ) + logger = ( + self._get_logger_cached(name, version, schema_url) + if attributes is None + else self._get_logger_no_cache(name, version, schema_url, attributes) + ) + with self._active_loggers_lock: + self._active_loggers.add(logger) + return logger + + def add_log_record_processor(self, log_record_processor: LogRecordProcessor): + """Registers a new :class:`LogRecordProcessor` for this `LoggerProvider` instance. + + The log processors are invoked in the same order they are registered. + """ + self._multi_log_record_processor.add_log_record_processor(log_record_processor) + + def _set_logger_configurator(self, *, logger_configurator: _LoggerConfiguratorT): + """Set a new LoggerConfigurator for this LoggerProvider. + + Setting a new LoggerConfigurator will result in the configurator being called + for each outstanding Logger and for any newly created loggers thereafter. + Therefore, it is important that the provided function returns quickly. + """ + self._logger_configurator = logger_configurator + with self._active_loggers_lock: + for logger in list(self._active_loggers): + # pylint: disable-next=protected-access + logger._set_logger_config(self._apply_logger_configurator(logger.instrumentation_scope)) + + def _apply_logger_configurator(self, instrumentation_scope: InstrumentationScope) -> _LoggerConfig: + try: + return self._logger_configurator(instrumentation_scope) + # pylint: disable-next=broad-exception-caught + except Exception: + _logger.exception( + "logger configurator failed for scope '%s', using default config", + instrumentation_scope.name, + ) + return _LoggerConfig.default() + + def shutdown(self) -> None: + """Shuts down the log processors.""" + self._multi_log_record_processor.shutdown() + if self._at_exit_handler is not None: + atexit.unregister(self._at_exit_handler) + self._at_exit_handler = None + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Force flush the log processors. + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. + + Returns: + True if all the log processors flushes the logs within timeout, + False otherwise. + """ + return self._multi_log_record_processor.force_flush(timeout_millis) + + +_STD_TO_OTEL = { + 10: SeverityNumber.DEBUG, + 11: SeverityNumber.DEBUG2, + 12: SeverityNumber.DEBUG3, + 13: SeverityNumber.DEBUG4, + 14: SeverityNumber.DEBUG4, + 15: SeverityNumber.DEBUG4, + 16: SeverityNumber.DEBUG4, + 17: SeverityNumber.DEBUG4, + 18: SeverityNumber.DEBUG4, + 19: SeverityNumber.DEBUG4, + 20: SeverityNumber.INFO, + 21: SeverityNumber.INFO2, + 22: SeverityNumber.INFO3, + 23: SeverityNumber.INFO4, + 24: SeverityNumber.INFO4, + 25: SeverityNumber.INFO4, + 26: SeverityNumber.INFO4, + 27: SeverityNumber.INFO4, + 28: SeverityNumber.INFO4, + 29: SeverityNumber.INFO4, + 30: SeverityNumber.WARN, + 31: SeverityNumber.WARN2, + 32: SeverityNumber.WARN3, + 33: SeverityNumber.WARN4, + 34: SeverityNumber.WARN4, + 35: SeverityNumber.WARN4, + 36: SeverityNumber.WARN4, + 37: SeverityNumber.WARN4, + 38: SeverityNumber.WARN4, + 39: SeverityNumber.WARN4, + 40: SeverityNumber.ERROR, + 41: SeverityNumber.ERROR2, + 42: SeverityNumber.ERROR3, + 43: SeverityNumber.ERROR4, + 44: SeverityNumber.ERROR4, + 45: SeverityNumber.ERROR4, + 46: SeverityNumber.ERROR4, + 47: SeverityNumber.ERROR4, + 48: SeverityNumber.ERROR4, + 49: SeverityNumber.ERROR4, + 50: SeverityNumber.FATAL, + 51: SeverityNumber.FATAL2, + 52: SeverityNumber.FATAL3, + 53: SeverityNumber.FATAL4, +} + + +def std_to_otel(levelno: int) -> SeverityNumber: + """ + Map python log levelno as defined in https://docs.python.org/3/library/logging.html#logging-levels + to OTel log severity number as defined here: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#field-severitynumber + """ + if levelno < 10: + return SeverityNumber.UNSPECIFIED + if levelno > 53: + return SeverityNumber.FATAL4 + return _STD_TO_OTEL[levelno] diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 6f47f213bff..bc09ae48e5d 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -1,1478 +1,1491 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# pylint: disable=too-many-lines -import abc -import atexit -import concurrent.futures -import json -import logging -import os -import threading -import traceback -import typing -import weakref -from collections.abc import ( - Callable, - Iterator, - Mapping, - MutableMapping, - Sequence, -) -from dataclasses import dataclass -from os import environ -from time import time_ns -from types import MappingProxyType, TracebackType -from typing import ( - Any, -) -from warnings import filterwarnings - -from typing_extensions import deprecated - -from opentelemetry import context as context_api -from opentelemetry import metrics as metrics_api -from opentelemetry import trace as trace_api -from opentelemetry.attributes import BoundedAttributes -from opentelemetry.sdk import util -from opentelemetry.sdk.environment_variables import ( - OTEL_ATTRIBUTE_COUNT_LIMIT, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, - OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, - OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, - OTEL_SDK_DISABLED, - OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_SPAN_EVENT_COUNT_LIMIT, - OTEL_SPAN_LINK_COUNT_LIMIT, -) -from opentelemetry.sdk.environment_variables._internal import ( - parse_boolean_environment_variable, -) -from opentelemetry.sdk.resources import ( - Resource, - _get_process_dependent_resource, -) -from opentelemetry.sdk.trace import sampling -from opentelemetry.sdk.trace._tracer_metrics import create_tracer_metrics -from opentelemetry.sdk.trace.id_generator import IdGenerator, RandomIdGenerator -from opentelemetry.sdk.util import BoundedList -from opentelemetry.sdk.util._configurator import RuleBasedConfigurator -from opentelemetry.sdk.util.instrumentation import ( - InstrumentationInfo, - InstrumentationScope, -) -from opentelemetry.semconv.attributes.exception_attributes import ( - EXCEPTION_ESCAPED, - EXCEPTION_MESSAGE, - EXCEPTION_STACKTRACE, - EXCEPTION_TYPE, -) -from opentelemetry.trace import NoOpTracer, SpanContext -from opentelemetry.trace.status import Status, StatusCode -from opentelemetry.util import types -from opentelemetry.util._decorator import _agnosticcontextmanager - -logger = logging.getLogger(__name__) - - -_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT = 128 -_DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT = 128 - - -_ENV_VALUE_UNSET = "" - - -class SpanProcessor: - """Interface which allows hooks for SDK's `Span` start and end method - invocations. - - Span processors can be registered directly using - :func:`TracerProvider.add_span_processor` and they are invoked - in the same order as they were registered. - """ - - def on_start( - self, - span: "Span", - parent_context: context_api.Context | None = None, - ) -> None: - """Called when a :class:`opentelemetry.trace.Span` is started. - - This method is called synchronously on the thread that starts the - span, therefore it should not block or throw an exception. - - Args: - span: The :class:`opentelemetry.trace.Span` that just started. - parent_context: The parent context of the span that just started. - """ - - def _on_ending(self, span: "Span") -> None: - """Called when a :class:`opentelemetry.trace.Span` is ending. - - This method is called synchronously on the thread that ends the - span, therefore it should not block or throw an exception. - - Args: - span: The :class:`opentelemetry.trace.Span` that is ending. - """ - - def on_end(self, span: "ReadableSpan") -> None: - """Called when a :class:`opentelemetry.trace.Span` is ended. - - This method is called synchronously on the thread that ends the - span, therefore it should not block or throw an exception. - - Args: - span: The :class:`opentelemetry.trace.Span` that just ended. - """ - - def shutdown(self) -> None: - """Called when a :class:`opentelemetry.sdk.trace.TracerProvider` is shutdown.""" - - def force_flush(self, timeout_millis: int = 30000) -> bool: # pylint: disable=no-self-use - """Export all ended spans to the configured Exporter that have not yet - been exported. - - Args: - timeout_millis: The maximum amount of time to wait for spans to be - exported. - - Returns: - False if the timeout is exceeded, True otherwise. - """ - return True - - -# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved -# pylint:disable=no-member -class SynchronousMultiSpanProcessor(SpanProcessor): - """Implementation of class:`SpanProcessor` that forwards all received - events to a list of span processors sequentially. - - The underlying span processors are called in sequential order as they were - added. - """ - - _span_processors: tuple[SpanProcessor, ...] - - def __init__(self): - # use a tuple to avoid race conditions when adding a new span and - # iterating through it on "on_start" and "on_end". - self._span_processors = () - self._lock = threading.Lock() - - def add_span_processor(self, span_processor: SpanProcessor) -> None: - """Adds a SpanProcessor to the list handled by this instance.""" - with self._lock: - self._span_processors += (span_processor,) - - def on_start( - self, - span: "Span", - parent_context: context_api.Context | None = None, - ) -> None: - for sp in self._span_processors: - sp.on_start(span, parent_context=parent_context) - - def _on_ending(self, span: "Span") -> None: - for sp in self._span_processors: - try: - # pylint: disable=protected-access - sp._on_ending(span) - except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while calling _on_ending on %s.", sp) - - def on_end(self, span: "ReadableSpan") -> None: - for sp in self._span_processors: - sp.on_end(span) - - def shutdown(self) -> None: - """Sequentially shuts down all underlying span processors.""" - for sp in self._span_processors: - sp.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Sequentially calls force_flush on all underlying - :class:`SpanProcessor` - - Args: - timeout_millis: The maximum amount of time over all span processors - to wait for spans to be exported. In case the first n span - processors exceeded the timeout followup span processors will be - skipped. - - Returns: - True if all span processors flushed their spans within the - given timeout, False otherwise. - """ - deadline_ns = time_ns() + timeout_millis * 1000000 - all_flushed = True - for sp in self._span_processors: - current_time_ns = time_ns() - if current_time_ns >= deadline_ns: - return False - - if sp.force_flush((deadline_ns - current_time_ns) // 1000000) is False: - all_flushed = False - - return all_flushed - - -class ConcurrentMultiSpanProcessor(SpanProcessor): - """Implementation of :class:`SpanProcessor` that forwards all received - events to a list of span processors in parallel. - - Calls to the underlying span processors are forwarded in parallel by - submitting them to a thread pool executor and waiting until each span - processor finished its work. The only exception is ``_on_ending``, which - is called synchronously on the thread ending the span, since that is the - only thread allowed to mutate the span while it is ending. - - Args: - num_threads: The number of threads managed by the thread pool executor - and thus defining how many span processors can work in parallel. - """ - - _span_processors: tuple[SpanProcessor, ...] - - def __init__(self, num_threads: int = 2): - # use a tuple to avoid race conditions when adding a new span and - # iterating through it on "on_start" and "on_end". - self._span_processors = () - self._lock = threading.Lock() - self._init_executor(num_threads) - if hasattr(os, "register_at_fork"): - # Only the main thread is kept in forked processed, the executor - # needs to be re-instantiated to get a fresh pool of threads: - weak_reinit = weakref.WeakMethod(self._init_executor) - - def _after_in_child() -> None: - reinit = weak_reinit() - if reinit is not None: - reinit(num_threads) - - os.register_at_fork(after_in_child=_after_in_child) - - def _init_executor(self, num_threads: int) -> None: - self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) - - def add_span_processor(self, span_processor: SpanProcessor) -> None: - """Adds a SpanProcessor to the list handled by this instance.""" - with self._lock: - self._span_processors += (span_processor,) - - def _submit_and_await( - self, - func: Callable[[SpanProcessor], Callable[..., None]], - *args: Any, - **kwargs: Any, - ): - futures = [] - for sp in self._span_processors: - future = self._executor.submit(func(sp), *args, **kwargs) - futures.append(future) - for future in futures: - future.result() - - def on_start( - self, - span: "Span", - parent_context: context_api.Context | None = None, - ) -> None: - self._submit_and_await(lambda sp: sp.on_start, span, parent_context=parent_context) - - def _on_ending(self, span: "Span") -> None: - # Unlike the other callbacks, _on_ending runs synchronously on the - # thread that ends the span: that is the only thread allowed to - # mutate the span while it is ending. - for sp in self._span_processors: - try: - # pylint: disable=protected-access - sp._on_ending(span) - except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while calling _on_ending on %s.", sp) - - def on_end(self, span: "ReadableSpan") -> None: - self._submit_and_await(lambda sp: sp.on_end, span) - - def shutdown(self) -> None: - """Shuts down all underlying span processors in parallel.""" - self._submit_and_await(lambda sp: sp.shutdown) - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Calls force_flush on all underlying span processors in parallel. - - Args: - timeout_millis: The maximum amount of time to wait for spans to be - exported. - - Returns: - True if all span processors flushed their spans within the given - timeout, False otherwise. - """ - futures = [] - for sp in self._span_processors: - future = self._executor.submit(sp.force_flush, timeout_millis) - futures.append(future) - - timeout_sec = timeout_millis / 1e3 - done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_sec) - if not_done_futures: - return False - - for future in done_futures: - if future.result() is False: - return False - - return True - - -class EventBase(abc.ABC): - def __init__(self, name: str, timestamp: int | None = None) -> None: - self._name = name - if timestamp is None: - self._timestamp = time_ns() - else: - self._timestamp = timestamp - - @property - def name(self) -> str: - return self._name - - @property - def timestamp(self) -> int: - return self._timestamp - - @property - @abc.abstractmethod - def attributes(self) -> types.Attributes: - pass - - -class Event(EventBase): - """A text annotation with a set of attributes. The attributes of an event - are immutable. - - Args: - name: Name of the event. - attributes: Attributes of the event. - timestamp: Timestamp of the event. If `None` it will filled - automatically. - """ - - def __init__( - self, - name: str, - attributes: types.Attributes = None, - timestamp: int | None = None, - limit: int | None = _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - ) -> None: - super().__init__(name, timestamp) - self._attributes = attributes - - @property - def attributes(self) -> types.Attributes: - return self._attributes - - @property - def dropped_attributes(self) -> int: - if isinstance(self._attributes, BoundedAttributes): - return self._attributes.dropped - return 0 - - -def _check_span_ended(func): - def wrapper(self, *args, **kwargs): - already_ended = False - with self._lock: # pylint: disable=protected-access - if self._is_mutable(): # pylint: disable=protected-access - func(self, *args, **kwargs) - else: - already_ended = True - - if already_ended: - logger.warning("Tried calling %s on an ended span.", func.__name__) - - return wrapper - - -def _is_valid_link(context: SpanContext, attributes: types.Attributes) -> bool: - return bool(context and (context.is_valid or (attributes or context.trace_state))) - - -class ReadableSpan: - """Provides read-only access to span attributes. - - Users should NOT be creating these objects directly. `ReadableSpan`s are created as - a direct result from using the tracing pipeline via the `Tracer`. - - """ - - def __init__( - self, - name: str, - context: trace_api.SpanContext | None = None, - parent: trace_api.SpanContext | None = None, - resource: Resource | None = None, - attributes: types.Attributes = None, - events: Sequence[Event] = (), - links: Sequence[trace_api.Link] = (), - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - instrumentation_info: InstrumentationInfo | None = None, - status: Status = Status(StatusCode.UNSET), - start_time: int | None = None, - end_time: int | None = None, - instrumentation_scope: InstrumentationScope | None = None, - ) -> None: - self._name = name - self._context = context - self._kind = kind - self._instrumentation_info = instrumentation_info - self._instrumentation_scope = instrumentation_scope - self._parent = parent - self._start_time = start_time - self._end_time = end_time - self._attributes = attributes - self._events = events - self._links = links - if resource is None: - self._resource = Resource.create({}) - else: - self._resource = resource - self._status = status - - @property - def dropped_attributes(self) -> int: - if isinstance(self._attributes, BoundedAttributes): - return self._attributes.dropped - return 0 - - @property - def dropped_events(self) -> int: - if isinstance(self._events, BoundedList): - return self._events.dropped - return 0 - - @property - def dropped_links(self) -> int: - if isinstance(self._links, BoundedList): - return self._links.dropped - return 0 - - @property - def name(self) -> str: - return self._name - - def get_span_context(self) -> trace_api.SpanContext | None: - return self._context - - @property - def context(self): - return self._context - - @property - def kind(self) -> trace_api.SpanKind: - return self._kind - - @property - def parent(self) -> trace_api.SpanContext | None: - return self._parent - - @property - def start_time(self) -> int | None: - return self._start_time - - @property - def end_time(self) -> int | None: - return self._end_time - - @property - def status(self) -> trace_api.Status: - return self._status - - @property - def attributes(self) -> types.Attributes: - return MappingProxyType(self._attributes or {}) - - @property - def events(self) -> Sequence[Event]: - return tuple(self._events) - - @property - def links(self) -> Sequence[trace_api.Link]: - return tuple(self._links) - - @property - def resource(self) -> Resource: - return self._resource - - @property - @deprecated("You should use instrumentation_scope. Deprecated since version 1.11.1.") - def instrumentation_info(self) -> InstrumentationInfo | None: - return self._instrumentation_info - - @property - def instrumentation_scope(self) -> InstrumentationScope | None: - return self._instrumentation_scope - - def to_json(self, indent: int | None = 4): - parent_id = None - if self.parent is not None: - parent_id = f"0x{trace_api.format_span_id(self.parent.span_id)}" - - start_time = None - if self._start_time: - start_time = util.ns_to_iso_str(self._start_time) - - end_time = None - if self._end_time: - end_time = util.ns_to_iso_str(self._end_time) - - status = { - "status_code": str(self._status.status_code.name), - } - if self._status.description: - status["description"] = self._status.description - - f_span = { - "name": self._name, - "context": (self._format_context(self._context) if self._context else None), - "kind": str(self.kind), - "parent_id": parent_id, - "start_time": start_time, - "end_time": end_time, - "status": status, - "attributes": self._format_attributes(self._attributes), - "events": self._format_events(self._events), - "links": self._format_links(self._links), - "resource": json.loads(self.resource.to_json()), - } - - return json.dumps(f_span, indent=indent) - - @staticmethod - def _format_context(context: SpanContext) -> dict[str, str]: - return { - "trace_id": f"0x{trace_api.format_trace_id(context.trace_id)}", - "span_id": f"0x{trace_api.format_span_id(context.span_id)}", - "trace_state": repr(context.trace_state), - } - - @staticmethod - def _format_attributes( - attributes: types.Attributes, - ) -> dict[str, Any] | None: - if attributes is not None and not isinstance(attributes, dict): - return dict(attributes) - return attributes - - @staticmethod - def _format_events(events: Sequence[Event]) -> list[dict[str, Any]]: - return [ - { - "name": event.name, - "timestamp": util.ns_to_iso_str(event.timestamp), - "attributes": Span._format_attributes( # pylint: disable=protected-access - event.attributes - ), - } - for event in events - ] - - @staticmethod - def _format_links(links: Sequence[trace_api.Link]) -> list[dict[str, Any]]: - return [ - { - "context": Span._format_context( # pylint: disable=protected-access - link.context - ), - "attributes": Span._format_attributes( # pylint: disable=protected-access - link.attributes - ), - } - for link in links - ] - - -class SpanLimits: - """The limits that should be enforced on recorded data such as events, links, attributes etc. - - This class does not enforce any limits itself. It only provides a way to read limits from env, - default values and from user provided arguments. - - All limit arguments must be either a non-negative integer, ``None`` or ``SpanLimits.UNSET``. - - - All limit arguments are optional. - - If a limit argument is not set, the class will try to read its value from the corresponding - environment variable. - - If the environment variable is not set, the default value, if any, will be used. - - Limit precedence: - - - If a model specific limit is set, it will be used. - - Else if the corresponding global limit is set, it will be used. - - Else if the model specific limit has a default value, the default value will be used. - - Else if the global limit has a default value, the default value will be used. - - Args: - max_attributes: Maximum number of attributes that can be added to a span, event, and link. - Environment variable: OTEL_ATTRIBUTE_COUNT_LIMIT - Default: {_DEFAULT_ATTRIBUTE_COUNT_LIMIT} - max_events: Maximum number of events that can be added to a Span. - Environment variable: OTEL_SPAN_EVENT_COUNT_LIMIT - Default: {_DEFAULT_SPAN_EVENT_COUNT_LIMIT} - max_links: Maximum number of links that can be added to a Span. - Environment variable: OTEL_SPAN_LINK_COUNT_LIMIT - Default: {_DEFAULT_SPAN_LINK_COUNT_LIMIT} - max_span_attributes: Maximum number of attributes that can be added to a Span. - Environment variable: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT - Default: {_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT} - max_event_attributes: Maximum number of attributes that can be added to an Event. - Default: {_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT} - max_link_attributes: Maximum number of attributes that can be added to a Link. - Default: {_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT} - max_attribute_length: Maximum length an attribute value can have. Values longer than - the specified length will be truncated. - max_span_attribute_length: Maximum length a span attribute value can have. Values longer than - the specified length will be truncated. - """ - - UNSET = -1 - - def __init__( - self, - max_attributes: int | None = None, - max_events: int | None = None, - max_links: int | None = None, - max_span_attributes: int | None = None, - max_event_attributes: int | None = None, - max_link_attributes: int | None = None, - max_attribute_length: int | None = None, - max_span_attribute_length: int | None = None, - ): - # span events and links count - self.max_events = self._from_env_if_absent( - max_events, - OTEL_SPAN_EVENT_COUNT_LIMIT, - _DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT, - ) - self.max_links = self._from_env_if_absent( - max_links, - OTEL_SPAN_LINK_COUNT_LIMIT, - _DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT, - ) - - # attribute count - global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) - self.max_attributes = ( - global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT - ) - - self.max_span_attributes = self._from_env_if_absent( - max_span_attributes, - OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT), - ) - self.max_event_attributes = self._from_env_if_absent( - max_event_attributes, - OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, - (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT), - ) - self.max_link_attributes = self._from_env_if_absent( - max_link_attributes, - OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, - (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT), - ) - - # attribute length - self.max_attribute_length = self._from_env_if_absent( - max_attribute_length, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - ) - self.max_span_attribute_length = self._from_env_if_absent( - max_span_attribute_length, - OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, - # use global attribute length limit as default - self.max_attribute_length, - ) - - def __repr__(self): - return f"{type(self).__name__}(max_span_attributes={self.max_span_attributes}, max_events_attributes={self.max_event_attributes}, max_link_attributes={self.max_link_attributes}, max_attributes={self.max_attributes}, max_events={self.max_events}, max_links={self.max_links}, max_attribute_length={self.max_attribute_length})" - - @classmethod - def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: - if value == cls.UNSET: - return None - - err_msg = "{} must be a non-negative integer but got {}" - - # if no value is provided for the limit, try to load it from env - if value is None: - # return default value if env var is not set - if env_var not in environ: - return default - - str_value = environ.get(env_var, "").strip().lower() - if str_value == _ENV_VALUE_UNSET: - return None - - try: - value = int(str_value) - except ValueError: - raise ValueError(err_msg.format(env_var, str_value)) - - if value < 0: - raise ValueError(err_msg.format(env_var, value)) - return value - - -_UnsetLimits = SpanLimits( - max_attributes=SpanLimits.UNSET, - max_events=SpanLimits.UNSET, - max_links=SpanLimits.UNSET, - max_span_attributes=SpanLimits.UNSET, - max_event_attributes=SpanLimits.UNSET, - max_link_attributes=SpanLimits.UNSET, - max_attribute_length=SpanLimits.UNSET, - max_span_attribute_length=SpanLimits.UNSET, -) - -# not removed for backward compat. please use SpanLimits instead. -SPAN_ATTRIBUTE_COUNT_LIMIT = SpanLimits._from_env_if_absent( # pylint: disable=protected-access - None, - OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, -) - - -class Span(trace_api.Span, ReadableSpan): - """See `opentelemetry.trace.Span`. - - Users should create `Span` objects via the `Tracer` instead of this - constructor. - - Args: - name: The name of the operation this span represents - context: The immutable span context - parent: This span's parent's `opentelemetry.trace.SpanContext`, or - None if this is a root span - sampler: The sampler used to create this span - trace_config: Unused. Originally intended for trace-level configuration - from the OpenTelemetry protocol, but the upstream ``TraceConfig`` - proto was removed. Retained for backwards compatibility. - resource: Entity producing telemetry - attributes: The span's attributes to be exported - events: Timestamped events to be exported - links: Links to other spans to be exported - span_processor: `SpanProcessor` to invoke when starting and ending - this `Span`. - limits: `SpanLimits` instance that was passed to the `TracerProvider` - """ - - def __new__(cls, *args, **kwargs): - if cls is Span: - raise TypeError("Span must be instantiated via a tracer.") - return super().__new__(cls) - - # pylint: disable=too-many-locals - def __init__( - self, - name: str, - context: trace_api.SpanContext, - parent: trace_api.SpanContext | None = None, - sampler: sampling.Sampler | None = None, - trace_config: None = None, - resource: Resource | None = None, - attributes: types.Attributes = None, - events: Sequence[Event] | None = None, - links: Sequence[trace_api.Link] = (), - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - span_processor: SpanProcessor = SpanProcessor(), - instrumentation_info: InstrumentationInfo | None = None, - record_exception: bool = True, - set_status_on_exception: bool = True, - limits=_UnsetLimits, - instrumentation_scope: InstrumentationScope | None = None, - *, - record_end_metrics: Callable[[], None] | None = None, - ) -> None: - if resource is None: - resource = Resource.create({}) - super().__init__( - name=name, - context=context, - parent=parent, - kind=kind, - resource=resource, - instrumentation_info=instrumentation_info, - instrumentation_scope=instrumentation_scope, - ) - self._sampler = sampler - self._trace_config = trace_config - self._record_exception = record_exception - self._set_status_on_exception = set_status_on_exception - self._span_processor = span_processor - self._limits = limits - self._lock = threading.Lock() - # Identifier of the thread running end() while the _on_ending - # callbacks execute: the end timestamp is already set, but the span - # must remain mutable from that thread until the callbacks return. - self._ending_thread: int | None = None - self._attributes = BoundedAttributes( - self._limits.max_span_attributes, - attributes, - immutable=False, - max_value_len=self._limits.max_span_attribute_length, - ) - self._events = self._new_events() - if events: - for event in events: - # Immutable set to true so attributes cannot be added or removed after creation. - event._attributes = BoundedAttributes( - self._limits.max_event_attributes, - event.attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - self._events.append(event) - - self._links = self._new_links(links) - - self._record_end_metrics = record_end_metrics - - def __repr__(self): - return f'{type(self).__name__}(name="{self._name}", context={self._context})' - - def _new_events(self): - return BoundedList(self._limits.max_events) - - def _new_links(self, links: Sequence[trace_api.Link]): - if not links: - return BoundedList(self._limits.max_links) - - valid_links = [] - for link in links: - if link and _is_valid_link(link.context, link.attributes): - # Immutable set to true so attributes cannot be added or removed after creation. - # pylint: disable=protected-access - link._attributes = BoundedAttributes( - self._limits.max_link_attributes, - link.attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - valid_links.append(link) - - return BoundedList.from_seq(self._limits.max_links, valid_links) - - def get_span_context(self) -> trace_api.SpanContext: - return typing.cast(trace_api.SpanContext, self._context) - - def set_attributes(self, attributes: Mapping[str, types.AnyValue]) -> None: - with self._lock: - if not self._is_mutable(): - logger.warning("Setting attribute on ended span.") - return - - self._attributes._set_items(attributes) # pylint: disable=protected-access - - def set_attribute(self, key: str, value: types.AnyValue) -> None: - with self._lock: - if not self._is_mutable(): - logger.warning("Setting attribute on ended span.") - return - - self._attributes[key] = value - - @_check_span_ended - def _add_event(self, event: EventBase) -> None: - self._events.append(event) - - def add_event( - self, - name: str, - attributes: types.Attributes = None, - timestamp: int | None = None, - ) -> None: - # Immutable set to true so attributes cannot be added or removed after creation. - attributes = BoundedAttributes( - self._limits.max_event_attributes, - attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - self._add_event( - Event( - name=name, - attributes=attributes, - timestamp=timestamp, - ) - ) - - @_check_span_ended - def _add_link(self, link: trace_api.Link) -> None: - self._links.append(link) - - def add_link( - self, - context: SpanContext, - attributes: types.Attributes = None, - ) -> None: - if not _is_valid_link(context, attributes): - return - # Immutable set to true so attributes cannot be added or removed after creation. - attributes = BoundedAttributes( - self._limits.max_link_attributes, - attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - self._add_link( - trace_api.Link( - context=context, - attributes=attributes, - ) - ) - - def _readable_span(self) -> ReadableSpan: - return ReadableSpan( - name=self._name, - context=self._context, - parent=self._parent, - resource=self._resource, - attributes=self._attributes, - events=self._events, - links=self._links, - kind=self.kind, - status=self._status, - start_time=self._start_time, - end_time=self._end_time, - instrumentation_info=self._instrumentation_info, - instrumentation_scope=self._instrumentation_scope, - ) - - def start( - self, - start_time: int | None = None, - parent_context: context_api.Context | None = None, - ) -> None: - with self._lock: - if self._start_time is not None: - logger.warning("Calling start() on a started span.") - return - self._start_time = start_time if start_time is not None else time_ns() - - self._span_processor.on_start(self, parent_context=parent_context) - - def end(self, end_time: int | None = None) -> None: - with self._lock: - if self._start_time is None: - raise RuntimeError("Calling end() on a not started span.") - if self._end_time is not None: - logger.warning("Calling end() on an ended span.") - return - - self._ending_thread = threading.get_ident() - self._end_time = end_time if end_time is not None else time_ns() - - # The span must remain mutable from this thread while the _on_ending - # callbacks run; it becomes immutable once they have returned. - # pylint: disable=protected-access - try: - if self._record_end_metrics: - self._record_end_metrics() - self._span_processor._on_ending(self) - finally: - with self._lock: - self._ending_thread = None - self._attributes._immutable = True - self._span_processor.on_end(self._readable_span()) - - @_check_span_ended - def update_name(self, name: str) -> None: - self._name = name - - def _is_mutable(self) -> bool: - return self._end_time is None or self._ending_thread == threading.get_ident() - - def is_recording(self) -> bool: - # Unlike the mutators, this check is deliberately not taken under - # self._lock: it does not modify the span, and the answer can become - # stale as soon as it is returned anyway, since another thread may - # end the span right after this call. - return self._is_mutable() - - @_check_span_ended - def set_status( - self, - status: Status | StatusCode, - description: str | None = None, - ) -> None: - if isinstance(status, Status): - if description is not None: - logger.warning( - "Description %s ignored. Use either `Status` or `(StatusCode, Description)`", - description, - ) - new_status = status - elif isinstance(status, StatusCode): - new_status = Status(status, description) - else: - return - - if self._accepts_status(new_status): - self._status = new_status - - def _accepts_status(self, new_status: Status) -> bool: - """Decide whether ``new_status`` may replace the status already recorded. - - The codes are totally ordered, ``Ok > Error > Unset``, an attempt to set - ``Unset`` is ignored, and ``Ok`` is final. Once those two are handled, a - differing code always outranks the one recorded. A repeat of the same code - gets through only when it fills in a description that is still missing, - and an empty description counts as missing. - """ - if new_status.status_code is StatusCode.UNSET: - return False - - current = self._status - if current is None: - return True - if current.status_code is StatusCode.OK: - return False - - # An empty description is equivalent to an absent one. - return current.status_code is not new_status.status_code or ( - not current.description and bool(new_status.description) - ) - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - """Ends context manager and calls `end` on the `Span`.""" - if exc_val is not None and self.is_recording(): - # Record the exception as an event - # pylint:disable=protected-access - if self._record_exception: - self.record_exception(exception=exc_val, escaped=True) - # Records status if span is used as context manager - # i.e. with tracer.start_span() as span: - if self._set_status_on_exception: - self.set_status( - Status( - status_code=StatusCode.ERROR, - description=(f"{type(exc_val).__name__}: {exc_val}"), - ) - ) - - super().__exit__(exc_type, exc_val, exc_tb) - - def record_exception( - self, - exception: BaseException, - attributes: types.Attributes = None, - timestamp: int | None = None, - escaped: bool = False, - ) -> None: - """Records an exception as a span event.""" - stacktrace = "".join(traceback.format_exception(exception)) - module = type(exception).__module__ - qualname = type(exception).__qualname__ - exception_type = f"{module}.{qualname}" if module and module != "builtins" else qualname - _attributes: MutableMapping[str, types.AnyValue] = { - EXCEPTION_TYPE: exception_type, - EXCEPTION_MESSAGE: str(exception), - EXCEPTION_STACKTRACE: stacktrace, - EXCEPTION_ESCAPED: str(escaped), - } - if attributes: - _attributes.update(attributes) - self.add_event(name="exception", attributes=_attributes, timestamp=timestamp) - - -class _Span(Span): - """Protected implementation of `opentelemetry.trace.Span`. - - This constructor exists to prevent the instantiation of the `Span` class - by other mechanisms than through the `Tracer`. - """ - - -@dataclass -class _TracerConfig: - is_enabled: bool - - @classmethod - def default(cls): - return cls(is_enabled=True) - - -class Tracer(trace_api.Tracer): - """See `opentelemetry.trace.Tracer`.""" - - def __init__( - self, - sampler: sampling.Sampler, - resource: Resource, - span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor, - id_generator: IdGenerator, - instrumentation_info: InstrumentationInfo, - span_limits: SpanLimits, - instrumentation_scope: InstrumentationScope, - *, - meter_provider: metrics_api.MeterProvider | None = None, - _tracer_config: _TracerConfig | None = None, - ) -> None: - self.sampler = sampler - self.resource = resource - self.span_processor = span_processor - self.id_generator = id_generator - self.instrumentation_info = instrumentation_info - self._span_limits = span_limits - self._instrumentation_scope = instrumentation_scope - self._tracer_config = _tracer_config or _TracerConfig.default() - - meter_provider = meter_provider or metrics_api.get_meter_provider() - self._tracer_metrics = create_tracer_metrics( - meter_provider, - parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), - ) - - def _set_tracer_config(self, tracer_config: _TracerConfig): - self._tracer_config = tracer_config - - def _set_resource(self, resource: Resource) -> None: - self.resource = resource - - def _is_enabled(self) -> bool: - """If the tracer is not enabled, start_span will create a NonRecordingSpan""" - return self._tracer_config.is_enabled - - @_agnosticcontextmanager # pylint: disable=protected-access - def start_as_current_span( - self, - name: str, - context: context_api.Context | None = None, - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - attributes: types.Attributes = None, - links: Sequence[trace_api.Link] | None = (), - start_time: int | None = None, - record_exception: bool = True, - set_status_on_exception: bool = True, - end_on_exit: bool = True, - ) -> Iterator[trace_api.Span]: - span = self.start_span( - name=name, - context=context, - kind=kind, - attributes=attributes, - links=links, - start_time=start_time, - record_exception=record_exception, - set_status_on_exception=set_status_on_exception, - ) - with trace_api.use_span( - span, - end_on_exit=end_on_exit, - record_exception=record_exception, - set_status_on_exception=set_status_on_exception, - ) as span: - yield span - - def start_span( # pylint: disable=too-many-locals - self, - name: str, - context: context_api.Context | None = None, - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - attributes: types.Attributes = None, - links: Sequence[trace_api.Link] | None = (), - start_time: int | None = None, - record_exception: bool = True, - set_status_on_exception: bool = True, - ) -> trace_api.Span: - links = links or () - parent_span_context = trace_api.get_current_span(context).get_span_context() - - if parent_span_context is not None and not isinstance(parent_span_context, trace_api.SpanContext): - raise TypeError("parent_span_context must be a SpanContext or None.") - - if not self._is_enabled(): - return trace_api.NonRecordingSpan(context=parent_span_context) - - # is_valid determines root span - if parent_span_context is None or not parent_span_context.is_valid: - parent_span_context = None - trace_id = self.id_generator.generate_trace_id() - else: - trace_id = parent_span_context.trace_id - - # The sampler decides whether to create a real or no-op span at the - # time of span creation. No-op spans do not record events, and are not - # exported. - # The sampler may also add attributes to the newly-created span, e.g. - # to include information about the sampling result. - # The sampler may also modify the parent span context's tracestate - sampling_result = self.sampler.should_sample(context, trace_id, name, kind, attributes, links) - - trace_flags = ( - trace_api.TraceFlags(trace_api.TraceFlags.SAMPLED) - if sampling_result.decision.is_sampled() - else trace_api.TraceFlags(trace_api.TraceFlags.DEFAULT) - ) - - if parent_span_context is None: - random_trace_id = self.id_generator.is_trace_id_random() - else: - random_trace_id = parent_span_context.trace_flags.random_trace_id - - if random_trace_id: - trace_flags = trace_api.TraceFlags(trace_flags | trace_api.TraceFlags.RANDOM_TRACE_ID) - - span_context = trace_api.SpanContext( - trace_id, - self.id_generator.generate_span_id(), - is_remote=False, - trace_flags=trace_flags, - trace_state=sampling_result.trace_state, - ) - - record_end_metrics = self._tracer_metrics.start_span(parent_span_context, sampling_result.decision) - - # Only record if is_recording() is true - if sampling_result.decision.is_recording(): - # pylint:disable=protected-access - span = _Span( - name=name, - context=span_context, - parent=parent_span_context, - sampler=self.sampler, - resource=self.resource, - attributes=sampling_result.attributes, - span_processor=self.span_processor, - kind=kind, - links=links, - instrumentation_info=self.instrumentation_info, - record_exception=record_exception, - set_status_on_exception=set_status_on_exception, - limits=self._span_limits, - instrumentation_scope=self._instrumentation_scope, - record_end_metrics=record_end_metrics, - ) - span.start(start_time=start_time, parent_context=context) - else: - span = trace_api.NonRecordingSpan(context=span_context) - return span - - -_TracerConfiguratorT = Callable[[InstrumentationScope], _TracerConfig] -_RuleBasedTracerConfigurator = RuleBasedConfigurator[_TracerConfig] - - -def _default_tracer_configurator( - tracer_scope: InstrumentationScope, -) -> _TracerConfig: - """Default Tracer Configurator implementation - - In order to update Tracers configs you need to call - TracerProvider._set_tracer_configurator with a function - implementing this interface returning a Tracer Config.""" - return _RuleBasedTracerConfigurator( - rules=[], - default_config=_TracerConfig.default(), - )(tracer_scope) - - -def _disable_tracer_configurator( - tracer_scope: InstrumentationScope, -) -> _TracerConfig: - return _RuleBasedTracerConfigurator( - rules=[], - default_config=_TracerConfig(is_enabled=False), - )(tracer_scope) - - -class TracerProvider(trace_api.TracerProvider): - """See `opentelemetry.trace.TracerProvider`.""" - - def __init__( - self, - sampler: sampling.Sampler | None = None, - resource: Resource | None = None, - shutdown_on_exit: bool = True, - active_span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor | None = None, - id_generator: IdGenerator | None = None, - span_limits: SpanLimits | None = None, - *, - meter_provider: metrics_api.MeterProvider | None = None, - _tracer_configurator: _TracerConfiguratorT | None = None, - ) -> None: - self._active_span_processor = active_span_processor or SynchronousMultiSpanProcessor() - if id_generator is None: - self.id_generator = RandomIdGenerator() - else: - self.id_generator = id_generator - if resource is None: - self._resource = Resource.create({}) - else: - self._resource = resource - if not sampler: - sampler = sampling._get_from_env_or_default() - self.sampler = sampler - self._span_limits = span_limits or SpanLimits() - disabled = environ.get(OTEL_SDK_DISABLED, "") - self._disabled = disabled.lower().strip() == "true" - self._atexit_handler = None - self._meter_provider = meter_provider - - if shutdown_on_exit: - self._atexit_handler = atexit.register(self.shutdown) - - self._tracer_configurator = _tracer_configurator or _default_tracer_configurator - self._tracers_lock = threading.Lock() - self._tracers: dict[InstrumentationScope, Tracer] = {} - if hasattr(os, "register_at_fork"): - weak_at_fork = weakref.WeakMethod(self._handle_fork) - - def _after_in_child() -> None: - if at_fork := weak_at_fork(): - at_fork() - - os.register_at_fork(after_in_child=_after_in_child) - - def _handle_fork(self) -> None: - self._tracers_lock = threading.Lock() - self._update_resource(_get_process_dependent_resource()) - - def _set_tracer_configurator(self, *, tracer_configurator: _TracerConfiguratorT): - """This is the function used to update the TracerProvider TracerConfigurator - - Setting a new TracerConfigurator for a TracerProvider will update the - TracerConfig of all Tracers create by this TracerProvider. - """ - self._tracer_configurator = tracer_configurator - with self._tracers_lock: - for instrumentation_scope, tracer in self._tracers.items(): - tracer_config = self._apply_tracer_configurator(instrumentation_scope) - # pylint: disable-next=protected-access - tracer._set_tracer_config(tracer_config) - - @property - def resource(self) -> Resource: - return self._resource - - def _update_resource(self, resource: Resource) -> None: - with self._tracers_lock: - self._resource = self._resource.merge(resource) - for tracer in self._tracers.values(): - tracer._set_resource(self._resource) # pylint: disable=protected-access - - def _apply_tracer_configurator(self, instrumentation_scope: InstrumentationScope): - try: - return self._tracer_configurator(instrumentation_scope) - except Exception: # pylint: disable=broad-exception-caught - logger.exception( - "Failed to create a Tracer Config for %s, using default Tracer config", - instrumentation_scope, - ) - return _TracerConfig.default() - - def get_tracer( - self, - instrumenting_module_name: str, - instrumenting_library_version: str | None = None, - schema_url: str | None = None, - attributes: types.Attributes = None, - ) -> "trace_api.Tracer": - if self._disabled: - return NoOpTracer() - if not instrumenting_module_name: # Reject empty strings too. - instrumenting_module_name = "" - logger.error("get_tracer called with missing module name.") - if instrumenting_library_version is None: - instrumenting_library_version = "" - - filterwarnings( - "ignore", - message=(r"You should use InstrumentationScope. Deprecated since version 1.11.1."), - category=DeprecationWarning, - module="opentelemetry.sdk.trace", - ) - - instrumentation_info = InstrumentationInfo( - instrumenting_module_name, - instrumenting_library_version, - schema_url, - ) - - instrumentation_scope = InstrumentationScope( - instrumenting_module_name, - instrumenting_library_version, - schema_url, - attributes, - ) - - with self._tracers_lock: - if instrumentation_scope in self._tracers: - return self._tracers[instrumentation_scope] - - tracer_config = self._apply_tracer_configurator(instrumentation_scope) - tracer = Tracer( - self.sampler, - self.resource, - self._active_span_processor, - self.id_generator, - instrumentation_info, - self._span_limits, - instrumentation_scope, - meter_provider=self._meter_provider, - _tracer_config=tracer_config, - ) - self._tracers[instrumentation_scope] = tracer - - return tracer - - def add_span_processor(self, span_processor: SpanProcessor) -> None: - """Registers a new :class:`SpanProcessor` for this `TracerProvider`. - - The span processors are invoked in the same order they are registered. - """ - - # no lock here because add_span_processor is thread safe for both - # SynchronousMultiSpanProcessor and ConcurrentMultiSpanProcessor. - self._active_span_processor.add_span_processor(span_processor) - - def shutdown(self) -> None: - """Shut down the span processors added to the tracer provider.""" - self._active_span_processor.shutdown() - if self._atexit_handler is not None: - atexit.unregister(self._atexit_handler) - self._atexit_handler = None - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Requests the active span processor to process all spans that have not - yet been processed. - - By default force flush is called sequentially on all added span - processors. This means that span processors further back in the list - have less time to flush their spans. - To have span processors flush their spans in parallel it is possible to - initialize the tracer provider with an instance of - `ConcurrentMultiSpanProcessor` at the cost of using multiple threads. - - Args: - timeout_millis: The maximum amount of time to wait for spans to be - processed. - - Returns: - False if the timeout is exceeded, True otherwise. - """ - return self._active_span_processor.force_flush(timeout_millis) +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=too-many-lines +import abc +import atexit +import concurrent.futures +import json +import logging +import os +import threading +import traceback +import typing +import weakref +from collections.abc import ( + Callable, + Iterator, + Mapping, + MutableMapping, + Sequence, +) +from dataclasses import dataclass +from os import environ +from time import time_ns +from types import MappingProxyType, TracebackType +from typing import ( + Any, +) +from warnings import filterwarnings + +from typing_extensions import deprecated + +from opentelemetry import context as context_api +from opentelemetry import metrics as metrics_api +from opentelemetry import trace as trace_api +from opentelemetry.attributes import BoundedAttributes +from opentelemetry.sdk import util +from opentelemetry.sdk.environment_variables import ( + OTEL_ATTRIBUTE_COUNT_LIMIT, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, + OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, + OTEL_SDK_DISABLED, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_SPAN_EVENT_COUNT_LIMIT, + OTEL_SPAN_LINK_COUNT_LIMIT, +) +from opentelemetry.sdk.environment_variables._internal import ( + parse_boolean_environment_variable, +) +from opentelemetry.sdk.resources import ( + Resource, + _get_process_dependent_resource, +) +from opentelemetry.sdk.trace import sampling +from opentelemetry.sdk.trace._tracer_metrics import create_tracer_metrics +from opentelemetry.sdk.trace.id_generator import IdGenerator, RandomIdGenerator +from opentelemetry.sdk.util import BoundedList +from opentelemetry.sdk.util._configurator import RuleBasedConfigurator +from opentelemetry.sdk.util.instrumentation import ( + InstrumentationInfo, + InstrumentationScope, +) +from opentelemetry.semconv.attributes.exception_attributes import ( + EXCEPTION_ESCAPED, + EXCEPTION_MESSAGE, + EXCEPTION_STACKTRACE, + EXCEPTION_TYPE, +) +from opentelemetry.trace import NoOpTracer, SpanContext +from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.util import types +from opentelemetry.util._decorator import _agnosticcontextmanager + +logger = logging.getLogger(__name__) + + +_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT = 128 +_DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT = 128 + + +_ENV_VALUE_UNSET = "" + + +class SpanProcessor: + """Interface which allows hooks for SDK's `Span` start and end method + invocations. + + Span processors can be registered directly using + :func:`TracerProvider.add_span_processor` and they are invoked + in the same order as they were registered. + """ + + def on_start( + self, + span: "Span", + parent_context: context_api.Context | None = None, + ) -> None: + """Called when a :class:`opentelemetry.trace.Span` is started. + + This method is called synchronously on the thread that starts the + span, therefore it should not block or throw an exception. + + Args: + span: The :class:`opentelemetry.trace.Span` that just started. + parent_context: The parent context of the span that just started. + """ + + def _on_ending(self, span: "Span") -> None: + """Called when a :class:`opentelemetry.trace.Span` is ending. + + This method is called synchronously on the thread that ends the + span, therefore it should not block or throw an exception. + + Args: + span: The :class:`opentelemetry.trace.Span` that is ending. + """ + + def on_end(self, span: "ReadableSpan") -> None: + """Called when a :class:`opentelemetry.trace.Span` is ended. + + This method is called synchronously on the thread that ends the + span, therefore it should not block or throw an exception. + + Args: + span: The :class:`opentelemetry.trace.Span` that just ended. + """ + + def shutdown(self) -> None: + """Called when a :class:`opentelemetry.sdk.trace.TracerProvider` is shutdown.""" + + def force_flush(self, timeout_millis: int = 30000) -> bool: # pylint: disable=no-self-use + """Export all ended spans to the configured Exporter that have not yet + been exported. + + Args: + timeout_millis: The maximum amount of time to wait for spans to be + exported. + + Returns: + False if the timeout is exceeded, True otherwise. + """ + return True + + +# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved +# pylint:disable=no-member +class SynchronousMultiSpanProcessor(SpanProcessor): + """Implementation of class:`SpanProcessor` that forwards all received + events to a list of span processors sequentially. + + The underlying span processors are called in sequential order as they were + added. + """ + + _span_processors: tuple[SpanProcessor, ...] + + def __init__(self): + # use a tuple to avoid race conditions when adding a new span and + # iterating through it on "on_start" and "on_end". + self._span_processors = () + self._lock = threading.Lock() + + def add_span_processor(self, span_processor: SpanProcessor) -> None: + """Adds a SpanProcessor to the list handled by this instance.""" + with self._lock: + self._span_processors += (span_processor,) + + def on_start( + self, + span: "Span", + parent_context: context_api.Context | None = None, + ) -> None: + for sp in self._span_processors: + sp.on_start(span, parent_context=parent_context) + + def _on_ending(self, span: "Span") -> None: + for sp in self._span_processors: + try: + # pylint: disable=protected-access + sp._on_ending(span) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while calling _on_ending on %s.", sp) + + def on_end(self, span: "ReadableSpan") -> None: + for sp in self._span_processors: + sp.on_end(span) + + def shutdown(self) -> None: + """Sequentially shuts down all underlying span processors.""" + for sp in self._span_processors: + sp.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Sequentially calls force_flush on all underlying + :class:`SpanProcessor` + + Args: + timeout_millis: The maximum amount of time over all span processors + to wait for spans to be exported. In case the first n span + processors exceeded the timeout followup span processors will be + skipped. + + Returns: + True if all span processors flushed their spans within the + given timeout, False otherwise. + """ + deadline_ns = time_ns() + timeout_millis * 1000000 + all_flushed = True + for sp in self._span_processors: + current_time_ns = time_ns() + if current_time_ns >= deadline_ns: + return False + + if sp.force_flush((deadline_ns - current_time_ns) // 1000000) is False: + all_flushed = False + + return all_flushed + + +class ConcurrentMultiSpanProcessor(SpanProcessor): + """Implementation of :class:`SpanProcessor` that forwards all received + events to a list of span processors in parallel. + + Calls to the underlying span processors are forwarded in parallel by + submitting them to a thread pool executor and waiting until each span + processor finished its work. The only exception is ``_on_ending``, which + is called synchronously on the thread ending the span, since that is the + only thread allowed to mutate the span while it is ending. + + Args: + num_threads: The number of threads managed by the thread pool executor + and thus defining how many span processors can work in parallel. + """ + + _span_processors: tuple[SpanProcessor, ...] + + def __init__(self, num_threads: int = 2): + # use a tuple to avoid race conditions when adding a new span and + # iterating through it on "on_start" and "on_end". + self._span_processors = () + self._lock = threading.Lock() + self._init_executor(num_threads) + if hasattr(os, "register_at_fork"): + # Only the main thread is kept in forked processed, the executor + # needs to be re-instantiated to get a fresh pool of threads: + weak_reinit = weakref.WeakMethod(self._init_executor) + + def _after_in_child() -> None: + reinit = weak_reinit() + if reinit is not None: + reinit(num_threads) + + os.register_at_fork(after_in_child=_after_in_child) + + def _init_executor(self, num_threads: int) -> None: + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) + + def add_span_processor(self, span_processor: SpanProcessor) -> None: + """Adds a SpanProcessor to the list handled by this instance.""" + with self._lock: + self._span_processors += (span_processor,) + + def _submit_and_await( + self, + func: Callable[[SpanProcessor], Callable[..., None]], + *args: Any, + **kwargs: Any, + ): + futures = [] + for sp in self._span_processors: + try: + future = self._executor.submit(func(sp), *args, **kwargs) + futures.append(future) + except RuntimeError: + # The executor may already be shut down when this runs from an + # atexit handler: CPython runs threading._shutdown() before the + # atexit queue, so the ThreadPoolExecutor is closed by the time + # TracerProvider.shutdown() runs. Run inline so buffered work is + # still flushed instead of being silently dropped. A + # RuntimeError raised by the callback itself is not caught here + # and still propagates via future.result() below. + logger.debug( + "Executor is shut down; running SpanProcessor callback inline.", + ) + func(sp)(*args, **kwargs) + for future in futures: + future.result() + + def on_start( + self, + span: "Span", + parent_context: context_api.Context | None = None, + ) -> None: + self._submit_and_await(lambda sp: sp.on_start, span, parent_context=parent_context) + + def _on_ending(self, span: "Span") -> None: + # Unlike the other callbacks, _on_ending runs synchronously on the + # thread that ends the span: that is the only thread allowed to + # mutate the span while it is ending. + for sp in self._span_processors: + try: + # pylint: disable=protected-access + sp._on_ending(span) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while calling _on_ending on %s.", sp) + + def on_end(self, span: "ReadableSpan") -> None: + self._submit_and_await(lambda sp: sp.on_end, span) + + def shutdown(self) -> None: + """Shuts down all underlying span processors in parallel.""" + self._submit_and_await(lambda sp: sp.shutdown) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Calls force_flush on all underlying span processors in parallel. + + Args: + timeout_millis: The maximum amount of time to wait for spans to be + exported. + + Returns: + True if all span processors flushed their spans within the given + timeout, False otherwise. + """ + futures = [] + for sp in self._span_processors: + future = self._executor.submit(sp.force_flush, timeout_millis) + futures.append(future) + + timeout_sec = timeout_millis / 1e3 + done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_sec) + if not_done_futures: + return False + + for future in done_futures: + if future.result() is False: + return False + + return True + + +class EventBase(abc.ABC): + def __init__(self, name: str, timestamp: int | None = None) -> None: + self._name = name + if timestamp is None: + self._timestamp = time_ns() + else: + self._timestamp = timestamp + + @property + def name(self) -> str: + return self._name + + @property + def timestamp(self) -> int: + return self._timestamp + + @property + @abc.abstractmethod + def attributes(self) -> types.Attributes: + pass + + +class Event(EventBase): + """A text annotation with a set of attributes. The attributes of an event + are immutable. + + Args: + name: Name of the event. + attributes: Attributes of the event. + timestamp: Timestamp of the event. If `None` it will filled + automatically. + """ + + def __init__( + self, + name: str, + attributes: types.Attributes = None, + timestamp: int | None = None, + limit: int | None = _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + ) -> None: + super().__init__(name, timestamp) + self._attributes = attributes + + @property + def attributes(self) -> types.Attributes: + return self._attributes + + @property + def dropped_attributes(self) -> int: + if isinstance(self._attributes, BoundedAttributes): + return self._attributes.dropped + return 0 + + +def _check_span_ended(func): + def wrapper(self, *args, **kwargs): + already_ended = False + with self._lock: # pylint: disable=protected-access + if self._is_mutable(): # pylint: disable=protected-access + func(self, *args, **kwargs) + else: + already_ended = True + + if already_ended: + logger.warning("Tried calling %s on an ended span.", func.__name__) + + return wrapper + + +def _is_valid_link(context: SpanContext, attributes: types.Attributes) -> bool: + return bool(context and (context.is_valid or (attributes or context.trace_state))) + + +class ReadableSpan: + """Provides read-only access to span attributes. + + Users should NOT be creating these objects directly. `ReadableSpan`s are created as + a direct result from using the tracing pipeline via the `Tracer`. + + """ + + def __init__( + self, + name: str, + context: trace_api.SpanContext | None = None, + parent: trace_api.SpanContext | None = None, + resource: Resource | None = None, + attributes: types.Attributes = None, + events: Sequence[Event] = (), + links: Sequence[trace_api.Link] = (), + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + instrumentation_info: InstrumentationInfo | None = None, + status: Status = Status(StatusCode.UNSET), + start_time: int | None = None, + end_time: int | None = None, + instrumentation_scope: InstrumentationScope | None = None, + ) -> None: + self._name = name + self._context = context + self._kind = kind + self._instrumentation_info = instrumentation_info + self._instrumentation_scope = instrumentation_scope + self._parent = parent + self._start_time = start_time + self._end_time = end_time + self._attributes = attributes + self._events = events + self._links = links + if resource is None: + self._resource = Resource.create({}) + else: + self._resource = resource + self._status = status + + @property + def dropped_attributes(self) -> int: + if isinstance(self._attributes, BoundedAttributes): + return self._attributes.dropped + return 0 + + @property + def dropped_events(self) -> int: + if isinstance(self._events, BoundedList): + return self._events.dropped + return 0 + + @property + def dropped_links(self) -> int: + if isinstance(self._links, BoundedList): + return self._links.dropped + return 0 + + @property + def name(self) -> str: + return self._name + + def get_span_context(self) -> trace_api.SpanContext | None: + return self._context + + @property + def context(self): + return self._context + + @property + def kind(self) -> trace_api.SpanKind: + return self._kind + + @property + def parent(self) -> trace_api.SpanContext | None: + return self._parent + + @property + def start_time(self) -> int | None: + return self._start_time + + @property + def end_time(self) -> int | None: + return self._end_time + + @property + def status(self) -> trace_api.Status: + return self._status + + @property + def attributes(self) -> types.Attributes: + return MappingProxyType(self._attributes or {}) + + @property + def events(self) -> Sequence[Event]: + return tuple(self._events) + + @property + def links(self) -> Sequence[trace_api.Link]: + return tuple(self._links) + + @property + def resource(self) -> Resource: + return self._resource + + @property + @deprecated("You should use instrumentation_scope. Deprecated since version 1.11.1.") + def instrumentation_info(self) -> InstrumentationInfo | None: + return self._instrumentation_info + + @property + def instrumentation_scope(self) -> InstrumentationScope | None: + return self._instrumentation_scope + + def to_json(self, indent: int | None = 4): + parent_id = None + if self.parent is not None: + parent_id = f"0x{trace_api.format_span_id(self.parent.span_id)}" + + start_time = None + if self._start_time: + start_time = util.ns_to_iso_str(self._start_time) + + end_time = None + if self._end_time: + end_time = util.ns_to_iso_str(self._end_time) + + status = { + "status_code": str(self._status.status_code.name), + } + if self._status.description: + status["description"] = self._status.description + + f_span = { + "name": self._name, + "context": (self._format_context(self._context) if self._context else None), + "kind": str(self.kind), + "parent_id": parent_id, + "start_time": start_time, + "end_time": end_time, + "status": status, + "attributes": self._format_attributes(self._attributes), + "events": self._format_events(self._events), + "links": self._format_links(self._links), + "resource": json.loads(self.resource.to_json()), + } + + return json.dumps(f_span, indent=indent) + + @staticmethod + def _format_context(context: SpanContext) -> dict[str, str]: + return { + "trace_id": f"0x{trace_api.format_trace_id(context.trace_id)}", + "span_id": f"0x{trace_api.format_span_id(context.span_id)}", + "trace_state": repr(context.trace_state), + } + + @staticmethod + def _format_attributes( + attributes: types.Attributes, + ) -> dict[str, Any] | None: + if attributes is not None and not isinstance(attributes, dict): + return dict(attributes) + return attributes + + @staticmethod + def _format_events(events: Sequence[Event]) -> list[dict[str, Any]]: + return [ + { + "name": event.name, + "timestamp": util.ns_to_iso_str(event.timestamp), + "attributes": Span._format_attributes( # pylint: disable=protected-access + event.attributes + ), + } + for event in events + ] + + @staticmethod + def _format_links(links: Sequence[trace_api.Link]) -> list[dict[str, Any]]: + return [ + { + "context": Span._format_context( # pylint: disable=protected-access + link.context + ), + "attributes": Span._format_attributes( # pylint: disable=protected-access + link.attributes + ), + } + for link in links + ] + + +class SpanLimits: + """The limits that should be enforced on recorded data such as events, links, attributes etc. + + This class does not enforce any limits itself. It only provides a way to read limits from env, + default values and from user provided arguments. + + All limit arguments must be either a non-negative integer, ``None`` or ``SpanLimits.UNSET``. + + - All limit arguments are optional. + - If a limit argument is not set, the class will try to read its value from the corresponding + environment variable. + - If the environment variable is not set, the default value, if any, will be used. + + Limit precedence: + + - If a model specific limit is set, it will be used. + - Else if the corresponding global limit is set, it will be used. + - Else if the model specific limit has a default value, the default value will be used. + - Else if the global limit has a default value, the default value will be used. + + Args: + max_attributes: Maximum number of attributes that can be added to a span, event, and link. + Environment variable: OTEL_ATTRIBUTE_COUNT_LIMIT + Default: {_DEFAULT_ATTRIBUTE_COUNT_LIMIT} + max_events: Maximum number of events that can be added to a Span. + Environment variable: OTEL_SPAN_EVENT_COUNT_LIMIT + Default: {_DEFAULT_SPAN_EVENT_COUNT_LIMIT} + max_links: Maximum number of links that can be added to a Span. + Environment variable: OTEL_SPAN_LINK_COUNT_LIMIT + Default: {_DEFAULT_SPAN_LINK_COUNT_LIMIT} + max_span_attributes: Maximum number of attributes that can be added to a Span. + Environment variable: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + Default: {_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT} + max_event_attributes: Maximum number of attributes that can be added to an Event. + Default: {_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT} + max_link_attributes: Maximum number of attributes that can be added to a Link. + Default: {_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT} + max_attribute_length: Maximum length an attribute value can have. Values longer than + the specified length will be truncated. + max_span_attribute_length: Maximum length a span attribute value can have. Values longer than + the specified length will be truncated. + """ + + UNSET = -1 + + def __init__( + self, + max_attributes: int | None = None, + max_events: int | None = None, + max_links: int | None = None, + max_span_attributes: int | None = None, + max_event_attributes: int | None = None, + max_link_attributes: int | None = None, + max_attribute_length: int | None = None, + max_span_attribute_length: int | None = None, + ): + # span events and links count + self.max_events = self._from_env_if_absent( + max_events, + OTEL_SPAN_EVENT_COUNT_LIMIT, + _DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT, + ) + self.max_links = self._from_env_if_absent( + max_links, + OTEL_SPAN_LINK_COUNT_LIMIT, + _DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT, + ) + + # attribute count + global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) + self.max_attributes = ( + global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT + ) + + self.max_span_attributes = self._from_env_if_absent( + max_span_attributes, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT), + ) + self.max_event_attributes = self._from_env_if_absent( + max_event_attributes, + OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, + (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT), + ) + self.max_link_attributes = self._from_env_if_absent( + max_link_attributes, + OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, + (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT), + ) + + # attribute length + self.max_attribute_length = self._from_env_if_absent( + max_attribute_length, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + ) + self.max_span_attribute_length = self._from_env_if_absent( + max_span_attribute_length, + OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, + # use global attribute length limit as default + self.max_attribute_length, + ) + + def __repr__(self): + return f"{type(self).__name__}(max_span_attributes={self.max_span_attributes}, max_events_attributes={self.max_event_attributes}, max_link_attributes={self.max_link_attributes}, max_attributes={self.max_attributes}, max_events={self.max_events}, max_links={self.max_links}, max_attribute_length={self.max_attribute_length})" + + @classmethod + def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: + if value == cls.UNSET: + return None + + err_msg = "{} must be a non-negative integer but got {}" + + # if no value is provided for the limit, try to load it from env + if value is None: + # return default value if env var is not set + if env_var not in environ: + return default + + str_value = environ.get(env_var, "").strip().lower() + if str_value == _ENV_VALUE_UNSET: + return None + + try: + value = int(str_value) + except ValueError: + raise ValueError(err_msg.format(env_var, str_value)) + + if value < 0: + raise ValueError(err_msg.format(env_var, value)) + return value + + +_UnsetLimits = SpanLimits( + max_attributes=SpanLimits.UNSET, + max_events=SpanLimits.UNSET, + max_links=SpanLimits.UNSET, + max_span_attributes=SpanLimits.UNSET, + max_event_attributes=SpanLimits.UNSET, + max_link_attributes=SpanLimits.UNSET, + max_attribute_length=SpanLimits.UNSET, + max_span_attribute_length=SpanLimits.UNSET, +) + +# not removed for backward compat. please use SpanLimits instead. +SPAN_ATTRIBUTE_COUNT_LIMIT = SpanLimits._from_env_if_absent( # pylint: disable=protected-access + None, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, +) + + +class Span(trace_api.Span, ReadableSpan): + """See `opentelemetry.trace.Span`. + + Users should create `Span` objects via the `Tracer` instead of this + constructor. + + Args: + name: The name of the operation this span represents + context: The immutable span context + parent: This span's parent's `opentelemetry.trace.SpanContext`, or + None if this is a root span + sampler: The sampler used to create this span + trace_config: Unused. Originally intended for trace-level configuration + from the OpenTelemetry protocol, but the upstream ``TraceConfig`` + proto was removed. Retained for backwards compatibility. + resource: Entity producing telemetry + attributes: The span's attributes to be exported + events: Timestamped events to be exported + links: Links to other spans to be exported + span_processor: `SpanProcessor` to invoke when starting and ending + this `Span`. + limits: `SpanLimits` instance that was passed to the `TracerProvider` + """ + + def __new__(cls, *args, **kwargs): + if cls is Span: + raise TypeError("Span must be instantiated via a tracer.") + return super().__new__(cls) + + # pylint: disable=too-many-locals + def __init__( + self, + name: str, + context: trace_api.SpanContext, + parent: trace_api.SpanContext | None = None, + sampler: sampling.Sampler | None = None, + trace_config: None = None, + resource: Resource | None = None, + attributes: types.Attributes = None, + events: Sequence[Event] | None = None, + links: Sequence[trace_api.Link] = (), + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + span_processor: SpanProcessor = SpanProcessor(), + instrumentation_info: InstrumentationInfo | None = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + limits=_UnsetLimits, + instrumentation_scope: InstrumentationScope | None = None, + *, + record_end_metrics: Callable[[], None] | None = None, + ) -> None: + if resource is None: + resource = Resource.create({}) + super().__init__( + name=name, + context=context, + parent=parent, + kind=kind, + resource=resource, + instrumentation_info=instrumentation_info, + instrumentation_scope=instrumentation_scope, + ) + self._sampler = sampler + self._trace_config = trace_config + self._record_exception = record_exception + self._set_status_on_exception = set_status_on_exception + self._span_processor = span_processor + self._limits = limits + self._lock = threading.Lock() + # Identifier of the thread running end() while the _on_ending + # callbacks execute: the end timestamp is already set, but the span + # must remain mutable from that thread until the callbacks return. + self._ending_thread: int | None = None + self._attributes = BoundedAttributes( + self._limits.max_span_attributes, + attributes, + immutable=False, + max_value_len=self._limits.max_span_attribute_length, + ) + self._events = self._new_events() + if events: + for event in events: + # Immutable set to true so attributes cannot be added or removed after creation. + event._attributes = BoundedAttributes( + self._limits.max_event_attributes, + event.attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + self._events.append(event) + + self._links = self._new_links(links) + + self._record_end_metrics = record_end_metrics + + def __repr__(self): + return f'{type(self).__name__}(name="{self._name}", context={self._context})' + + def _new_events(self): + return BoundedList(self._limits.max_events) + + def _new_links(self, links: Sequence[trace_api.Link]): + if not links: + return BoundedList(self._limits.max_links) + + valid_links = [] + for link in links: + if link and _is_valid_link(link.context, link.attributes): + # Immutable set to true so attributes cannot be added or removed after creation. + # pylint: disable=protected-access + link._attributes = BoundedAttributes( + self._limits.max_link_attributes, + link.attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + valid_links.append(link) + + return BoundedList.from_seq(self._limits.max_links, valid_links) + + def get_span_context(self) -> trace_api.SpanContext: + return typing.cast(trace_api.SpanContext, self._context) + + def set_attributes(self, attributes: Mapping[str, types.AnyValue]) -> None: + with self._lock: + if not self._is_mutable(): + logger.warning("Setting attribute on ended span.") + return + + self._attributes._set_items(attributes) # pylint: disable=protected-access + + def set_attribute(self, key: str, value: types.AnyValue) -> None: + with self._lock: + if not self._is_mutable(): + logger.warning("Setting attribute on ended span.") + return + + self._attributes[key] = value + + @_check_span_ended + def _add_event(self, event: EventBase) -> None: + self._events.append(event) + + def add_event( + self, + name: str, + attributes: types.Attributes = None, + timestamp: int | None = None, + ) -> None: + # Immutable set to true so attributes cannot be added or removed after creation. + attributes = BoundedAttributes( + self._limits.max_event_attributes, + attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + self._add_event( + Event( + name=name, + attributes=attributes, + timestamp=timestamp, + ) + ) + + @_check_span_ended + def _add_link(self, link: trace_api.Link) -> None: + self._links.append(link) + + def add_link( + self, + context: SpanContext, + attributes: types.Attributes = None, + ) -> None: + if not _is_valid_link(context, attributes): + return + # Immutable set to true so attributes cannot be added or removed after creation. + attributes = BoundedAttributes( + self._limits.max_link_attributes, + attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + self._add_link( + trace_api.Link( + context=context, + attributes=attributes, + ) + ) + + def _readable_span(self) -> ReadableSpan: + return ReadableSpan( + name=self._name, + context=self._context, + parent=self._parent, + resource=self._resource, + attributes=self._attributes, + events=self._events, + links=self._links, + kind=self.kind, + status=self._status, + start_time=self._start_time, + end_time=self._end_time, + instrumentation_info=self._instrumentation_info, + instrumentation_scope=self._instrumentation_scope, + ) + + def start( + self, + start_time: int | None = None, + parent_context: context_api.Context | None = None, + ) -> None: + with self._lock: + if self._start_time is not None: + logger.warning("Calling start() on a started span.") + return + self._start_time = start_time if start_time is not None else time_ns() + + self._span_processor.on_start(self, parent_context=parent_context) + + def end(self, end_time: int | None = None) -> None: + with self._lock: + if self._start_time is None: + raise RuntimeError("Calling end() on a not started span.") + if self._end_time is not None: + logger.warning("Calling end() on an ended span.") + return + + self._ending_thread = threading.get_ident() + self._end_time = end_time if end_time is not None else time_ns() + + # The span must remain mutable from this thread while the _on_ending + # callbacks run; it becomes immutable once they have returned. + # pylint: disable=protected-access + try: + if self._record_end_metrics: + self._record_end_metrics() + self._span_processor._on_ending(self) + finally: + with self._lock: + self._ending_thread = None + self._attributes._immutable = True + self._span_processor.on_end(self._readable_span()) + + @_check_span_ended + def update_name(self, name: str) -> None: + self._name = name + + def _is_mutable(self) -> bool: + return self._end_time is None or self._ending_thread == threading.get_ident() + + def is_recording(self) -> bool: + # Unlike the mutators, this check is deliberately not taken under + # self._lock: it does not modify the span, and the answer can become + # stale as soon as it is returned anyway, since another thread may + # end the span right after this call. + return self._is_mutable() + + @_check_span_ended + def set_status( + self, + status: Status | StatusCode, + description: str | None = None, + ) -> None: + if isinstance(status, Status): + if description is not None: + logger.warning( + "Description %s ignored. Use either `Status` or `(StatusCode, Description)`", + description, + ) + new_status = status + elif isinstance(status, StatusCode): + new_status = Status(status, description) + else: + return + + if self._accepts_status(new_status): + self._status = new_status + + def _accepts_status(self, new_status: Status) -> bool: + """Decide whether ``new_status`` may replace the status already recorded. + + The codes are totally ordered, ``Ok > Error > Unset``, an attempt to set + ``Unset`` is ignored, and ``Ok`` is final. Once those two are handled, a + differing code always outranks the one recorded. A repeat of the same code + gets through only when it fills in a description that is still missing, + and an empty description counts as missing. + """ + if new_status.status_code is StatusCode.UNSET: + return False + + current = self._status + if current is None: + return True + if current.status_code is StatusCode.OK: + return False + + # An empty description is equivalent to an absent one. + return current.status_code is not new_status.status_code or ( + not current.description and bool(new_status.description) + ) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + """Ends context manager and calls `end` on the `Span`.""" + if exc_val is not None and self.is_recording(): + # Record the exception as an event + # pylint:disable=protected-access + if self._record_exception: + self.record_exception(exception=exc_val, escaped=True) + # Records status if span is used as context manager + # i.e. with tracer.start_span() as span: + if self._set_status_on_exception: + self.set_status( + Status( + status_code=StatusCode.ERROR, + description=(f"{type(exc_val).__name__}: {exc_val}"), + ) + ) + + super().__exit__(exc_type, exc_val, exc_tb) + + def record_exception( + self, + exception: BaseException, + attributes: types.Attributes = None, + timestamp: int | None = None, + escaped: bool = False, + ) -> None: + """Records an exception as a span event.""" + stacktrace = "".join(traceback.format_exception(exception)) + module = type(exception).__module__ + qualname = type(exception).__qualname__ + exception_type = f"{module}.{qualname}" if module and module != "builtins" else qualname + _attributes: MutableMapping[str, types.AnyValue] = { + EXCEPTION_TYPE: exception_type, + EXCEPTION_MESSAGE: str(exception), + EXCEPTION_STACKTRACE: stacktrace, + EXCEPTION_ESCAPED: str(escaped), + } + if attributes: + _attributes.update(attributes) + self.add_event(name="exception", attributes=_attributes, timestamp=timestamp) + + +class _Span(Span): + """Protected implementation of `opentelemetry.trace.Span`. + + This constructor exists to prevent the instantiation of the `Span` class + by other mechanisms than through the `Tracer`. + """ + + +@dataclass +class _TracerConfig: + is_enabled: bool + + @classmethod + def default(cls): + return cls(is_enabled=True) + + +class Tracer(trace_api.Tracer): + """See `opentelemetry.trace.Tracer`.""" + + def __init__( + self, + sampler: sampling.Sampler, + resource: Resource, + span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor, + id_generator: IdGenerator, + instrumentation_info: InstrumentationInfo, + span_limits: SpanLimits, + instrumentation_scope: InstrumentationScope, + *, + meter_provider: metrics_api.MeterProvider | None = None, + _tracer_config: _TracerConfig | None = None, + ) -> None: + self.sampler = sampler + self.resource = resource + self.span_processor = span_processor + self.id_generator = id_generator + self.instrumentation_info = instrumentation_info + self._span_limits = span_limits + self._instrumentation_scope = instrumentation_scope + self._tracer_config = _tracer_config or _TracerConfig.default() + + meter_provider = meter_provider or metrics_api.get_meter_provider() + self._tracer_metrics = create_tracer_metrics( + meter_provider, + parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), + ) + + def _set_tracer_config(self, tracer_config: _TracerConfig): + self._tracer_config = tracer_config + + def _set_resource(self, resource: Resource) -> None: + self.resource = resource + + def _is_enabled(self) -> bool: + """If the tracer is not enabled, start_span will create a NonRecordingSpan""" + return self._tracer_config.is_enabled + + @_agnosticcontextmanager # pylint: disable=protected-access + def start_as_current_span( + self, + name: str, + context: context_api.Context | None = None, + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + attributes: types.Attributes = None, + links: Sequence[trace_api.Link] | None = (), + start_time: int | None = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + end_on_exit: bool = True, + ) -> Iterator[trace_api.Span]: + span = self.start_span( + name=name, + context=context, + kind=kind, + attributes=attributes, + links=links, + start_time=start_time, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + ) + with trace_api.use_span( + span, + end_on_exit=end_on_exit, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + ) as span: + yield span + + def start_span( # pylint: disable=too-many-locals + self, + name: str, + context: context_api.Context | None = None, + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + attributes: types.Attributes = None, + links: Sequence[trace_api.Link] | None = (), + start_time: int | None = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + ) -> trace_api.Span: + links = links or () + parent_span_context = trace_api.get_current_span(context).get_span_context() + + if parent_span_context is not None and not isinstance(parent_span_context, trace_api.SpanContext): + raise TypeError("parent_span_context must be a SpanContext or None.") + + if not self._is_enabled(): + return trace_api.NonRecordingSpan(context=parent_span_context) + + # is_valid determines root span + if parent_span_context is None or not parent_span_context.is_valid: + parent_span_context = None + trace_id = self.id_generator.generate_trace_id() + else: + trace_id = parent_span_context.trace_id + + # The sampler decides whether to create a real or no-op span at the + # time of span creation. No-op spans do not record events, and are not + # exported. + # The sampler may also add attributes to the newly-created span, e.g. + # to include information about the sampling result. + # The sampler may also modify the parent span context's tracestate + sampling_result = self.sampler.should_sample(context, trace_id, name, kind, attributes, links) + + trace_flags = ( + trace_api.TraceFlags(trace_api.TraceFlags.SAMPLED) + if sampling_result.decision.is_sampled() + else trace_api.TraceFlags(trace_api.TraceFlags.DEFAULT) + ) + + if parent_span_context is None: + random_trace_id = self.id_generator.is_trace_id_random() + else: + random_trace_id = parent_span_context.trace_flags.random_trace_id + + if random_trace_id: + trace_flags = trace_api.TraceFlags(trace_flags | trace_api.TraceFlags.RANDOM_TRACE_ID) + + span_context = trace_api.SpanContext( + trace_id, + self.id_generator.generate_span_id(), + is_remote=False, + trace_flags=trace_flags, + trace_state=sampling_result.trace_state, + ) + + record_end_metrics = self._tracer_metrics.start_span(parent_span_context, sampling_result.decision) + + # Only record if is_recording() is true + if sampling_result.decision.is_recording(): + # pylint:disable=protected-access + span = _Span( + name=name, + context=span_context, + parent=parent_span_context, + sampler=self.sampler, + resource=self.resource, + attributes=sampling_result.attributes, + span_processor=self.span_processor, + kind=kind, + links=links, + instrumentation_info=self.instrumentation_info, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + limits=self._span_limits, + instrumentation_scope=self._instrumentation_scope, + record_end_metrics=record_end_metrics, + ) + span.start(start_time=start_time, parent_context=context) + else: + span = trace_api.NonRecordingSpan(context=span_context) + return span + + +_TracerConfiguratorT = Callable[[InstrumentationScope], _TracerConfig] +_RuleBasedTracerConfigurator = RuleBasedConfigurator[_TracerConfig] + + +def _default_tracer_configurator( + tracer_scope: InstrumentationScope, +) -> _TracerConfig: + """Default Tracer Configurator implementation + + In order to update Tracers configs you need to call + TracerProvider._set_tracer_configurator with a function + implementing this interface returning a Tracer Config.""" + return _RuleBasedTracerConfigurator( + rules=[], + default_config=_TracerConfig.default(), + )(tracer_scope) + + +def _disable_tracer_configurator( + tracer_scope: InstrumentationScope, +) -> _TracerConfig: + return _RuleBasedTracerConfigurator( + rules=[], + default_config=_TracerConfig(is_enabled=False), + )(tracer_scope) + + +class TracerProvider(trace_api.TracerProvider): + """See `opentelemetry.trace.TracerProvider`.""" + + def __init__( + self, + sampler: sampling.Sampler | None = None, + resource: Resource | None = None, + shutdown_on_exit: bool = True, + active_span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor | None = None, + id_generator: IdGenerator | None = None, + span_limits: SpanLimits | None = None, + *, + meter_provider: metrics_api.MeterProvider | None = None, + _tracer_configurator: _TracerConfiguratorT | None = None, + ) -> None: + self._active_span_processor = active_span_processor or SynchronousMultiSpanProcessor() + if id_generator is None: + self.id_generator = RandomIdGenerator() + else: + self.id_generator = id_generator + if resource is None: + self._resource = Resource.create({}) + else: + self._resource = resource + if not sampler: + sampler = sampling._get_from_env_or_default() + self.sampler = sampler + self._span_limits = span_limits or SpanLimits() + disabled = environ.get(OTEL_SDK_DISABLED, "") + self._disabled = disabled.lower().strip() == "true" + self._atexit_handler = None + self._meter_provider = meter_provider + + if shutdown_on_exit: + self._atexit_handler = atexit.register(self.shutdown) + + self._tracer_configurator = _tracer_configurator or _default_tracer_configurator + self._tracers_lock = threading.Lock() + self._tracers: dict[InstrumentationScope, Tracer] = {} + if hasattr(os, "register_at_fork"): + weak_at_fork = weakref.WeakMethod(self._handle_fork) + + def _after_in_child() -> None: + if at_fork := weak_at_fork(): + at_fork() + + os.register_at_fork(after_in_child=_after_in_child) + + def _handle_fork(self) -> None: + self._tracers_lock = threading.Lock() + self._update_resource(_get_process_dependent_resource()) + + def _set_tracer_configurator(self, *, tracer_configurator: _TracerConfiguratorT): + """This is the function used to update the TracerProvider TracerConfigurator + + Setting a new TracerConfigurator for a TracerProvider will update the + TracerConfig of all Tracers create by this TracerProvider. + """ + self._tracer_configurator = tracer_configurator + with self._tracers_lock: + for instrumentation_scope, tracer in self._tracers.items(): + tracer_config = self._apply_tracer_configurator(instrumentation_scope) + # pylint: disable-next=protected-access + tracer._set_tracer_config(tracer_config) + + @property + def resource(self) -> Resource: + return self._resource + + def _update_resource(self, resource: Resource) -> None: + with self._tracers_lock: + self._resource = self._resource.merge(resource) + for tracer in self._tracers.values(): + tracer._set_resource(self._resource) # pylint: disable=protected-access + + def _apply_tracer_configurator(self, instrumentation_scope: InstrumentationScope): + try: + return self._tracer_configurator(instrumentation_scope) + except Exception: # pylint: disable=broad-exception-caught + logger.exception( + "Failed to create a Tracer Config for %s, using default Tracer config", + instrumentation_scope, + ) + return _TracerConfig.default() + + def get_tracer( + self, + instrumenting_module_name: str, + instrumenting_library_version: str | None = None, + schema_url: str | None = None, + attributes: types.Attributes = None, + ) -> "trace_api.Tracer": + if self._disabled: + return NoOpTracer() + if not instrumenting_module_name: # Reject empty strings too. + instrumenting_module_name = "" + logger.error("get_tracer called with missing module name.") + if instrumenting_library_version is None: + instrumenting_library_version = "" + + filterwarnings( + "ignore", + message=(r"You should use InstrumentationScope. Deprecated since version 1.11.1."), + category=DeprecationWarning, + module="opentelemetry.sdk.trace", + ) + + instrumentation_info = InstrumentationInfo( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + ) + + instrumentation_scope = InstrumentationScope( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + attributes, + ) + + with self._tracers_lock: + if instrumentation_scope in self._tracers: + return self._tracers[instrumentation_scope] + + tracer_config = self._apply_tracer_configurator(instrumentation_scope) + tracer = Tracer( + self.sampler, + self.resource, + self._active_span_processor, + self.id_generator, + instrumentation_info, + self._span_limits, + instrumentation_scope, + meter_provider=self._meter_provider, + _tracer_config=tracer_config, + ) + self._tracers[instrumentation_scope] = tracer + + return tracer + + def add_span_processor(self, span_processor: SpanProcessor) -> None: + """Registers a new :class:`SpanProcessor` for this `TracerProvider`. + + The span processors are invoked in the same order they are registered. + """ + + # no lock here because add_span_processor is thread safe for both + # SynchronousMultiSpanProcessor and ConcurrentMultiSpanProcessor. + self._active_span_processor.add_span_processor(span_processor) + + def shutdown(self) -> None: + """Shut down the span processors added to the tracer provider.""" + self._active_span_processor.shutdown() + if self._atexit_handler is not None: + atexit.unregister(self._atexit_handler) + self._atexit_handler = None + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Requests the active span processor to process all spans that have not + yet been processed. + + By default force flush is called sequentially on all added span + processors. This means that span processors further back in the list + have less time to flush their spans. + To have span processors flush their spans in parallel it is possible to + initialize the tracer provider with an instance of + `ConcurrentMultiSpanProcessor` at the cost of using multiple threads. + + Args: + timeout_millis: The maximum amount of time to wait for spans to be + processed. + + Returns: + False if the timeout is exceeded, True otherwise. + """ + return self._active_span_processor.force_flush(timeout_millis) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index e88854fc7c4..7652a807cc5 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -1,253 +1,279 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# pylint:disable=protected-access,no-self-use,no-member - -import logging -import threading -import time -import unittest -from abc import ABC, abstractmethod -from unittest.mock import Mock - -from opentelemetry._logs import LogRecord, SeverityNumber -from opentelemetry.sdk._logs._internal import ( - ConcurrentMultiLogRecordProcessor, - LoggerProvider, - LoggingHandler, - LogRecordProcessor, - ReadWriteLogRecord, - SynchronousMultiLogRecordProcessor, -) - - -class AnotherLogRecordProcessor(LogRecordProcessor): - def __init__(self, exporter, logs_list): - self._exporter = exporter - self._log_list = logs_list - self._closed = False - - def on_emit(self, log_record: ReadWriteLogRecord): - if self._closed: - return - self._log_list.append( - ( - log_record.log_record.body, - log_record.log_record.severity_text, - ) - ) - - def shutdown(self): - self._closed = True - self._exporter.shutdown() - - def force_flush(self, timeout_millis=30000): - self._log_list.clear() - return True - - -class TestLogRecordProcessor(unittest.TestCase): - def test_log_record_processor(self): - provider = LoggerProvider() - handler = LoggingHandler(logger_provider=provider) - - logs_list_1 = [] - processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) - logs_list_2 = [] - processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) - - logger = logging.getLogger("test.span.processor") - logger.addHandler(handler) - - # Test no proessor added - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Odisha, we have another major cyclone") - - self.assertEqual(len(logs_list_1), 0) - self.assertEqual(len(logs_list_2), 0) - - # Add one processor - provider.add_log_record_processor(processor1) - with self.assertLogs(level=logging.WARNING): - logger.warning("Brace yourself") - with self.assertLogs(level=logging.ERROR): - logger.error("Some error message") - - expected_list_1 = [ - ("Brace yourself", "WARN"), - ("Some error message", "ERROR"), - ] - self.assertEqual(logs_list_1, expected_list_1) - - # Add another processor - provider.add_log_record_processor(processor2) - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Something disastrous") - expected_list_1.append(("Something disastrous", "FATAL")) - - expected_list_2 = [("Something disastrous", "FATAL")] - - self.assertEqual(logs_list_1, expected_list_1) - self.assertEqual(logs_list_2, expected_list_2) - - -class MultiLogRecordProcessorTestBase(ABC): - @abstractmethod - def _get_multi_log_record_processor(self): - pass - - def make_record(self): - return ReadWriteLogRecord( - LogRecord( - timestamp=1622300111608942000, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Warning message", - ) - ) - - def test_on_emit(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - record = self.make_record() - multi_log_record_processor.on_emit(record) - for mock in mocks: - mock.on_emit.assert_called_with(record) - multi_log_record_processor.shutdown() - - def test_on_shutdown(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - multi_log_record_processor.shutdown() - for mock in mocks: - mock.shutdown.assert_called_once_with() - - def test_on_force_flush(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - ret_value = multi_log_record_processor.force_flush(100) - - self.assertTrue(ret_value) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return SynchronousMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - def delay(_): - time.sleep(0.09) - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(side_effect=delay) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 0) - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=False) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=None) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertTrue(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - -class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return ConcurrentMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - wait_event = threading.Event() - - def delay(_): - wait_event.wait() - - mock1 = Mock(spec=LogRecordProcessor) - mock1.force_flush = Mock(side_effect=delay) - mocks = [Mock(LogRecordProcessor) for _ in range(5)] - mocks = [mock1] + mocks - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush(50) - wait_event.set() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - false_mock = Mock(spec=LogRecordProcessor) - false_mock.force_flush = Mock(return_value=False) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, false_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - none_mock = Mock(spec=LogRecordProcessor) - none_mock.force_flush = Mock(return_value=None) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertTrue(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint:disable=protected-access,no-self-use,no-member + +import logging +import threading +import time +import unittest +from abc import ABC, abstractmethod +from unittest.mock import Mock + +from opentelemetry._logs import LogRecord, SeverityNumber +from opentelemetry.sdk._logs._internal import ( + ConcurrentMultiLogRecordProcessor, + LoggerProvider, + LoggingHandler, + LogRecordProcessor, + ReadWriteLogRecord, + SynchronousMultiLogRecordProcessor, +) + + +class AnotherLogRecordProcessor(LogRecordProcessor): + def __init__(self, exporter, logs_list): + self._exporter = exporter + self._log_list = logs_list + self._closed = False + + def on_emit(self, log_record: ReadWriteLogRecord): + if self._closed: + return + self._log_list.append( + ( + log_record.log_record.body, + log_record.log_record.severity_text, + ) + ) + + def shutdown(self): + self._closed = True + self._exporter.shutdown() + + def force_flush(self, timeout_millis=30000): + self._log_list.clear() + return True + + +class TestLogRecordProcessor(unittest.TestCase): + def test_log_record_processor(self): + provider = LoggerProvider() + handler = LoggingHandler(logger_provider=provider) + + logs_list_1 = [] + processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) + logs_list_2 = [] + processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) + + logger = logging.getLogger("test.span.processor") + logger.addHandler(handler) + + # Test no proessor added + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Odisha, we have another major cyclone") + + self.assertEqual(len(logs_list_1), 0) + self.assertEqual(len(logs_list_2), 0) + + # Add one processor + provider.add_log_record_processor(processor1) + with self.assertLogs(level=logging.WARNING): + logger.warning("Brace yourself") + with self.assertLogs(level=logging.ERROR): + logger.error("Some error message") + + expected_list_1 = [ + ("Brace yourself", "WARN"), + ("Some error message", "ERROR"), + ] + self.assertEqual(logs_list_1, expected_list_1) + + # Add another processor + provider.add_log_record_processor(processor2) + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Something disastrous") + expected_list_1.append(("Something disastrous", "FATAL")) + + expected_list_2 = [("Something disastrous", "FATAL")] + + self.assertEqual(logs_list_1, expected_list_1) + self.assertEqual(logs_list_2, expected_list_2) + + +class MultiLogRecordProcessorTestBase(ABC): + @abstractmethod + def _get_multi_log_record_processor(self): + pass + + def make_record(self): + return ReadWriteLogRecord( + LogRecord( + timestamp=1622300111608942000, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Warning message", + ) + ) + + def test_on_emit(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + record = self.make_record() + multi_log_record_processor.on_emit(record) + for mock in mocks: + mock.on_emit.assert_called_with(record) + multi_log_record_processor.shutdown() + + def test_on_shutdown(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + multi_log_record_processor.shutdown() + for mock in mocks: + mock.shutdown.assert_called_once_with() + + def test_on_force_flush(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + ret_value = multi_log_record_processor.force_flush(100) + + self.assertTrue(ret_value) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return SynchronousMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + def delay(_): + time.sleep(0.09) + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(side_effect=delay) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 0) + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=False) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=None) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertTrue(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + +class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return ConcurrentMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + wait_event = threading.Event() + + def delay(_): + wait_event.wait() + + mock1 = Mock(spec=LogRecordProcessor) + mock1.force_flush = Mock(side_effect=delay) + mocks = [Mock(LogRecordProcessor) for _ in range(5)] + mocks = [mock1] + mocks + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush(50) + wait_event.set() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + false_mock = Mock(spec=LogRecordProcessor) + false_mock.force_flush = Mock(return_value=False) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, false_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + none_mock = Mock(spec=LogRecordProcessor) + none_mock.force_flush = Mock(return_value=None) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertTrue(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time LoggerProvider.shutdown() runs. Underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + mock_processor = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor) + + # pylint: disable=protected-access + multi_log_record_processor._executor.shutdown(wait=True) + + multi_log_record_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_callback_runtime_error_propagates_not_swallowed(self): + # A RuntimeError raised by a processor callback (while the executor is + # alive) must still propagate out of shutdown(). + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + raising_processor = Mock(spec=LogRecordProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + multi_log_record_processor.add_log_record_processor(raising_processor) + + with self.assertRaises(RuntimeError): + multi_log_record_processor.shutdown() diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index f2595b9669d..a08e7a8e3ef 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -1,757 +1,796 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -import abc -import gc -import multiprocessing -import os -import time -import unittest -import weakref -from threading import Event, Thread -from unittest import mock - -from opentelemetry import trace as trace_api -from opentelemetry.context import Context -from opentelemetry.sdk import trace -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, -) - - -def span_event_start_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":start" - - -def span_event_ending_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":ending" - - -def span_event_end_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":end" - - -class MySpanProcessor(trace.SpanProcessor): - def __init__(self, name, span_list): - self.name = name - self.span_list = span_list - - def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: - self.span_list.append(span_event_start_fmt(self.name, span.name)) - - def on_end(self, span: "trace.Span") -> None: - self.span_list.append(span_event_end_fmt(self.name, span.name)) - - -class MyExtendedSpanProcessor(MySpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - self.span_list.append(span_event_ending_fmt(self.name, span.name)) - - -class TestSpanProcessor(unittest.TestCase): - def test_span_processor(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MySpanProcessor("SP1", spans_calls_list) - sp2 = MySpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - # pylint: disable=too-many-statements - def test_span_processor_with_on_ending(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) - sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_ending_fmt("SP2", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_ending_fmt("SP2", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_ending_fmt("SP2", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - def test_add_span_processor_after_span_creation(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp = MySpanProcessor("SP1", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - # add span processor after spans have been created - tracer_provider.add_span_processor(sp) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Does not implement _on_ending - sp = MySpanProcessor("SP1", spans_calls_list) - tracer_provider.add_span_processor(sp) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - self.fail("_on_ending() should not raise an exception") - - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - assert span.is_recording() - assert span.end_time is not None - span.update_name("renamed") - span.set_attribute("attribute", "value") - span.set_attributes({"attributes": "value"}) - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(MutatingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - tracer.start_span("foo").end() - - (span,) = exporter.get_finished_spans() - self.assertEqual(span.name, "renamed") - self.assertEqual(span.attributes["attribute"], "value") - self.assertEqual(span.attributes["attributes"], "value") - self.assertEqual(span.events[0].name, "event") - self.assertEqual(len(span.links), 1) - self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) - - def test_end_during_on_ending_is_ignored(self): - exporter = InMemorySpanExporter() - on_ending_calls = [] - - class ReentrantSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - on_ending_calls.append(span) - span.end() - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(ReentrantSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - span = tracer.start_span("foo") - span.end() - - self.assertEqual(len(on_ending_calls), 1) - self.assertEqual(len(exporter.get_finished_spans()), 1) - - # After end() returns the span is immutable again. - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - (exported,) = exporter.get_finished_spans() - self.assertNotIn("late", exported.attributes) - - def test_other_thread_cannot_mutate_during_on_ending(self): - exporter = InMemorySpanExporter() - inside_on_ending = Event() - other_thread_done = Event() - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def mutate_from_another_thread(): - inside_on_ending.wait(5) - span.set_attribute("other.attribute", "value") - span.set_attributes({"other.attributes": "value"}) - span.update_name("renamed") - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - other_thread_done.set() - - thread = Thread(target=mutate_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(inside_on_ending.is_set()) - self.assertTrue(other_thread_done.is_set()) - - (exported,) = exporter.get_finished_spans() - self.assertNotIn("other.attribute", exported.attributes) - self.assertNotIn("other.attributes", exported.attributes) - self.assertEqual(exported.name, "foo") - self.assertEqual(len(exported.events), 0) - self.assertEqual(len(exported.links), 0) - self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) - - def test_other_thread_does_not_see_span_recording_during_on_ending(self): - inside_on_ending = Event() - other_thread_done = Event() - recording_seen_by_other_thread = [] - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def observe_from_another_thread(): - inside_on_ending.wait(5) - recording_seen_by_other_thread.append(span.is_recording()) - other_thread_done.set() - - thread = Thread(target=observe_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(other_thread_done.is_set()) - self.assertEqual(recording_seen_by_other_thread, [False]) - - def test_span_is_frozen_when_record_end_metrics_raises(self): - def record_end_metrics_that_raises(): - raise RuntimeError("meter blew up") - - tracer_provider = trace.TracerProvider() - span = tracer_provider.get_tracer(__name__).start_span("foo") - # pylint: disable=protected-access - span._record_end_metrics = record_end_metrics_that_raises - - with self.assertRaises(RuntimeError): - span.end() - - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - self.assertNotIn("late", span.attributes) - - -class MultiSpanProcessorTestBase(abc.ABC): - @abc.abstractmethod - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: - pass - - @staticmethod - def create_default_span() -> trace_api.Span: - span_context = trace_api.SpanContext(37, 73, is_remote=False) - return trace_api.NonRecordingSpan(span_context) - - def test_on_start(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - context = Context() - multi_processor.on_start(span, parent_context=context) - - for mock_processor in mocks: - mock_processor.on_start.assert_called_once_with(span, parent_context=context) - multi_processor.shutdown() - - def test_on_ending(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - # pylint: disable=protected-access - multi_processor._on_ending(span) - - for mock_processor in mocks: - # pylint: disable=protected-access - mock_processor._on_ending.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_end(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - multi_processor.on_end(span) - - for mock_processor in mocks: - mock_processor.on_end.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_shutdown(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - multi_processor.shutdown() - - for mock_processor in mocks: - mock_processor.shutdown.assert_called_once_with() - - def test_force_flush(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - timeout_millis = 100 - - flushed = multi_processor.force_flush(timeout_millis) - - # pylint: disable=no-member - self.assertTrue(flushed) - for mock_processor in mocks: - # pylint: disable=no-member - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - multi_processor = self.create_multi_span_processor() - # Does not implement _on_ending - multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) - - tracer_provider.add_span_processor(multi_processor) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - # pylint: disable=no-member - self.fail("_on_ending() should not raise an exception") - - # pylint: disable=no-member - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - tracer_provider.get_tracer(__name__).start_span("foo").end() - - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - def test_on_ending_exception_does_not_stop_other_processors(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class FailingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - raise RuntimeError("_on_ending failed") - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(FailingSpanProcessor()) - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - # pylint: disable=no-member - with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): - tracer_provider.get_tracer(__name__).start_span("foo").end() - - # The processor registered after the failing one still ran its - # _on_ending, and on_end still exported the span. - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - -class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor: - return trace.SynchronousMultiSpanProcessor() - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - def delayed_flush(_): - time.sleep(0.055) - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(0, mock_processor2.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=False) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=None) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - mock_processor2.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_default_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - default_processor = trace.SpanProcessor() - multi_processor.add_span_processor(default_processor) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - mock_processor.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.ConcurrentMultiSpanProcessor: - return trace.ConcurrentMultiSpanProcessor(3) - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - wait_event = Event() - - def delayed_flush(_): - wait_event.wait() - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(side_effect=delayed_flush) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(timeout_millis=25) - # let the thread executing the late_mock continue - wait_event.set() - multi_processor.shutdown() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(return_value=False) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - none_mock = mock.Mock(spec=trace.SpanProcessor) - none_mock.force_flush = mock.Mock(return_value=None) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertTrue(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_processor_gc(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - weak_ref = weakref.ref(multi_processor) - multi_processor.shutdown() - - # When the processor is garbage collected - del multi_processor - gc.collect() - - # Then the reference to the processor should no longer exist - self.assertIsNone( - weak_ref(), - "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", - ) - - @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") - def test_batch_span_processor_fork(self): - multiprocessing_context = multiprocessing.get_context("fork") - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - exporter = InMemorySpanExporter() - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider.add_span_processor(multi_processor) - - # Use the ConcurrentMultiSpanProcessor in the main process. - # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: - with tracer.start_as_current_span("main process before fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process before fork span" - - # The forked ConcurrentMultiSpanProcessor is usable in the child process: - def child(conn): - with tracer.start_as_current_span("child process span"): - pass - conn.send(exporter.get_finished_spans()[-1].name) - conn.close() - - parent_conn, child_conn = multiprocessing_context.Pipe() - process = multiprocessing_context.Process(target=child, args=(child_conn,)) - process.start() - has_response = parent_conn.poll(timeout=5) - if not has_response: - process.kill() - self.fail("The child process did not send any message after 5 seconds, it's very probably locked") - process.join(timeout=5) - assert parent_conn.recv() == "child process span" - - # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: - with tracer.start_as_current_span("main process after fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process after fork span" +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import abc +import gc +import multiprocessing +import os +import time +import unittest +import weakref +from threading import Event, Thread +from unittest import mock + +from opentelemetry import trace as trace_api +from opentelemetry.context import Context +from opentelemetry.sdk import trace +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +def span_event_start_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":start" + + +def span_event_ending_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":ending" + + +def span_event_end_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":end" + + +class MySpanProcessor(trace.SpanProcessor): + def __init__(self, name, span_list): + self.name = name + self.span_list = span_list + + def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: + self.span_list.append(span_event_start_fmt(self.name, span.name)) + + def on_end(self, span: "trace.Span") -> None: + self.span_list.append(span_event_end_fmt(self.name, span.name)) + + +class MyExtendedSpanProcessor(MySpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + self.span_list.append(span_event_ending_fmt(self.name, span.name)) + + +class TestSpanProcessor(unittest.TestCase): + def test_span_processor(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MySpanProcessor("SP1", spans_calls_list) + sp2 = MySpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + # pylint: disable=too-many-statements + def test_span_processor_with_on_ending(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) + sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_ending_fmt("SP2", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_ending_fmt("SP2", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_ending_fmt("SP2", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + def test_add_span_processor_after_span_creation(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp = MySpanProcessor("SP1", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + # add span processor after spans have been created + tracer_provider.add_span_processor(sp) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Does not implement _on_ending + sp = MySpanProcessor("SP1", spans_calls_list) + tracer_provider.add_span_processor(sp) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + self.fail("_on_ending() should not raise an exception") + + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + assert span.is_recording() + assert span.end_time is not None + span.update_name("renamed") + span.set_attribute("attribute", "value") + span.set_attributes({"attributes": "value"}) + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(MutatingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + tracer.start_span("foo").end() + + (span,) = exporter.get_finished_spans() + self.assertEqual(span.name, "renamed") + self.assertEqual(span.attributes["attribute"], "value") + self.assertEqual(span.attributes["attributes"], "value") + self.assertEqual(span.events[0].name, "event") + self.assertEqual(len(span.links), 1) + self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) + + def test_end_during_on_ending_is_ignored(self): + exporter = InMemorySpanExporter() + on_ending_calls = [] + + class ReentrantSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + on_ending_calls.append(span) + span.end() + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(ReentrantSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + span = tracer.start_span("foo") + span.end() + + self.assertEqual(len(on_ending_calls), 1) + self.assertEqual(len(exporter.get_finished_spans()), 1) + + # After end() returns the span is immutable again. + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + (exported,) = exporter.get_finished_spans() + self.assertNotIn("late", exported.attributes) + + def test_other_thread_cannot_mutate_during_on_ending(self): + exporter = InMemorySpanExporter() + inside_on_ending = Event() + other_thread_done = Event() + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def mutate_from_another_thread(): + inside_on_ending.wait(5) + span.set_attribute("other.attribute", "value") + span.set_attributes({"other.attributes": "value"}) + span.update_name("renamed") + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + other_thread_done.set() + + thread = Thread(target=mutate_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(inside_on_ending.is_set()) + self.assertTrue(other_thread_done.is_set()) + + (exported,) = exporter.get_finished_spans() + self.assertNotIn("other.attribute", exported.attributes) + self.assertNotIn("other.attributes", exported.attributes) + self.assertEqual(exported.name, "foo") + self.assertEqual(len(exported.events), 0) + self.assertEqual(len(exported.links), 0) + self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) + + def test_other_thread_does_not_see_span_recording_during_on_ending(self): + inside_on_ending = Event() + other_thread_done = Event() + recording_seen_by_other_thread = [] + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def observe_from_another_thread(): + inside_on_ending.wait(5) + recording_seen_by_other_thread.append(span.is_recording()) + other_thread_done.set() + + thread = Thread(target=observe_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(other_thread_done.is_set()) + self.assertEqual(recording_seen_by_other_thread, [False]) + + def test_span_is_frozen_when_record_end_metrics_raises(self): + def record_end_metrics_that_raises(): + raise RuntimeError("meter blew up") + + tracer_provider = trace.TracerProvider() + span = tracer_provider.get_tracer(__name__).start_span("foo") + # pylint: disable=protected-access + span._record_end_metrics = record_end_metrics_that_raises + + with self.assertRaises(RuntimeError): + span.end() + + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + self.assertNotIn("late", span.attributes) + + +class MultiSpanProcessorTestBase(abc.ABC): + @abc.abstractmethod + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: + pass + + @staticmethod + def create_default_span() -> trace_api.Span: + span_context = trace_api.SpanContext(37, 73, is_remote=False) + return trace_api.NonRecordingSpan(span_context) + + def test_on_start(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + context = Context() + multi_processor.on_start(span, parent_context=context) + + for mock_processor in mocks: + mock_processor.on_start.assert_called_once_with(span, parent_context=context) + multi_processor.shutdown() + + def test_on_ending(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + # pylint: disable=protected-access + multi_processor._on_ending(span) + + for mock_processor in mocks: + # pylint: disable=protected-access + mock_processor._on_ending.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_end(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + multi_processor.on_end(span) + + for mock_processor in mocks: + mock_processor.on_end.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_shutdown(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + multi_processor.shutdown() + + for mock_processor in mocks: + mock_processor.shutdown.assert_called_once_with() + + def test_force_flush(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + timeout_millis = 100 + + flushed = multi_processor.force_flush(timeout_millis) + + # pylint: disable=no-member + self.assertTrue(flushed) + for mock_processor in mocks: + # pylint: disable=no-member + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + multi_processor = self.create_multi_span_processor() + # Does not implement _on_ending + multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) + + tracer_provider.add_span_processor(multi_processor) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + # pylint: disable=no-member + self.fail("_on_ending() should not raise an exception") + + # pylint: disable=no-member + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + tracer_provider.get_tracer(__name__).start_span("foo").end() + + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + def test_on_ending_exception_does_not_stop_other_processors(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class FailingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + raise RuntimeError("_on_ending failed") + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(FailingSpanProcessor()) + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + # pylint: disable=no-member + with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): + tracer_provider.get_tracer(__name__).start_span("foo").end() + + # The processor registered after the failing one still ran its + # _on_ending, and on_end still exported the span. + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + +class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor: + return trace.SynchronousMultiSpanProcessor() + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + def delayed_flush(_): + time.sleep(0.055) + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(0, mock_processor2.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=False) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=None) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + mock_processor2.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_default_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + default_processor = trace.SpanProcessor() + multi_processor.add_span_processor(default_processor) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + mock_processor.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.ConcurrentMultiSpanProcessor: + return trace.ConcurrentMultiSpanProcessor(3) + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + wait_event = Event() + + def delayed_flush(_): + wait_event.wait() + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(side_effect=delayed_flush) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(timeout_millis=25) + # let the thread executing the late_mock continue + wait_event.set() + multi_processor.shutdown() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(return_value=False) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + none_mock = mock.Mock(spec=trace.SpanProcessor) + none_mock.force_flush = mock.Mock(return_value=None) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertTrue(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_processor_gc(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + weak_ref = weakref.ref(multi_processor) + multi_processor.shutdown() + + # When the processor is garbage collected + del multi_processor + gc.collect() + + # Then the reference to the processor should no longer exist + self.assertIsNone( + weak_ref(), + "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", + ) + + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time TracerProvider.shutdown() runs. The underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + multi_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_on_end_runs_inline_after_executor_shutdown(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + span = self.create_default_span() + multi_processor.on_end(span) + mock_processor.on_end.assert_called_once_with(span) + + def test_callback_runtime_error_propagates_not_swallowed(self): + # A RuntimeError raised by a processor callback (while the executor is + # alive) must still propagate out of shutdown(); the inline fallback + # must not mask unrelated callback failures. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + raising_processor = mock.Mock(spec=trace.SpanProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + multi_processor.add_span_processor(raising_processor) + + with self.assertRaises(RuntimeError): + multi_processor.shutdown() + + @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") + def test_batch_span_processor_fork(self): + multiprocessing_context = multiprocessing.get_context("fork") + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + exporter = InMemorySpanExporter() + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider.add_span_processor(multi_processor) + + # Use the ConcurrentMultiSpanProcessor in the main process. + # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: + with tracer.start_as_current_span("main process before fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process before fork span" + + # The forked ConcurrentMultiSpanProcessor is usable in the child process: + def child(conn): + with tracer.start_as_current_span("child process span"): + pass + conn.send(exporter.get_finished_spans()[-1].name) + conn.close() + + parent_conn, child_conn = multiprocessing_context.Pipe() + process = multiprocessing_context.Process(target=child, args=(child_conn,)) + process.start() + has_response = parent_conn.poll(timeout=5) + if not has_response: + process.kill() + self.fail("The child process did not send any message after 5 seconds, it's very probably locked") + process.join(timeout=5) + assert parent_conn.recv() == "child process span" + + # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: + with tracer.start_as_current_span("main process after fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process after fork span" From 1b89bcf7e1958690101f247416a0f223372632f1 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:11:47 +0800 Subject: [PATCH 02/26] fix(sdk): use LF line endings to match repo convention --- .../sdk/_logs/_internal/__init__.py | 2210 ++++++------ .../src/opentelemetry/sdk/trace/__init__.py | 2982 ++++++++--------- .../tests/logs/test_multi_log_processor.py | 558 +-- .../tests/trace/test_span_processor.py | 1592 ++++----- 4 files changed, 3671 insertions(+), 3671 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 42e329e7c32..6378a40cecd 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -1,1105 +1,1105 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -import abc -import atexit -import base64 -import concurrent.futures -import json -import logging -import os -import threading -import traceback -import warnings -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, field -from os import environ -from threading import Lock -from time import time_ns -from types import NoneType -from typing import ( # noqa - Any, - Tuple, - Union, - cast, - overload, -) -from weakref import WeakMethod, WeakSet - -from typing_extensions import deprecated - -from opentelemetry._logs import Logger as APILogger -from opentelemetry._logs import LoggerProvider as APILoggerProvider -from opentelemetry._logs import ( - LogRecord, - NoOpLogger, - SeverityNumber, - get_logger, - get_logger_provider, -) -from opentelemetry.attributes import BoundedAttributes -from opentelemetry.context import get_current -from opentelemetry.context.context import Context -from opentelemetry.metrics import MeterProvider, get_meter_provider -from opentelemetry.sdk._logs._internal._exceptions import ( - _copy_log_record_with_exception, - _create_log_record_with_exception, - _set_log_record_exception_attributes, -) -from opentelemetry.sdk._logs._internal._logger_metrics import ( - LoggerMetricsT, - create_logger_metrics, -) -from opentelemetry.sdk.environment_variables import ( - OTEL_ATTRIBUTE_COUNT_LIMIT, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, - OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, - OTEL_SDK_DISABLED, -) -from opentelemetry.sdk.environment_variables._internal import ( - parse_boolean_environment_variable, -) -from opentelemetry.sdk.resources import ( - Resource, - _get_process_dependent_resource, -) -from opentelemetry.sdk.util import ns_to_iso_str -from opentelemetry.sdk.util._configurator import RuleBasedConfigurator -from opentelemetry.sdk.util.instrumentation import ( - InstrumentationScope, -) -from opentelemetry.semconv.attributes import ( - code_attributes, - exception_attributes, -) -from opentelemetry.trace import ( - format_span_id, - format_trace_id, -) -from opentelemetry.util.types import AnyValue, Attributes - -# pylint: disable=too-many-lines - -_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 -_ENV_VALUE_UNSET = "" - -_logger = logging.getLogger(__name__) - - -class BytesEncoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, bytes): - return base64.b64encode(o).decode() - return super().default(o) - - -class LogRecordDroppedAttributesWarning(UserWarning): - """Custom warning to indicate dropped log attributes due to limits. - - This class is used to filter and handle these specific warnings separately - from other warnings, ensuring that they are only shown once without - interfering with default user warnings. - """ - - -warnings.simplefilter("once", LogRecordDroppedAttributesWarning) - - -@deprecated( - "Use LogRecordDroppedAttributesWarning. Since logs are not stable yet this WILL be removed in future releases." -) -class LogDroppedAttributesWarning(LogRecordDroppedAttributesWarning): - pass - - -class LogRecordLimits: - """This class is based on a SpanLimits class in the Tracing module. - - This class represents the limits that should be enforced on recorded data such as events, links, attributes etc. - - This class does not enforce any limits itself. It only provides a way to read limits from env, - default values and from user provided arguments. - - All limit arguments must be either a non-negative integer, ``None`` or ``LogRecordLimits.UNSET``. - - - All limit arguments are optional. - - If a limit argument is not set, the class will try to read its value from the corresponding - environment variable. - - If the environment variable is not set, the default value, if any, will be used. - - Limit precedence: - - - If a model specific limit is set, it will be used. - - Else if the corresponding global limit is set, it will be used. - - Else if the model specific limit has a default value, the default value will be used. - - Else if the global limit has a default value, the default value will be used. - - Args: - max_attributes: Maximum number of attributes that can be added to a log record (global fallback). - Environment variable: ``OTEL_ATTRIBUTE_COUNT_LIMIT`` - Default: {_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT} - max_attribute_length: Maximum length an attribute value can have (global fallback). Values - longer than the specified length will be truncated. - max_log_record_attributes: Maximum number of attributes that can be added to a log record. - Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT`` - Falls back to ``max_attributes`` when unset. - max_log_record_attribute_length: Maximum length a log record attribute value can have. - Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT`` - Falls back to ``max_attribute_length`` when unset. - """ - - UNSET = -1 - - def __init__( - self, - max_attributes: int | None = None, - max_attribute_length: int | None = None, - max_log_record_attributes: int | None = None, - max_log_record_attribute_length: int | None = None, - ): - # global attribute count - global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) - self.max_attributes = ( - global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT - ) - - # global attribute length - self.max_attribute_length = self._from_env_if_absent( - max_attribute_length, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - ) - - self.max_log_record_attributes = self._from_env_if_absent( - max_log_record_attributes, - OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, - default=self.max_attributes, - ) - - self.max_log_record_attribute_length = self._from_env_if_absent( - max_log_record_attribute_length, - OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, - default=self.max_attribute_length, - ) - - def __repr__(self): - return ( - f"{type(self).__name__}(" - f"max_attributes={self.max_attributes}, " - f"max_attribute_length={self.max_attribute_length}, " - f"max_log_record_attributes={self.max_log_record_attributes}, " - f"max_log_record_attribute_length={self.max_log_record_attribute_length})" - ) - - @classmethod - def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: - if value == cls.UNSET: - return None - - err_msg = "{} must be a non-negative integer but got {}" - - # if no value is provided for the limit, try to load it from env - if value is None: - # return default value if env var is not set - if env_var not in environ: - return default - - str_value = environ.get(env_var, "").strip().lower() - if str_value == _ENV_VALUE_UNSET: - return None - - try: - value = int(str_value) - except ValueError: - raise ValueError(err_msg.format(env_var, str_value)) - - if value < 0: - raise ValueError(err_msg.format(env_var, value)) - return value - - -@deprecated("Use LogRecordLimits. Since logs are not stable yet this WILL be removed in future releases.") -class LogLimits(LogRecordLimits): - pass - - -@dataclass(frozen=True) -class ReadableLogRecord: - """Readable LogRecord should be kept exactly in-sync with ReadWriteLogRecord, only difference is the frozen=True param.""" - - log_record: LogRecord - resource: Resource - instrumentation_scope: InstrumentationScope | None = None - limits: LogRecordLimits | None = None - - @property - def dropped_attributes(self) -> int: - if isinstance(self.log_record.attributes, BoundedAttributes): - return self.log_record.attributes.dropped - return 0 - - def to_json(self, indent: int | None = 4) -> str: - return json.dumps( - { - "body": self.log_record.body, - "severity_number": self.log_record.severity_number.value - if self.log_record.severity_number is not None - else None, - "severity_text": self.log_record.severity_text, - "attributes": (dict(self.log_record.attributes) if bool(self.log_record.attributes) else None), - "dropped_attributes": self.dropped_attributes, - "timestamp": ns_to_iso_str(self.log_record.timestamp) - if self.log_record.timestamp is not None - else None, - "observed_timestamp": ns_to_iso_str(self.log_record.observed_timestamp), - "trace_id": ( - f"0x{format_trace_id(self.log_record.trace_id)}" if self.log_record.trace_id is not None else "" - ), - "span_id": ( - f"0x{format_span_id(self.log_record.span_id)}" if self.log_record.span_id is not None else "" - ), - "trace_flags": self.log_record.trace_flags, - "resource": json.loads(self.resource.to_json()), - "event_name": self.log_record.event_name if self.log_record.event_name else "", - }, - indent=indent, - cls=BytesEncoder, - ) - - -@dataclass -class ReadWriteLogRecord: - """A ReadWriteLogRecord instance represents an event being logged. - ReadWriteLogRecord instances are created and emitted via `Logger` - every time something is logged. They contain all the information - pertinent to the event being logged. - """ - - log_record: LogRecord - resource: Resource | None = Resource.create({}) - instrumentation_scope: InstrumentationScope | None = None - limits: LogRecordLimits = field(default_factory=LogRecordLimits) - - def __post_init__(self): - self.log_record.attributes = BoundedAttributes( - maxlen=self.limits.max_log_record_attributes, - attributes=self.log_record.attributes if self.log_record.attributes else None, - immutable=False, - max_value_len=self.limits.max_log_record_attribute_length, - extended_attributes=True, - ) - if self.dropped_attributes > 0: - warnings.warn( - "Log record attributes were dropped due to limits", - LogRecordDroppedAttributesWarning, - stacklevel=2, - ) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, ReadWriteLogRecord): - return NotImplemented - return self.__dict__ == other.__dict__ - - @property - def dropped_attributes(self) -> int: - if isinstance(self.log_record.attributes, BoundedAttributes): - return self.log_record.attributes.dropped - return 0 - - @classmethod - def _from_api_log_record( - cls, - *, - record: LogRecord, - resource: Resource, - instrumentation_scope: InstrumentationScope | None = None, - limits: LogRecordLimits | None = None, - ) -> ReadWriteLogRecord: - return cls( - log_record=record, - resource=resource, - instrumentation_scope=instrumentation_scope, - **({} if limits is None else {"limits": limits}), - ) - - -class LogRecordProcessor(abc.ABC): - """Interface to hook the log record emitting action. - - Log processors can be registered directly using - :func:`LoggerProvider.add_log_record_processor` and they are invoked - in the same order as they were registered. - - Implementers of custom log processors should be aware of the following: - - Error Handling - -------------- - According to the OpenTelemetry error handling principles, the SDK should - not throw unhandled exceptions at runtime. When implementing a custom - ``LogRecordProcessor``, it is the **processor's responsibility** to handle - any exceptions that may be raised by the exporter's ``export()`` method. - - The ``LogRecordExporter.export()`` method may raise exceptions (e.g., - network errors, timeouts). If these exceptions are not caught, they will - propagate up and potentially crash the application. - - Custom processor implementations should wrap exporter calls in a - try/except block. See ``SimpleLogRecordProcessor`` for a reference - implementation:: - - def on_emit(self, log_record: ReadWriteLogRecord): - try: - self._exporter.export((log_record,)) - except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while exporting logs.") - - The ``BatchLogRecordProcessor`` handles this implicitly since export - operations occur in a background thread where exceptions cannot bubble - up to the caller. - """ - - @abc.abstractmethod - def on_emit(self, log_record: ReadWriteLogRecord) -> None: - """Emits the ``ReadWriteLogRecord``. - - Implementers should handle any exceptions raised during log processing - to prevent application crashes. See the class docstring for details - on error handling expectations. - """ - - def enabled( # pylint: disable=no-self-use,unused-argument - self, - *, - context: Context | None = None, - instrumentation_scope: InstrumentationScope | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - """Returns whether this processor would emit a record for the given arguments. - - Processors that support filtering MAY override this method. The default - implementation returns ``True``. - """ - return True - - @abc.abstractmethod - def shutdown(self) -> None: - """Called when a :class:`opentelemetry.sdk._logs.Logger` is shutdown""" - - @abc.abstractmethod - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Export all the received logs to the configured Exporter that have not yet - been exported. - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. - - Returns: - False if the timeout is exceeded, True otherwise. - """ - - -# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved -# pylint:disable=no-member -class SynchronousMultiLogRecordProcessor(LogRecordProcessor): - """Implementation of class:`LogRecordProcessor` that forwards all received - events to a list of log processors sequentially. - - The underlying log processors are called in sequential order as they were - added. - """ - - def __init__(self) -> None: - # use a tuple to avoid race conditions when adding a new log and - # iterating through it on "emit". - self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] - self._lock = threading.Lock() - - def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: - """Adds a Logprocessor to the list of log processors handled by this instance""" - with self._lock: - self._log_record_processors += (log_record_processor,) - - def on_emit(self, log_record: ReadWriteLogRecord) -> None: - for lp in self._log_record_processors: - lp.on_emit(log_record) - - def enabled( - self, - *, - context: Context | None = None, - instrumentation_scope: InstrumentationScope | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - processors = self._log_record_processors - return any( - lp.enabled( - context=context, - instrumentation_scope=instrumentation_scope, - severity_number=severity_number, - event_name=event_name, - ) - for lp in processors - ) - - def shutdown(self) -> None: - """Shutdown the log processors one by one""" - for lp in self._log_record_processors: - lp.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Force flush the log processors one by one - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. If the first n log processors exceeded the timeout - then remaining log processors will not be flushed. - - Returns: - True if all the log processors flushes the logs within timeout, - False otherwise. - """ - deadline_ns = time_ns() + timeout_millis * 1000000 - all_flushed = True - for lp in self._log_record_processors: - current_ts = time_ns() - if current_ts >= deadline_ns: - return False - - if lp.force_flush((deadline_ns - current_ts) // 1000000) is False: - all_flushed = False - - return all_flushed - - -class ConcurrentMultiLogRecordProcessor(LogRecordProcessor): - """Implementation of :class:`LogRecordProcessor` that forwards all received - events to a list of log processors in parallel. - - Calls to the underlying log processors are forwarded in parallel by - submitting them to a thread pool executor and waiting until each log - processor finished its work. - - Args: - max_workers: The number of threads managed by the thread pool executor - and thus defining how many log processors can work in parallel. - """ - - def __init__(self, max_workers: int = 2) -> None: - # use a tuple to avoid race conditions when adding a new log and - # iterating through it on "emit". - self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] - self._lock = threading.Lock() - self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) - - def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: - with self._lock: - self._log_record_processors += (log_record_processor,) - - def _submit_and_wait( - self, - func: Callable[[LogRecordProcessor], Callable[..., None]], - *args: Any, - **kwargs: Any, - ): - futures = [] - for lp in self._log_record_processors: - try: - future = self._executor.submit(func(lp), *args, **kwargs) - futures.append(future) - except RuntimeError: - # The executor may already be shut down when this runs from an - # atexit handler: CPython runs threading._shutdown() before the - # atexit queue, so the ThreadPoolExecutor is closed by the time - # LoggerProvider.shutdown() runs. Run inline so buffered work is - # still flushed instead of being silently dropped. A - # RuntimeError raised by the callback itself is not caught here - # and still propagates via future.result() below. - _logger.debug( - "Executor is shut down; running LogRecordProcessor callback inline.", - ) - func(lp)(*args, **kwargs) - for future in futures: - future.result() - - def on_emit(self, log_record: ReadWriteLogRecord) -> None: - self._submit_and_wait(lambda lp: lp.on_emit, log_record) - - def enabled( - self, - *, - context: Context | None = None, - instrumentation_scope: InstrumentationScope | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - processors = self._log_record_processors - return any( - lp.enabled( - context=context, - instrumentation_scope=instrumentation_scope, - severity_number=severity_number, - event_name=event_name, - ) - for lp in processors - ) - - def shutdown(self) -> None: - self._submit_and_wait(lambda lp: lp.shutdown) - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Force flush the log processors in parallel. - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. - - Returns: - True if all the log processors flushes the logs within timeout, - False otherwise. - """ - futures = [] - for lp in self._log_record_processors: - future = self._executor.submit(lp.force_flush, timeout_millis) - futures.append(future) - - done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_millis / 1e3) - - if not_done_futures: - return False - - for future in done_futures: - if future.result() is False: - return False - - return True - - -# skip natural LogRecord attributes -# http://docs.python.org/library/logging.html#logrecord-attributes -_RESERVED_ATTRS = frozenset( - ( - "asctime", - "args", - "created", - "exc_info", - "exc_text", - "filename", - "funcName", - "getMessage", - "message", - "levelname", - "levelno", - "lineno", - "module", - "msecs", - "msg", - "name", - "pathname", - "process", - "processName", - "relativeCreated", - "stack_info", - "thread", - "threadName", - "taskName", - ) -) - - -class LoggingHandler(logging.Handler): - """A handler class which writes logging records, in OTLP format, to - a network destination or file. Supports signals from the `logging` module. - https://docs.python.org/3/library/logging.html - """ - - def __init__( - self, - level: int = logging.NOTSET, - logger_provider: APILoggerProvider | None = None, - ) -> None: - super().__init__(level=level) - self._logger_provider = logger_provider or get_logger_provider() - - warnings.warn( - "`LoggingHandler` in `opentelemetry-sdk` is deprecated. Use the " - "handler from `opentelemetry-instrumentation-logging` instead.", - DeprecationWarning, - ) - - @staticmethod - def _get_attributes(record: logging.LogRecord) -> Attributes: - attributes = {k: v for k, v in vars(record).items() if k not in _RESERVED_ATTRS} - - # Add standard code attributes for logs. - attributes[code_attributes.CODE_FILE_PATH] = record.pathname - attributes[code_attributes.CODE_FUNCTION_NAME] = record.funcName - attributes[code_attributes.CODE_LINE_NUMBER] = record.lineno - - if record.exc_info: - exctype, value, tb = record.exc_info - if exctype is not None: - attributes[exception_attributes.EXCEPTION_TYPE] = exctype.__name__ - if value is not None and value.args: - attributes[exception_attributes.EXCEPTION_MESSAGE] = str(value.args[0]) - if tb is not None: - # https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans/#stacktrace-representation - attributes[exception_attributes.EXCEPTION_STACKTRACE] = "".join( - traceback.format_exception(*record.exc_info) - ) - return attributes - - def _translate(self, record: logging.LogRecord) -> LogRecord: - timestamp = int(record.created * 1e9) - observered_timestamp = time_ns() - attributes = self._get_attributes(record) - severity_number = std_to_otel(record.levelno) - if self.formatter: - body = self.format(record) - else: - # `record.getMessage()` uses `record.msg` as a template to format - # `record.args` into. There is a special case in `record.getMessage()` - # where it will only attempt formatting if args are provided, - # otherwise, it just stringifies `record.msg`. - # - # Since the OTLP body field has a type of 'any' and the logging module - # is sometimes used in such a way that objects incorrectly end up - # set as record.msg, in those cases we would like to bypass - # `record.getMessage()` completely and set the body to the object - # itself instead of its string representation. - # For more background, see: https://github.com/open-telemetry/opentelemetry-python/pull/4216 - if not record.args and not isinstance(record.msg, str): - # if record.msg is not a value we can export, cast it to string - # TODO: https://github.com/open-telemetry/opentelemetry-python/issues/5304 - do something better - # than just casting to a string. - if not isinstance( - record.msg, - ( - NoneType, - bool, - bytes, - int, - float, - str, - Sequence, - Mapping, - ), - ): - body = str(record.msg) - else: - body = record.msg - else: - body = record.getMessage() - - # Map Python log level names to OTel severity text as defined in - # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity - # Python "WARNING" -> OTel "WARN" (see #3548) - # Python "CRITICAL" -> OTel "FATAL" (see #4984) - _python_to_otel_severity_text = { - "WARNING": "WARN", - "CRITICAL": "FATAL", - } - level_name = _python_to_otel_severity_text.get(record.levelname, record.levelname) - - return LogRecord( - timestamp=timestamp, - observed_timestamp=observered_timestamp, - context=get_current() or None, - severity_text=level_name, - severity_number=severity_number, - body=body, - attributes=attributes, - ) - - def emit(self, record: logging.LogRecord) -> None: - """ - Emit a record. Skip emitting if logger is NoOp. - - The record is translated to OTel format, and then sent across the pipeline. - """ - logger = get_logger(record.name, logger_provider=self._logger_provider) - if not isinstance(logger, NoOpLogger): - logger.emit(self._translate(record)) - - def flush(self) -> None: - """ - Flushes the logging output. Skip flushing if logging_provider has no force_flush method. - """ - if hasattr(self._logger_provider, "force_flush") and callable( - self._logger_provider.force_flush # type: ignore[reportAttributeAccessIssue] - ): - # This is done in a separate thread to avoid a potential deadlock, for - # details see https://github.com/open-telemetry/opentelemetry-python/pull/4636. - thread = threading.Thread(target=self._logger_provider.force_flush) # type: ignore[reportAttributeAccessIssue] - thread.start() - - -@dataclass -class _LoggerConfig: - is_enabled: bool = True - - @classmethod - def default(cls) -> _LoggerConfig: - return _LoggerConfig() - - -class Logger(APILogger): - def __init__( - self, - resource: Resource, - multi_log_record_processor: SynchronousMultiLogRecordProcessor | ConcurrentMultiLogRecordProcessor, - instrumentation_scope: InstrumentationScope, - *, - logger_metrics: LoggerMetricsT, - log_record_limits: LogRecordLimits | None = None, - _logger_config: _LoggerConfig, - ): - super().__init__( - instrumentation_scope.name, - instrumentation_scope.version, - instrumentation_scope.schema_url, - instrumentation_scope.attributes, - ) - self._resource = resource - self._multi_log_record_processor = multi_log_record_processor - self._instrumentation_scope = instrumentation_scope - self._logger_metrics = logger_metrics - self._logger_config = _logger_config - self._log_record_limits = log_record_limits or LogRecordLimits() - - def _is_enabled(self) -> bool: - return self._logger_config.is_enabled - - def enabled( - self, - *, - context: Context | None = None, - severity_number: SeverityNumber | None = None, - event_name: str | None = None, - ) -> bool: - if not self._is_enabled(): - return False - return self._multi_log_record_processor.enabled( - context=context, - instrumentation_scope=self._instrumentation_scope, - severity_number=severity_number, - event_name=event_name, - ) - - def _set_logger_config(self, logger_config: _LoggerConfig) -> None: - self._logger_config = logger_config - - def _set_resource(self, resource: Resource) -> None: - self._resource = resource - - @property - def instrumentation_scope(self): - return self._instrumentation_scope - - @property - def resource(self): - return self._resource - - # pylint: disable=arguments-differ - def emit( - self, - record: LogRecord | None = None, - *, - timestamp: int | None = None, - observed_timestamp: int | None = None, - context: Context | None = None, - severity_number: SeverityNumber | None = None, - severity_text: str | None = None, - body: AnyValue | None = None, - attributes: Attributes = None, - event_name: str | None = None, - exception: BaseException | None = None, - ) -> None: - """Emits the :class:`ReadWriteLogRecord` by setting instrumentation scope - and forwarding to the processor. - """ - if not self._is_enabled(): - return - # If a record is provided, use it directly - if record is not None: - if not isinstance(record, ReadWriteLogRecord): - if record.exception is not None: - record = _copy_log_record_with_exception(record) - # pylint:disable=protected-access - writable_record = ReadWriteLogRecord._from_api_log_record( - record=record, - resource=self._resource, - instrumentation_scope=self._instrumentation_scope, - limits=self._log_record_limits, - ) - else: - _set_log_record_exception_attributes(record.log_record) - writable_record = record - else: - # Create a record from individual parameters - log_record = _create_log_record_with_exception( - timestamp=timestamp, - observed_timestamp=observed_timestamp, - context=context, - severity_number=severity_number, - severity_text=severity_text, - body=body, - attributes=attributes, - event_name=event_name, - exception=exception, - ) - # pylint:disable=protected-access - writable_record = ReadWriteLogRecord._from_api_log_record( - record=log_record, - resource=self._resource, - instrumentation_scope=self._instrumentation_scope, - limits=self._log_record_limits, - ) - - self._logger_metrics.emit_log() - self._multi_log_record_processor.on_emit(writable_record) - - -_LoggerConfiguratorT = Callable[[InstrumentationScope], _LoggerConfig] -_RuleBasedLoggerConfigurator = RuleBasedConfigurator[_LoggerConfig] - - -def _default_logger_configurator( - _logger_scope: InstrumentationScope, -) -> _LoggerConfig: - return _LoggerConfig.default() - - -def _disable_logger_configurator( - _logger_scope: InstrumentationScope, -) -> _LoggerConfig: - return _LoggerConfig(is_enabled=False) - - -class LoggerProvider(APILoggerProvider): - def __init__( - self, - resource: Resource | None = None, - shutdown_on_exit: bool = True, - multi_log_record_processor: SynchronousMultiLogRecordProcessor - | ConcurrentMultiLogRecordProcessor - | None = None, - *, - meter_provider: MeterProvider | None = None, - log_record_limits: LogRecordLimits | None = None, - _logger_configurator: _LoggerConfiguratorT | None = None, - ): - if resource is None: - self._resource = Resource.create({}) - else: - self._resource = resource - self._multi_log_record_processor = multi_log_record_processor or SynchronousMultiLogRecordProcessor() - self._logger_metrics = create_logger_metrics( - meter_provider or get_meter_provider(), - parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), - ) - disabled = environ.get(OTEL_SDK_DISABLED, "") - self._disabled = disabled.lower().strip() == "true" - self._logger_configurator = _logger_configurator or _default_logger_configurator - self._log_record_limits = log_record_limits or LogRecordLimits() - self._at_exit_handler = None - if shutdown_on_exit: - self._at_exit_handler = atexit.register(self.shutdown) - self._logger_cache = {} - self._logger_cache_lock = Lock() - self._active_loggers: WeakSet[Logger] = WeakSet() - self._active_loggers_lock = Lock() - if hasattr(os, "register_at_fork"): - weak_at_fork = WeakMethod(self._handle_fork) - - def _after_in_child() -> None: - if at_fork := weak_at_fork(): - at_fork() - - os.register_at_fork(after_in_child=_after_in_child) - - def _handle_fork(self) -> None: - self._logger_cache_lock = Lock() - self._active_loggers_lock = Lock() - self._update_resource(_get_process_dependent_resource()) - - @property - def resource(self): - return self._resource - - def _update_resource(self, resource: Resource) -> None: - with self._active_loggers_lock: - self._resource = self._resource.merge(resource) - for logger in list(self._active_loggers): - # pylint: disable-next=protected-access - logger._set_resource(self._resource) - - def _get_logger_no_cache( - self, - name: str, - version: str | None = None, - schema_url: str | None = None, - attributes: Attributes = None, - ) -> Logger: - scope = InstrumentationScope(name, version, schema_url, attributes) - - return Logger( - self._resource, - self._multi_log_record_processor, - scope, - logger_metrics=self._logger_metrics, - _logger_config=self._apply_logger_configurator(scope), - log_record_limits=self._log_record_limits, - ) - - def _get_logger_cached( - self, - name: str, - version: str | None = None, - schema_url: str | None = None, - ) -> Logger: - with self._logger_cache_lock: - key = (name, version, schema_url) - if key in self._logger_cache: - return self._logger_cache[key] - - self._logger_cache[key] = self._get_logger_no_cache(name, version, schema_url) - return self._logger_cache[key] - - def get_logger( - self, - name: str, - version: str | None = None, - schema_url: str | None = None, - attributes: Attributes = None, - ) -> APILogger: - if self._disabled: - return NoOpLogger( - name, - version=version, - schema_url=schema_url, - attributes=attributes, - ) - logger = ( - self._get_logger_cached(name, version, schema_url) - if attributes is None - else self._get_logger_no_cache(name, version, schema_url, attributes) - ) - with self._active_loggers_lock: - self._active_loggers.add(logger) - return logger - - def add_log_record_processor(self, log_record_processor: LogRecordProcessor): - """Registers a new :class:`LogRecordProcessor` for this `LoggerProvider` instance. - - The log processors are invoked in the same order they are registered. - """ - self._multi_log_record_processor.add_log_record_processor(log_record_processor) - - def _set_logger_configurator(self, *, logger_configurator: _LoggerConfiguratorT): - """Set a new LoggerConfigurator for this LoggerProvider. - - Setting a new LoggerConfigurator will result in the configurator being called - for each outstanding Logger and for any newly created loggers thereafter. - Therefore, it is important that the provided function returns quickly. - """ - self._logger_configurator = logger_configurator - with self._active_loggers_lock: - for logger in list(self._active_loggers): - # pylint: disable-next=protected-access - logger._set_logger_config(self._apply_logger_configurator(logger.instrumentation_scope)) - - def _apply_logger_configurator(self, instrumentation_scope: InstrumentationScope) -> _LoggerConfig: - try: - return self._logger_configurator(instrumentation_scope) - # pylint: disable-next=broad-exception-caught - except Exception: - _logger.exception( - "logger configurator failed for scope '%s', using default config", - instrumentation_scope.name, - ) - return _LoggerConfig.default() - - def shutdown(self) -> None: - """Shuts down the log processors.""" - self._multi_log_record_processor.shutdown() - if self._at_exit_handler is not None: - atexit.unregister(self._at_exit_handler) - self._at_exit_handler = None - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Force flush the log processors. - - Args: - timeout_millis: The maximum amount of time to wait for logs to be - exported. - - Returns: - True if all the log processors flushes the logs within timeout, - False otherwise. - """ - return self._multi_log_record_processor.force_flush(timeout_millis) - - -_STD_TO_OTEL = { - 10: SeverityNumber.DEBUG, - 11: SeverityNumber.DEBUG2, - 12: SeverityNumber.DEBUG3, - 13: SeverityNumber.DEBUG4, - 14: SeverityNumber.DEBUG4, - 15: SeverityNumber.DEBUG4, - 16: SeverityNumber.DEBUG4, - 17: SeverityNumber.DEBUG4, - 18: SeverityNumber.DEBUG4, - 19: SeverityNumber.DEBUG4, - 20: SeverityNumber.INFO, - 21: SeverityNumber.INFO2, - 22: SeverityNumber.INFO3, - 23: SeverityNumber.INFO4, - 24: SeverityNumber.INFO4, - 25: SeverityNumber.INFO4, - 26: SeverityNumber.INFO4, - 27: SeverityNumber.INFO4, - 28: SeverityNumber.INFO4, - 29: SeverityNumber.INFO4, - 30: SeverityNumber.WARN, - 31: SeverityNumber.WARN2, - 32: SeverityNumber.WARN3, - 33: SeverityNumber.WARN4, - 34: SeverityNumber.WARN4, - 35: SeverityNumber.WARN4, - 36: SeverityNumber.WARN4, - 37: SeverityNumber.WARN4, - 38: SeverityNumber.WARN4, - 39: SeverityNumber.WARN4, - 40: SeverityNumber.ERROR, - 41: SeverityNumber.ERROR2, - 42: SeverityNumber.ERROR3, - 43: SeverityNumber.ERROR4, - 44: SeverityNumber.ERROR4, - 45: SeverityNumber.ERROR4, - 46: SeverityNumber.ERROR4, - 47: SeverityNumber.ERROR4, - 48: SeverityNumber.ERROR4, - 49: SeverityNumber.ERROR4, - 50: SeverityNumber.FATAL, - 51: SeverityNumber.FATAL2, - 52: SeverityNumber.FATAL3, - 53: SeverityNumber.FATAL4, -} - - -def std_to_otel(levelno: int) -> SeverityNumber: - """ - Map python log levelno as defined in https://docs.python.org/3/library/logging.html#logging-levels - to OTel log severity number as defined here: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#field-severitynumber - """ - if levelno < 10: - return SeverityNumber.UNSPECIFIED - if levelno > 53: - return SeverityNumber.FATAL4 - return _STD_TO_OTEL[levelno] +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import abc +import atexit +import base64 +import concurrent.futures +import json +import logging +import os +import threading +import traceback +import warnings +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from os import environ +from threading import Lock +from time import time_ns +from types import NoneType +from typing import ( # noqa + Any, + Tuple, + Union, + cast, + overload, +) +from weakref import WeakMethod, WeakSet + +from typing_extensions import deprecated + +from opentelemetry._logs import Logger as APILogger +from opentelemetry._logs import LoggerProvider as APILoggerProvider +from opentelemetry._logs import ( + LogRecord, + NoOpLogger, + SeverityNumber, + get_logger, + get_logger_provider, +) +from opentelemetry.attributes import BoundedAttributes +from opentelemetry.context import get_current +from opentelemetry.context.context import Context +from opentelemetry.metrics import MeterProvider, get_meter_provider +from opentelemetry.sdk._logs._internal._exceptions import ( + _copy_log_record_with_exception, + _create_log_record_with_exception, + _set_log_record_exception_attributes, +) +from opentelemetry.sdk._logs._internal._logger_metrics import ( + LoggerMetricsT, + create_logger_metrics, +) +from opentelemetry.sdk.environment_variables import ( + OTEL_ATTRIBUTE_COUNT_LIMIT, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, + OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, + OTEL_SDK_DISABLED, +) +from opentelemetry.sdk.environment_variables._internal import ( + parse_boolean_environment_variable, +) +from opentelemetry.sdk.resources import ( + Resource, + _get_process_dependent_resource, +) +from opentelemetry.sdk.util import ns_to_iso_str +from opentelemetry.sdk.util._configurator import RuleBasedConfigurator +from opentelemetry.sdk.util.instrumentation import ( + InstrumentationScope, +) +from opentelemetry.semconv.attributes import ( + code_attributes, + exception_attributes, +) +from opentelemetry.trace import ( + format_span_id, + format_trace_id, +) +from opentelemetry.util.types import AnyValue, Attributes + +# pylint: disable=too-many-lines + +_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 +_ENV_VALUE_UNSET = "" + +_logger = logging.getLogger(__name__) + + +class BytesEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, bytes): + return base64.b64encode(o).decode() + return super().default(o) + + +class LogRecordDroppedAttributesWarning(UserWarning): + """Custom warning to indicate dropped log attributes due to limits. + + This class is used to filter and handle these specific warnings separately + from other warnings, ensuring that they are only shown once without + interfering with default user warnings. + """ + + +warnings.simplefilter("once", LogRecordDroppedAttributesWarning) + + +@deprecated( + "Use LogRecordDroppedAttributesWarning. Since logs are not stable yet this WILL be removed in future releases." +) +class LogDroppedAttributesWarning(LogRecordDroppedAttributesWarning): + pass + + +class LogRecordLimits: + """This class is based on a SpanLimits class in the Tracing module. + + This class represents the limits that should be enforced on recorded data such as events, links, attributes etc. + + This class does not enforce any limits itself. It only provides a way to read limits from env, + default values and from user provided arguments. + + All limit arguments must be either a non-negative integer, ``None`` or ``LogRecordLimits.UNSET``. + + - All limit arguments are optional. + - If a limit argument is not set, the class will try to read its value from the corresponding + environment variable. + - If the environment variable is not set, the default value, if any, will be used. + + Limit precedence: + + - If a model specific limit is set, it will be used. + - Else if the corresponding global limit is set, it will be used. + - Else if the model specific limit has a default value, the default value will be used. + - Else if the global limit has a default value, the default value will be used. + + Args: + max_attributes: Maximum number of attributes that can be added to a log record (global fallback). + Environment variable: ``OTEL_ATTRIBUTE_COUNT_LIMIT`` + Default: {_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT} + max_attribute_length: Maximum length an attribute value can have (global fallback). Values + longer than the specified length will be truncated. + max_log_record_attributes: Maximum number of attributes that can be added to a log record. + Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT`` + Falls back to ``max_attributes`` when unset. + max_log_record_attribute_length: Maximum length a log record attribute value can have. + Environment variable: ``OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT`` + Falls back to ``max_attribute_length`` when unset. + """ + + UNSET = -1 + + def __init__( + self, + max_attributes: int | None = None, + max_attribute_length: int | None = None, + max_log_record_attributes: int | None = None, + max_log_record_attribute_length: int | None = None, + ): + # global attribute count + global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) + self.max_attributes = ( + global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT + ) + + # global attribute length + self.max_attribute_length = self._from_env_if_absent( + max_attribute_length, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + ) + + self.max_log_record_attributes = self._from_env_if_absent( + max_log_record_attributes, + OTEL_LOGRECORD_ATTRIBUTE_COUNT_LIMIT, + default=self.max_attributes, + ) + + self.max_log_record_attribute_length = self._from_env_if_absent( + max_log_record_attribute_length, + OTEL_LOGRECORD_ATTRIBUTE_VALUE_LENGTH_LIMIT, + default=self.max_attribute_length, + ) + + def __repr__(self): + return ( + f"{type(self).__name__}(" + f"max_attributes={self.max_attributes}, " + f"max_attribute_length={self.max_attribute_length}, " + f"max_log_record_attributes={self.max_log_record_attributes}, " + f"max_log_record_attribute_length={self.max_log_record_attribute_length})" + ) + + @classmethod + def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: + if value == cls.UNSET: + return None + + err_msg = "{} must be a non-negative integer but got {}" + + # if no value is provided for the limit, try to load it from env + if value is None: + # return default value if env var is not set + if env_var not in environ: + return default + + str_value = environ.get(env_var, "").strip().lower() + if str_value == _ENV_VALUE_UNSET: + return None + + try: + value = int(str_value) + except ValueError: + raise ValueError(err_msg.format(env_var, str_value)) + + if value < 0: + raise ValueError(err_msg.format(env_var, value)) + return value + + +@deprecated("Use LogRecordLimits. Since logs are not stable yet this WILL be removed in future releases.") +class LogLimits(LogRecordLimits): + pass + + +@dataclass(frozen=True) +class ReadableLogRecord: + """Readable LogRecord should be kept exactly in-sync with ReadWriteLogRecord, only difference is the frozen=True param.""" + + log_record: LogRecord + resource: Resource + instrumentation_scope: InstrumentationScope | None = None + limits: LogRecordLimits | None = None + + @property + def dropped_attributes(self) -> int: + if isinstance(self.log_record.attributes, BoundedAttributes): + return self.log_record.attributes.dropped + return 0 + + def to_json(self, indent: int | None = 4) -> str: + return json.dumps( + { + "body": self.log_record.body, + "severity_number": self.log_record.severity_number.value + if self.log_record.severity_number is not None + else None, + "severity_text": self.log_record.severity_text, + "attributes": (dict(self.log_record.attributes) if bool(self.log_record.attributes) else None), + "dropped_attributes": self.dropped_attributes, + "timestamp": ns_to_iso_str(self.log_record.timestamp) + if self.log_record.timestamp is not None + else None, + "observed_timestamp": ns_to_iso_str(self.log_record.observed_timestamp), + "trace_id": ( + f"0x{format_trace_id(self.log_record.trace_id)}" if self.log_record.trace_id is not None else "" + ), + "span_id": ( + f"0x{format_span_id(self.log_record.span_id)}" if self.log_record.span_id is not None else "" + ), + "trace_flags": self.log_record.trace_flags, + "resource": json.loads(self.resource.to_json()), + "event_name": self.log_record.event_name if self.log_record.event_name else "", + }, + indent=indent, + cls=BytesEncoder, + ) + + +@dataclass +class ReadWriteLogRecord: + """A ReadWriteLogRecord instance represents an event being logged. + ReadWriteLogRecord instances are created and emitted via `Logger` + every time something is logged. They contain all the information + pertinent to the event being logged. + """ + + log_record: LogRecord + resource: Resource | None = Resource.create({}) + instrumentation_scope: InstrumentationScope | None = None + limits: LogRecordLimits = field(default_factory=LogRecordLimits) + + def __post_init__(self): + self.log_record.attributes = BoundedAttributes( + maxlen=self.limits.max_log_record_attributes, + attributes=self.log_record.attributes if self.log_record.attributes else None, + immutable=False, + max_value_len=self.limits.max_log_record_attribute_length, + extended_attributes=True, + ) + if self.dropped_attributes > 0: + warnings.warn( + "Log record attributes were dropped due to limits", + LogRecordDroppedAttributesWarning, + stacklevel=2, + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, ReadWriteLogRecord): + return NotImplemented + return self.__dict__ == other.__dict__ + + @property + def dropped_attributes(self) -> int: + if isinstance(self.log_record.attributes, BoundedAttributes): + return self.log_record.attributes.dropped + return 0 + + @classmethod + def _from_api_log_record( + cls, + *, + record: LogRecord, + resource: Resource, + instrumentation_scope: InstrumentationScope | None = None, + limits: LogRecordLimits | None = None, + ) -> ReadWriteLogRecord: + return cls( + log_record=record, + resource=resource, + instrumentation_scope=instrumentation_scope, + **({} if limits is None else {"limits": limits}), + ) + + +class LogRecordProcessor(abc.ABC): + """Interface to hook the log record emitting action. + + Log processors can be registered directly using + :func:`LoggerProvider.add_log_record_processor` and they are invoked + in the same order as they were registered. + + Implementers of custom log processors should be aware of the following: + + Error Handling + -------------- + According to the OpenTelemetry error handling principles, the SDK should + not throw unhandled exceptions at runtime. When implementing a custom + ``LogRecordProcessor``, it is the **processor's responsibility** to handle + any exceptions that may be raised by the exporter's ``export()`` method. + + The ``LogRecordExporter.export()`` method may raise exceptions (e.g., + network errors, timeouts). If these exceptions are not caught, they will + propagate up and potentially crash the application. + + Custom processor implementations should wrap exporter calls in a + try/except block. See ``SimpleLogRecordProcessor`` for a reference + implementation:: + + def on_emit(self, log_record: ReadWriteLogRecord): + try: + self._exporter.export((log_record,)) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while exporting logs.") + + The ``BatchLogRecordProcessor`` handles this implicitly since export + operations occur in a background thread where exceptions cannot bubble + up to the caller. + """ + + @abc.abstractmethod + def on_emit(self, log_record: ReadWriteLogRecord) -> None: + """Emits the ``ReadWriteLogRecord``. + + Implementers should handle any exceptions raised during log processing + to prevent application crashes. See the class docstring for details + on error handling expectations. + """ + + def enabled( # pylint: disable=no-self-use,unused-argument + self, + *, + context: Context | None = None, + instrumentation_scope: InstrumentationScope | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + """Returns whether this processor would emit a record for the given arguments. + + Processors that support filtering MAY override this method. The default + implementation returns ``True``. + """ + return True + + @abc.abstractmethod + def shutdown(self) -> None: + """Called when a :class:`opentelemetry.sdk._logs.Logger` is shutdown""" + + @abc.abstractmethod + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Export all the received logs to the configured Exporter that have not yet + been exported. + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. + + Returns: + False if the timeout is exceeded, True otherwise. + """ + + +# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved +# pylint:disable=no-member +class SynchronousMultiLogRecordProcessor(LogRecordProcessor): + """Implementation of class:`LogRecordProcessor` that forwards all received + events to a list of log processors sequentially. + + The underlying log processors are called in sequential order as they were + added. + """ + + def __init__(self) -> None: + # use a tuple to avoid race conditions when adding a new log and + # iterating through it on "emit". + self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] + self._lock = threading.Lock() + + def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: + """Adds a Logprocessor to the list of log processors handled by this instance""" + with self._lock: + self._log_record_processors += (log_record_processor,) + + def on_emit(self, log_record: ReadWriteLogRecord) -> None: + for lp in self._log_record_processors: + lp.on_emit(log_record) + + def enabled( + self, + *, + context: Context | None = None, + instrumentation_scope: InstrumentationScope | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + processors = self._log_record_processors + return any( + lp.enabled( + context=context, + instrumentation_scope=instrumentation_scope, + severity_number=severity_number, + event_name=event_name, + ) + for lp in processors + ) + + def shutdown(self) -> None: + """Shutdown the log processors one by one""" + for lp in self._log_record_processors: + lp.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Force flush the log processors one by one + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. If the first n log processors exceeded the timeout + then remaining log processors will not be flushed. + + Returns: + True if all the log processors flushes the logs within timeout, + False otherwise. + """ + deadline_ns = time_ns() + timeout_millis * 1000000 + all_flushed = True + for lp in self._log_record_processors: + current_ts = time_ns() + if current_ts >= deadline_ns: + return False + + if lp.force_flush((deadline_ns - current_ts) // 1000000) is False: + all_flushed = False + + return all_flushed + + +class ConcurrentMultiLogRecordProcessor(LogRecordProcessor): + """Implementation of :class:`LogRecordProcessor` that forwards all received + events to a list of log processors in parallel. + + Calls to the underlying log processors are forwarded in parallel by + submitting them to a thread pool executor and waiting until each log + processor finished its work. + + Args: + max_workers: The number of threads managed by the thread pool executor + and thus defining how many log processors can work in parallel. + """ + + def __init__(self, max_workers: int = 2) -> None: + # use a tuple to avoid race conditions when adding a new log and + # iterating through it on "emit". + self._log_record_processors = () # type: tuple[LogRecordProcessor, ...] + self._lock = threading.Lock() + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) + + def add_log_record_processor(self, log_record_processor: LogRecordProcessor) -> None: + with self._lock: + self._log_record_processors += (log_record_processor,) + + def _submit_and_wait( + self, + func: Callable[[LogRecordProcessor], Callable[..., None]], + *args: Any, + **kwargs: Any, + ): + futures = [] + for lp in self._log_record_processors: + try: + future = self._executor.submit(func(lp), *args, **kwargs) + futures.append(future) + except RuntimeError: + # The executor may already be shut down when this runs from an + # atexit handler: CPython runs threading._shutdown() before the + # atexit queue, so the ThreadPoolExecutor is closed by the time + # LoggerProvider.shutdown() runs. Run inline so buffered work is + # still flushed instead of being silently dropped. A + # RuntimeError raised by the callback itself is not caught here + # and still propagates via future.result() below. + _logger.debug( + "Executor is shut down; running LogRecordProcessor callback inline.", + ) + func(lp)(*args, **kwargs) + for future in futures: + future.result() + + def on_emit(self, log_record: ReadWriteLogRecord) -> None: + self._submit_and_wait(lambda lp: lp.on_emit, log_record) + + def enabled( + self, + *, + context: Context | None = None, + instrumentation_scope: InstrumentationScope | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + processors = self._log_record_processors + return any( + lp.enabled( + context=context, + instrumentation_scope=instrumentation_scope, + severity_number=severity_number, + event_name=event_name, + ) + for lp in processors + ) + + def shutdown(self) -> None: + self._submit_and_wait(lambda lp: lp.shutdown) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Force flush the log processors in parallel. + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. + + Returns: + True if all the log processors flushes the logs within timeout, + False otherwise. + """ + futures = [] + for lp in self._log_record_processors: + future = self._executor.submit(lp.force_flush, timeout_millis) + futures.append(future) + + done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_millis / 1e3) + + if not_done_futures: + return False + + for future in done_futures: + if future.result() is False: + return False + + return True + + +# skip natural LogRecord attributes +# http://docs.python.org/library/logging.html#logrecord-attributes +_RESERVED_ATTRS = frozenset( + ( + "asctime", + "args", + "created", + "exc_info", + "exc_text", + "filename", + "funcName", + "getMessage", + "message", + "levelname", + "levelno", + "lineno", + "module", + "msecs", + "msg", + "name", + "pathname", + "process", + "processName", + "relativeCreated", + "stack_info", + "thread", + "threadName", + "taskName", + ) +) + + +class LoggingHandler(logging.Handler): + """A handler class which writes logging records, in OTLP format, to + a network destination or file. Supports signals from the `logging` module. + https://docs.python.org/3/library/logging.html + """ + + def __init__( + self, + level: int = logging.NOTSET, + logger_provider: APILoggerProvider | None = None, + ) -> None: + super().__init__(level=level) + self._logger_provider = logger_provider or get_logger_provider() + + warnings.warn( + "`LoggingHandler` in `opentelemetry-sdk` is deprecated. Use the " + "handler from `opentelemetry-instrumentation-logging` instead.", + DeprecationWarning, + ) + + @staticmethod + def _get_attributes(record: logging.LogRecord) -> Attributes: + attributes = {k: v for k, v in vars(record).items() if k not in _RESERVED_ATTRS} + + # Add standard code attributes for logs. + attributes[code_attributes.CODE_FILE_PATH] = record.pathname + attributes[code_attributes.CODE_FUNCTION_NAME] = record.funcName + attributes[code_attributes.CODE_LINE_NUMBER] = record.lineno + + if record.exc_info: + exctype, value, tb = record.exc_info + if exctype is not None: + attributes[exception_attributes.EXCEPTION_TYPE] = exctype.__name__ + if value is not None and value.args: + attributes[exception_attributes.EXCEPTION_MESSAGE] = str(value.args[0]) + if tb is not None: + # https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-spans/#stacktrace-representation + attributes[exception_attributes.EXCEPTION_STACKTRACE] = "".join( + traceback.format_exception(*record.exc_info) + ) + return attributes + + def _translate(self, record: logging.LogRecord) -> LogRecord: + timestamp = int(record.created * 1e9) + observered_timestamp = time_ns() + attributes = self._get_attributes(record) + severity_number = std_to_otel(record.levelno) + if self.formatter: + body = self.format(record) + else: + # `record.getMessage()` uses `record.msg` as a template to format + # `record.args` into. There is a special case in `record.getMessage()` + # where it will only attempt formatting if args are provided, + # otherwise, it just stringifies `record.msg`. + # + # Since the OTLP body field has a type of 'any' and the logging module + # is sometimes used in such a way that objects incorrectly end up + # set as record.msg, in those cases we would like to bypass + # `record.getMessage()` completely and set the body to the object + # itself instead of its string representation. + # For more background, see: https://github.com/open-telemetry/opentelemetry-python/pull/4216 + if not record.args and not isinstance(record.msg, str): + # if record.msg is not a value we can export, cast it to string + # TODO: https://github.com/open-telemetry/opentelemetry-python/issues/5304 - do something better + # than just casting to a string. + if not isinstance( + record.msg, + ( + NoneType, + bool, + bytes, + int, + float, + str, + Sequence, + Mapping, + ), + ): + body = str(record.msg) + else: + body = record.msg + else: + body = record.getMessage() + + # Map Python log level names to OTel severity text as defined in + # https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#displaying-severity + # Python "WARNING" -> OTel "WARN" (see #3548) + # Python "CRITICAL" -> OTel "FATAL" (see #4984) + _python_to_otel_severity_text = { + "WARNING": "WARN", + "CRITICAL": "FATAL", + } + level_name = _python_to_otel_severity_text.get(record.levelname, record.levelname) + + return LogRecord( + timestamp=timestamp, + observed_timestamp=observered_timestamp, + context=get_current() or None, + severity_text=level_name, + severity_number=severity_number, + body=body, + attributes=attributes, + ) + + def emit(self, record: logging.LogRecord) -> None: + """ + Emit a record. Skip emitting if logger is NoOp. + + The record is translated to OTel format, and then sent across the pipeline. + """ + logger = get_logger(record.name, logger_provider=self._logger_provider) + if not isinstance(logger, NoOpLogger): + logger.emit(self._translate(record)) + + def flush(self) -> None: + """ + Flushes the logging output. Skip flushing if logging_provider has no force_flush method. + """ + if hasattr(self._logger_provider, "force_flush") and callable( + self._logger_provider.force_flush # type: ignore[reportAttributeAccessIssue] + ): + # This is done in a separate thread to avoid a potential deadlock, for + # details see https://github.com/open-telemetry/opentelemetry-python/pull/4636. + thread = threading.Thread(target=self._logger_provider.force_flush) # type: ignore[reportAttributeAccessIssue] + thread.start() + + +@dataclass +class _LoggerConfig: + is_enabled: bool = True + + @classmethod + def default(cls) -> _LoggerConfig: + return _LoggerConfig() + + +class Logger(APILogger): + def __init__( + self, + resource: Resource, + multi_log_record_processor: SynchronousMultiLogRecordProcessor | ConcurrentMultiLogRecordProcessor, + instrumentation_scope: InstrumentationScope, + *, + logger_metrics: LoggerMetricsT, + log_record_limits: LogRecordLimits | None = None, + _logger_config: _LoggerConfig, + ): + super().__init__( + instrumentation_scope.name, + instrumentation_scope.version, + instrumentation_scope.schema_url, + instrumentation_scope.attributes, + ) + self._resource = resource + self._multi_log_record_processor = multi_log_record_processor + self._instrumentation_scope = instrumentation_scope + self._logger_metrics = logger_metrics + self._logger_config = _logger_config + self._log_record_limits = log_record_limits or LogRecordLimits() + + def _is_enabled(self) -> bool: + return self._logger_config.is_enabled + + def enabled( + self, + *, + context: Context | None = None, + severity_number: SeverityNumber | None = None, + event_name: str | None = None, + ) -> bool: + if not self._is_enabled(): + return False + return self._multi_log_record_processor.enabled( + context=context, + instrumentation_scope=self._instrumentation_scope, + severity_number=severity_number, + event_name=event_name, + ) + + def _set_logger_config(self, logger_config: _LoggerConfig) -> None: + self._logger_config = logger_config + + def _set_resource(self, resource: Resource) -> None: + self._resource = resource + + @property + def instrumentation_scope(self): + return self._instrumentation_scope + + @property + def resource(self): + return self._resource + + # pylint: disable=arguments-differ + def emit( + self, + record: LogRecord | None = None, + *, + timestamp: int | None = None, + observed_timestamp: int | None = None, + context: Context | None = None, + severity_number: SeverityNumber | None = None, + severity_text: str | None = None, + body: AnyValue | None = None, + attributes: Attributes = None, + event_name: str | None = None, + exception: BaseException | None = None, + ) -> None: + """Emits the :class:`ReadWriteLogRecord` by setting instrumentation scope + and forwarding to the processor. + """ + if not self._is_enabled(): + return + # If a record is provided, use it directly + if record is not None: + if not isinstance(record, ReadWriteLogRecord): + if record.exception is not None: + record = _copy_log_record_with_exception(record) + # pylint:disable=protected-access + writable_record = ReadWriteLogRecord._from_api_log_record( + record=record, + resource=self._resource, + instrumentation_scope=self._instrumentation_scope, + limits=self._log_record_limits, + ) + else: + _set_log_record_exception_attributes(record.log_record) + writable_record = record + else: + # Create a record from individual parameters + log_record = _create_log_record_with_exception( + timestamp=timestamp, + observed_timestamp=observed_timestamp, + context=context, + severity_number=severity_number, + severity_text=severity_text, + body=body, + attributes=attributes, + event_name=event_name, + exception=exception, + ) + # pylint:disable=protected-access + writable_record = ReadWriteLogRecord._from_api_log_record( + record=log_record, + resource=self._resource, + instrumentation_scope=self._instrumentation_scope, + limits=self._log_record_limits, + ) + + self._logger_metrics.emit_log() + self._multi_log_record_processor.on_emit(writable_record) + + +_LoggerConfiguratorT = Callable[[InstrumentationScope], _LoggerConfig] +_RuleBasedLoggerConfigurator = RuleBasedConfigurator[_LoggerConfig] + + +def _default_logger_configurator( + _logger_scope: InstrumentationScope, +) -> _LoggerConfig: + return _LoggerConfig.default() + + +def _disable_logger_configurator( + _logger_scope: InstrumentationScope, +) -> _LoggerConfig: + return _LoggerConfig(is_enabled=False) + + +class LoggerProvider(APILoggerProvider): + def __init__( + self, + resource: Resource | None = None, + shutdown_on_exit: bool = True, + multi_log_record_processor: SynchronousMultiLogRecordProcessor + | ConcurrentMultiLogRecordProcessor + | None = None, + *, + meter_provider: MeterProvider | None = None, + log_record_limits: LogRecordLimits | None = None, + _logger_configurator: _LoggerConfiguratorT | None = None, + ): + if resource is None: + self._resource = Resource.create({}) + else: + self._resource = resource + self._multi_log_record_processor = multi_log_record_processor or SynchronousMultiLogRecordProcessor() + self._logger_metrics = create_logger_metrics( + meter_provider or get_meter_provider(), + parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), + ) + disabled = environ.get(OTEL_SDK_DISABLED, "") + self._disabled = disabled.lower().strip() == "true" + self._logger_configurator = _logger_configurator or _default_logger_configurator + self._log_record_limits = log_record_limits or LogRecordLimits() + self._at_exit_handler = None + if shutdown_on_exit: + self._at_exit_handler = atexit.register(self.shutdown) + self._logger_cache = {} + self._logger_cache_lock = Lock() + self._active_loggers: WeakSet[Logger] = WeakSet() + self._active_loggers_lock = Lock() + if hasattr(os, "register_at_fork"): + weak_at_fork = WeakMethod(self._handle_fork) + + def _after_in_child() -> None: + if at_fork := weak_at_fork(): + at_fork() + + os.register_at_fork(after_in_child=_after_in_child) + + def _handle_fork(self) -> None: + self._logger_cache_lock = Lock() + self._active_loggers_lock = Lock() + self._update_resource(_get_process_dependent_resource()) + + @property + def resource(self): + return self._resource + + def _update_resource(self, resource: Resource) -> None: + with self._active_loggers_lock: + self._resource = self._resource.merge(resource) + for logger in list(self._active_loggers): + # pylint: disable-next=protected-access + logger._set_resource(self._resource) + + def _get_logger_no_cache( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes = None, + ) -> Logger: + scope = InstrumentationScope(name, version, schema_url, attributes) + + return Logger( + self._resource, + self._multi_log_record_processor, + scope, + logger_metrics=self._logger_metrics, + _logger_config=self._apply_logger_configurator(scope), + log_record_limits=self._log_record_limits, + ) + + def _get_logger_cached( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + ) -> Logger: + with self._logger_cache_lock: + key = (name, version, schema_url) + if key in self._logger_cache: + return self._logger_cache[key] + + self._logger_cache[key] = self._get_logger_no_cache(name, version, schema_url) + return self._logger_cache[key] + + def get_logger( + self, + name: str, + version: str | None = None, + schema_url: str | None = None, + attributes: Attributes = None, + ) -> APILogger: + if self._disabled: + return NoOpLogger( + name, + version=version, + schema_url=schema_url, + attributes=attributes, + ) + logger = ( + self._get_logger_cached(name, version, schema_url) + if attributes is None + else self._get_logger_no_cache(name, version, schema_url, attributes) + ) + with self._active_loggers_lock: + self._active_loggers.add(logger) + return logger + + def add_log_record_processor(self, log_record_processor: LogRecordProcessor): + """Registers a new :class:`LogRecordProcessor` for this `LoggerProvider` instance. + + The log processors are invoked in the same order they are registered. + """ + self._multi_log_record_processor.add_log_record_processor(log_record_processor) + + def _set_logger_configurator(self, *, logger_configurator: _LoggerConfiguratorT): + """Set a new LoggerConfigurator for this LoggerProvider. + + Setting a new LoggerConfigurator will result in the configurator being called + for each outstanding Logger and for any newly created loggers thereafter. + Therefore, it is important that the provided function returns quickly. + """ + self._logger_configurator = logger_configurator + with self._active_loggers_lock: + for logger in list(self._active_loggers): + # pylint: disable-next=protected-access + logger._set_logger_config(self._apply_logger_configurator(logger.instrumentation_scope)) + + def _apply_logger_configurator(self, instrumentation_scope: InstrumentationScope) -> _LoggerConfig: + try: + return self._logger_configurator(instrumentation_scope) + # pylint: disable-next=broad-exception-caught + except Exception: + _logger.exception( + "logger configurator failed for scope '%s', using default config", + instrumentation_scope.name, + ) + return _LoggerConfig.default() + + def shutdown(self) -> None: + """Shuts down the log processors.""" + self._multi_log_record_processor.shutdown() + if self._at_exit_handler is not None: + atexit.unregister(self._at_exit_handler) + self._at_exit_handler = None + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Force flush the log processors. + + Args: + timeout_millis: The maximum amount of time to wait for logs to be + exported. + + Returns: + True if all the log processors flushes the logs within timeout, + False otherwise. + """ + return self._multi_log_record_processor.force_flush(timeout_millis) + + +_STD_TO_OTEL = { + 10: SeverityNumber.DEBUG, + 11: SeverityNumber.DEBUG2, + 12: SeverityNumber.DEBUG3, + 13: SeverityNumber.DEBUG4, + 14: SeverityNumber.DEBUG4, + 15: SeverityNumber.DEBUG4, + 16: SeverityNumber.DEBUG4, + 17: SeverityNumber.DEBUG4, + 18: SeverityNumber.DEBUG4, + 19: SeverityNumber.DEBUG4, + 20: SeverityNumber.INFO, + 21: SeverityNumber.INFO2, + 22: SeverityNumber.INFO3, + 23: SeverityNumber.INFO4, + 24: SeverityNumber.INFO4, + 25: SeverityNumber.INFO4, + 26: SeverityNumber.INFO4, + 27: SeverityNumber.INFO4, + 28: SeverityNumber.INFO4, + 29: SeverityNumber.INFO4, + 30: SeverityNumber.WARN, + 31: SeverityNumber.WARN2, + 32: SeverityNumber.WARN3, + 33: SeverityNumber.WARN4, + 34: SeverityNumber.WARN4, + 35: SeverityNumber.WARN4, + 36: SeverityNumber.WARN4, + 37: SeverityNumber.WARN4, + 38: SeverityNumber.WARN4, + 39: SeverityNumber.WARN4, + 40: SeverityNumber.ERROR, + 41: SeverityNumber.ERROR2, + 42: SeverityNumber.ERROR3, + 43: SeverityNumber.ERROR4, + 44: SeverityNumber.ERROR4, + 45: SeverityNumber.ERROR4, + 46: SeverityNumber.ERROR4, + 47: SeverityNumber.ERROR4, + 48: SeverityNumber.ERROR4, + 49: SeverityNumber.ERROR4, + 50: SeverityNumber.FATAL, + 51: SeverityNumber.FATAL2, + 52: SeverityNumber.FATAL3, + 53: SeverityNumber.FATAL4, +} + + +def std_to_otel(levelno: int) -> SeverityNumber: + """ + Map python log levelno as defined in https://docs.python.org/3/library/logging.html#logging-levels + to OTel log severity number as defined here: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/logs/data-model.md#field-severitynumber + """ + if levelno < 10: + return SeverityNumber.UNSPECIFIED + if levelno > 53: + return SeverityNumber.FATAL4 + return _STD_TO_OTEL[levelno] diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index bc09ae48e5d..11cad9d587d 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -1,1491 +1,1491 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# pylint: disable=too-many-lines -import abc -import atexit -import concurrent.futures -import json -import logging -import os -import threading -import traceback -import typing -import weakref -from collections.abc import ( - Callable, - Iterator, - Mapping, - MutableMapping, - Sequence, -) -from dataclasses import dataclass -from os import environ -from time import time_ns -from types import MappingProxyType, TracebackType -from typing import ( - Any, -) -from warnings import filterwarnings - -from typing_extensions import deprecated - -from opentelemetry import context as context_api -from opentelemetry import metrics as metrics_api -from opentelemetry import trace as trace_api -from opentelemetry.attributes import BoundedAttributes -from opentelemetry.sdk import util -from opentelemetry.sdk.environment_variables import ( - OTEL_ATTRIBUTE_COUNT_LIMIT, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, - OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, - OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, - OTEL_SDK_DISABLED, - OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, - OTEL_SPAN_EVENT_COUNT_LIMIT, - OTEL_SPAN_LINK_COUNT_LIMIT, -) -from opentelemetry.sdk.environment_variables._internal import ( - parse_boolean_environment_variable, -) -from opentelemetry.sdk.resources import ( - Resource, - _get_process_dependent_resource, -) -from opentelemetry.sdk.trace import sampling -from opentelemetry.sdk.trace._tracer_metrics import create_tracer_metrics -from opentelemetry.sdk.trace.id_generator import IdGenerator, RandomIdGenerator -from opentelemetry.sdk.util import BoundedList -from opentelemetry.sdk.util._configurator import RuleBasedConfigurator -from opentelemetry.sdk.util.instrumentation import ( - InstrumentationInfo, - InstrumentationScope, -) -from opentelemetry.semconv.attributes.exception_attributes import ( - EXCEPTION_ESCAPED, - EXCEPTION_MESSAGE, - EXCEPTION_STACKTRACE, - EXCEPTION_TYPE, -) -from opentelemetry.trace import NoOpTracer, SpanContext -from opentelemetry.trace.status import Status, StatusCode -from opentelemetry.util import types -from opentelemetry.util._decorator import _agnosticcontextmanager - -logger = logging.getLogger(__name__) - - -_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT = 128 -_DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT = 128 -_DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT = 128 - - -_ENV_VALUE_UNSET = "" - - -class SpanProcessor: - """Interface which allows hooks for SDK's `Span` start and end method - invocations. - - Span processors can be registered directly using - :func:`TracerProvider.add_span_processor` and they are invoked - in the same order as they were registered. - """ - - def on_start( - self, - span: "Span", - parent_context: context_api.Context | None = None, - ) -> None: - """Called when a :class:`opentelemetry.trace.Span` is started. - - This method is called synchronously on the thread that starts the - span, therefore it should not block or throw an exception. - - Args: - span: The :class:`opentelemetry.trace.Span` that just started. - parent_context: The parent context of the span that just started. - """ - - def _on_ending(self, span: "Span") -> None: - """Called when a :class:`opentelemetry.trace.Span` is ending. - - This method is called synchronously on the thread that ends the - span, therefore it should not block or throw an exception. - - Args: - span: The :class:`opentelemetry.trace.Span` that is ending. - """ - - def on_end(self, span: "ReadableSpan") -> None: - """Called when a :class:`opentelemetry.trace.Span` is ended. - - This method is called synchronously on the thread that ends the - span, therefore it should not block or throw an exception. - - Args: - span: The :class:`opentelemetry.trace.Span` that just ended. - """ - - def shutdown(self) -> None: - """Called when a :class:`opentelemetry.sdk.trace.TracerProvider` is shutdown.""" - - def force_flush(self, timeout_millis: int = 30000) -> bool: # pylint: disable=no-self-use - """Export all ended spans to the configured Exporter that have not yet - been exported. - - Args: - timeout_millis: The maximum amount of time to wait for spans to be - exported. - - Returns: - False if the timeout is exceeded, True otherwise. - """ - return True - - -# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved -# pylint:disable=no-member -class SynchronousMultiSpanProcessor(SpanProcessor): - """Implementation of class:`SpanProcessor` that forwards all received - events to a list of span processors sequentially. - - The underlying span processors are called in sequential order as they were - added. - """ - - _span_processors: tuple[SpanProcessor, ...] - - def __init__(self): - # use a tuple to avoid race conditions when adding a new span and - # iterating through it on "on_start" and "on_end". - self._span_processors = () - self._lock = threading.Lock() - - def add_span_processor(self, span_processor: SpanProcessor) -> None: - """Adds a SpanProcessor to the list handled by this instance.""" - with self._lock: - self._span_processors += (span_processor,) - - def on_start( - self, - span: "Span", - parent_context: context_api.Context | None = None, - ) -> None: - for sp in self._span_processors: - sp.on_start(span, parent_context=parent_context) - - def _on_ending(self, span: "Span") -> None: - for sp in self._span_processors: - try: - # pylint: disable=protected-access - sp._on_ending(span) - except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while calling _on_ending on %s.", sp) - - def on_end(self, span: "ReadableSpan") -> None: - for sp in self._span_processors: - sp.on_end(span) - - def shutdown(self) -> None: - """Sequentially shuts down all underlying span processors.""" - for sp in self._span_processors: - sp.shutdown() - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Sequentially calls force_flush on all underlying - :class:`SpanProcessor` - - Args: - timeout_millis: The maximum amount of time over all span processors - to wait for spans to be exported. In case the first n span - processors exceeded the timeout followup span processors will be - skipped. - - Returns: - True if all span processors flushed their spans within the - given timeout, False otherwise. - """ - deadline_ns = time_ns() + timeout_millis * 1000000 - all_flushed = True - for sp in self._span_processors: - current_time_ns = time_ns() - if current_time_ns >= deadline_ns: - return False - - if sp.force_flush((deadline_ns - current_time_ns) // 1000000) is False: - all_flushed = False - - return all_flushed - - -class ConcurrentMultiSpanProcessor(SpanProcessor): - """Implementation of :class:`SpanProcessor` that forwards all received - events to a list of span processors in parallel. - - Calls to the underlying span processors are forwarded in parallel by - submitting them to a thread pool executor and waiting until each span - processor finished its work. The only exception is ``_on_ending``, which - is called synchronously on the thread ending the span, since that is the - only thread allowed to mutate the span while it is ending. - - Args: - num_threads: The number of threads managed by the thread pool executor - and thus defining how many span processors can work in parallel. - """ - - _span_processors: tuple[SpanProcessor, ...] - - def __init__(self, num_threads: int = 2): - # use a tuple to avoid race conditions when adding a new span and - # iterating through it on "on_start" and "on_end". - self._span_processors = () - self._lock = threading.Lock() - self._init_executor(num_threads) - if hasattr(os, "register_at_fork"): - # Only the main thread is kept in forked processed, the executor - # needs to be re-instantiated to get a fresh pool of threads: - weak_reinit = weakref.WeakMethod(self._init_executor) - - def _after_in_child() -> None: - reinit = weak_reinit() - if reinit is not None: - reinit(num_threads) - - os.register_at_fork(after_in_child=_after_in_child) - - def _init_executor(self, num_threads: int) -> None: - self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) - - def add_span_processor(self, span_processor: SpanProcessor) -> None: - """Adds a SpanProcessor to the list handled by this instance.""" - with self._lock: - self._span_processors += (span_processor,) - - def _submit_and_await( - self, - func: Callable[[SpanProcessor], Callable[..., None]], - *args: Any, - **kwargs: Any, - ): - futures = [] - for sp in self._span_processors: - try: - future = self._executor.submit(func(sp), *args, **kwargs) - futures.append(future) - except RuntimeError: - # The executor may already be shut down when this runs from an - # atexit handler: CPython runs threading._shutdown() before the - # atexit queue, so the ThreadPoolExecutor is closed by the time - # TracerProvider.shutdown() runs. Run inline so buffered work is - # still flushed instead of being silently dropped. A - # RuntimeError raised by the callback itself is not caught here - # and still propagates via future.result() below. - logger.debug( - "Executor is shut down; running SpanProcessor callback inline.", - ) - func(sp)(*args, **kwargs) - for future in futures: - future.result() - - def on_start( - self, - span: "Span", - parent_context: context_api.Context | None = None, - ) -> None: - self._submit_and_await(lambda sp: sp.on_start, span, parent_context=parent_context) - - def _on_ending(self, span: "Span") -> None: - # Unlike the other callbacks, _on_ending runs synchronously on the - # thread that ends the span: that is the only thread allowed to - # mutate the span while it is ending. - for sp in self._span_processors: - try: - # pylint: disable=protected-access - sp._on_ending(span) - except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while calling _on_ending on %s.", sp) - - def on_end(self, span: "ReadableSpan") -> None: - self._submit_and_await(lambda sp: sp.on_end, span) - - def shutdown(self) -> None: - """Shuts down all underlying span processors in parallel.""" - self._submit_and_await(lambda sp: sp.shutdown) - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Calls force_flush on all underlying span processors in parallel. - - Args: - timeout_millis: The maximum amount of time to wait for spans to be - exported. - - Returns: - True if all span processors flushed their spans within the given - timeout, False otherwise. - """ - futures = [] - for sp in self._span_processors: - future = self._executor.submit(sp.force_flush, timeout_millis) - futures.append(future) - - timeout_sec = timeout_millis / 1e3 - done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_sec) - if not_done_futures: - return False - - for future in done_futures: - if future.result() is False: - return False - - return True - - -class EventBase(abc.ABC): - def __init__(self, name: str, timestamp: int | None = None) -> None: - self._name = name - if timestamp is None: - self._timestamp = time_ns() - else: - self._timestamp = timestamp - - @property - def name(self) -> str: - return self._name - - @property - def timestamp(self) -> int: - return self._timestamp - - @property - @abc.abstractmethod - def attributes(self) -> types.Attributes: - pass - - -class Event(EventBase): - """A text annotation with a set of attributes. The attributes of an event - are immutable. - - Args: - name: Name of the event. - attributes: Attributes of the event. - timestamp: Timestamp of the event. If `None` it will filled - automatically. - """ - - def __init__( - self, - name: str, - attributes: types.Attributes = None, - timestamp: int | None = None, - limit: int | None = _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - ) -> None: - super().__init__(name, timestamp) - self._attributes = attributes - - @property - def attributes(self) -> types.Attributes: - return self._attributes - - @property - def dropped_attributes(self) -> int: - if isinstance(self._attributes, BoundedAttributes): - return self._attributes.dropped - return 0 - - -def _check_span_ended(func): - def wrapper(self, *args, **kwargs): - already_ended = False - with self._lock: # pylint: disable=protected-access - if self._is_mutable(): # pylint: disable=protected-access - func(self, *args, **kwargs) - else: - already_ended = True - - if already_ended: - logger.warning("Tried calling %s on an ended span.", func.__name__) - - return wrapper - - -def _is_valid_link(context: SpanContext, attributes: types.Attributes) -> bool: - return bool(context and (context.is_valid or (attributes or context.trace_state))) - - -class ReadableSpan: - """Provides read-only access to span attributes. - - Users should NOT be creating these objects directly. `ReadableSpan`s are created as - a direct result from using the tracing pipeline via the `Tracer`. - - """ - - def __init__( - self, - name: str, - context: trace_api.SpanContext | None = None, - parent: trace_api.SpanContext | None = None, - resource: Resource | None = None, - attributes: types.Attributes = None, - events: Sequence[Event] = (), - links: Sequence[trace_api.Link] = (), - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - instrumentation_info: InstrumentationInfo | None = None, - status: Status = Status(StatusCode.UNSET), - start_time: int | None = None, - end_time: int | None = None, - instrumentation_scope: InstrumentationScope | None = None, - ) -> None: - self._name = name - self._context = context - self._kind = kind - self._instrumentation_info = instrumentation_info - self._instrumentation_scope = instrumentation_scope - self._parent = parent - self._start_time = start_time - self._end_time = end_time - self._attributes = attributes - self._events = events - self._links = links - if resource is None: - self._resource = Resource.create({}) - else: - self._resource = resource - self._status = status - - @property - def dropped_attributes(self) -> int: - if isinstance(self._attributes, BoundedAttributes): - return self._attributes.dropped - return 0 - - @property - def dropped_events(self) -> int: - if isinstance(self._events, BoundedList): - return self._events.dropped - return 0 - - @property - def dropped_links(self) -> int: - if isinstance(self._links, BoundedList): - return self._links.dropped - return 0 - - @property - def name(self) -> str: - return self._name - - def get_span_context(self) -> trace_api.SpanContext | None: - return self._context - - @property - def context(self): - return self._context - - @property - def kind(self) -> trace_api.SpanKind: - return self._kind - - @property - def parent(self) -> trace_api.SpanContext | None: - return self._parent - - @property - def start_time(self) -> int | None: - return self._start_time - - @property - def end_time(self) -> int | None: - return self._end_time - - @property - def status(self) -> trace_api.Status: - return self._status - - @property - def attributes(self) -> types.Attributes: - return MappingProxyType(self._attributes or {}) - - @property - def events(self) -> Sequence[Event]: - return tuple(self._events) - - @property - def links(self) -> Sequence[trace_api.Link]: - return tuple(self._links) - - @property - def resource(self) -> Resource: - return self._resource - - @property - @deprecated("You should use instrumentation_scope. Deprecated since version 1.11.1.") - def instrumentation_info(self) -> InstrumentationInfo | None: - return self._instrumentation_info - - @property - def instrumentation_scope(self) -> InstrumentationScope | None: - return self._instrumentation_scope - - def to_json(self, indent: int | None = 4): - parent_id = None - if self.parent is not None: - parent_id = f"0x{trace_api.format_span_id(self.parent.span_id)}" - - start_time = None - if self._start_time: - start_time = util.ns_to_iso_str(self._start_time) - - end_time = None - if self._end_time: - end_time = util.ns_to_iso_str(self._end_time) - - status = { - "status_code": str(self._status.status_code.name), - } - if self._status.description: - status["description"] = self._status.description - - f_span = { - "name": self._name, - "context": (self._format_context(self._context) if self._context else None), - "kind": str(self.kind), - "parent_id": parent_id, - "start_time": start_time, - "end_time": end_time, - "status": status, - "attributes": self._format_attributes(self._attributes), - "events": self._format_events(self._events), - "links": self._format_links(self._links), - "resource": json.loads(self.resource.to_json()), - } - - return json.dumps(f_span, indent=indent) - - @staticmethod - def _format_context(context: SpanContext) -> dict[str, str]: - return { - "trace_id": f"0x{trace_api.format_trace_id(context.trace_id)}", - "span_id": f"0x{trace_api.format_span_id(context.span_id)}", - "trace_state": repr(context.trace_state), - } - - @staticmethod - def _format_attributes( - attributes: types.Attributes, - ) -> dict[str, Any] | None: - if attributes is not None and not isinstance(attributes, dict): - return dict(attributes) - return attributes - - @staticmethod - def _format_events(events: Sequence[Event]) -> list[dict[str, Any]]: - return [ - { - "name": event.name, - "timestamp": util.ns_to_iso_str(event.timestamp), - "attributes": Span._format_attributes( # pylint: disable=protected-access - event.attributes - ), - } - for event in events - ] - - @staticmethod - def _format_links(links: Sequence[trace_api.Link]) -> list[dict[str, Any]]: - return [ - { - "context": Span._format_context( # pylint: disable=protected-access - link.context - ), - "attributes": Span._format_attributes( # pylint: disable=protected-access - link.attributes - ), - } - for link in links - ] - - -class SpanLimits: - """The limits that should be enforced on recorded data such as events, links, attributes etc. - - This class does not enforce any limits itself. It only provides a way to read limits from env, - default values and from user provided arguments. - - All limit arguments must be either a non-negative integer, ``None`` or ``SpanLimits.UNSET``. - - - All limit arguments are optional. - - If a limit argument is not set, the class will try to read its value from the corresponding - environment variable. - - If the environment variable is not set, the default value, if any, will be used. - - Limit precedence: - - - If a model specific limit is set, it will be used. - - Else if the corresponding global limit is set, it will be used. - - Else if the model specific limit has a default value, the default value will be used. - - Else if the global limit has a default value, the default value will be used. - - Args: - max_attributes: Maximum number of attributes that can be added to a span, event, and link. - Environment variable: OTEL_ATTRIBUTE_COUNT_LIMIT - Default: {_DEFAULT_ATTRIBUTE_COUNT_LIMIT} - max_events: Maximum number of events that can be added to a Span. - Environment variable: OTEL_SPAN_EVENT_COUNT_LIMIT - Default: {_DEFAULT_SPAN_EVENT_COUNT_LIMIT} - max_links: Maximum number of links that can be added to a Span. - Environment variable: OTEL_SPAN_LINK_COUNT_LIMIT - Default: {_DEFAULT_SPAN_LINK_COUNT_LIMIT} - max_span_attributes: Maximum number of attributes that can be added to a Span. - Environment variable: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT - Default: {_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT} - max_event_attributes: Maximum number of attributes that can be added to an Event. - Default: {_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT} - max_link_attributes: Maximum number of attributes that can be added to a Link. - Default: {_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT} - max_attribute_length: Maximum length an attribute value can have. Values longer than - the specified length will be truncated. - max_span_attribute_length: Maximum length a span attribute value can have. Values longer than - the specified length will be truncated. - """ - - UNSET = -1 - - def __init__( - self, - max_attributes: int | None = None, - max_events: int | None = None, - max_links: int | None = None, - max_span_attributes: int | None = None, - max_event_attributes: int | None = None, - max_link_attributes: int | None = None, - max_attribute_length: int | None = None, - max_span_attribute_length: int | None = None, - ): - # span events and links count - self.max_events = self._from_env_if_absent( - max_events, - OTEL_SPAN_EVENT_COUNT_LIMIT, - _DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT, - ) - self.max_links = self._from_env_if_absent( - max_links, - OTEL_SPAN_LINK_COUNT_LIMIT, - _DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT, - ) - - # attribute count - global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) - self.max_attributes = ( - global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT - ) - - self.max_span_attributes = self._from_env_if_absent( - max_span_attributes, - OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT), - ) - self.max_event_attributes = self._from_env_if_absent( - max_event_attributes, - OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, - (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT), - ) - self.max_link_attributes = self._from_env_if_absent( - max_link_attributes, - OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, - (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT), - ) - - # attribute length - self.max_attribute_length = self._from_env_if_absent( - max_attribute_length, - OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, - ) - self.max_span_attribute_length = self._from_env_if_absent( - max_span_attribute_length, - OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, - # use global attribute length limit as default - self.max_attribute_length, - ) - - def __repr__(self): - return f"{type(self).__name__}(max_span_attributes={self.max_span_attributes}, max_events_attributes={self.max_event_attributes}, max_link_attributes={self.max_link_attributes}, max_attributes={self.max_attributes}, max_events={self.max_events}, max_links={self.max_links}, max_attribute_length={self.max_attribute_length})" - - @classmethod - def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: - if value == cls.UNSET: - return None - - err_msg = "{} must be a non-negative integer but got {}" - - # if no value is provided for the limit, try to load it from env - if value is None: - # return default value if env var is not set - if env_var not in environ: - return default - - str_value = environ.get(env_var, "").strip().lower() - if str_value == _ENV_VALUE_UNSET: - return None - - try: - value = int(str_value) - except ValueError: - raise ValueError(err_msg.format(env_var, str_value)) - - if value < 0: - raise ValueError(err_msg.format(env_var, value)) - return value - - -_UnsetLimits = SpanLimits( - max_attributes=SpanLimits.UNSET, - max_events=SpanLimits.UNSET, - max_links=SpanLimits.UNSET, - max_span_attributes=SpanLimits.UNSET, - max_event_attributes=SpanLimits.UNSET, - max_link_attributes=SpanLimits.UNSET, - max_attribute_length=SpanLimits.UNSET, - max_span_attribute_length=SpanLimits.UNSET, -) - -# not removed for backward compat. please use SpanLimits instead. -SPAN_ATTRIBUTE_COUNT_LIMIT = SpanLimits._from_env_if_absent( # pylint: disable=protected-access - None, - OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, - _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, -) - - -class Span(trace_api.Span, ReadableSpan): - """See `opentelemetry.trace.Span`. - - Users should create `Span` objects via the `Tracer` instead of this - constructor. - - Args: - name: The name of the operation this span represents - context: The immutable span context - parent: This span's parent's `opentelemetry.trace.SpanContext`, or - None if this is a root span - sampler: The sampler used to create this span - trace_config: Unused. Originally intended for trace-level configuration - from the OpenTelemetry protocol, but the upstream ``TraceConfig`` - proto was removed. Retained for backwards compatibility. - resource: Entity producing telemetry - attributes: The span's attributes to be exported - events: Timestamped events to be exported - links: Links to other spans to be exported - span_processor: `SpanProcessor` to invoke when starting and ending - this `Span`. - limits: `SpanLimits` instance that was passed to the `TracerProvider` - """ - - def __new__(cls, *args, **kwargs): - if cls is Span: - raise TypeError("Span must be instantiated via a tracer.") - return super().__new__(cls) - - # pylint: disable=too-many-locals - def __init__( - self, - name: str, - context: trace_api.SpanContext, - parent: trace_api.SpanContext | None = None, - sampler: sampling.Sampler | None = None, - trace_config: None = None, - resource: Resource | None = None, - attributes: types.Attributes = None, - events: Sequence[Event] | None = None, - links: Sequence[trace_api.Link] = (), - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - span_processor: SpanProcessor = SpanProcessor(), - instrumentation_info: InstrumentationInfo | None = None, - record_exception: bool = True, - set_status_on_exception: bool = True, - limits=_UnsetLimits, - instrumentation_scope: InstrumentationScope | None = None, - *, - record_end_metrics: Callable[[], None] | None = None, - ) -> None: - if resource is None: - resource = Resource.create({}) - super().__init__( - name=name, - context=context, - parent=parent, - kind=kind, - resource=resource, - instrumentation_info=instrumentation_info, - instrumentation_scope=instrumentation_scope, - ) - self._sampler = sampler - self._trace_config = trace_config - self._record_exception = record_exception - self._set_status_on_exception = set_status_on_exception - self._span_processor = span_processor - self._limits = limits - self._lock = threading.Lock() - # Identifier of the thread running end() while the _on_ending - # callbacks execute: the end timestamp is already set, but the span - # must remain mutable from that thread until the callbacks return. - self._ending_thread: int | None = None - self._attributes = BoundedAttributes( - self._limits.max_span_attributes, - attributes, - immutable=False, - max_value_len=self._limits.max_span_attribute_length, - ) - self._events = self._new_events() - if events: - for event in events: - # Immutable set to true so attributes cannot be added or removed after creation. - event._attributes = BoundedAttributes( - self._limits.max_event_attributes, - event.attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - self._events.append(event) - - self._links = self._new_links(links) - - self._record_end_metrics = record_end_metrics - - def __repr__(self): - return f'{type(self).__name__}(name="{self._name}", context={self._context})' - - def _new_events(self): - return BoundedList(self._limits.max_events) - - def _new_links(self, links: Sequence[trace_api.Link]): - if not links: - return BoundedList(self._limits.max_links) - - valid_links = [] - for link in links: - if link and _is_valid_link(link.context, link.attributes): - # Immutable set to true so attributes cannot be added or removed after creation. - # pylint: disable=protected-access - link._attributes = BoundedAttributes( - self._limits.max_link_attributes, - link.attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - valid_links.append(link) - - return BoundedList.from_seq(self._limits.max_links, valid_links) - - def get_span_context(self) -> trace_api.SpanContext: - return typing.cast(trace_api.SpanContext, self._context) - - def set_attributes(self, attributes: Mapping[str, types.AnyValue]) -> None: - with self._lock: - if not self._is_mutable(): - logger.warning("Setting attribute on ended span.") - return - - self._attributes._set_items(attributes) # pylint: disable=protected-access - - def set_attribute(self, key: str, value: types.AnyValue) -> None: - with self._lock: - if not self._is_mutable(): - logger.warning("Setting attribute on ended span.") - return - - self._attributes[key] = value - - @_check_span_ended - def _add_event(self, event: EventBase) -> None: - self._events.append(event) - - def add_event( - self, - name: str, - attributes: types.Attributes = None, - timestamp: int | None = None, - ) -> None: - # Immutable set to true so attributes cannot be added or removed after creation. - attributes = BoundedAttributes( - self._limits.max_event_attributes, - attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - self._add_event( - Event( - name=name, - attributes=attributes, - timestamp=timestamp, - ) - ) - - @_check_span_ended - def _add_link(self, link: trace_api.Link) -> None: - self._links.append(link) - - def add_link( - self, - context: SpanContext, - attributes: types.Attributes = None, - ) -> None: - if not _is_valid_link(context, attributes): - return - # Immutable set to true so attributes cannot be added or removed after creation. - attributes = BoundedAttributes( - self._limits.max_link_attributes, - attributes, - max_value_len=self._limits.max_attribute_length, - immutable=True, - ) - self._add_link( - trace_api.Link( - context=context, - attributes=attributes, - ) - ) - - def _readable_span(self) -> ReadableSpan: - return ReadableSpan( - name=self._name, - context=self._context, - parent=self._parent, - resource=self._resource, - attributes=self._attributes, - events=self._events, - links=self._links, - kind=self.kind, - status=self._status, - start_time=self._start_time, - end_time=self._end_time, - instrumentation_info=self._instrumentation_info, - instrumentation_scope=self._instrumentation_scope, - ) - - def start( - self, - start_time: int | None = None, - parent_context: context_api.Context | None = None, - ) -> None: - with self._lock: - if self._start_time is not None: - logger.warning("Calling start() on a started span.") - return - self._start_time = start_time if start_time is not None else time_ns() - - self._span_processor.on_start(self, parent_context=parent_context) - - def end(self, end_time: int | None = None) -> None: - with self._lock: - if self._start_time is None: - raise RuntimeError("Calling end() on a not started span.") - if self._end_time is not None: - logger.warning("Calling end() on an ended span.") - return - - self._ending_thread = threading.get_ident() - self._end_time = end_time if end_time is not None else time_ns() - - # The span must remain mutable from this thread while the _on_ending - # callbacks run; it becomes immutable once they have returned. - # pylint: disable=protected-access - try: - if self._record_end_metrics: - self._record_end_metrics() - self._span_processor._on_ending(self) - finally: - with self._lock: - self._ending_thread = None - self._attributes._immutable = True - self._span_processor.on_end(self._readable_span()) - - @_check_span_ended - def update_name(self, name: str) -> None: - self._name = name - - def _is_mutable(self) -> bool: - return self._end_time is None or self._ending_thread == threading.get_ident() - - def is_recording(self) -> bool: - # Unlike the mutators, this check is deliberately not taken under - # self._lock: it does not modify the span, and the answer can become - # stale as soon as it is returned anyway, since another thread may - # end the span right after this call. - return self._is_mutable() - - @_check_span_ended - def set_status( - self, - status: Status | StatusCode, - description: str | None = None, - ) -> None: - if isinstance(status, Status): - if description is not None: - logger.warning( - "Description %s ignored. Use either `Status` or `(StatusCode, Description)`", - description, - ) - new_status = status - elif isinstance(status, StatusCode): - new_status = Status(status, description) - else: - return - - if self._accepts_status(new_status): - self._status = new_status - - def _accepts_status(self, new_status: Status) -> bool: - """Decide whether ``new_status`` may replace the status already recorded. - - The codes are totally ordered, ``Ok > Error > Unset``, an attempt to set - ``Unset`` is ignored, and ``Ok`` is final. Once those two are handled, a - differing code always outranks the one recorded. A repeat of the same code - gets through only when it fills in a description that is still missing, - and an empty description counts as missing. - """ - if new_status.status_code is StatusCode.UNSET: - return False - - current = self._status - if current is None: - return True - if current.status_code is StatusCode.OK: - return False - - # An empty description is equivalent to an absent one. - return current.status_code is not new_status.status_code or ( - not current.description and bool(new_status.description) - ) - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - """Ends context manager and calls `end` on the `Span`.""" - if exc_val is not None and self.is_recording(): - # Record the exception as an event - # pylint:disable=protected-access - if self._record_exception: - self.record_exception(exception=exc_val, escaped=True) - # Records status if span is used as context manager - # i.e. with tracer.start_span() as span: - if self._set_status_on_exception: - self.set_status( - Status( - status_code=StatusCode.ERROR, - description=(f"{type(exc_val).__name__}: {exc_val}"), - ) - ) - - super().__exit__(exc_type, exc_val, exc_tb) - - def record_exception( - self, - exception: BaseException, - attributes: types.Attributes = None, - timestamp: int | None = None, - escaped: bool = False, - ) -> None: - """Records an exception as a span event.""" - stacktrace = "".join(traceback.format_exception(exception)) - module = type(exception).__module__ - qualname = type(exception).__qualname__ - exception_type = f"{module}.{qualname}" if module and module != "builtins" else qualname - _attributes: MutableMapping[str, types.AnyValue] = { - EXCEPTION_TYPE: exception_type, - EXCEPTION_MESSAGE: str(exception), - EXCEPTION_STACKTRACE: stacktrace, - EXCEPTION_ESCAPED: str(escaped), - } - if attributes: - _attributes.update(attributes) - self.add_event(name="exception", attributes=_attributes, timestamp=timestamp) - - -class _Span(Span): - """Protected implementation of `opentelemetry.trace.Span`. - - This constructor exists to prevent the instantiation of the `Span` class - by other mechanisms than through the `Tracer`. - """ - - -@dataclass -class _TracerConfig: - is_enabled: bool - - @classmethod - def default(cls): - return cls(is_enabled=True) - - -class Tracer(trace_api.Tracer): - """See `opentelemetry.trace.Tracer`.""" - - def __init__( - self, - sampler: sampling.Sampler, - resource: Resource, - span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor, - id_generator: IdGenerator, - instrumentation_info: InstrumentationInfo, - span_limits: SpanLimits, - instrumentation_scope: InstrumentationScope, - *, - meter_provider: metrics_api.MeterProvider | None = None, - _tracer_config: _TracerConfig | None = None, - ) -> None: - self.sampler = sampler - self.resource = resource - self.span_processor = span_processor - self.id_generator = id_generator - self.instrumentation_info = instrumentation_info - self._span_limits = span_limits - self._instrumentation_scope = instrumentation_scope - self._tracer_config = _tracer_config or _TracerConfig.default() - - meter_provider = meter_provider or metrics_api.get_meter_provider() - self._tracer_metrics = create_tracer_metrics( - meter_provider, - parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), - ) - - def _set_tracer_config(self, tracer_config: _TracerConfig): - self._tracer_config = tracer_config - - def _set_resource(self, resource: Resource) -> None: - self.resource = resource - - def _is_enabled(self) -> bool: - """If the tracer is not enabled, start_span will create a NonRecordingSpan""" - return self._tracer_config.is_enabled - - @_agnosticcontextmanager # pylint: disable=protected-access - def start_as_current_span( - self, - name: str, - context: context_api.Context | None = None, - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - attributes: types.Attributes = None, - links: Sequence[trace_api.Link] | None = (), - start_time: int | None = None, - record_exception: bool = True, - set_status_on_exception: bool = True, - end_on_exit: bool = True, - ) -> Iterator[trace_api.Span]: - span = self.start_span( - name=name, - context=context, - kind=kind, - attributes=attributes, - links=links, - start_time=start_time, - record_exception=record_exception, - set_status_on_exception=set_status_on_exception, - ) - with trace_api.use_span( - span, - end_on_exit=end_on_exit, - record_exception=record_exception, - set_status_on_exception=set_status_on_exception, - ) as span: - yield span - - def start_span( # pylint: disable=too-many-locals - self, - name: str, - context: context_api.Context | None = None, - kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, - attributes: types.Attributes = None, - links: Sequence[trace_api.Link] | None = (), - start_time: int | None = None, - record_exception: bool = True, - set_status_on_exception: bool = True, - ) -> trace_api.Span: - links = links or () - parent_span_context = trace_api.get_current_span(context).get_span_context() - - if parent_span_context is not None and not isinstance(parent_span_context, trace_api.SpanContext): - raise TypeError("parent_span_context must be a SpanContext or None.") - - if not self._is_enabled(): - return trace_api.NonRecordingSpan(context=parent_span_context) - - # is_valid determines root span - if parent_span_context is None or not parent_span_context.is_valid: - parent_span_context = None - trace_id = self.id_generator.generate_trace_id() - else: - trace_id = parent_span_context.trace_id - - # The sampler decides whether to create a real or no-op span at the - # time of span creation. No-op spans do not record events, and are not - # exported. - # The sampler may also add attributes to the newly-created span, e.g. - # to include information about the sampling result. - # The sampler may also modify the parent span context's tracestate - sampling_result = self.sampler.should_sample(context, trace_id, name, kind, attributes, links) - - trace_flags = ( - trace_api.TraceFlags(trace_api.TraceFlags.SAMPLED) - if sampling_result.decision.is_sampled() - else trace_api.TraceFlags(trace_api.TraceFlags.DEFAULT) - ) - - if parent_span_context is None: - random_trace_id = self.id_generator.is_trace_id_random() - else: - random_trace_id = parent_span_context.trace_flags.random_trace_id - - if random_trace_id: - trace_flags = trace_api.TraceFlags(trace_flags | trace_api.TraceFlags.RANDOM_TRACE_ID) - - span_context = trace_api.SpanContext( - trace_id, - self.id_generator.generate_span_id(), - is_remote=False, - trace_flags=trace_flags, - trace_state=sampling_result.trace_state, - ) - - record_end_metrics = self._tracer_metrics.start_span(parent_span_context, sampling_result.decision) - - # Only record if is_recording() is true - if sampling_result.decision.is_recording(): - # pylint:disable=protected-access - span = _Span( - name=name, - context=span_context, - parent=parent_span_context, - sampler=self.sampler, - resource=self.resource, - attributes=sampling_result.attributes, - span_processor=self.span_processor, - kind=kind, - links=links, - instrumentation_info=self.instrumentation_info, - record_exception=record_exception, - set_status_on_exception=set_status_on_exception, - limits=self._span_limits, - instrumentation_scope=self._instrumentation_scope, - record_end_metrics=record_end_metrics, - ) - span.start(start_time=start_time, parent_context=context) - else: - span = trace_api.NonRecordingSpan(context=span_context) - return span - - -_TracerConfiguratorT = Callable[[InstrumentationScope], _TracerConfig] -_RuleBasedTracerConfigurator = RuleBasedConfigurator[_TracerConfig] - - -def _default_tracer_configurator( - tracer_scope: InstrumentationScope, -) -> _TracerConfig: - """Default Tracer Configurator implementation - - In order to update Tracers configs you need to call - TracerProvider._set_tracer_configurator with a function - implementing this interface returning a Tracer Config.""" - return _RuleBasedTracerConfigurator( - rules=[], - default_config=_TracerConfig.default(), - )(tracer_scope) - - -def _disable_tracer_configurator( - tracer_scope: InstrumentationScope, -) -> _TracerConfig: - return _RuleBasedTracerConfigurator( - rules=[], - default_config=_TracerConfig(is_enabled=False), - )(tracer_scope) - - -class TracerProvider(trace_api.TracerProvider): - """See `opentelemetry.trace.TracerProvider`.""" - - def __init__( - self, - sampler: sampling.Sampler | None = None, - resource: Resource | None = None, - shutdown_on_exit: bool = True, - active_span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor | None = None, - id_generator: IdGenerator | None = None, - span_limits: SpanLimits | None = None, - *, - meter_provider: metrics_api.MeterProvider | None = None, - _tracer_configurator: _TracerConfiguratorT | None = None, - ) -> None: - self._active_span_processor = active_span_processor or SynchronousMultiSpanProcessor() - if id_generator is None: - self.id_generator = RandomIdGenerator() - else: - self.id_generator = id_generator - if resource is None: - self._resource = Resource.create({}) - else: - self._resource = resource - if not sampler: - sampler = sampling._get_from_env_or_default() - self.sampler = sampler - self._span_limits = span_limits or SpanLimits() - disabled = environ.get(OTEL_SDK_DISABLED, "") - self._disabled = disabled.lower().strip() == "true" - self._atexit_handler = None - self._meter_provider = meter_provider - - if shutdown_on_exit: - self._atexit_handler = atexit.register(self.shutdown) - - self._tracer_configurator = _tracer_configurator or _default_tracer_configurator - self._tracers_lock = threading.Lock() - self._tracers: dict[InstrumentationScope, Tracer] = {} - if hasattr(os, "register_at_fork"): - weak_at_fork = weakref.WeakMethod(self._handle_fork) - - def _after_in_child() -> None: - if at_fork := weak_at_fork(): - at_fork() - - os.register_at_fork(after_in_child=_after_in_child) - - def _handle_fork(self) -> None: - self._tracers_lock = threading.Lock() - self._update_resource(_get_process_dependent_resource()) - - def _set_tracer_configurator(self, *, tracer_configurator: _TracerConfiguratorT): - """This is the function used to update the TracerProvider TracerConfigurator - - Setting a new TracerConfigurator for a TracerProvider will update the - TracerConfig of all Tracers create by this TracerProvider. - """ - self._tracer_configurator = tracer_configurator - with self._tracers_lock: - for instrumentation_scope, tracer in self._tracers.items(): - tracer_config = self._apply_tracer_configurator(instrumentation_scope) - # pylint: disable-next=protected-access - tracer._set_tracer_config(tracer_config) - - @property - def resource(self) -> Resource: - return self._resource - - def _update_resource(self, resource: Resource) -> None: - with self._tracers_lock: - self._resource = self._resource.merge(resource) - for tracer in self._tracers.values(): - tracer._set_resource(self._resource) # pylint: disable=protected-access - - def _apply_tracer_configurator(self, instrumentation_scope: InstrumentationScope): - try: - return self._tracer_configurator(instrumentation_scope) - except Exception: # pylint: disable=broad-exception-caught - logger.exception( - "Failed to create a Tracer Config for %s, using default Tracer config", - instrumentation_scope, - ) - return _TracerConfig.default() - - def get_tracer( - self, - instrumenting_module_name: str, - instrumenting_library_version: str | None = None, - schema_url: str | None = None, - attributes: types.Attributes = None, - ) -> "trace_api.Tracer": - if self._disabled: - return NoOpTracer() - if not instrumenting_module_name: # Reject empty strings too. - instrumenting_module_name = "" - logger.error("get_tracer called with missing module name.") - if instrumenting_library_version is None: - instrumenting_library_version = "" - - filterwarnings( - "ignore", - message=(r"You should use InstrumentationScope. Deprecated since version 1.11.1."), - category=DeprecationWarning, - module="opentelemetry.sdk.trace", - ) - - instrumentation_info = InstrumentationInfo( - instrumenting_module_name, - instrumenting_library_version, - schema_url, - ) - - instrumentation_scope = InstrumentationScope( - instrumenting_module_name, - instrumenting_library_version, - schema_url, - attributes, - ) - - with self._tracers_lock: - if instrumentation_scope in self._tracers: - return self._tracers[instrumentation_scope] - - tracer_config = self._apply_tracer_configurator(instrumentation_scope) - tracer = Tracer( - self.sampler, - self.resource, - self._active_span_processor, - self.id_generator, - instrumentation_info, - self._span_limits, - instrumentation_scope, - meter_provider=self._meter_provider, - _tracer_config=tracer_config, - ) - self._tracers[instrumentation_scope] = tracer - - return tracer - - def add_span_processor(self, span_processor: SpanProcessor) -> None: - """Registers a new :class:`SpanProcessor` for this `TracerProvider`. - - The span processors are invoked in the same order they are registered. - """ - - # no lock here because add_span_processor is thread safe for both - # SynchronousMultiSpanProcessor and ConcurrentMultiSpanProcessor. - self._active_span_processor.add_span_processor(span_processor) - - def shutdown(self) -> None: - """Shut down the span processors added to the tracer provider.""" - self._active_span_processor.shutdown() - if self._atexit_handler is not None: - atexit.unregister(self._atexit_handler) - self._atexit_handler = None - - def force_flush(self, timeout_millis: int = 30000) -> bool: - """Requests the active span processor to process all spans that have not - yet been processed. - - By default force flush is called sequentially on all added span - processors. This means that span processors further back in the list - have less time to flush their spans. - To have span processors flush their spans in parallel it is possible to - initialize the tracer provider with an instance of - `ConcurrentMultiSpanProcessor` at the cost of using multiple threads. - - Args: - timeout_millis: The maximum amount of time to wait for spans to be - processed. - - Returns: - False if the timeout is exceeded, True otherwise. - """ - return self._active_span_processor.force_flush(timeout_millis) +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint: disable=too-many-lines +import abc +import atexit +import concurrent.futures +import json +import logging +import os +import threading +import traceback +import typing +import weakref +from collections.abc import ( + Callable, + Iterator, + Mapping, + MutableMapping, + Sequence, +) +from dataclasses import dataclass +from os import environ +from time import time_ns +from types import MappingProxyType, TracebackType +from typing import ( + Any, +) +from warnings import filterwarnings + +from typing_extensions import deprecated + +from opentelemetry import context as context_api +from opentelemetry import metrics as metrics_api +from opentelemetry import trace as trace_api +from opentelemetry.attributes import BoundedAttributes +from opentelemetry.sdk import util +from opentelemetry.sdk.environment_variables import ( + OTEL_ATTRIBUTE_COUNT_LIMIT, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, + OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, + OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED, + OTEL_SDK_DISABLED, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, + OTEL_SPAN_EVENT_COUNT_LIMIT, + OTEL_SPAN_LINK_COUNT_LIMIT, +) +from opentelemetry.sdk.environment_variables._internal import ( + parse_boolean_environment_variable, +) +from opentelemetry.sdk.resources import ( + Resource, + _get_process_dependent_resource, +) +from opentelemetry.sdk.trace import sampling +from opentelemetry.sdk.trace._tracer_metrics import create_tracer_metrics +from opentelemetry.sdk.trace.id_generator import IdGenerator, RandomIdGenerator +from opentelemetry.sdk.util import BoundedList +from opentelemetry.sdk.util._configurator import RuleBasedConfigurator +from opentelemetry.sdk.util.instrumentation import ( + InstrumentationInfo, + InstrumentationScope, +) +from opentelemetry.semconv.attributes.exception_attributes import ( + EXCEPTION_ESCAPED, + EXCEPTION_MESSAGE, + EXCEPTION_STACKTRACE, + EXCEPTION_TYPE, +) +from opentelemetry.trace import NoOpTracer, SpanContext +from opentelemetry.trace.status import Status, StatusCode +from opentelemetry.util import types +from opentelemetry.util._decorator import _agnosticcontextmanager + +logger = logging.getLogger(__name__) + + +_DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT = 128 +_DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT = 128 +_DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT = 128 + + +_ENV_VALUE_UNSET = "" + + +class SpanProcessor: + """Interface which allows hooks for SDK's `Span` start and end method + invocations. + + Span processors can be registered directly using + :func:`TracerProvider.add_span_processor` and they are invoked + in the same order as they were registered. + """ + + def on_start( + self, + span: "Span", + parent_context: context_api.Context | None = None, + ) -> None: + """Called when a :class:`opentelemetry.trace.Span` is started. + + This method is called synchronously on the thread that starts the + span, therefore it should not block or throw an exception. + + Args: + span: The :class:`opentelemetry.trace.Span` that just started. + parent_context: The parent context of the span that just started. + """ + + def _on_ending(self, span: "Span") -> None: + """Called when a :class:`opentelemetry.trace.Span` is ending. + + This method is called synchronously on the thread that ends the + span, therefore it should not block or throw an exception. + + Args: + span: The :class:`opentelemetry.trace.Span` that is ending. + """ + + def on_end(self, span: "ReadableSpan") -> None: + """Called when a :class:`opentelemetry.trace.Span` is ended. + + This method is called synchronously on the thread that ends the + span, therefore it should not block or throw an exception. + + Args: + span: The :class:`opentelemetry.trace.Span` that just ended. + """ + + def shutdown(self) -> None: + """Called when a :class:`opentelemetry.sdk.trace.TracerProvider` is shutdown.""" + + def force_flush(self, timeout_millis: int = 30000) -> bool: # pylint: disable=no-self-use + """Export all ended spans to the configured Exporter that have not yet + been exported. + + Args: + timeout_millis: The maximum amount of time to wait for spans to be + exported. + + Returns: + False if the timeout is exceeded, True otherwise. + """ + return True + + +# Temporary fix until https://github.com/PyCQA/pylint/issues/4098 is resolved +# pylint:disable=no-member +class SynchronousMultiSpanProcessor(SpanProcessor): + """Implementation of class:`SpanProcessor` that forwards all received + events to a list of span processors sequentially. + + The underlying span processors are called in sequential order as they were + added. + """ + + _span_processors: tuple[SpanProcessor, ...] + + def __init__(self): + # use a tuple to avoid race conditions when adding a new span and + # iterating through it on "on_start" and "on_end". + self._span_processors = () + self._lock = threading.Lock() + + def add_span_processor(self, span_processor: SpanProcessor) -> None: + """Adds a SpanProcessor to the list handled by this instance.""" + with self._lock: + self._span_processors += (span_processor,) + + def on_start( + self, + span: "Span", + parent_context: context_api.Context | None = None, + ) -> None: + for sp in self._span_processors: + sp.on_start(span, parent_context=parent_context) + + def _on_ending(self, span: "Span") -> None: + for sp in self._span_processors: + try: + # pylint: disable=protected-access + sp._on_ending(span) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while calling _on_ending on %s.", sp) + + def on_end(self, span: "ReadableSpan") -> None: + for sp in self._span_processors: + sp.on_end(span) + + def shutdown(self) -> None: + """Sequentially shuts down all underlying span processors.""" + for sp in self._span_processors: + sp.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Sequentially calls force_flush on all underlying + :class:`SpanProcessor` + + Args: + timeout_millis: The maximum amount of time over all span processors + to wait for spans to be exported. In case the first n span + processors exceeded the timeout followup span processors will be + skipped. + + Returns: + True if all span processors flushed their spans within the + given timeout, False otherwise. + """ + deadline_ns = time_ns() + timeout_millis * 1000000 + all_flushed = True + for sp in self._span_processors: + current_time_ns = time_ns() + if current_time_ns >= deadline_ns: + return False + + if sp.force_flush((deadline_ns - current_time_ns) // 1000000) is False: + all_flushed = False + + return all_flushed + + +class ConcurrentMultiSpanProcessor(SpanProcessor): + """Implementation of :class:`SpanProcessor` that forwards all received + events to a list of span processors in parallel. + + Calls to the underlying span processors are forwarded in parallel by + submitting them to a thread pool executor and waiting until each span + processor finished its work. The only exception is ``_on_ending``, which + is called synchronously on the thread ending the span, since that is the + only thread allowed to mutate the span while it is ending. + + Args: + num_threads: The number of threads managed by the thread pool executor + and thus defining how many span processors can work in parallel. + """ + + _span_processors: tuple[SpanProcessor, ...] + + def __init__(self, num_threads: int = 2): + # use a tuple to avoid race conditions when adding a new span and + # iterating through it on "on_start" and "on_end". + self._span_processors = () + self._lock = threading.Lock() + self._init_executor(num_threads) + if hasattr(os, "register_at_fork"): + # Only the main thread is kept in forked processed, the executor + # needs to be re-instantiated to get a fresh pool of threads: + weak_reinit = weakref.WeakMethod(self._init_executor) + + def _after_in_child() -> None: + reinit = weak_reinit() + if reinit is not None: + reinit(num_threads) + + os.register_at_fork(after_in_child=_after_in_child) + + def _init_executor(self, num_threads: int) -> None: + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) + + def add_span_processor(self, span_processor: SpanProcessor) -> None: + """Adds a SpanProcessor to the list handled by this instance.""" + with self._lock: + self._span_processors += (span_processor,) + + def _submit_and_await( + self, + func: Callable[[SpanProcessor], Callable[..., None]], + *args: Any, + **kwargs: Any, + ): + futures = [] + for sp in self._span_processors: + try: + future = self._executor.submit(func(sp), *args, **kwargs) + futures.append(future) + except RuntimeError: + # The executor may already be shut down when this runs from an + # atexit handler: CPython runs threading._shutdown() before the + # atexit queue, so the ThreadPoolExecutor is closed by the time + # TracerProvider.shutdown() runs. Run inline so buffered work is + # still flushed instead of being silently dropped. A + # RuntimeError raised by the callback itself is not caught here + # and still propagates via future.result() below. + logger.debug( + "Executor is shut down; running SpanProcessor callback inline.", + ) + func(sp)(*args, **kwargs) + for future in futures: + future.result() + + def on_start( + self, + span: "Span", + parent_context: context_api.Context | None = None, + ) -> None: + self._submit_and_await(lambda sp: sp.on_start, span, parent_context=parent_context) + + def _on_ending(self, span: "Span") -> None: + # Unlike the other callbacks, _on_ending runs synchronously on the + # thread that ends the span: that is the only thread allowed to + # mutate the span while it is ending. + for sp in self._span_processors: + try: + # pylint: disable=protected-access + sp._on_ending(span) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while calling _on_ending on %s.", sp) + + def on_end(self, span: "ReadableSpan") -> None: + self._submit_and_await(lambda sp: sp.on_end, span) + + def shutdown(self) -> None: + """Shuts down all underlying span processors in parallel.""" + self._submit_and_await(lambda sp: sp.shutdown) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Calls force_flush on all underlying span processors in parallel. + + Args: + timeout_millis: The maximum amount of time to wait for spans to be + exported. + + Returns: + True if all span processors flushed their spans within the given + timeout, False otherwise. + """ + futures = [] + for sp in self._span_processors: + future = self._executor.submit(sp.force_flush, timeout_millis) + futures.append(future) + + timeout_sec = timeout_millis / 1e3 + done_futures, not_done_futures = concurrent.futures.wait(futures, timeout_sec) + if not_done_futures: + return False + + for future in done_futures: + if future.result() is False: + return False + + return True + + +class EventBase(abc.ABC): + def __init__(self, name: str, timestamp: int | None = None) -> None: + self._name = name + if timestamp is None: + self._timestamp = time_ns() + else: + self._timestamp = timestamp + + @property + def name(self) -> str: + return self._name + + @property + def timestamp(self) -> int: + return self._timestamp + + @property + @abc.abstractmethod + def attributes(self) -> types.Attributes: + pass + + +class Event(EventBase): + """A text annotation with a set of attributes. The attributes of an event + are immutable. + + Args: + name: Name of the event. + attributes: Attributes of the event. + timestamp: Timestamp of the event. If `None` it will filled + automatically. + """ + + def __init__( + self, + name: str, + attributes: types.Attributes = None, + timestamp: int | None = None, + limit: int | None = _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + ) -> None: + super().__init__(name, timestamp) + self._attributes = attributes + + @property + def attributes(self) -> types.Attributes: + return self._attributes + + @property + def dropped_attributes(self) -> int: + if isinstance(self._attributes, BoundedAttributes): + return self._attributes.dropped + return 0 + + +def _check_span_ended(func): + def wrapper(self, *args, **kwargs): + already_ended = False + with self._lock: # pylint: disable=protected-access + if self._is_mutable(): # pylint: disable=protected-access + func(self, *args, **kwargs) + else: + already_ended = True + + if already_ended: + logger.warning("Tried calling %s on an ended span.", func.__name__) + + return wrapper + + +def _is_valid_link(context: SpanContext, attributes: types.Attributes) -> bool: + return bool(context and (context.is_valid or (attributes or context.trace_state))) + + +class ReadableSpan: + """Provides read-only access to span attributes. + + Users should NOT be creating these objects directly. `ReadableSpan`s are created as + a direct result from using the tracing pipeline via the `Tracer`. + + """ + + def __init__( + self, + name: str, + context: trace_api.SpanContext | None = None, + parent: trace_api.SpanContext | None = None, + resource: Resource | None = None, + attributes: types.Attributes = None, + events: Sequence[Event] = (), + links: Sequence[trace_api.Link] = (), + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + instrumentation_info: InstrumentationInfo | None = None, + status: Status = Status(StatusCode.UNSET), + start_time: int | None = None, + end_time: int | None = None, + instrumentation_scope: InstrumentationScope | None = None, + ) -> None: + self._name = name + self._context = context + self._kind = kind + self._instrumentation_info = instrumentation_info + self._instrumentation_scope = instrumentation_scope + self._parent = parent + self._start_time = start_time + self._end_time = end_time + self._attributes = attributes + self._events = events + self._links = links + if resource is None: + self._resource = Resource.create({}) + else: + self._resource = resource + self._status = status + + @property + def dropped_attributes(self) -> int: + if isinstance(self._attributes, BoundedAttributes): + return self._attributes.dropped + return 0 + + @property + def dropped_events(self) -> int: + if isinstance(self._events, BoundedList): + return self._events.dropped + return 0 + + @property + def dropped_links(self) -> int: + if isinstance(self._links, BoundedList): + return self._links.dropped + return 0 + + @property + def name(self) -> str: + return self._name + + def get_span_context(self) -> trace_api.SpanContext | None: + return self._context + + @property + def context(self): + return self._context + + @property + def kind(self) -> trace_api.SpanKind: + return self._kind + + @property + def parent(self) -> trace_api.SpanContext | None: + return self._parent + + @property + def start_time(self) -> int | None: + return self._start_time + + @property + def end_time(self) -> int | None: + return self._end_time + + @property + def status(self) -> trace_api.Status: + return self._status + + @property + def attributes(self) -> types.Attributes: + return MappingProxyType(self._attributes or {}) + + @property + def events(self) -> Sequence[Event]: + return tuple(self._events) + + @property + def links(self) -> Sequence[trace_api.Link]: + return tuple(self._links) + + @property + def resource(self) -> Resource: + return self._resource + + @property + @deprecated("You should use instrumentation_scope. Deprecated since version 1.11.1.") + def instrumentation_info(self) -> InstrumentationInfo | None: + return self._instrumentation_info + + @property + def instrumentation_scope(self) -> InstrumentationScope | None: + return self._instrumentation_scope + + def to_json(self, indent: int | None = 4): + parent_id = None + if self.parent is not None: + parent_id = f"0x{trace_api.format_span_id(self.parent.span_id)}" + + start_time = None + if self._start_time: + start_time = util.ns_to_iso_str(self._start_time) + + end_time = None + if self._end_time: + end_time = util.ns_to_iso_str(self._end_time) + + status = { + "status_code": str(self._status.status_code.name), + } + if self._status.description: + status["description"] = self._status.description + + f_span = { + "name": self._name, + "context": (self._format_context(self._context) if self._context else None), + "kind": str(self.kind), + "parent_id": parent_id, + "start_time": start_time, + "end_time": end_time, + "status": status, + "attributes": self._format_attributes(self._attributes), + "events": self._format_events(self._events), + "links": self._format_links(self._links), + "resource": json.loads(self.resource.to_json()), + } + + return json.dumps(f_span, indent=indent) + + @staticmethod + def _format_context(context: SpanContext) -> dict[str, str]: + return { + "trace_id": f"0x{trace_api.format_trace_id(context.trace_id)}", + "span_id": f"0x{trace_api.format_span_id(context.span_id)}", + "trace_state": repr(context.trace_state), + } + + @staticmethod + def _format_attributes( + attributes: types.Attributes, + ) -> dict[str, Any] | None: + if attributes is not None and not isinstance(attributes, dict): + return dict(attributes) + return attributes + + @staticmethod + def _format_events(events: Sequence[Event]) -> list[dict[str, Any]]: + return [ + { + "name": event.name, + "timestamp": util.ns_to_iso_str(event.timestamp), + "attributes": Span._format_attributes( # pylint: disable=protected-access + event.attributes + ), + } + for event in events + ] + + @staticmethod + def _format_links(links: Sequence[trace_api.Link]) -> list[dict[str, Any]]: + return [ + { + "context": Span._format_context( # pylint: disable=protected-access + link.context + ), + "attributes": Span._format_attributes( # pylint: disable=protected-access + link.attributes + ), + } + for link in links + ] + + +class SpanLimits: + """The limits that should be enforced on recorded data such as events, links, attributes etc. + + This class does not enforce any limits itself. It only provides a way to read limits from env, + default values and from user provided arguments. + + All limit arguments must be either a non-negative integer, ``None`` or ``SpanLimits.UNSET``. + + - All limit arguments are optional. + - If a limit argument is not set, the class will try to read its value from the corresponding + environment variable. + - If the environment variable is not set, the default value, if any, will be used. + + Limit precedence: + + - If a model specific limit is set, it will be used. + - Else if the corresponding global limit is set, it will be used. + - Else if the model specific limit has a default value, the default value will be used. + - Else if the global limit has a default value, the default value will be used. + + Args: + max_attributes: Maximum number of attributes that can be added to a span, event, and link. + Environment variable: OTEL_ATTRIBUTE_COUNT_LIMIT + Default: {_DEFAULT_ATTRIBUTE_COUNT_LIMIT} + max_events: Maximum number of events that can be added to a Span. + Environment variable: OTEL_SPAN_EVENT_COUNT_LIMIT + Default: {_DEFAULT_SPAN_EVENT_COUNT_LIMIT} + max_links: Maximum number of links that can be added to a Span. + Environment variable: OTEL_SPAN_LINK_COUNT_LIMIT + Default: {_DEFAULT_SPAN_LINK_COUNT_LIMIT} + max_span_attributes: Maximum number of attributes that can be added to a Span. + Environment variable: OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT + Default: {_DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT} + max_event_attributes: Maximum number of attributes that can be added to an Event. + Default: {_DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT} + max_link_attributes: Maximum number of attributes that can be added to a Link. + Default: {_DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT} + max_attribute_length: Maximum length an attribute value can have. Values longer than + the specified length will be truncated. + max_span_attribute_length: Maximum length a span attribute value can have. Values longer than + the specified length will be truncated. + """ + + UNSET = -1 + + def __init__( + self, + max_attributes: int | None = None, + max_events: int | None = None, + max_links: int | None = None, + max_span_attributes: int | None = None, + max_event_attributes: int | None = None, + max_link_attributes: int | None = None, + max_attribute_length: int | None = None, + max_span_attribute_length: int | None = None, + ): + # span events and links count + self.max_events = self._from_env_if_absent( + max_events, + OTEL_SPAN_EVENT_COUNT_LIMIT, + _DEFAULT_OTEL_SPAN_EVENT_COUNT_LIMIT, + ) + self.max_links = self._from_env_if_absent( + max_links, + OTEL_SPAN_LINK_COUNT_LIMIT, + _DEFAULT_OTEL_SPAN_LINK_COUNT_LIMIT, + ) + + # attribute count + global_max_attributes = self._from_env_if_absent(max_attributes, OTEL_ATTRIBUTE_COUNT_LIMIT) + self.max_attributes = ( + global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT + ) + + self.max_span_attributes = self._from_env_if_absent( + max_span_attributes, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT), + ) + self.max_event_attributes = self._from_env_if_absent( + max_event_attributes, + OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT, + (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT), + ) + self.max_link_attributes = self._from_env_if_absent( + max_link_attributes, + OTEL_LINK_ATTRIBUTE_COUNT_LIMIT, + (global_max_attributes if global_max_attributes is not None else _DEFAULT_OTEL_LINK_ATTRIBUTE_COUNT_LIMIT), + ) + + # attribute length + self.max_attribute_length = self._from_env_if_absent( + max_attribute_length, + OTEL_ATTRIBUTE_VALUE_LENGTH_LIMIT, + ) + self.max_span_attribute_length = self._from_env_if_absent( + max_span_attribute_length, + OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT, + # use global attribute length limit as default + self.max_attribute_length, + ) + + def __repr__(self): + return f"{type(self).__name__}(max_span_attributes={self.max_span_attributes}, max_events_attributes={self.max_event_attributes}, max_link_attributes={self.max_link_attributes}, max_attributes={self.max_attributes}, max_events={self.max_events}, max_links={self.max_links}, max_attribute_length={self.max_attribute_length})" + + @classmethod + def _from_env_if_absent(cls, value: int | None, env_var: str, default: int | None = None) -> int | None: + if value == cls.UNSET: + return None + + err_msg = "{} must be a non-negative integer but got {}" + + # if no value is provided for the limit, try to load it from env + if value is None: + # return default value if env var is not set + if env_var not in environ: + return default + + str_value = environ.get(env_var, "").strip().lower() + if str_value == _ENV_VALUE_UNSET: + return None + + try: + value = int(str_value) + except ValueError: + raise ValueError(err_msg.format(env_var, str_value)) + + if value < 0: + raise ValueError(err_msg.format(env_var, value)) + return value + + +_UnsetLimits = SpanLimits( + max_attributes=SpanLimits.UNSET, + max_events=SpanLimits.UNSET, + max_links=SpanLimits.UNSET, + max_span_attributes=SpanLimits.UNSET, + max_event_attributes=SpanLimits.UNSET, + max_link_attributes=SpanLimits.UNSET, + max_attribute_length=SpanLimits.UNSET, + max_span_attribute_length=SpanLimits.UNSET, +) + +# not removed for backward compat. please use SpanLimits instead. +SPAN_ATTRIBUTE_COUNT_LIMIT = SpanLimits._from_env_if_absent( # pylint: disable=protected-access + None, + OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, + _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT, +) + + +class Span(trace_api.Span, ReadableSpan): + """See `opentelemetry.trace.Span`. + + Users should create `Span` objects via the `Tracer` instead of this + constructor. + + Args: + name: The name of the operation this span represents + context: The immutable span context + parent: This span's parent's `opentelemetry.trace.SpanContext`, or + None if this is a root span + sampler: The sampler used to create this span + trace_config: Unused. Originally intended for trace-level configuration + from the OpenTelemetry protocol, but the upstream ``TraceConfig`` + proto was removed. Retained for backwards compatibility. + resource: Entity producing telemetry + attributes: The span's attributes to be exported + events: Timestamped events to be exported + links: Links to other spans to be exported + span_processor: `SpanProcessor` to invoke when starting and ending + this `Span`. + limits: `SpanLimits` instance that was passed to the `TracerProvider` + """ + + def __new__(cls, *args, **kwargs): + if cls is Span: + raise TypeError("Span must be instantiated via a tracer.") + return super().__new__(cls) + + # pylint: disable=too-many-locals + def __init__( + self, + name: str, + context: trace_api.SpanContext, + parent: trace_api.SpanContext | None = None, + sampler: sampling.Sampler | None = None, + trace_config: None = None, + resource: Resource | None = None, + attributes: types.Attributes = None, + events: Sequence[Event] | None = None, + links: Sequence[trace_api.Link] = (), + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + span_processor: SpanProcessor = SpanProcessor(), + instrumentation_info: InstrumentationInfo | None = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + limits=_UnsetLimits, + instrumentation_scope: InstrumentationScope | None = None, + *, + record_end_metrics: Callable[[], None] | None = None, + ) -> None: + if resource is None: + resource = Resource.create({}) + super().__init__( + name=name, + context=context, + parent=parent, + kind=kind, + resource=resource, + instrumentation_info=instrumentation_info, + instrumentation_scope=instrumentation_scope, + ) + self._sampler = sampler + self._trace_config = trace_config + self._record_exception = record_exception + self._set_status_on_exception = set_status_on_exception + self._span_processor = span_processor + self._limits = limits + self._lock = threading.Lock() + # Identifier of the thread running end() while the _on_ending + # callbacks execute: the end timestamp is already set, but the span + # must remain mutable from that thread until the callbacks return. + self._ending_thread: int | None = None + self._attributes = BoundedAttributes( + self._limits.max_span_attributes, + attributes, + immutable=False, + max_value_len=self._limits.max_span_attribute_length, + ) + self._events = self._new_events() + if events: + for event in events: + # Immutable set to true so attributes cannot be added or removed after creation. + event._attributes = BoundedAttributes( + self._limits.max_event_attributes, + event.attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + self._events.append(event) + + self._links = self._new_links(links) + + self._record_end_metrics = record_end_metrics + + def __repr__(self): + return f'{type(self).__name__}(name="{self._name}", context={self._context})' + + def _new_events(self): + return BoundedList(self._limits.max_events) + + def _new_links(self, links: Sequence[trace_api.Link]): + if not links: + return BoundedList(self._limits.max_links) + + valid_links = [] + for link in links: + if link and _is_valid_link(link.context, link.attributes): + # Immutable set to true so attributes cannot be added or removed after creation. + # pylint: disable=protected-access + link._attributes = BoundedAttributes( + self._limits.max_link_attributes, + link.attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + valid_links.append(link) + + return BoundedList.from_seq(self._limits.max_links, valid_links) + + def get_span_context(self) -> trace_api.SpanContext: + return typing.cast(trace_api.SpanContext, self._context) + + def set_attributes(self, attributes: Mapping[str, types.AnyValue]) -> None: + with self._lock: + if not self._is_mutable(): + logger.warning("Setting attribute on ended span.") + return + + self._attributes._set_items(attributes) # pylint: disable=protected-access + + def set_attribute(self, key: str, value: types.AnyValue) -> None: + with self._lock: + if not self._is_mutable(): + logger.warning("Setting attribute on ended span.") + return + + self._attributes[key] = value + + @_check_span_ended + def _add_event(self, event: EventBase) -> None: + self._events.append(event) + + def add_event( + self, + name: str, + attributes: types.Attributes = None, + timestamp: int | None = None, + ) -> None: + # Immutable set to true so attributes cannot be added or removed after creation. + attributes = BoundedAttributes( + self._limits.max_event_attributes, + attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + self._add_event( + Event( + name=name, + attributes=attributes, + timestamp=timestamp, + ) + ) + + @_check_span_ended + def _add_link(self, link: trace_api.Link) -> None: + self._links.append(link) + + def add_link( + self, + context: SpanContext, + attributes: types.Attributes = None, + ) -> None: + if not _is_valid_link(context, attributes): + return + # Immutable set to true so attributes cannot be added or removed after creation. + attributes = BoundedAttributes( + self._limits.max_link_attributes, + attributes, + max_value_len=self._limits.max_attribute_length, + immutable=True, + ) + self._add_link( + trace_api.Link( + context=context, + attributes=attributes, + ) + ) + + def _readable_span(self) -> ReadableSpan: + return ReadableSpan( + name=self._name, + context=self._context, + parent=self._parent, + resource=self._resource, + attributes=self._attributes, + events=self._events, + links=self._links, + kind=self.kind, + status=self._status, + start_time=self._start_time, + end_time=self._end_time, + instrumentation_info=self._instrumentation_info, + instrumentation_scope=self._instrumentation_scope, + ) + + def start( + self, + start_time: int | None = None, + parent_context: context_api.Context | None = None, + ) -> None: + with self._lock: + if self._start_time is not None: + logger.warning("Calling start() on a started span.") + return + self._start_time = start_time if start_time is not None else time_ns() + + self._span_processor.on_start(self, parent_context=parent_context) + + def end(self, end_time: int | None = None) -> None: + with self._lock: + if self._start_time is None: + raise RuntimeError("Calling end() on a not started span.") + if self._end_time is not None: + logger.warning("Calling end() on an ended span.") + return + + self._ending_thread = threading.get_ident() + self._end_time = end_time if end_time is not None else time_ns() + + # The span must remain mutable from this thread while the _on_ending + # callbacks run; it becomes immutable once they have returned. + # pylint: disable=protected-access + try: + if self._record_end_metrics: + self._record_end_metrics() + self._span_processor._on_ending(self) + finally: + with self._lock: + self._ending_thread = None + self._attributes._immutable = True + self._span_processor.on_end(self._readable_span()) + + @_check_span_ended + def update_name(self, name: str) -> None: + self._name = name + + def _is_mutable(self) -> bool: + return self._end_time is None or self._ending_thread == threading.get_ident() + + def is_recording(self) -> bool: + # Unlike the mutators, this check is deliberately not taken under + # self._lock: it does not modify the span, and the answer can become + # stale as soon as it is returned anyway, since another thread may + # end the span right after this call. + return self._is_mutable() + + @_check_span_ended + def set_status( + self, + status: Status | StatusCode, + description: str | None = None, + ) -> None: + if isinstance(status, Status): + if description is not None: + logger.warning( + "Description %s ignored. Use either `Status` or `(StatusCode, Description)`", + description, + ) + new_status = status + elif isinstance(status, StatusCode): + new_status = Status(status, description) + else: + return + + if self._accepts_status(new_status): + self._status = new_status + + def _accepts_status(self, new_status: Status) -> bool: + """Decide whether ``new_status`` may replace the status already recorded. + + The codes are totally ordered, ``Ok > Error > Unset``, an attempt to set + ``Unset`` is ignored, and ``Ok`` is final. Once those two are handled, a + differing code always outranks the one recorded. A repeat of the same code + gets through only when it fills in a description that is still missing, + and an empty description counts as missing. + """ + if new_status.status_code is StatusCode.UNSET: + return False + + current = self._status + if current is None: + return True + if current.status_code is StatusCode.OK: + return False + + # An empty description is equivalent to an absent one. + return current.status_code is not new_status.status_code or ( + not current.description and bool(new_status.description) + ) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + """Ends context manager and calls `end` on the `Span`.""" + if exc_val is not None and self.is_recording(): + # Record the exception as an event + # pylint:disable=protected-access + if self._record_exception: + self.record_exception(exception=exc_val, escaped=True) + # Records status if span is used as context manager + # i.e. with tracer.start_span() as span: + if self._set_status_on_exception: + self.set_status( + Status( + status_code=StatusCode.ERROR, + description=(f"{type(exc_val).__name__}: {exc_val}"), + ) + ) + + super().__exit__(exc_type, exc_val, exc_tb) + + def record_exception( + self, + exception: BaseException, + attributes: types.Attributes = None, + timestamp: int | None = None, + escaped: bool = False, + ) -> None: + """Records an exception as a span event.""" + stacktrace = "".join(traceback.format_exception(exception)) + module = type(exception).__module__ + qualname = type(exception).__qualname__ + exception_type = f"{module}.{qualname}" if module and module != "builtins" else qualname + _attributes: MutableMapping[str, types.AnyValue] = { + EXCEPTION_TYPE: exception_type, + EXCEPTION_MESSAGE: str(exception), + EXCEPTION_STACKTRACE: stacktrace, + EXCEPTION_ESCAPED: str(escaped), + } + if attributes: + _attributes.update(attributes) + self.add_event(name="exception", attributes=_attributes, timestamp=timestamp) + + +class _Span(Span): + """Protected implementation of `opentelemetry.trace.Span`. + + This constructor exists to prevent the instantiation of the `Span` class + by other mechanisms than through the `Tracer`. + """ + + +@dataclass +class _TracerConfig: + is_enabled: bool + + @classmethod + def default(cls): + return cls(is_enabled=True) + + +class Tracer(trace_api.Tracer): + """See `opentelemetry.trace.Tracer`.""" + + def __init__( + self, + sampler: sampling.Sampler, + resource: Resource, + span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor, + id_generator: IdGenerator, + instrumentation_info: InstrumentationInfo, + span_limits: SpanLimits, + instrumentation_scope: InstrumentationScope, + *, + meter_provider: metrics_api.MeterProvider | None = None, + _tracer_config: _TracerConfig | None = None, + ) -> None: + self.sampler = sampler + self.resource = resource + self.span_processor = span_processor + self.id_generator = id_generator + self.instrumentation_info = instrumentation_info + self._span_limits = span_limits + self._instrumentation_scope = instrumentation_scope + self._tracer_config = _tracer_config or _TracerConfig.default() + + meter_provider = meter_provider or metrics_api.get_meter_provider() + self._tracer_metrics = create_tracer_metrics( + meter_provider, + parse_boolean_environment_variable(OTEL_PYTHON_SDK_INTERNAL_METRICS_ENABLED), + ) + + def _set_tracer_config(self, tracer_config: _TracerConfig): + self._tracer_config = tracer_config + + def _set_resource(self, resource: Resource) -> None: + self.resource = resource + + def _is_enabled(self) -> bool: + """If the tracer is not enabled, start_span will create a NonRecordingSpan""" + return self._tracer_config.is_enabled + + @_agnosticcontextmanager # pylint: disable=protected-access + def start_as_current_span( + self, + name: str, + context: context_api.Context | None = None, + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + attributes: types.Attributes = None, + links: Sequence[trace_api.Link] | None = (), + start_time: int | None = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + end_on_exit: bool = True, + ) -> Iterator[trace_api.Span]: + span = self.start_span( + name=name, + context=context, + kind=kind, + attributes=attributes, + links=links, + start_time=start_time, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + ) + with trace_api.use_span( + span, + end_on_exit=end_on_exit, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + ) as span: + yield span + + def start_span( # pylint: disable=too-many-locals + self, + name: str, + context: context_api.Context | None = None, + kind: trace_api.SpanKind = trace_api.SpanKind.INTERNAL, + attributes: types.Attributes = None, + links: Sequence[trace_api.Link] | None = (), + start_time: int | None = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + ) -> trace_api.Span: + links = links or () + parent_span_context = trace_api.get_current_span(context).get_span_context() + + if parent_span_context is not None and not isinstance(parent_span_context, trace_api.SpanContext): + raise TypeError("parent_span_context must be a SpanContext or None.") + + if not self._is_enabled(): + return trace_api.NonRecordingSpan(context=parent_span_context) + + # is_valid determines root span + if parent_span_context is None or not parent_span_context.is_valid: + parent_span_context = None + trace_id = self.id_generator.generate_trace_id() + else: + trace_id = parent_span_context.trace_id + + # The sampler decides whether to create a real or no-op span at the + # time of span creation. No-op spans do not record events, and are not + # exported. + # The sampler may also add attributes to the newly-created span, e.g. + # to include information about the sampling result. + # The sampler may also modify the parent span context's tracestate + sampling_result = self.sampler.should_sample(context, trace_id, name, kind, attributes, links) + + trace_flags = ( + trace_api.TraceFlags(trace_api.TraceFlags.SAMPLED) + if sampling_result.decision.is_sampled() + else trace_api.TraceFlags(trace_api.TraceFlags.DEFAULT) + ) + + if parent_span_context is None: + random_trace_id = self.id_generator.is_trace_id_random() + else: + random_trace_id = parent_span_context.trace_flags.random_trace_id + + if random_trace_id: + trace_flags = trace_api.TraceFlags(trace_flags | trace_api.TraceFlags.RANDOM_TRACE_ID) + + span_context = trace_api.SpanContext( + trace_id, + self.id_generator.generate_span_id(), + is_remote=False, + trace_flags=trace_flags, + trace_state=sampling_result.trace_state, + ) + + record_end_metrics = self._tracer_metrics.start_span(parent_span_context, sampling_result.decision) + + # Only record if is_recording() is true + if sampling_result.decision.is_recording(): + # pylint:disable=protected-access + span = _Span( + name=name, + context=span_context, + parent=parent_span_context, + sampler=self.sampler, + resource=self.resource, + attributes=sampling_result.attributes, + span_processor=self.span_processor, + kind=kind, + links=links, + instrumentation_info=self.instrumentation_info, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception, + limits=self._span_limits, + instrumentation_scope=self._instrumentation_scope, + record_end_metrics=record_end_metrics, + ) + span.start(start_time=start_time, parent_context=context) + else: + span = trace_api.NonRecordingSpan(context=span_context) + return span + + +_TracerConfiguratorT = Callable[[InstrumentationScope], _TracerConfig] +_RuleBasedTracerConfigurator = RuleBasedConfigurator[_TracerConfig] + + +def _default_tracer_configurator( + tracer_scope: InstrumentationScope, +) -> _TracerConfig: + """Default Tracer Configurator implementation + + In order to update Tracers configs you need to call + TracerProvider._set_tracer_configurator with a function + implementing this interface returning a Tracer Config.""" + return _RuleBasedTracerConfigurator( + rules=[], + default_config=_TracerConfig.default(), + )(tracer_scope) + + +def _disable_tracer_configurator( + tracer_scope: InstrumentationScope, +) -> _TracerConfig: + return _RuleBasedTracerConfigurator( + rules=[], + default_config=_TracerConfig(is_enabled=False), + )(tracer_scope) + + +class TracerProvider(trace_api.TracerProvider): + """See `opentelemetry.trace.TracerProvider`.""" + + def __init__( + self, + sampler: sampling.Sampler | None = None, + resource: Resource | None = None, + shutdown_on_exit: bool = True, + active_span_processor: SynchronousMultiSpanProcessor | ConcurrentMultiSpanProcessor | None = None, + id_generator: IdGenerator | None = None, + span_limits: SpanLimits | None = None, + *, + meter_provider: metrics_api.MeterProvider | None = None, + _tracer_configurator: _TracerConfiguratorT | None = None, + ) -> None: + self._active_span_processor = active_span_processor or SynchronousMultiSpanProcessor() + if id_generator is None: + self.id_generator = RandomIdGenerator() + else: + self.id_generator = id_generator + if resource is None: + self._resource = Resource.create({}) + else: + self._resource = resource + if not sampler: + sampler = sampling._get_from_env_or_default() + self.sampler = sampler + self._span_limits = span_limits or SpanLimits() + disabled = environ.get(OTEL_SDK_DISABLED, "") + self._disabled = disabled.lower().strip() == "true" + self._atexit_handler = None + self._meter_provider = meter_provider + + if shutdown_on_exit: + self._atexit_handler = atexit.register(self.shutdown) + + self._tracer_configurator = _tracer_configurator or _default_tracer_configurator + self._tracers_lock = threading.Lock() + self._tracers: dict[InstrumentationScope, Tracer] = {} + if hasattr(os, "register_at_fork"): + weak_at_fork = weakref.WeakMethod(self._handle_fork) + + def _after_in_child() -> None: + if at_fork := weak_at_fork(): + at_fork() + + os.register_at_fork(after_in_child=_after_in_child) + + def _handle_fork(self) -> None: + self._tracers_lock = threading.Lock() + self._update_resource(_get_process_dependent_resource()) + + def _set_tracer_configurator(self, *, tracer_configurator: _TracerConfiguratorT): + """This is the function used to update the TracerProvider TracerConfigurator + + Setting a new TracerConfigurator for a TracerProvider will update the + TracerConfig of all Tracers create by this TracerProvider. + """ + self._tracer_configurator = tracer_configurator + with self._tracers_lock: + for instrumentation_scope, tracer in self._tracers.items(): + tracer_config = self._apply_tracer_configurator(instrumentation_scope) + # pylint: disable-next=protected-access + tracer._set_tracer_config(tracer_config) + + @property + def resource(self) -> Resource: + return self._resource + + def _update_resource(self, resource: Resource) -> None: + with self._tracers_lock: + self._resource = self._resource.merge(resource) + for tracer in self._tracers.values(): + tracer._set_resource(self._resource) # pylint: disable=protected-access + + def _apply_tracer_configurator(self, instrumentation_scope: InstrumentationScope): + try: + return self._tracer_configurator(instrumentation_scope) + except Exception: # pylint: disable=broad-exception-caught + logger.exception( + "Failed to create a Tracer Config for %s, using default Tracer config", + instrumentation_scope, + ) + return _TracerConfig.default() + + def get_tracer( + self, + instrumenting_module_name: str, + instrumenting_library_version: str | None = None, + schema_url: str | None = None, + attributes: types.Attributes = None, + ) -> "trace_api.Tracer": + if self._disabled: + return NoOpTracer() + if not instrumenting_module_name: # Reject empty strings too. + instrumenting_module_name = "" + logger.error("get_tracer called with missing module name.") + if instrumenting_library_version is None: + instrumenting_library_version = "" + + filterwarnings( + "ignore", + message=(r"You should use InstrumentationScope. Deprecated since version 1.11.1."), + category=DeprecationWarning, + module="opentelemetry.sdk.trace", + ) + + instrumentation_info = InstrumentationInfo( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + ) + + instrumentation_scope = InstrumentationScope( + instrumenting_module_name, + instrumenting_library_version, + schema_url, + attributes, + ) + + with self._tracers_lock: + if instrumentation_scope in self._tracers: + return self._tracers[instrumentation_scope] + + tracer_config = self._apply_tracer_configurator(instrumentation_scope) + tracer = Tracer( + self.sampler, + self.resource, + self._active_span_processor, + self.id_generator, + instrumentation_info, + self._span_limits, + instrumentation_scope, + meter_provider=self._meter_provider, + _tracer_config=tracer_config, + ) + self._tracers[instrumentation_scope] = tracer + + return tracer + + def add_span_processor(self, span_processor: SpanProcessor) -> None: + """Registers a new :class:`SpanProcessor` for this `TracerProvider`. + + The span processors are invoked in the same order they are registered. + """ + + # no lock here because add_span_processor is thread safe for both + # SynchronousMultiSpanProcessor and ConcurrentMultiSpanProcessor. + self._active_span_processor.add_span_processor(span_processor) + + def shutdown(self) -> None: + """Shut down the span processors added to the tracer provider.""" + self._active_span_processor.shutdown() + if self._atexit_handler is not None: + atexit.unregister(self._atexit_handler) + self._atexit_handler = None + + def force_flush(self, timeout_millis: int = 30000) -> bool: + """Requests the active span processor to process all spans that have not + yet been processed. + + By default force flush is called sequentially on all added span + processors. This means that span processors further back in the list + have less time to flush their spans. + To have span processors flush their spans in parallel it is possible to + initialize the tracer provider with an instance of + `ConcurrentMultiSpanProcessor` at the cost of using multiple threads. + + Args: + timeout_millis: The maximum amount of time to wait for spans to be + processed. + + Returns: + False if the timeout is exceeded, True otherwise. + """ + return self._active_span_processor.force_flush(timeout_millis) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 7652a807cc5..c5552a887dd 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -1,279 +1,279 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# pylint:disable=protected-access,no-self-use,no-member - -import logging -import threading -import time -import unittest -from abc import ABC, abstractmethod -from unittest.mock import Mock - -from opentelemetry._logs import LogRecord, SeverityNumber -from opentelemetry.sdk._logs._internal import ( - ConcurrentMultiLogRecordProcessor, - LoggerProvider, - LoggingHandler, - LogRecordProcessor, - ReadWriteLogRecord, - SynchronousMultiLogRecordProcessor, -) - - -class AnotherLogRecordProcessor(LogRecordProcessor): - def __init__(self, exporter, logs_list): - self._exporter = exporter - self._log_list = logs_list - self._closed = False - - def on_emit(self, log_record: ReadWriteLogRecord): - if self._closed: - return - self._log_list.append( - ( - log_record.log_record.body, - log_record.log_record.severity_text, - ) - ) - - def shutdown(self): - self._closed = True - self._exporter.shutdown() - - def force_flush(self, timeout_millis=30000): - self._log_list.clear() - return True - - -class TestLogRecordProcessor(unittest.TestCase): - def test_log_record_processor(self): - provider = LoggerProvider() - handler = LoggingHandler(logger_provider=provider) - - logs_list_1 = [] - processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) - logs_list_2 = [] - processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) - - logger = logging.getLogger("test.span.processor") - logger.addHandler(handler) - - # Test no proessor added - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Odisha, we have another major cyclone") - - self.assertEqual(len(logs_list_1), 0) - self.assertEqual(len(logs_list_2), 0) - - # Add one processor - provider.add_log_record_processor(processor1) - with self.assertLogs(level=logging.WARNING): - logger.warning("Brace yourself") - with self.assertLogs(level=logging.ERROR): - logger.error("Some error message") - - expected_list_1 = [ - ("Brace yourself", "WARN"), - ("Some error message", "ERROR"), - ] - self.assertEqual(logs_list_1, expected_list_1) - - # Add another processor - provider.add_log_record_processor(processor2) - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Something disastrous") - expected_list_1.append(("Something disastrous", "FATAL")) - - expected_list_2 = [("Something disastrous", "FATAL")] - - self.assertEqual(logs_list_1, expected_list_1) - self.assertEqual(logs_list_2, expected_list_2) - - -class MultiLogRecordProcessorTestBase(ABC): - @abstractmethod - def _get_multi_log_record_processor(self): - pass - - def make_record(self): - return ReadWriteLogRecord( - LogRecord( - timestamp=1622300111608942000, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Warning message", - ) - ) - - def test_on_emit(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - record = self.make_record() - multi_log_record_processor.on_emit(record) - for mock in mocks: - mock.on_emit.assert_called_with(record) - multi_log_record_processor.shutdown() - - def test_on_shutdown(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - multi_log_record_processor.shutdown() - for mock in mocks: - mock.shutdown.assert_called_once_with() - - def test_on_force_flush(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - ret_value = multi_log_record_processor.force_flush(100) - - self.assertTrue(ret_value) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return SynchronousMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - def delay(_): - time.sleep(0.09) - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(side_effect=delay) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 0) - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=False) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=None) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertTrue(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - -class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return ConcurrentMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - wait_event = threading.Event() - - def delay(_): - wait_event.wait() - - mock1 = Mock(spec=LogRecordProcessor) - mock1.force_flush = Mock(side_effect=delay) - mocks = [Mock(LogRecordProcessor) for _ in range(5)] - mocks = [mock1] + mocks - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush(50) - wait_event.set() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - false_mock = Mock(spec=LogRecordProcessor) - false_mock.force_flush = Mock(return_value=False) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, false_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - none_mock = Mock(spec=LogRecordProcessor) - none_mock.force_flush = Mock(return_value=None) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertTrue(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_shutdown_runs_inline_after_executor_shutdown(self): - # Simulate the atexit scenario: CPython runs threading._shutdown() - # before the atexit queue, so the ThreadPoolExecutor is closed by the - # time LoggerProvider.shutdown() runs. Underlying processors must - # still be shut down inline so buffered work is not dropped. - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - mock_processor = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(mock_processor) - - # pylint: disable=protected-access - multi_log_record_processor._executor.shutdown(wait=True) - - multi_log_record_processor.shutdown() - mock_processor.shutdown.assert_called_once_with() - - def test_callback_runtime_error_propagates_not_swallowed(self): - # A RuntimeError raised by a processor callback (while the executor is - # alive) must still propagate out of shutdown(). - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - raising_processor = Mock(spec=LogRecordProcessor) - raising_processor.shutdown.side_effect = RuntimeError("processor boom") - multi_log_record_processor.add_log_record_processor(raising_processor) - - with self.assertRaises(RuntimeError): - multi_log_record_processor.shutdown() +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint:disable=protected-access,no-self-use,no-member + +import logging +import threading +import time +import unittest +from abc import ABC, abstractmethod +from unittest.mock import Mock + +from opentelemetry._logs import LogRecord, SeverityNumber +from opentelemetry.sdk._logs._internal import ( + ConcurrentMultiLogRecordProcessor, + LoggerProvider, + LoggingHandler, + LogRecordProcessor, + ReadWriteLogRecord, + SynchronousMultiLogRecordProcessor, +) + + +class AnotherLogRecordProcessor(LogRecordProcessor): + def __init__(self, exporter, logs_list): + self._exporter = exporter + self._log_list = logs_list + self._closed = False + + def on_emit(self, log_record: ReadWriteLogRecord): + if self._closed: + return + self._log_list.append( + ( + log_record.log_record.body, + log_record.log_record.severity_text, + ) + ) + + def shutdown(self): + self._closed = True + self._exporter.shutdown() + + def force_flush(self, timeout_millis=30000): + self._log_list.clear() + return True + + +class TestLogRecordProcessor(unittest.TestCase): + def test_log_record_processor(self): + provider = LoggerProvider() + handler = LoggingHandler(logger_provider=provider) + + logs_list_1 = [] + processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) + logs_list_2 = [] + processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) + + logger = logging.getLogger("test.span.processor") + logger.addHandler(handler) + + # Test no proessor added + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Odisha, we have another major cyclone") + + self.assertEqual(len(logs_list_1), 0) + self.assertEqual(len(logs_list_2), 0) + + # Add one processor + provider.add_log_record_processor(processor1) + with self.assertLogs(level=logging.WARNING): + logger.warning("Brace yourself") + with self.assertLogs(level=logging.ERROR): + logger.error("Some error message") + + expected_list_1 = [ + ("Brace yourself", "WARN"), + ("Some error message", "ERROR"), + ] + self.assertEqual(logs_list_1, expected_list_1) + + # Add another processor + provider.add_log_record_processor(processor2) + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Something disastrous") + expected_list_1.append(("Something disastrous", "FATAL")) + + expected_list_2 = [("Something disastrous", "FATAL")] + + self.assertEqual(logs_list_1, expected_list_1) + self.assertEqual(logs_list_2, expected_list_2) + + +class MultiLogRecordProcessorTestBase(ABC): + @abstractmethod + def _get_multi_log_record_processor(self): + pass + + def make_record(self): + return ReadWriteLogRecord( + LogRecord( + timestamp=1622300111608942000, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Warning message", + ) + ) + + def test_on_emit(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + record = self.make_record() + multi_log_record_processor.on_emit(record) + for mock in mocks: + mock.on_emit.assert_called_with(record) + multi_log_record_processor.shutdown() + + def test_on_shutdown(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + multi_log_record_processor.shutdown() + for mock in mocks: + mock.shutdown.assert_called_once_with() + + def test_on_force_flush(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + ret_value = multi_log_record_processor.force_flush(100) + + self.assertTrue(ret_value) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return SynchronousMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + def delay(_): + time.sleep(0.09) + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(side_effect=delay) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 0) + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=False) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=None) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertTrue(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + +class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return ConcurrentMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + wait_event = threading.Event() + + def delay(_): + wait_event.wait() + + mock1 = Mock(spec=LogRecordProcessor) + mock1.force_flush = Mock(side_effect=delay) + mocks = [Mock(LogRecordProcessor) for _ in range(5)] + mocks = [mock1] + mocks + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush(50) + wait_event.set() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + false_mock = Mock(spec=LogRecordProcessor) + false_mock.force_flush = Mock(return_value=False) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, false_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + none_mock = Mock(spec=LogRecordProcessor) + none_mock.force_flush = Mock(return_value=None) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertTrue(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time LoggerProvider.shutdown() runs. Underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + mock_processor = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor) + + # pylint: disable=protected-access + multi_log_record_processor._executor.shutdown(wait=True) + + multi_log_record_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_callback_runtime_error_propagates_not_swallowed(self): + # A RuntimeError raised by a processor callback (while the executor is + # alive) must still propagate out of shutdown(). + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + raising_processor = Mock(spec=LogRecordProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + multi_log_record_processor.add_log_record_processor(raising_processor) + + with self.assertRaises(RuntimeError): + multi_log_record_processor.shutdown() diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index a08e7a8e3ef..c878dc210c5 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -1,796 +1,796 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -import abc -import gc -import multiprocessing -import os -import time -import unittest -import weakref -from threading import Event, Thread -from unittest import mock - -from opentelemetry import trace as trace_api -from opentelemetry.context import Context -from opentelemetry.sdk import trace -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, -) - - -def span_event_start_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":start" - - -def span_event_ending_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":ending" - - -def span_event_end_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":end" - - -class MySpanProcessor(trace.SpanProcessor): - def __init__(self, name, span_list): - self.name = name - self.span_list = span_list - - def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: - self.span_list.append(span_event_start_fmt(self.name, span.name)) - - def on_end(self, span: "trace.Span") -> None: - self.span_list.append(span_event_end_fmt(self.name, span.name)) - - -class MyExtendedSpanProcessor(MySpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - self.span_list.append(span_event_ending_fmt(self.name, span.name)) - - -class TestSpanProcessor(unittest.TestCase): - def test_span_processor(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MySpanProcessor("SP1", spans_calls_list) - sp2 = MySpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - # pylint: disable=too-many-statements - def test_span_processor_with_on_ending(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) - sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_ending_fmt("SP2", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_ending_fmt("SP2", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_ending_fmt("SP2", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - def test_add_span_processor_after_span_creation(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp = MySpanProcessor("SP1", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - # add span processor after spans have been created - tracer_provider.add_span_processor(sp) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Does not implement _on_ending - sp = MySpanProcessor("SP1", spans_calls_list) - tracer_provider.add_span_processor(sp) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - self.fail("_on_ending() should not raise an exception") - - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - assert span.is_recording() - assert span.end_time is not None - span.update_name("renamed") - span.set_attribute("attribute", "value") - span.set_attributes({"attributes": "value"}) - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(MutatingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - tracer.start_span("foo").end() - - (span,) = exporter.get_finished_spans() - self.assertEqual(span.name, "renamed") - self.assertEqual(span.attributes["attribute"], "value") - self.assertEqual(span.attributes["attributes"], "value") - self.assertEqual(span.events[0].name, "event") - self.assertEqual(len(span.links), 1) - self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) - - def test_end_during_on_ending_is_ignored(self): - exporter = InMemorySpanExporter() - on_ending_calls = [] - - class ReentrantSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - on_ending_calls.append(span) - span.end() - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(ReentrantSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - span = tracer.start_span("foo") - span.end() - - self.assertEqual(len(on_ending_calls), 1) - self.assertEqual(len(exporter.get_finished_spans()), 1) - - # After end() returns the span is immutable again. - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - (exported,) = exporter.get_finished_spans() - self.assertNotIn("late", exported.attributes) - - def test_other_thread_cannot_mutate_during_on_ending(self): - exporter = InMemorySpanExporter() - inside_on_ending = Event() - other_thread_done = Event() - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def mutate_from_another_thread(): - inside_on_ending.wait(5) - span.set_attribute("other.attribute", "value") - span.set_attributes({"other.attributes": "value"}) - span.update_name("renamed") - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - other_thread_done.set() - - thread = Thread(target=mutate_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(inside_on_ending.is_set()) - self.assertTrue(other_thread_done.is_set()) - - (exported,) = exporter.get_finished_spans() - self.assertNotIn("other.attribute", exported.attributes) - self.assertNotIn("other.attributes", exported.attributes) - self.assertEqual(exported.name, "foo") - self.assertEqual(len(exported.events), 0) - self.assertEqual(len(exported.links), 0) - self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) - - def test_other_thread_does_not_see_span_recording_during_on_ending(self): - inside_on_ending = Event() - other_thread_done = Event() - recording_seen_by_other_thread = [] - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def observe_from_another_thread(): - inside_on_ending.wait(5) - recording_seen_by_other_thread.append(span.is_recording()) - other_thread_done.set() - - thread = Thread(target=observe_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(other_thread_done.is_set()) - self.assertEqual(recording_seen_by_other_thread, [False]) - - def test_span_is_frozen_when_record_end_metrics_raises(self): - def record_end_metrics_that_raises(): - raise RuntimeError("meter blew up") - - tracer_provider = trace.TracerProvider() - span = tracer_provider.get_tracer(__name__).start_span("foo") - # pylint: disable=protected-access - span._record_end_metrics = record_end_metrics_that_raises - - with self.assertRaises(RuntimeError): - span.end() - - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - self.assertNotIn("late", span.attributes) - - -class MultiSpanProcessorTestBase(abc.ABC): - @abc.abstractmethod - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: - pass - - @staticmethod - def create_default_span() -> trace_api.Span: - span_context = trace_api.SpanContext(37, 73, is_remote=False) - return trace_api.NonRecordingSpan(span_context) - - def test_on_start(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - context = Context() - multi_processor.on_start(span, parent_context=context) - - for mock_processor in mocks: - mock_processor.on_start.assert_called_once_with(span, parent_context=context) - multi_processor.shutdown() - - def test_on_ending(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - # pylint: disable=protected-access - multi_processor._on_ending(span) - - for mock_processor in mocks: - # pylint: disable=protected-access - mock_processor._on_ending.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_end(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - multi_processor.on_end(span) - - for mock_processor in mocks: - mock_processor.on_end.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_shutdown(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - multi_processor.shutdown() - - for mock_processor in mocks: - mock_processor.shutdown.assert_called_once_with() - - def test_force_flush(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - timeout_millis = 100 - - flushed = multi_processor.force_flush(timeout_millis) - - # pylint: disable=no-member - self.assertTrue(flushed) - for mock_processor in mocks: - # pylint: disable=no-member - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - multi_processor = self.create_multi_span_processor() - # Does not implement _on_ending - multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) - - tracer_provider.add_span_processor(multi_processor) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - # pylint: disable=no-member - self.fail("_on_ending() should not raise an exception") - - # pylint: disable=no-member - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - tracer_provider.get_tracer(__name__).start_span("foo").end() - - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - def test_on_ending_exception_does_not_stop_other_processors(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class FailingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - raise RuntimeError("_on_ending failed") - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(FailingSpanProcessor()) - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - # pylint: disable=no-member - with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): - tracer_provider.get_tracer(__name__).start_span("foo").end() - - # The processor registered after the failing one still ran its - # _on_ending, and on_end still exported the span. - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - -class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor: - return trace.SynchronousMultiSpanProcessor() - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - def delayed_flush(_): - time.sleep(0.055) - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(0, mock_processor2.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=False) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=None) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - mock_processor2.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_default_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - default_processor = trace.SpanProcessor() - multi_processor.add_span_processor(default_processor) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - mock_processor.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.ConcurrentMultiSpanProcessor: - return trace.ConcurrentMultiSpanProcessor(3) - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - wait_event = Event() - - def delayed_flush(_): - wait_event.wait() - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(side_effect=delayed_flush) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(timeout_millis=25) - # let the thread executing the late_mock continue - wait_event.set() - multi_processor.shutdown() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(return_value=False) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - none_mock = mock.Mock(spec=trace.SpanProcessor) - none_mock.force_flush = mock.Mock(return_value=None) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertTrue(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_processor_gc(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - weak_ref = weakref.ref(multi_processor) - multi_processor.shutdown() - - # When the processor is garbage collected - del multi_processor - gc.collect() - - # Then the reference to the processor should no longer exist - self.assertIsNone( - weak_ref(), - "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", - ) - - def test_shutdown_runs_inline_after_executor_shutdown(self): - # Simulate the atexit scenario: CPython runs threading._shutdown() - # before the atexit queue, so the ThreadPoolExecutor is closed by the - # time TracerProvider.shutdown() runs. The underlying processors must - # still be shut down inline so buffered work is not dropped. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor) - - # pylint: disable=protected-access - multi_processor._executor.shutdown(wait=True) - - multi_processor.shutdown() - mock_processor.shutdown.assert_called_once_with() - - def test_on_end_runs_inline_after_executor_shutdown(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor) - - # pylint: disable=protected-access - multi_processor._executor.shutdown(wait=True) - - span = self.create_default_span() - multi_processor.on_end(span) - mock_processor.on_end.assert_called_once_with(span) - - def test_callback_runtime_error_propagates_not_swallowed(self): - # A RuntimeError raised by a processor callback (while the executor is - # alive) must still propagate out of shutdown(); the inline fallback - # must not mask unrelated callback failures. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - raising_processor = mock.Mock(spec=trace.SpanProcessor) - raising_processor.shutdown.side_effect = RuntimeError("processor boom") - multi_processor.add_span_processor(raising_processor) - - with self.assertRaises(RuntimeError): - multi_processor.shutdown() - - @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") - def test_batch_span_processor_fork(self): - multiprocessing_context = multiprocessing.get_context("fork") - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - exporter = InMemorySpanExporter() - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider.add_span_processor(multi_processor) - - # Use the ConcurrentMultiSpanProcessor in the main process. - # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: - with tracer.start_as_current_span("main process before fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process before fork span" - - # The forked ConcurrentMultiSpanProcessor is usable in the child process: - def child(conn): - with tracer.start_as_current_span("child process span"): - pass - conn.send(exporter.get_finished_spans()[-1].name) - conn.close() - - parent_conn, child_conn = multiprocessing_context.Pipe() - process = multiprocessing_context.Process(target=child, args=(child_conn,)) - process.start() - has_response = parent_conn.poll(timeout=5) - if not has_response: - process.kill() - self.fail("The child process did not send any message after 5 seconds, it's very probably locked") - process.join(timeout=5) - assert parent_conn.recv() == "child process span" - - # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: - with tracer.start_as_current_span("main process after fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process after fork span" +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import abc +import gc +import multiprocessing +import os +import time +import unittest +import weakref +from threading import Event, Thread +from unittest import mock + +from opentelemetry import trace as trace_api +from opentelemetry.context import Context +from opentelemetry.sdk import trace +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +def span_event_start_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":start" + + +def span_event_ending_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":ending" + + +def span_event_end_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":end" + + +class MySpanProcessor(trace.SpanProcessor): + def __init__(self, name, span_list): + self.name = name + self.span_list = span_list + + def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: + self.span_list.append(span_event_start_fmt(self.name, span.name)) + + def on_end(self, span: "trace.Span") -> None: + self.span_list.append(span_event_end_fmt(self.name, span.name)) + + +class MyExtendedSpanProcessor(MySpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + self.span_list.append(span_event_ending_fmt(self.name, span.name)) + + +class TestSpanProcessor(unittest.TestCase): + def test_span_processor(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MySpanProcessor("SP1", spans_calls_list) + sp2 = MySpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + # pylint: disable=too-many-statements + def test_span_processor_with_on_ending(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) + sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_ending_fmt("SP2", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_ending_fmt("SP2", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_ending_fmt("SP2", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + def test_add_span_processor_after_span_creation(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp = MySpanProcessor("SP1", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + # add span processor after spans have been created + tracer_provider.add_span_processor(sp) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Does not implement _on_ending + sp = MySpanProcessor("SP1", spans_calls_list) + tracer_provider.add_span_processor(sp) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + self.fail("_on_ending() should not raise an exception") + + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + assert span.is_recording() + assert span.end_time is not None + span.update_name("renamed") + span.set_attribute("attribute", "value") + span.set_attributes({"attributes": "value"}) + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(MutatingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + tracer.start_span("foo").end() + + (span,) = exporter.get_finished_spans() + self.assertEqual(span.name, "renamed") + self.assertEqual(span.attributes["attribute"], "value") + self.assertEqual(span.attributes["attributes"], "value") + self.assertEqual(span.events[0].name, "event") + self.assertEqual(len(span.links), 1) + self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) + + def test_end_during_on_ending_is_ignored(self): + exporter = InMemorySpanExporter() + on_ending_calls = [] + + class ReentrantSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + on_ending_calls.append(span) + span.end() + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(ReentrantSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + span = tracer.start_span("foo") + span.end() + + self.assertEqual(len(on_ending_calls), 1) + self.assertEqual(len(exporter.get_finished_spans()), 1) + + # After end() returns the span is immutable again. + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + (exported,) = exporter.get_finished_spans() + self.assertNotIn("late", exported.attributes) + + def test_other_thread_cannot_mutate_during_on_ending(self): + exporter = InMemorySpanExporter() + inside_on_ending = Event() + other_thread_done = Event() + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def mutate_from_another_thread(): + inside_on_ending.wait(5) + span.set_attribute("other.attribute", "value") + span.set_attributes({"other.attributes": "value"}) + span.update_name("renamed") + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + other_thread_done.set() + + thread = Thread(target=mutate_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(inside_on_ending.is_set()) + self.assertTrue(other_thread_done.is_set()) + + (exported,) = exporter.get_finished_spans() + self.assertNotIn("other.attribute", exported.attributes) + self.assertNotIn("other.attributes", exported.attributes) + self.assertEqual(exported.name, "foo") + self.assertEqual(len(exported.events), 0) + self.assertEqual(len(exported.links), 0) + self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) + + def test_other_thread_does_not_see_span_recording_during_on_ending(self): + inside_on_ending = Event() + other_thread_done = Event() + recording_seen_by_other_thread = [] + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def observe_from_another_thread(): + inside_on_ending.wait(5) + recording_seen_by_other_thread.append(span.is_recording()) + other_thread_done.set() + + thread = Thread(target=observe_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(other_thread_done.is_set()) + self.assertEqual(recording_seen_by_other_thread, [False]) + + def test_span_is_frozen_when_record_end_metrics_raises(self): + def record_end_metrics_that_raises(): + raise RuntimeError("meter blew up") + + tracer_provider = trace.TracerProvider() + span = tracer_provider.get_tracer(__name__).start_span("foo") + # pylint: disable=protected-access + span._record_end_metrics = record_end_metrics_that_raises + + with self.assertRaises(RuntimeError): + span.end() + + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + self.assertNotIn("late", span.attributes) + + +class MultiSpanProcessorTestBase(abc.ABC): + @abc.abstractmethod + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: + pass + + @staticmethod + def create_default_span() -> trace_api.Span: + span_context = trace_api.SpanContext(37, 73, is_remote=False) + return trace_api.NonRecordingSpan(span_context) + + def test_on_start(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + context = Context() + multi_processor.on_start(span, parent_context=context) + + for mock_processor in mocks: + mock_processor.on_start.assert_called_once_with(span, parent_context=context) + multi_processor.shutdown() + + def test_on_ending(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + # pylint: disable=protected-access + multi_processor._on_ending(span) + + for mock_processor in mocks: + # pylint: disable=protected-access + mock_processor._on_ending.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_end(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + multi_processor.on_end(span) + + for mock_processor in mocks: + mock_processor.on_end.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_shutdown(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + multi_processor.shutdown() + + for mock_processor in mocks: + mock_processor.shutdown.assert_called_once_with() + + def test_force_flush(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + timeout_millis = 100 + + flushed = multi_processor.force_flush(timeout_millis) + + # pylint: disable=no-member + self.assertTrue(flushed) + for mock_processor in mocks: + # pylint: disable=no-member + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + multi_processor = self.create_multi_span_processor() + # Does not implement _on_ending + multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) + + tracer_provider.add_span_processor(multi_processor) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + # pylint: disable=no-member + self.fail("_on_ending() should not raise an exception") + + # pylint: disable=no-member + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + tracer_provider.get_tracer(__name__).start_span("foo").end() + + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + def test_on_ending_exception_does_not_stop_other_processors(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class FailingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + raise RuntimeError("_on_ending failed") + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(FailingSpanProcessor()) + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + # pylint: disable=no-member + with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): + tracer_provider.get_tracer(__name__).start_span("foo").end() + + # The processor registered after the failing one still ran its + # _on_ending, and on_end still exported the span. + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + +class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor: + return trace.SynchronousMultiSpanProcessor() + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + def delayed_flush(_): + time.sleep(0.055) + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(0, mock_processor2.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=False) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=None) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + mock_processor2.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_default_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + default_processor = trace.SpanProcessor() + multi_processor.add_span_processor(default_processor) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + mock_processor.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.ConcurrentMultiSpanProcessor: + return trace.ConcurrentMultiSpanProcessor(3) + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + wait_event = Event() + + def delayed_flush(_): + wait_event.wait() + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(side_effect=delayed_flush) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(timeout_millis=25) + # let the thread executing the late_mock continue + wait_event.set() + multi_processor.shutdown() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(return_value=False) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + none_mock = mock.Mock(spec=trace.SpanProcessor) + none_mock.force_flush = mock.Mock(return_value=None) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertTrue(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_processor_gc(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + weak_ref = weakref.ref(multi_processor) + multi_processor.shutdown() + + # When the processor is garbage collected + del multi_processor + gc.collect() + + # Then the reference to the processor should no longer exist + self.assertIsNone( + weak_ref(), + "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", + ) + + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time TracerProvider.shutdown() runs. The underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + multi_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_on_end_runs_inline_after_executor_shutdown(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + span = self.create_default_span() + multi_processor.on_end(span) + mock_processor.on_end.assert_called_once_with(span) + + def test_callback_runtime_error_propagates_not_swallowed(self): + # A RuntimeError raised by a processor callback (while the executor is + # alive) must still propagate out of shutdown(); the inline fallback + # must not mask unrelated callback failures. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + raising_processor = mock.Mock(spec=trace.SpanProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + multi_processor.add_span_processor(raising_processor) + + with self.assertRaises(RuntimeError): + multi_processor.shutdown() + + @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") + def test_batch_span_processor_fork(self): + multiprocessing_context = multiprocessing.get_context("fork") + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + exporter = InMemorySpanExporter() + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider.add_span_processor(multi_processor) + + # Use the ConcurrentMultiSpanProcessor in the main process. + # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: + with tracer.start_as_current_span("main process before fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process before fork span" + + # The forked ConcurrentMultiSpanProcessor is usable in the child process: + def child(conn): + with tracer.start_as_current_span("child process span"): + pass + conn.send(exporter.get_finished_spans()[-1].name) + conn.close() + + parent_conn, child_conn = multiprocessing_context.Pipe() + process = multiprocessing_context.Process(target=child, args=(child_conn,)) + process.start() + has_response = parent_conn.poll(timeout=5) + if not has_response: + process.kill() + self.fail("The child process did not send any message after 5 seconds, it's very probably locked") + process.join(timeout=5) + assert parent_conn.recv() == "child process span" + + # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: + with tracer.start_as_current_span("main process after fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process after fork span" From c3d00ea729d5f9648e02892b2fec5932c3965261 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:16:43 +0800 Subject: [PATCH 03/26] changelog: rename fragment to match PR number --- .changelog/{5568.fixed => 5835.fixed} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .changelog/{5568.fixed => 5835.fixed} (100%) diff --git a/.changelog/5568.fixed b/.changelog/5835.fixed similarity index 100% rename from .changelog/5568.fixed rename to .changelog/5835.fixed From 28a7abec14f2ba7db51bb688208278352810e421 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:17:05 +0800 Subject: [PATCH 04/26] changelog: strip BOM from fragment --- .changelog/5835.fixed | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changelog/5835.fixed b/.changelog/5835.fixed index e91f3dae308..355432d60b3 100644 --- a/.changelog/5835.fixed +++ b/.changelog/5835.fixed @@ -1 +1 @@ -opentelemetry-sdk: run concurrent span/log record processor callbacks inline when the executor is already shut down at interpreter exit, so buffered telemetry is flushed instead of being dropped \ No newline at end of file +opentelemetry-sdk: run concurrent span/log record processor callbacks inline when the executor is already shut down at interpreter exit, so buffered telemetry is flushed instead of being dropped \ No newline at end of file From 071c23e7498405d6aeee16c596ad136ec3118e3f Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 20:22:13 +0800 Subject: [PATCH 05/26] tests: disable pylint no-self-use on inline fallback test --- opentelemetry-sdk/tests/logs/test_multi_log_processor.py | 1 + opentelemetry-sdk/tests/trace/test_span_processor.py | 1 + 2 files changed, 2 insertions(+) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index c5552a887dd..49b19b7acf9 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -252,6 +252,7 @@ def test_force_flush_processor_returns_none(self): self.assertEqual(1, mock.force_flush.call_count) multi_log_record_processor.shutdown() + # pylint: disable=no-self-use def test_shutdown_runs_inline_after_executor_shutdown(self): # Simulate the atexit scenario: CPython runs threading._shutdown() # before the atexit queue, so the ThreadPoolExecutor is closed by the diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index c878dc210c5..9df7a67cd64 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -718,6 +718,7 @@ def test_processor_gc(self): "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", ) + # pylint: disable=no-self-use def test_shutdown_runs_inline_after_executor_shutdown(self): # Simulate the atexit scenario: CPython runs threading._shutdown() # before the atexit queue, so the ThreadPoolExecutor is closed by the From 961ab09bb5e79753cacfa5f9e1cabda5d704e840 Mon Sep 17 00:00:00 2001 From: RichardoMrMu Date: Mon, 26 Oct 2026 22:08:42 +0800 Subject: [PATCH 06/26] fix(sdk): contain concurrent processor callback errors and prevent double-invoke on submit RuntimeError Adopt the contain+log error-handling contract for ConcurrentMultiSpanProcessor and ConcurrentMultiLogRecordProcessor: - Add a per-dispatch _CallbackAttempt that shares a lock-guarded 'started' claim between the queue worker and the inline fallback taken when executor.submit raises RuntimeError. This prevents the callback from running twice when Thread.start fails after the work item was already enqueued (the previous code ran it inline AND the queued worker ran it). - Contain callback exceptions per processor (logger.exception) so a faulty processor does not block healthy siblings, leak out of shutdown(), or replace the caller's own exception. - Wait for claimed attempts to finish before returning. Tests: - Replace the propagation assertion with containment + sibling-continues + application-exception-preserved assertions. - Add a Thread.start-failure no-double-invoke test for trace and logs. - Real subprocess atexit exit verified: a faulty sibling no longer blocks the healthy Batch processor's EXPORT/SHUTDOWN at interpreter exit. Known limitations: no retry/delivery guarantee, no global exactly-once, no bounded shutdown wait; force_flush path unchanged; logs self-diagnostics recursion not separately exercised. --- .../sdk/_logs/_internal/__init__.py | 66 ++++++++++++++--- .../src/opentelemetry/sdk/trace/__init__.py | 73 +++++++++++++++++-- .../tests/logs/test_multi_log_processor.py | 39 +++++++++- .../tests/trace/test_span_processor.py | 67 +++++++++++++++-- 4 files changed, 219 insertions(+), 26 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 6378a40cecd..e3d02f185b5 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -476,6 +476,41 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: return all_flushed +class _CallbackAttempt: + """Per-dispatch at-most-once wrapper for a log record processor callback. + + See :class:`opentelemetry.sdk.trace._CallbackAttempt` for the rationale. + """ + + __slots__ = ("_callback", "_args", "_kwargs", "_lock", "_started", "_finished") + + def __init__(self, callback: Callable[..., None], args: tuple[Any, ...], kwargs: dict[str, Any]): + self._callback = callback + self._args = args + self._kwargs = kwargs + self._lock = threading.Lock() + self._started = False + self._finished = threading.Event() + + def run(self) -> None: + with self._lock: + if self._started: + return + self._started = True + try: + self._callback(*self._args, **self._kwargs) + except Exception: # pylint: disable=broad-exception-caught + _logger.exception("Exception while calling LogRecordProcessor callback.") + finally: + self._finished.set() + + def wait_if_started(self) -> None: + with self._lock: + started = self._started + if started: + self._finished.wait() + + class ConcurrentMultiLogRecordProcessor(LogRecordProcessor): """Implementation of :class:`LogRecordProcessor` that forwards all received events to a list of log processors in parallel. @@ -506,25 +541,38 @@ def _submit_and_wait( *args: Any, **kwargs: Any, ): + attempts = [] futures = [] for lp in self._log_record_processors: try: - future = self._executor.submit(func(lp), *args, **kwargs) - futures.append(future) + callback = func(lp) + except Exception: # pylint: disable=broad-exception-caught + _logger.exception("Exception while resolving LogRecordProcessor callback.") + continue + attempt = _CallbackAttempt(callback, args, kwargs) + attempts.append(attempt) + try: + futures.append(self._executor.submit(attempt.run)) except RuntimeError: # The executor may already be shut down when this runs from an # atexit handler: CPython runs threading._shutdown() before the - # atexit queue, so the ThreadPoolExecutor is closed by the time - # LoggerProvider.shutdown() runs. Run inline so buffered work is - # still flushed instead of being silently dropped. A - # RuntimeError raised by the callback itself is not caught here - # and still propagates via future.result() below. + # atexit queue, so the LoggerProvider.shutdown() runs. Run + # inline through the SAME attempt; the shared lock prevents + # double invocation even if the work item was already enqueued + # before Thread.start failed. _logger.debug( "Executor is shut down; running LogRecordProcessor callback inline.", ) - func(lp)(*args, **kwargs) + attempt.run() + except Exception: # pylint: disable=broad-exception-caught + _logger.exception("Exception while submitting LogRecordProcessor callback.") for future in futures: - future.result() + try: + future.result() + except Exception: # pylint: disable=broad-exception-caught + _logger.exception("Exception in LogRecordProcessor worker.") + for attempt in attempts: + attempt.wait_if_started() def on_emit(self, log_record: ReadWriteLogRecord) -> None: self._submit_and_wait(lambda lp: lp.on_emit, log_record) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 11cad9d587d..573ea377731 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -224,6 +224,50 @@ def force_flush(self, timeout_millis: int = 30000) -> bool: return all_flushed +class _CallbackAttempt: + """Per-dispatch at-most-once wrapper for a processor callback. + + The queue worker and the inline fallback (when ``executor.submit`` raises + ``RuntimeError``) share the same attempt. A lock-guarded claim ensures the + underlying callback runs at most once even when ``Thread.start`` fails + after the work item was already enqueued, which previously caused the + callback to run twice. Callback exceptions are contained and logged so a + faulty processor does not block healthy siblings or replace the caller's + own exception. + + This scopes to a single dispatch entry; it does not promise global + exactly-once delivery, retry, or bounded shutdown. + """ + + __slots__ = ("_callback", "_args", "_kwargs", "_lock", "_started", "_finished") + + def __init__(self, callback: Callable[..., None], args: tuple[Any, ...], kwargs: dict[str, Any]): + self._callback = callback + self._args = args + self._kwargs = kwargs + self._lock = threading.Lock() + self._started = False + self._finished = threading.Event() + + def run(self) -> None: + with self._lock: + if self._started: + return + self._started = True + try: + self._callback(*self._args, **self._kwargs) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while calling SpanProcessor callback.") + finally: + self._finished.set() + + def wait_if_started(self) -> None: + with self._lock: + started = self._started + if started: + self._finished.wait() + + class ConcurrentMultiSpanProcessor(SpanProcessor): """Implementation of :class:`SpanProcessor` that forwards all received events to a list of span processors in parallel. @@ -273,25 +317,38 @@ def _submit_and_await( *args: Any, **kwargs: Any, ): + attempts = [] futures = [] for sp in self._span_processors: try: - future = self._executor.submit(func(sp), *args, **kwargs) - futures.append(future) + callback = func(sp) + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while resolving SpanProcessor callback.") + continue + attempt = _CallbackAttempt(callback, args, kwargs) + attempts.append(attempt) + try: + futures.append(self._executor.submit(attempt.run)) except RuntimeError: # The executor may already be shut down when this runs from an # atexit handler: CPython runs threading._shutdown() before the # atexit queue, so the ThreadPoolExecutor is closed by the time - # TracerProvider.shutdown() runs. Run inline so buffered work is - # still flushed instead of being silently dropped. A - # RuntimeError raised by the callback itself is not caught here - # and still propagates via future.result() below. + # TracerProvider.shutdown() runs. Run inline through the SAME + # attempt; the shared lock prevents double invocation even if + # the work item was already enqueued before Thread.start failed. logger.debug( "Executor is shut down; running SpanProcessor callback inline.", ) - func(sp)(*args, **kwargs) + attempt.run() + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception while submitting SpanProcessor callback.") for future in futures: - future.result() + try: + future.result() + except Exception: # pylint: disable=broad-exception-caught + logger.exception("Exception in SpanProcessor worker.") + for attempt in attempts: + attempt.wait_if_started() def on_start( self, diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 49b19b7acf9..3e3b95222fc 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -8,6 +8,7 @@ import time import unittest from abc import ABC, abstractmethod +from unittest import mock from unittest.mock import Mock from opentelemetry._logs import LogRecord, SeverityNumber @@ -268,13 +269,43 @@ def test_shutdown_runs_inline_after_executor_shutdown(self): multi_log_record_processor.shutdown() mock_processor.shutdown.assert_called_once_with() - def test_callback_runtime_error_propagates_not_swallowed(self): - # A RuntimeError raised by a processor callback (while the executor is - # alive) must still propagate out of shutdown(). + def test_callback_exception_does_not_block_siblings(self): + # A LogRecordProcessor callback failure must not raise out of + # shutdown() and must not prevent healthy siblings from running. multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) raising_processor = Mock(spec=LogRecordProcessor) raising_processor.shutdown.side_effect = RuntimeError("processor boom") + healthy_processor = Mock(spec=LogRecordProcessor) multi_log_record_processor.add_log_record_processor(raising_processor) + multi_log_record_processor.add_log_record_processor(healthy_processor) - with self.assertRaises(RuntimeError): + with self.assertLogs("opentelemetry.sdk._logs._internal", level="ERROR"): multi_log_record_processor.shutdown() + + raising_processor.shutdown.assert_called_once_with() + healthy_processor.shutdown.assert_called_once_with() + + def test_submit_runtime_error_does_not_double_invoke(self): + # If Thread.start fails AFTER the work item was enqueued, the inline + # fallback must not run the callback a second time. + import threading + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + a = Mock(spec=LogRecordProcessor) + b = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(a) + multi_log_record_processor.add_log_record_processor(b) + + real_start = threading.Thread.start + calls = {"n": 0} + + def first_start_fails(t): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("cannot start new thread") + return real_start(t) + + with mock.patch.object(threading.Thread, "start", first_start_fails): + multi_log_record_processor.shutdown() + + self.assertEqual(a.shutdown.call_count, 1) + self.assertEqual(b.shutdown.call_count, 1) diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 9df7a67cd64..25eb316a9bf 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -746,18 +746,75 @@ def test_on_end_runs_inline_after_executor_shutdown(self): multi_processor.on_end(span) mock_processor.on_end.assert_called_once_with(span) - def test_callback_runtime_error_propagates_not_swallowed(self): - # A RuntimeError raised by a processor callback (while the executor is - # alive) must still propagate out of shutdown(); the inline fallback - # must not mask unrelated callback failures. + def test_callback_exception_does_not_block_siblings(self): + # Per OTel error-handling spec, a processor callback must not raise out + # of shutdown() and must not prevent healthy siblings from running. multi_processor = trace.ConcurrentMultiSpanProcessor(2) raising_processor = mock.Mock(spec=trace.SpanProcessor) raising_processor.shutdown.side_effect = RuntimeError("processor boom") + healthy_processor = mock.Mock(spec=trace.SpanProcessor) multi_processor.add_span_processor(raising_processor) + multi_processor.add_span_processor(healthy_processor) - with self.assertRaises(RuntimeError): + with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): multi_processor.shutdown() + raising_processor.shutdown.assert_called_once_with() + healthy_processor.shutdown.assert_called_once_with() + + def test_submit_runtime_error_does_not_double_invoke(self): + # If Thread.start fails AFTER the work item was enqueued, the inline + # fallback must not run the callback a second time: the queue worker + # will pick up the enqueued item. A shared claim per dispatch ensures + # at-most-once. + import threading + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + a = mock.Mock(spec=trace.SpanProcessor) + b = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(a) + multi_processor.add_span_processor(b) + + real_start = threading.Thread.start + calls = {"n": 0} + + def first_start_fails(t): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("cannot start new thread") + return real_start(t) + + try: + with mock.patch.object(threading.Thread, "start", first_start_fails): + multi_processor.shutdown() + finally: + pass + + self.assertEqual(a.shutdown.call_count, 1) + self.assertEqual(b.shutdown.call_count, 1) + + def test_callback_exception_preserves_application_exception(self): + # A processor callback failure must not replace the caller's own + # exception when on_end runs on the caller's thread. + from opentelemetry.sdk.trace import TracerProvider + + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + faulty = mock.Mock(spec=trace.SpanProcessor) + faulty.on_end.side_effect = RuntimeError("telemetry fault") + healthy = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(faulty) + multi_processor.add_span_processor(healthy) + + provider = TracerProvider(active_span_processor=multi_processor, shutdown_on_exit=False) + app_error = ValueError("business fault") + try: + with self.assertRaises(ValueError) as caught: + with provider.get_tracer("test").start_as_current_span("x"): + raise app_error + self.assertIs(caught.exception, app_error) + finally: + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") def test_batch_span_processor_fork(self): multiprocessing_context = multiprocessing.get_context("fork") From c7446c1dc46e8905ca69d2cfa3e8b3b75226163f Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:32:13 +0800 Subject: [PATCH 07/26] test(sdk): address pylint/ruff nits in CallbackAttempt tests --- .../tests/logs/test_multi_log_processor.py | 620 +++--- .../tests/trace/test_span_processor.py | 1703 ++++++++--------- 2 files changed, 1158 insertions(+), 1165 deletions(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 3e3b95222fc..3581d61bdc5 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -1,311 +1,309 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# pylint:disable=protected-access,no-self-use,no-member - -import logging -import threading -import time -import unittest -from abc import ABC, abstractmethod -from unittest import mock -from unittest.mock import Mock - -from opentelemetry._logs import LogRecord, SeverityNumber -from opentelemetry.sdk._logs._internal import ( - ConcurrentMultiLogRecordProcessor, - LoggerProvider, - LoggingHandler, - LogRecordProcessor, - ReadWriteLogRecord, - SynchronousMultiLogRecordProcessor, -) - - -class AnotherLogRecordProcessor(LogRecordProcessor): - def __init__(self, exporter, logs_list): - self._exporter = exporter - self._log_list = logs_list - self._closed = False - - def on_emit(self, log_record: ReadWriteLogRecord): - if self._closed: - return - self._log_list.append( - ( - log_record.log_record.body, - log_record.log_record.severity_text, - ) - ) - - def shutdown(self): - self._closed = True - self._exporter.shutdown() - - def force_flush(self, timeout_millis=30000): - self._log_list.clear() - return True - - -class TestLogRecordProcessor(unittest.TestCase): - def test_log_record_processor(self): - provider = LoggerProvider() - handler = LoggingHandler(logger_provider=provider) - - logs_list_1 = [] - processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) - logs_list_2 = [] - processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) - - logger = logging.getLogger("test.span.processor") - logger.addHandler(handler) - - # Test no proessor added - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Odisha, we have another major cyclone") - - self.assertEqual(len(logs_list_1), 0) - self.assertEqual(len(logs_list_2), 0) - - # Add one processor - provider.add_log_record_processor(processor1) - with self.assertLogs(level=logging.WARNING): - logger.warning("Brace yourself") - with self.assertLogs(level=logging.ERROR): - logger.error("Some error message") - - expected_list_1 = [ - ("Brace yourself", "WARN"), - ("Some error message", "ERROR"), - ] - self.assertEqual(logs_list_1, expected_list_1) - - # Add another processor - provider.add_log_record_processor(processor2) - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Something disastrous") - expected_list_1.append(("Something disastrous", "FATAL")) - - expected_list_2 = [("Something disastrous", "FATAL")] - - self.assertEqual(logs_list_1, expected_list_1) - self.assertEqual(logs_list_2, expected_list_2) - - -class MultiLogRecordProcessorTestBase(ABC): - @abstractmethod - def _get_multi_log_record_processor(self): - pass - - def make_record(self): - return ReadWriteLogRecord( - LogRecord( - timestamp=1622300111608942000, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Warning message", - ) - ) - - def test_on_emit(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - record = self.make_record() - multi_log_record_processor.on_emit(record) - for mock in mocks: - mock.on_emit.assert_called_with(record) - multi_log_record_processor.shutdown() - - def test_on_shutdown(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - multi_log_record_processor.shutdown() - for mock in mocks: - mock.shutdown.assert_called_once_with() - - def test_on_force_flush(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - ret_value = multi_log_record_processor.force_flush(100) - - self.assertTrue(ret_value) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return SynchronousMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - def delay(_): - time.sleep(0.09) - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(side_effect=delay) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 0) - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=False) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=None) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertTrue(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - -class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return ConcurrentMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - wait_event = threading.Event() - - def delay(_): - wait_event.wait() - - mock1 = Mock(spec=LogRecordProcessor) - mock1.force_flush = Mock(side_effect=delay) - mocks = [Mock(LogRecordProcessor) for _ in range(5)] - mocks = [mock1] + mocks - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush(50) - wait_event.set() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - false_mock = Mock(spec=LogRecordProcessor) - false_mock.force_flush = Mock(return_value=False) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, false_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - none_mock = Mock(spec=LogRecordProcessor) - none_mock.force_flush = Mock(return_value=None) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertTrue(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - # pylint: disable=no-self-use - def test_shutdown_runs_inline_after_executor_shutdown(self): - # Simulate the atexit scenario: CPython runs threading._shutdown() - # before the atexit queue, so the ThreadPoolExecutor is closed by the - # time LoggerProvider.shutdown() runs. Underlying processors must - # still be shut down inline so buffered work is not dropped. - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - mock_processor = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(mock_processor) - - # pylint: disable=protected-access - multi_log_record_processor._executor.shutdown(wait=True) - - multi_log_record_processor.shutdown() - mock_processor.shutdown.assert_called_once_with() - - def test_callback_exception_does_not_block_siblings(self): - # A LogRecordProcessor callback failure must not raise out of - # shutdown() and must not prevent healthy siblings from running. - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - raising_processor = Mock(spec=LogRecordProcessor) - raising_processor.shutdown.side_effect = RuntimeError("processor boom") - healthy_processor = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(raising_processor) - multi_log_record_processor.add_log_record_processor(healthy_processor) - - with self.assertLogs("opentelemetry.sdk._logs._internal", level="ERROR"): - multi_log_record_processor.shutdown() - - raising_processor.shutdown.assert_called_once_with() - healthy_processor.shutdown.assert_called_once_with() - - def test_submit_runtime_error_does_not_double_invoke(self): - # If Thread.start fails AFTER the work item was enqueued, the inline - # fallback must not run the callback a second time. - import threading - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - a = Mock(spec=LogRecordProcessor) - b = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(a) - multi_log_record_processor.add_log_record_processor(b) - - real_start = threading.Thread.start - calls = {"n": 0} - - def first_start_fails(t): - calls["n"] += 1 - if calls["n"] == 1: - raise RuntimeError("cannot start new thread") - return real_start(t) - - with mock.patch.object(threading.Thread, "start", first_start_fails): - multi_log_record_processor.shutdown() - - self.assertEqual(a.shutdown.call_count, 1) - self.assertEqual(b.shutdown.call_count, 1) +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint:disable=protected-access,no-self-use,no-member + +import logging +import threading +import time +import unittest +from abc import ABC, abstractmethod +from unittest.mock import Mock, patch + +from opentelemetry._logs import LogRecord, SeverityNumber +from opentelemetry.sdk._logs._internal import ( + ConcurrentMultiLogRecordProcessor, + LoggerProvider, + LoggingHandler, + LogRecordProcessor, + ReadWriteLogRecord, + SynchronousMultiLogRecordProcessor, +) + + +class AnotherLogRecordProcessor(LogRecordProcessor): + def __init__(self, exporter, logs_list): + self._exporter = exporter + self._log_list = logs_list + self._closed = False + + def on_emit(self, log_record: ReadWriteLogRecord): + if self._closed: + return + self._log_list.append( + ( + log_record.log_record.body, + log_record.log_record.severity_text, + ) + ) + + def shutdown(self): + self._closed = True + self._exporter.shutdown() + + def force_flush(self, timeout_millis=30000): + self._log_list.clear() + return True + + +class TestLogRecordProcessor(unittest.TestCase): + def test_log_record_processor(self): + provider = LoggerProvider() + handler = LoggingHandler(logger_provider=provider) + + logs_list_1 = [] + processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) + logs_list_2 = [] + processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) + + logger = logging.getLogger("test.span.processor") + logger.addHandler(handler) + + # Test no proessor added + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Odisha, we have another major cyclone") + + self.assertEqual(len(logs_list_1), 0) + self.assertEqual(len(logs_list_2), 0) + + # Add one processor + provider.add_log_record_processor(processor1) + with self.assertLogs(level=logging.WARNING): + logger.warning("Brace yourself") + with self.assertLogs(level=logging.ERROR): + logger.error("Some error message") + + expected_list_1 = [ + ("Brace yourself", "WARN"), + ("Some error message", "ERROR"), + ] + self.assertEqual(logs_list_1, expected_list_1) + + # Add another processor + provider.add_log_record_processor(processor2) + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Something disastrous") + expected_list_1.append(("Something disastrous", "FATAL")) + + expected_list_2 = [("Something disastrous", "FATAL")] + + self.assertEqual(logs_list_1, expected_list_1) + self.assertEqual(logs_list_2, expected_list_2) + + +class MultiLogRecordProcessorTestBase(ABC): + @abstractmethod + def _get_multi_log_record_processor(self): + pass + + def make_record(self): + return ReadWriteLogRecord( + LogRecord( + timestamp=1622300111608942000, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Warning message", + ) + ) + + def test_on_emit(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + record = self.make_record() + multi_log_record_processor.on_emit(record) + for mock in mocks: + mock.on_emit.assert_called_with(record) + multi_log_record_processor.shutdown() + + def test_on_shutdown(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + multi_log_record_processor.shutdown() + for mock in mocks: + mock.shutdown.assert_called_once_with() + + def test_on_force_flush(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + ret_value = multi_log_record_processor.force_flush(100) + + self.assertTrue(ret_value) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return SynchronousMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + def delay(_): + time.sleep(0.09) + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(side_effect=delay) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 0) + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=False) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=None) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertTrue(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + +class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return ConcurrentMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + wait_event = threading.Event() + + def delay(_): + wait_event.wait() + + mock1 = Mock(spec=LogRecordProcessor) + mock1.force_flush = Mock(side_effect=delay) + mocks = [Mock(LogRecordProcessor) for _ in range(5)] + mocks = [mock1] + mocks + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush(50) + wait_event.set() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + false_mock = Mock(spec=LogRecordProcessor) + false_mock.force_flush = Mock(return_value=False) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, false_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + none_mock = Mock(spec=LogRecordProcessor) + none_mock.force_flush = Mock(return_value=None) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertTrue(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + # pylint: disable=no-self-use + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time LoggerProvider.shutdown() runs. Underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + mock_processor = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor) + + # pylint: disable=protected-access + multi_log_record_processor._executor.shutdown(wait=True) + + multi_log_record_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_callback_exception_does_not_block_siblings(self): + # A LogRecordProcessor callback failure must not raise out of + # shutdown() and must not prevent healthy siblings from running. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + raising_processor = Mock(spec=LogRecordProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + healthy_processor = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(raising_processor) + multi_log_record_processor.add_log_record_processor(healthy_processor) + + with self.assertLogs("opentelemetry.sdk._logs._internal", level="ERROR"): + multi_log_record_processor.shutdown() + + raising_processor.shutdown.assert_called_once_with() + healthy_processor.shutdown.assert_called_once_with() + + def test_submit_runtime_error_does_not_double_invoke(self): + # If Thread.start fails AFTER the work item was enqueued, the inline + # fallback must not run the callback a second time. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + processor_a = Mock(spec=LogRecordProcessor) + processor_b = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(processor_a) + multi_log_record_processor.add_log_record_processor(processor_b) + + real_start = threading.Thread.start + calls = {"n": 0} + + def first_start_fails(target): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("cannot start new thread") + return real_start(target) + + with patch.object(threading.Thread, "start", first_start_fails): + multi_log_record_processor.shutdown() + + self.assertEqual(processor_a.shutdown.call_count, 1) + self.assertEqual(processor_b.shutdown.call_count, 1) diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 25eb316a9bf..f89dcadd2a2 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -1,854 +1,849 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -import abc -import gc -import multiprocessing -import os -import time -import unittest -import weakref -from threading import Event, Thread -from unittest import mock - -from opentelemetry import trace as trace_api -from opentelemetry.context import Context -from opentelemetry.sdk import trace -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, -) - - -def span_event_start_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":start" - - -def span_event_ending_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":ending" - - -def span_event_end_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":end" - - -class MySpanProcessor(trace.SpanProcessor): - def __init__(self, name, span_list): - self.name = name - self.span_list = span_list - - def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: - self.span_list.append(span_event_start_fmt(self.name, span.name)) - - def on_end(self, span: "trace.Span") -> None: - self.span_list.append(span_event_end_fmt(self.name, span.name)) - - -class MyExtendedSpanProcessor(MySpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - self.span_list.append(span_event_ending_fmt(self.name, span.name)) - - -class TestSpanProcessor(unittest.TestCase): - def test_span_processor(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MySpanProcessor("SP1", spans_calls_list) - sp2 = MySpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - # pylint: disable=too-many-statements - def test_span_processor_with_on_ending(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) - sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_ending_fmt("SP2", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_ending_fmt("SP2", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_ending_fmt("SP2", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - def test_add_span_processor_after_span_creation(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp = MySpanProcessor("SP1", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - # add span processor after spans have been created - tracer_provider.add_span_processor(sp) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Does not implement _on_ending - sp = MySpanProcessor("SP1", spans_calls_list) - tracer_provider.add_span_processor(sp) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - self.fail("_on_ending() should not raise an exception") - - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - assert span.is_recording() - assert span.end_time is not None - span.update_name("renamed") - span.set_attribute("attribute", "value") - span.set_attributes({"attributes": "value"}) - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(MutatingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - tracer.start_span("foo").end() - - (span,) = exporter.get_finished_spans() - self.assertEqual(span.name, "renamed") - self.assertEqual(span.attributes["attribute"], "value") - self.assertEqual(span.attributes["attributes"], "value") - self.assertEqual(span.events[0].name, "event") - self.assertEqual(len(span.links), 1) - self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) - - def test_end_during_on_ending_is_ignored(self): - exporter = InMemorySpanExporter() - on_ending_calls = [] - - class ReentrantSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - on_ending_calls.append(span) - span.end() - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(ReentrantSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - span = tracer.start_span("foo") - span.end() - - self.assertEqual(len(on_ending_calls), 1) - self.assertEqual(len(exporter.get_finished_spans()), 1) - - # After end() returns the span is immutable again. - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - (exported,) = exporter.get_finished_spans() - self.assertNotIn("late", exported.attributes) - - def test_other_thread_cannot_mutate_during_on_ending(self): - exporter = InMemorySpanExporter() - inside_on_ending = Event() - other_thread_done = Event() - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def mutate_from_another_thread(): - inside_on_ending.wait(5) - span.set_attribute("other.attribute", "value") - span.set_attributes({"other.attributes": "value"}) - span.update_name("renamed") - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - other_thread_done.set() - - thread = Thread(target=mutate_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(inside_on_ending.is_set()) - self.assertTrue(other_thread_done.is_set()) - - (exported,) = exporter.get_finished_spans() - self.assertNotIn("other.attribute", exported.attributes) - self.assertNotIn("other.attributes", exported.attributes) - self.assertEqual(exported.name, "foo") - self.assertEqual(len(exported.events), 0) - self.assertEqual(len(exported.links), 0) - self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) - - def test_other_thread_does_not_see_span_recording_during_on_ending(self): - inside_on_ending = Event() - other_thread_done = Event() - recording_seen_by_other_thread = [] - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def observe_from_another_thread(): - inside_on_ending.wait(5) - recording_seen_by_other_thread.append(span.is_recording()) - other_thread_done.set() - - thread = Thread(target=observe_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(other_thread_done.is_set()) - self.assertEqual(recording_seen_by_other_thread, [False]) - - def test_span_is_frozen_when_record_end_metrics_raises(self): - def record_end_metrics_that_raises(): - raise RuntimeError("meter blew up") - - tracer_provider = trace.TracerProvider() - span = tracer_provider.get_tracer(__name__).start_span("foo") - # pylint: disable=protected-access - span._record_end_metrics = record_end_metrics_that_raises - - with self.assertRaises(RuntimeError): - span.end() - - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - self.assertNotIn("late", span.attributes) - - -class MultiSpanProcessorTestBase(abc.ABC): - @abc.abstractmethod - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: - pass - - @staticmethod - def create_default_span() -> trace_api.Span: - span_context = trace_api.SpanContext(37, 73, is_remote=False) - return trace_api.NonRecordingSpan(span_context) - - def test_on_start(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - context = Context() - multi_processor.on_start(span, parent_context=context) - - for mock_processor in mocks: - mock_processor.on_start.assert_called_once_with(span, parent_context=context) - multi_processor.shutdown() - - def test_on_ending(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - # pylint: disable=protected-access - multi_processor._on_ending(span) - - for mock_processor in mocks: - # pylint: disable=protected-access - mock_processor._on_ending.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_end(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - multi_processor.on_end(span) - - for mock_processor in mocks: - mock_processor.on_end.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_shutdown(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - multi_processor.shutdown() - - for mock_processor in mocks: - mock_processor.shutdown.assert_called_once_with() - - def test_force_flush(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - timeout_millis = 100 - - flushed = multi_processor.force_flush(timeout_millis) - - # pylint: disable=no-member - self.assertTrue(flushed) - for mock_processor in mocks: - # pylint: disable=no-member - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - multi_processor = self.create_multi_span_processor() - # Does not implement _on_ending - multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) - - tracer_provider.add_span_processor(multi_processor) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - # pylint: disable=no-member - self.fail("_on_ending() should not raise an exception") - - # pylint: disable=no-member - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - tracer_provider.get_tracer(__name__).start_span("foo").end() - - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - def test_on_ending_exception_does_not_stop_other_processors(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class FailingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - raise RuntimeError("_on_ending failed") - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(FailingSpanProcessor()) - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - # pylint: disable=no-member - with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): - tracer_provider.get_tracer(__name__).start_span("foo").end() - - # The processor registered after the failing one still ran its - # _on_ending, and on_end still exported the span. - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - -class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor: - return trace.SynchronousMultiSpanProcessor() - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - def delayed_flush(_): - time.sleep(0.055) - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(0, mock_processor2.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=False) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=None) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - mock_processor2.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_default_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - default_processor = trace.SpanProcessor() - multi_processor.add_span_processor(default_processor) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - mock_processor.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.ConcurrentMultiSpanProcessor: - return trace.ConcurrentMultiSpanProcessor(3) - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - wait_event = Event() - - def delayed_flush(_): - wait_event.wait() - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(side_effect=delayed_flush) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(timeout_millis=25) - # let the thread executing the late_mock continue - wait_event.set() - multi_processor.shutdown() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(return_value=False) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - none_mock = mock.Mock(spec=trace.SpanProcessor) - none_mock.force_flush = mock.Mock(return_value=None) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertTrue(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_processor_gc(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - weak_ref = weakref.ref(multi_processor) - multi_processor.shutdown() - - # When the processor is garbage collected - del multi_processor - gc.collect() - - # Then the reference to the processor should no longer exist - self.assertIsNone( - weak_ref(), - "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", - ) - - # pylint: disable=no-self-use - def test_shutdown_runs_inline_after_executor_shutdown(self): - # Simulate the atexit scenario: CPython runs threading._shutdown() - # before the atexit queue, so the ThreadPoolExecutor is closed by the - # time TracerProvider.shutdown() runs. The underlying processors must - # still be shut down inline so buffered work is not dropped. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor) - - # pylint: disable=protected-access - multi_processor._executor.shutdown(wait=True) - - multi_processor.shutdown() - mock_processor.shutdown.assert_called_once_with() - - def test_on_end_runs_inline_after_executor_shutdown(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor) - - # pylint: disable=protected-access - multi_processor._executor.shutdown(wait=True) - - span = self.create_default_span() - multi_processor.on_end(span) - mock_processor.on_end.assert_called_once_with(span) - - def test_callback_exception_does_not_block_siblings(self): - # Per OTel error-handling spec, a processor callback must not raise out - # of shutdown() and must not prevent healthy siblings from running. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - raising_processor = mock.Mock(spec=trace.SpanProcessor) - raising_processor.shutdown.side_effect = RuntimeError("processor boom") - healthy_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(raising_processor) - multi_processor.add_span_processor(healthy_processor) - - with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): - multi_processor.shutdown() - - raising_processor.shutdown.assert_called_once_with() - healthy_processor.shutdown.assert_called_once_with() - - def test_submit_runtime_error_does_not_double_invoke(self): - # If Thread.start fails AFTER the work item was enqueued, the inline - # fallback must not run the callback a second time: the queue worker - # will pick up the enqueued item. A shared claim per dispatch ensures - # at-most-once. - import threading - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - a = mock.Mock(spec=trace.SpanProcessor) - b = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(a) - multi_processor.add_span_processor(b) - - real_start = threading.Thread.start - calls = {"n": 0} - - def first_start_fails(t): - calls["n"] += 1 - if calls["n"] == 1: - raise RuntimeError("cannot start new thread") - return real_start(t) - - try: - with mock.patch.object(threading.Thread, "start", first_start_fails): - multi_processor.shutdown() - finally: - pass - - self.assertEqual(a.shutdown.call_count, 1) - self.assertEqual(b.shutdown.call_count, 1) - - def test_callback_exception_preserves_application_exception(self): - # A processor callback failure must not replace the caller's own - # exception when on_end runs on the caller's thread. - from opentelemetry.sdk.trace import TracerProvider - - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - faulty = mock.Mock(spec=trace.SpanProcessor) - faulty.on_end.side_effect = RuntimeError("telemetry fault") - healthy = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(faulty) - multi_processor.add_span_processor(healthy) - - provider = TracerProvider(active_span_processor=multi_processor, shutdown_on_exit=False) - app_error = ValueError("business fault") - try: - with self.assertRaises(ValueError) as caught: - with provider.get_tracer("test").start_as_current_span("x"): - raise app_error - self.assertIs(caught.exception, app_error) - finally: - # pylint: disable=protected-access - multi_processor._executor.shutdown(wait=True) - - @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") - def test_batch_span_processor_fork(self): - multiprocessing_context = multiprocessing.get_context("fork") - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - exporter = InMemorySpanExporter() - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider.add_span_processor(multi_processor) - - # Use the ConcurrentMultiSpanProcessor in the main process. - # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: - with tracer.start_as_current_span("main process before fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process before fork span" - - # The forked ConcurrentMultiSpanProcessor is usable in the child process: - def child(conn): - with tracer.start_as_current_span("child process span"): - pass - conn.send(exporter.get_finished_spans()[-1].name) - conn.close() - - parent_conn, child_conn = multiprocessing_context.Pipe() - process = multiprocessing_context.Process(target=child, args=(child_conn,)) - process.start() - has_response = parent_conn.poll(timeout=5) - if not has_response: - process.kill() - self.fail("The child process did not send any message after 5 seconds, it's very probably locked") - process.join(timeout=5) - assert parent_conn.recv() == "child process span" - - # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: - with tracer.start_as_current_span("main process after fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process after fork span" +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import abc +import gc +import multiprocessing +import os +import time +import unittest +import weakref +from threading import Event, Thread +from unittest import mock + +from opentelemetry import trace as trace_api +from opentelemetry.context import Context +from opentelemetry.sdk import trace +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +def span_event_start_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":start" + + +def span_event_ending_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":ending" + + +def span_event_end_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":end" + + +class MySpanProcessor(trace.SpanProcessor): + def __init__(self, name, span_list): + self.name = name + self.span_list = span_list + + def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: + self.span_list.append(span_event_start_fmt(self.name, span.name)) + + def on_end(self, span: "trace.Span") -> None: + self.span_list.append(span_event_end_fmt(self.name, span.name)) + + +class MyExtendedSpanProcessor(MySpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + self.span_list.append(span_event_ending_fmt(self.name, span.name)) + + +class TestSpanProcessor(unittest.TestCase): + def test_span_processor(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MySpanProcessor("SP1", spans_calls_list) + sp2 = MySpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + # pylint: disable=too-many-statements + def test_span_processor_with_on_ending(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) + sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_ending_fmt("SP2", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_ending_fmt("SP2", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_ending_fmt("SP2", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + def test_add_span_processor_after_span_creation(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp = MySpanProcessor("SP1", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + # add span processor after spans have been created + tracer_provider.add_span_processor(sp) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Does not implement _on_ending + sp = MySpanProcessor("SP1", spans_calls_list) + tracer_provider.add_span_processor(sp) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + self.fail("_on_ending() should not raise an exception") + + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + assert span.is_recording() + assert span.end_time is not None + span.update_name("renamed") + span.set_attribute("attribute", "value") + span.set_attributes({"attributes": "value"}) + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(MutatingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + tracer.start_span("foo").end() + + (span,) = exporter.get_finished_spans() + self.assertEqual(span.name, "renamed") + self.assertEqual(span.attributes["attribute"], "value") + self.assertEqual(span.attributes["attributes"], "value") + self.assertEqual(span.events[0].name, "event") + self.assertEqual(len(span.links), 1) + self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) + + def test_end_during_on_ending_is_ignored(self): + exporter = InMemorySpanExporter() + on_ending_calls = [] + + class ReentrantSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + on_ending_calls.append(span) + span.end() + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(ReentrantSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + span = tracer.start_span("foo") + span.end() + + self.assertEqual(len(on_ending_calls), 1) + self.assertEqual(len(exporter.get_finished_spans()), 1) + + # After end() returns the span is immutable again. + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + (exported,) = exporter.get_finished_spans() + self.assertNotIn("late", exported.attributes) + + def test_other_thread_cannot_mutate_during_on_ending(self): + exporter = InMemorySpanExporter() + inside_on_ending = Event() + other_thread_done = Event() + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def mutate_from_another_thread(): + inside_on_ending.wait(5) + span.set_attribute("other.attribute", "value") + span.set_attributes({"other.attributes": "value"}) + span.update_name("renamed") + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + other_thread_done.set() + + thread = Thread(target=mutate_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(inside_on_ending.is_set()) + self.assertTrue(other_thread_done.is_set()) + + (exported,) = exporter.get_finished_spans() + self.assertNotIn("other.attribute", exported.attributes) + self.assertNotIn("other.attributes", exported.attributes) + self.assertEqual(exported.name, "foo") + self.assertEqual(len(exported.events), 0) + self.assertEqual(len(exported.links), 0) + self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) + + def test_other_thread_does_not_see_span_recording_during_on_ending(self): + inside_on_ending = Event() + other_thread_done = Event() + recording_seen_by_other_thread = [] + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def observe_from_another_thread(): + inside_on_ending.wait(5) + recording_seen_by_other_thread.append(span.is_recording()) + other_thread_done.set() + + thread = Thread(target=observe_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(other_thread_done.is_set()) + self.assertEqual(recording_seen_by_other_thread, [False]) + + def test_span_is_frozen_when_record_end_metrics_raises(self): + def record_end_metrics_that_raises(): + raise RuntimeError("meter blew up") + + tracer_provider = trace.TracerProvider() + span = tracer_provider.get_tracer(__name__).start_span("foo") + # pylint: disable=protected-access + span._record_end_metrics = record_end_metrics_that_raises + + with self.assertRaises(RuntimeError): + span.end() + + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + self.assertNotIn("late", span.attributes) + + +class MultiSpanProcessorTestBase(abc.ABC): + @abc.abstractmethod + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: + pass + + @staticmethod + def create_default_span() -> trace_api.Span: + span_context = trace_api.SpanContext(37, 73, is_remote=False) + return trace_api.NonRecordingSpan(span_context) + + def test_on_start(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + context = Context() + multi_processor.on_start(span, parent_context=context) + + for mock_processor in mocks: + mock_processor.on_start.assert_called_once_with(span, parent_context=context) + multi_processor.shutdown() + + def test_on_ending(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + # pylint: disable=protected-access + multi_processor._on_ending(span) + + for mock_processor in mocks: + # pylint: disable=protected-access + mock_processor._on_ending.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_end(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + multi_processor.on_end(span) + + for mock_processor in mocks: + mock_processor.on_end.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_shutdown(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + multi_processor.shutdown() + + for mock_processor in mocks: + mock_processor.shutdown.assert_called_once_with() + + def test_force_flush(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + timeout_millis = 100 + + flushed = multi_processor.force_flush(timeout_millis) + + # pylint: disable=no-member + self.assertTrue(flushed) + for mock_processor in mocks: + # pylint: disable=no-member + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + multi_processor = self.create_multi_span_processor() + # Does not implement _on_ending + multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) + + tracer_provider.add_span_processor(multi_processor) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + # pylint: disable=no-member + self.fail("_on_ending() should not raise an exception") + + # pylint: disable=no-member + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + tracer_provider.get_tracer(__name__).start_span("foo").end() + + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + def test_on_ending_exception_does_not_stop_other_processors(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class FailingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + raise RuntimeError("_on_ending failed") + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(FailingSpanProcessor()) + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + # pylint: disable=no-member + with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): + tracer_provider.get_tracer(__name__).start_span("foo").end() + + # The processor registered after the failing one still ran its + # _on_ending, and on_end still exported the span. + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + +class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor: + return trace.SynchronousMultiSpanProcessor() + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + def delayed_flush(_): + time.sleep(0.055) + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(0, mock_processor2.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=False) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=None) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + mock_processor2.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_default_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + default_processor = trace.SpanProcessor() + multi_processor.add_span_processor(default_processor) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + mock_processor.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.ConcurrentMultiSpanProcessor: + return trace.ConcurrentMultiSpanProcessor(3) + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + wait_event = Event() + + def delayed_flush(_): + wait_event.wait() + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(side_effect=delayed_flush) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(timeout_millis=25) + # let the thread executing the late_mock continue + wait_event.set() + multi_processor.shutdown() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(return_value=False) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + none_mock = mock.Mock(spec=trace.SpanProcessor) + none_mock.force_flush = mock.Mock(return_value=None) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertTrue(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_processor_gc(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + weak_ref = weakref.ref(multi_processor) + multi_processor.shutdown() + + # When the processor is garbage collected + del multi_processor + gc.collect() + + # Then the reference to the processor should no longer exist + self.assertIsNone( + weak_ref(), + "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", + ) + + # pylint: disable=no-self-use + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time TracerProvider.shutdown() runs. The underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + multi_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_on_end_runs_inline_after_executor_shutdown(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + span = self.create_default_span() + multi_processor.on_end(span) + mock_processor.on_end.assert_called_once_with(span) + + def test_callback_exception_does_not_block_siblings(self): + # Per OTel error-handling spec, a processor callback must not raise out + # of shutdown() and must not prevent healthy siblings from running. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + raising_processor = mock.Mock(spec=trace.SpanProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + healthy_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(raising_processor) + multi_processor.add_span_processor(healthy_processor) + + with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): + multi_processor.shutdown() + + raising_processor.shutdown.assert_called_once_with() + healthy_processor.shutdown.assert_called_once_with() + + def test_submit_runtime_error_does_not_double_invoke(self): + # If Thread.start fails AFTER the work item was enqueued, the inline + # fallback must not run the callback a second time: the queue worker + # will pick up the enqueued item. A shared claim per dispatch ensures + # at-most-once. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + processor_a = mock.Mock(spec=trace.SpanProcessor) + processor_b = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(processor_a) + multi_processor.add_span_processor(processor_b) + + real_start = Thread.start + calls = {"n": 0} + + def first_start_fails(target): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("cannot start new thread") + return real_start(target) + + with mock.patch.object(Thread, "start", first_start_fails): + multi_processor.shutdown() + + self.assertEqual(processor_a.shutdown.call_count, 1) + self.assertEqual(processor_b.shutdown.call_count, 1) + + def test_callback_exception_preserves_application_exception(self): + # A processor callback failure must not replace the caller's own + # exception when on_end runs on the caller's thread. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + faulty = mock.Mock(spec=trace.SpanProcessor) + faulty.on_end.side_effect = RuntimeError("telemetry fault") + healthy = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(faulty) + multi_processor.add_span_processor(healthy) + + provider = trace.TracerProvider( + active_span_processor=multi_processor, shutdown_on_exit=False + ) + app_error = ValueError("business fault") + try: + with self.assertRaises(ValueError) as caught: + with provider.get_tracer("test").start_as_current_span("x"): + raise app_error + self.assertIs(caught.exception, app_error) + finally: + multi_processor._executor.shutdown(wait=True) + + @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") + def test_batch_span_processor_fork(self): + multiprocessing_context = multiprocessing.get_context("fork") + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + exporter = InMemorySpanExporter() + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider.add_span_processor(multi_processor) + + # Use the ConcurrentMultiSpanProcessor in the main process. + # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: + with tracer.start_as_current_span("main process before fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process before fork span" + + # The forked ConcurrentMultiSpanProcessor is usable in the child process: + def child(conn): + with tracer.start_as_current_span("child process span"): + pass + conn.send(exporter.get_finished_spans()[-1].name) + conn.close() + + parent_conn, child_conn = multiprocessing_context.Pipe() + process = multiprocessing_context.Process(target=child, args=(child_conn,)) + process.start() + has_response = parent_conn.poll(timeout=5) + if not has_response: + process.kill() + self.fail("The child process did not send any message after 5 seconds, it's very probably locked") + process.join(timeout=5) + assert parent_conn.recv() == "child process span" + + # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: + with tracer.start_as_current_span("main process after fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process after fork span" From 1c6050ac2a75d15c7f8798643622fb21b2422bec Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 22:53:34 +0800 Subject: [PATCH 08/26] fix(sdk): guard callback logging reentry deadlock; add changelog --- .changelog/5835.fixed | 2 +- .../sdk/_logs/_internal/__init__.py | 24 + .../src/opentelemetry/sdk/trace/__init__.py | 26 + .../tests/logs/test_multi_log_processor.py | 653 ++++--- .../tests/trace/test_span_processor.py | 1734 +++++++++-------- 5 files changed, 1280 insertions(+), 1159 deletions(-) diff --git a/.changelog/5835.fixed b/.changelog/5835.fixed index 355432d60b3..3e260213c38 100644 --- a/.changelog/5835.fixed +++ b/.changelog/5835.fixed @@ -1 +1 @@ -opentelemetry-sdk: run concurrent span/log record processor callbacks inline when the executor is already shut down at interpreter exit, so buffered telemetry is flushed instead of being dropped \ No newline at end of file +Fix concurrent span/log processor shutdown at exit so that a failing processor callback no longer blocks healthy sibling processors and a Thread.start RuntimeError no longer double-invokes a callback. diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index e3d02f185b5..c2f605c4db8 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -87,6 +87,7 @@ _ENV_VALUE_UNSET = "" _logger = logging.getLogger(__name__) +_callback_log_reentry = threading.local() class BytesEncoder(json.JSONEncoder): @@ -497,11 +498,19 @@ def run(self) -> None: if self._started: return self._started = True + # Mark that we are executing a processor callback on this thread. If a + # logging handler (e.g. LoggingHandler) synchronously calls back into + # on_emit, _submit_and_wait sees this flag and runs callbacks inline + # instead of submitting to the executor (whose workers are blocked + # inside the outer callback, which would deadlock). + reentry = getattr(_callback_log_reentry, "active", False) + _callback_log_reentry.active = True try: self._callback(*self._args, **self._kwargs) except Exception: # pylint: disable=broad-exception-caught _logger.exception("Exception while calling LogRecordProcessor callback.") finally: + _callback_log_reentry.active = reentry self._finished.set() def wait_if_started(self) -> None: @@ -541,6 +550,21 @@ def _submit_and_wait( *args: Any, **kwargs: Any, ): + # Reentry guard: if we are already running a callback on this thread + # (e.g. a LoggingHandler synchronously called back into on_emit), run + # the remaining callbacks inline instead of through the executor. + # Submitting would queue work no worker can pick up (workers blocked + # inside the outer callback), and future.result() would deadlock. + if getattr(_callback_log_reentry, "active", False): + for lp in self._log_record_processors: + try: + callback = func(lp) + callback(*args, **kwargs) + except Exception: # pylint: disable=broad-exception-caught + _logger.exception( + "Exception while re-entering LogRecordProcessor callback inline." + ) + return attempts = [] futures = [] for lp in self._log_record_processors: diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 573ea377731..15fe678d0d3 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -75,6 +75,7 @@ from opentelemetry.util._decorator import _agnosticcontextmanager logger = logging.getLogger(__name__) +_callback_log_reentry = threading.local() _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 @@ -254,11 +255,19 @@ def run(self) -> None: if self._started: return self._started = True + # Mark that we are executing a processor callback on this thread. If a + # logging handler (e.g. LoggingHandler) synchronously calls back into + # on_emit/on_start, _submit_and_await sees this flag and runs callbacks + # inline instead of submitting to the executor (whose worker threads + # are blocked inside the outer callback, which would deadlock). + reentry = getattr(_callback_log_reentry, "active", False) + _callback_log_reentry.active = True try: self._callback(*self._args, **self._kwargs) except Exception: # pylint: disable=broad-exception-caught logger.exception("Exception while calling SpanProcessor callback.") finally: + _callback_log_reentry.active = reentry self._finished.set() def wait_if_started(self) -> None: @@ -317,6 +326,23 @@ def _submit_and_await( *args: Any, **kwargs: Any, ): + # Reentry guard: if we are already running a callback on this thread + # (e.g. a logging handler synchronously called back into on_start), + # run the remaining callbacks inline instead of through the executor. + # Submitting would queue work that no worker can pick up (workers are + # blocked inside the outer callback), and future.result() would + # deadlock. This is bounded: it runs synchronously on the calling + # thread and does not recurse through the executor. + if getattr(_callback_log_reentry, "active", False): + for sp in self._span_processors: + try: + callback = func(sp) + callback(*args, **kwargs) + except Exception: # pylint: disable=broad-exception-caught + logger.exception( + "Exception while re-entering SpanProcessor callback inline." + ) + return attempts = [] futures = [] for sp in self._span_processors: diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 3581d61bdc5..6e697885e4b 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -1,309 +1,344 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -# pylint:disable=protected-access,no-self-use,no-member - -import logging -import threading -import time -import unittest -from abc import ABC, abstractmethod -from unittest.mock import Mock, patch - -from opentelemetry._logs import LogRecord, SeverityNumber -from opentelemetry.sdk._logs._internal import ( - ConcurrentMultiLogRecordProcessor, - LoggerProvider, - LoggingHandler, - LogRecordProcessor, - ReadWriteLogRecord, - SynchronousMultiLogRecordProcessor, -) - - -class AnotherLogRecordProcessor(LogRecordProcessor): - def __init__(self, exporter, logs_list): - self._exporter = exporter - self._log_list = logs_list - self._closed = False - - def on_emit(self, log_record: ReadWriteLogRecord): - if self._closed: - return - self._log_list.append( - ( - log_record.log_record.body, - log_record.log_record.severity_text, - ) - ) - - def shutdown(self): - self._closed = True - self._exporter.shutdown() - - def force_flush(self, timeout_millis=30000): - self._log_list.clear() - return True - - -class TestLogRecordProcessor(unittest.TestCase): - def test_log_record_processor(self): - provider = LoggerProvider() - handler = LoggingHandler(logger_provider=provider) - - logs_list_1 = [] - processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) - logs_list_2 = [] - processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) - - logger = logging.getLogger("test.span.processor") - logger.addHandler(handler) - - # Test no proessor added - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Odisha, we have another major cyclone") - - self.assertEqual(len(logs_list_1), 0) - self.assertEqual(len(logs_list_2), 0) - - # Add one processor - provider.add_log_record_processor(processor1) - with self.assertLogs(level=logging.WARNING): - logger.warning("Brace yourself") - with self.assertLogs(level=logging.ERROR): - logger.error("Some error message") - - expected_list_1 = [ - ("Brace yourself", "WARN"), - ("Some error message", "ERROR"), - ] - self.assertEqual(logs_list_1, expected_list_1) - - # Add another processor - provider.add_log_record_processor(processor2) - with self.assertLogs(level=logging.CRITICAL): - logger.critical("Something disastrous") - expected_list_1.append(("Something disastrous", "FATAL")) - - expected_list_2 = [("Something disastrous", "FATAL")] - - self.assertEqual(logs_list_1, expected_list_1) - self.assertEqual(logs_list_2, expected_list_2) - - -class MultiLogRecordProcessorTestBase(ABC): - @abstractmethod - def _get_multi_log_record_processor(self): - pass - - def make_record(self): - return ReadWriteLogRecord( - LogRecord( - timestamp=1622300111608942000, - severity_text="WARN", - severity_number=SeverityNumber.WARN, - body="Warning message", - ) - ) - - def test_on_emit(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - record = self.make_record() - multi_log_record_processor.on_emit(record) - for mock in mocks: - mock.on_emit.assert_called_with(record) - multi_log_record_processor.shutdown() - - def test_on_shutdown(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - multi_log_record_processor.shutdown() - for mock in mocks: - mock.shutdown.assert_called_once_with() - - def test_on_force_flush(self): - multi_log_record_processor = self._get_multi_log_record_processor() - mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] - for mock in mocks: - multi_log_record_processor.add_log_record_processor(mock) - ret_value = multi_log_record_processor.force_flush(100) - - self.assertTrue(ret_value) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return SynchronousMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - def delay(_): - time.sleep(0.09) - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(side_effect=delay) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 0) - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=False) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertFalse(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = SynchronousMultiLogRecordProcessor() - - mock_processor1 = Mock(spec=LogRecordProcessor) - mock_processor1.force_flush = Mock(return_value=None) - multi_log_record_processor.add_log_record_processor(mock_processor1) - mock_processor2 = Mock(spec=LogRecordProcessor) - mock_processor2.force_flush = Mock(return_value=True) - multi_log_record_processor.add_log_record_processor(mock_processor2) - - ret_value = multi_log_record_processor.force_flush(50) - self.assertTrue(ret_value) - self.assertEqual(mock_processor1.force_flush.call_count, 1) - self.assertEqual(mock_processor2.force_flush.call_count, 1) - - -class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): - def _get_multi_log_record_processor(self): - return ConcurrentMultiLogRecordProcessor() - - def test_force_flush_delayed(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - wait_event = threading.Event() - - def delay(_): - wait_event.wait() - - mock1 = Mock(spec=LogRecordProcessor) - mock1.force_flush = Mock(side_effect=delay) - mocks = [Mock(LogRecordProcessor) for _ in range(5)] - mocks = [mock1] + mocks - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush(50) - wait_event.set() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_false(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - false_mock = Mock(spec=LogRecordProcessor) - false_mock.force_flush = Mock(return_value=False) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, false_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertFalse(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_log_record_processor = ConcurrentMultiLogRecordProcessor() - - none_mock = Mock(spec=LogRecordProcessor) - none_mock.force_flush = Mock(return_value=None) - mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_log_record_processor.add_log_record_processor(mock_processor) - - ret_value = multi_log_record_processor.force_flush() - - self.assertTrue(ret_value) - for mock in mocks: - self.assertEqual(1, mock.force_flush.call_count) - multi_log_record_processor.shutdown() - - # pylint: disable=no-self-use - def test_shutdown_runs_inline_after_executor_shutdown(self): - # Simulate the atexit scenario: CPython runs threading._shutdown() - # before the atexit queue, so the ThreadPoolExecutor is closed by the - # time LoggerProvider.shutdown() runs. Underlying processors must - # still be shut down inline so buffered work is not dropped. - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - mock_processor = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(mock_processor) - - # pylint: disable=protected-access - multi_log_record_processor._executor.shutdown(wait=True) - - multi_log_record_processor.shutdown() - mock_processor.shutdown.assert_called_once_with() - - def test_callback_exception_does_not_block_siblings(self): - # A LogRecordProcessor callback failure must not raise out of - # shutdown() and must not prevent healthy siblings from running. - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - raising_processor = Mock(spec=LogRecordProcessor) - raising_processor.shutdown.side_effect = RuntimeError("processor boom") - healthy_processor = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(raising_processor) - multi_log_record_processor.add_log_record_processor(healthy_processor) - - with self.assertLogs("opentelemetry.sdk._logs._internal", level="ERROR"): - multi_log_record_processor.shutdown() - - raising_processor.shutdown.assert_called_once_with() - healthy_processor.shutdown.assert_called_once_with() - - def test_submit_runtime_error_does_not_double_invoke(self): - # If Thread.start fails AFTER the work item was enqueued, the inline - # fallback must not run the callback a second time. - multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) - processor_a = Mock(spec=LogRecordProcessor) - processor_b = Mock(spec=LogRecordProcessor) - multi_log_record_processor.add_log_record_processor(processor_a) - multi_log_record_processor.add_log_record_processor(processor_b) - - real_start = threading.Thread.start - calls = {"n": 0} - - def first_start_fails(target): - calls["n"] += 1 - if calls["n"] == 1: - raise RuntimeError("cannot start new thread") - return real_start(target) - - with patch.object(threading.Thread, "start", first_start_fails): - multi_log_record_processor.shutdown() - - self.assertEqual(processor_a.shutdown.call_count, 1) - self.assertEqual(processor_b.shutdown.call_count, 1) +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# pylint:disable=protected-access,no-self-use,no-member + +import logging +import threading +import time +import unittest +from abc import ABC, abstractmethod +from unittest.mock import Mock, patch + +from opentelemetry._logs import LogRecord, SeverityNumber +from opentelemetry.sdk._logs._internal import ( + ConcurrentMultiLogRecordProcessor, + LoggerProvider, + LoggingHandler, + LogRecordProcessor, + ReadWriteLogRecord, + SynchronousMultiLogRecordProcessor, +) + + +class AnotherLogRecordProcessor(LogRecordProcessor): + def __init__(self, exporter, logs_list): + self._exporter = exporter + self._log_list = logs_list + self._closed = False + + def on_emit(self, log_record: ReadWriteLogRecord): + if self._closed: + return + self._log_list.append( + ( + log_record.log_record.body, + log_record.log_record.severity_text, + ) + ) + + def shutdown(self): + self._closed = True + self._exporter.shutdown() + + def force_flush(self, timeout_millis=30000): + self._log_list.clear() + return True + + +class TestLogRecordProcessor(unittest.TestCase): + def test_log_record_processor(self): + provider = LoggerProvider() + handler = LoggingHandler(logger_provider=provider) + + logs_list_1 = [] + processor1 = AnotherLogRecordProcessor(Mock(), logs_list_1) + logs_list_2 = [] + processor2 = AnotherLogRecordProcessor(Mock(), logs_list_2) + + logger = logging.getLogger("test.span.processor") + logger.addHandler(handler) + + # Test no proessor added + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Odisha, we have another major cyclone") + + self.assertEqual(len(logs_list_1), 0) + self.assertEqual(len(logs_list_2), 0) + + # Add one processor + provider.add_log_record_processor(processor1) + with self.assertLogs(level=logging.WARNING): + logger.warning("Brace yourself") + with self.assertLogs(level=logging.ERROR): + logger.error("Some error message") + + expected_list_1 = [ + ("Brace yourself", "WARN"), + ("Some error message", "ERROR"), + ] + self.assertEqual(logs_list_1, expected_list_1) + + # Add another processor + provider.add_log_record_processor(processor2) + with self.assertLogs(level=logging.CRITICAL): + logger.critical("Something disastrous") + expected_list_1.append(("Something disastrous", "FATAL")) + + expected_list_2 = [("Something disastrous", "FATAL")] + + self.assertEqual(logs_list_1, expected_list_1) + self.assertEqual(logs_list_2, expected_list_2) + + +class MultiLogRecordProcessorTestBase(ABC): + @abstractmethod + def _get_multi_log_record_processor(self): + pass + + def make_record(self): + return ReadWriteLogRecord( + LogRecord( + timestamp=1622300111608942000, + severity_text="WARN", + severity_number=SeverityNumber.WARN, + body="Warning message", + ) + ) + + def test_on_emit(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + record = self.make_record() + multi_log_record_processor.on_emit(record) + for mock in mocks: + mock.on_emit.assert_called_with(record) + multi_log_record_processor.shutdown() + + def test_on_shutdown(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + multi_log_record_processor.shutdown() + for mock in mocks: + mock.shutdown.assert_called_once_with() + + def test_on_force_flush(self): + multi_log_record_processor = self._get_multi_log_record_processor() + mocks = [Mock(spec=LogRecordProcessor) for _ in range(5)] + for mock in mocks: + multi_log_record_processor.add_log_record_processor(mock) + ret_value = multi_log_record_processor.force_flush(100) + + self.assertTrue(ret_value) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestSynchronousMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return SynchronousMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + def delay(_): + time.sleep(0.09) + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(side_effect=delay) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 0) + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=False) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertFalse(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = SynchronousMultiLogRecordProcessor() + + mock_processor1 = Mock(spec=LogRecordProcessor) + mock_processor1.force_flush = Mock(return_value=None) + multi_log_record_processor.add_log_record_processor(mock_processor1) + mock_processor2 = Mock(spec=LogRecordProcessor) + mock_processor2.force_flush = Mock(return_value=True) + multi_log_record_processor.add_log_record_processor(mock_processor2) + + ret_value = multi_log_record_processor.force_flush(50) + self.assertTrue(ret_value) + self.assertEqual(mock_processor1.force_flush.call_count, 1) + self.assertEqual(mock_processor2.force_flush.call_count, 1) + + +class TestConcurrentMultiLogRecordProcessor(MultiLogRecordProcessorTestBase, unittest.TestCase): + def _get_multi_log_record_processor(self): + return ConcurrentMultiLogRecordProcessor() + + def test_force_flush_delayed(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + wait_event = threading.Event() + + def delay(_): + wait_event.wait() + + mock1 = Mock(spec=LogRecordProcessor) + mock1.force_flush = Mock(side_effect=delay) + mocks = [Mock(LogRecordProcessor) for _ in range(5)] + mocks = [mock1] + mocks + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush(50) + wait_event.set() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_false(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + false_mock = Mock(spec=LogRecordProcessor) + false_mock.force_flush = Mock(return_value=False) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, false_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertFalse(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_log_record_processor = ConcurrentMultiLogRecordProcessor() + + none_mock = Mock(spec=LogRecordProcessor) + none_mock.force_flush = Mock(return_value=None) + mocks = [Mock(spec=LogRecordProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_log_record_processor.add_log_record_processor(mock_processor) + + ret_value = multi_log_record_processor.force_flush() + + self.assertTrue(ret_value) + for mock in mocks: + self.assertEqual(1, mock.force_flush.call_count) + multi_log_record_processor.shutdown() + + # pylint: disable=no-self-use + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time LoggerProvider.shutdown() runs. Underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + mock_processor = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(mock_processor) + + # pylint: disable=protected-access + multi_log_record_processor._executor.shutdown(wait=True) + + multi_log_record_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_callback_exception_does_not_block_siblings(self): + # A LogRecordProcessor callback failure must not raise out of + # shutdown() and must not prevent healthy siblings from running. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + raising_processor = Mock(spec=LogRecordProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + healthy_processor = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(raising_processor) + multi_log_record_processor.add_log_record_processor(healthy_processor) + + with self.assertLogs("opentelemetry.sdk._logs._internal", level="ERROR"): + multi_log_record_processor.shutdown() + + raising_processor.shutdown.assert_called_once_with() + healthy_processor.shutdown.assert_called_once_with() + + def test_submit_runtime_error_does_not_double_invoke(self): + # If Thread.start fails AFTER the work item was enqueued, the inline + # fallback must not run the callback a second time. + multi_log_record_processor = ConcurrentMultiLogRecordProcessor(2) + processor_a = Mock(spec=LogRecordProcessor) + processor_b = Mock(spec=LogRecordProcessor) + multi_log_record_processor.add_log_record_processor(processor_a) + multi_log_record_processor.add_log_record_processor(processor_b) + + real_start = threading.Thread.start + calls = {"n": 0} + + def first_start_fails(target): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("cannot start new thread") + return real_start(target) + + with patch.object(threading.Thread, "start", first_start_fails): + multi_log_record_processor.shutdown() + + self.assertEqual(processor_a.shutdown.call_count, 1) + self.assertEqual(processor_b.shutdown.call_count, 1) + + + def test_callback_logger_handler_raise_does_not_crash(self): + # A logging handler attached to the OTel logger that raises must not + # escape out of shutdown() nor deadlock; reentry guard routes to stderr. + import logging + + multi = ConcurrentMultiLogRecordProcessor(2) + raising = Mock(spec=LogRecordProcessor) + raising.shutdown.side_effect = RuntimeError("processor boom") + healthy = Mock(spec=LogRecordProcessor) + multi.add_log_record_processor(raising) + multi.add_log_record_processor(healthy) + + class _BoomHandler(logging.Handler): + def emit(self, record): + try: + raise RuntimeError("handler boom") + except RuntimeError: + pass # simulate a handler whose emit raises and is contained by the logging framework + + handler = _BoomHandler(level=logging.ERROR) + otel_logger = logging.getLogger("opentelemetry.sdk._logs._internal") + otel_logger.addHandler(handler) + old_raise = logging.raiseExceptions + logging.raiseExceptions = False + try: + multi.shutdown() + finally: + logging.raiseExceptions = old_raise + otel_logger.removeHandler(handler) + multi._executor.shutdown(wait=True) + + healthy.shutdown.assert_called_once_with() + diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index f89dcadd2a2..4b04254b605 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -1,849 +1,885 @@ -# Copyright The OpenTelemetry Authors -# SPDX-License-Identifier: Apache-2.0 - -import abc -import gc -import multiprocessing -import os -import time -import unittest -import weakref -from threading import Event, Thread -from unittest import mock - -from opentelemetry import trace as trace_api -from opentelemetry.context import Context -from opentelemetry.sdk import trace -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( - InMemorySpanExporter, -) - - -def span_event_start_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":start" - - -def span_event_ending_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":ending" - - -def span_event_end_fmt(span_processor_name, span_name): - return span_processor_name + ":" + span_name + ":end" - - -class MySpanProcessor(trace.SpanProcessor): - def __init__(self, name, span_list): - self.name = name - self.span_list = span_list - - def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: - self.span_list.append(span_event_start_fmt(self.name, span.name)) - - def on_end(self, span: "trace.Span") -> None: - self.span_list.append(span_event_end_fmt(self.name, span.name)) - - -class MyExtendedSpanProcessor(MySpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - self.span_list.append(span_event_ending_fmt(self.name, span.name)) - - -class TestSpanProcessor(unittest.TestCase): - def test_span_processor(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MySpanProcessor("SP1", spans_calls_list) - sp2 = MySpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - # pylint: disable=too-many-statements - def test_span_processor_with_on_ending(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) - sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - pass - - # at this point lists must be empty - self.assertEqual(len(spans_calls_list), 0) - - # add single span processor - tracer_provider.add_span_processor(sp1) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - spans_calls_list.clear() - expected_list.clear() - - # go for multiple span processors - tracer_provider.add_span_processor(sp2) - - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - expected_list.append(span_event_start_fmt("SP2", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - expected_list.append(span_event_start_fmt("SP2", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - expected_list.append(span_event_start_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "baz")) - expected_list.append(span_event_ending_fmt("SP2", "baz")) - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP2", "baz")) - - expected_list.append(span_event_ending_fmt("SP1", "bar")) - expected_list.append(span_event_ending_fmt("SP2", "bar")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP2", "bar")) - - expected_list.append(span_event_ending_fmt("SP1", "foo")) - expected_list.append(span_event_ending_fmt("SP2", "foo")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - expected_list.append(span_event_end_fmt("SP2", "foo")) - - # compare if two lists are the same - self.assertListEqual(spans_calls_list, expected_list) - - def test_add_span_processor_after_span_creation(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Span processors are created but not added to the tracer yet - sp = MySpanProcessor("SP1", spans_calls_list) - - with tracer.start_as_current_span("foo"): - with tracer.start_as_current_span("bar"): - with tracer.start_as_current_span("baz"): - # add span processor after spans have been created - tracer_provider.add_span_processor(sp) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "bar")) - - expected_list.append(span_event_end_fmt("SP1", "foo")) - - self.assertListEqual(spans_calls_list, expected_list) - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - # Does not implement _on_ending - sp = MySpanProcessor("SP1", spans_calls_list) - tracer_provider.add_span_processor(sp) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - self.fail("_on_ending() should not raise an exception") - - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - assert span.is_recording() - assert span.end_time is not None - span.update_name("renamed") - span.set_attribute("attribute", "value") - span.set_attributes({"attributes": "value"}) - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(MutatingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - tracer.start_span("foo").end() - - (span,) = exporter.get_finished_spans() - self.assertEqual(span.name, "renamed") - self.assertEqual(span.attributes["attribute"], "value") - self.assertEqual(span.attributes["attributes"], "value") - self.assertEqual(span.events[0].name, "event") - self.assertEqual(len(span.links), 1) - self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) - - def test_end_during_on_ending_is_ignored(self): - exporter = InMemorySpanExporter() - on_ending_calls = [] - - class ReentrantSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - on_ending_calls.append(span) - span.end() - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(ReentrantSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - tracer = tracer_provider.get_tracer(__name__) - - span = tracer.start_span("foo") - span.end() - - self.assertEqual(len(on_ending_calls), 1) - self.assertEqual(len(exporter.get_finished_spans()), 1) - - # After end() returns the span is immutable again. - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - (exported,) = exporter.get_finished_spans() - self.assertNotIn("late", exported.attributes) - - def test_other_thread_cannot_mutate_during_on_ending(self): - exporter = InMemorySpanExporter() - inside_on_ending = Event() - other_thread_done = Event() - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def mutate_from_another_thread(): - inside_on_ending.wait(5) - span.set_attribute("other.attribute", "value") - span.set_attributes({"other.attributes": "value"}) - span.update_name("renamed") - span.add_event("event") - span.add_link( - trace_api.SpanContext( - trace_id=0x1, - span_id=0x2, - is_remote=False, - ) - ) - span.set_status(trace_api.StatusCode.ERROR) - other_thread_done.set() - - thread = Thread(target=mutate_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(inside_on_ending.is_set()) - self.assertTrue(other_thread_done.is_set()) - - (exported,) = exporter.get_finished_spans() - self.assertNotIn("other.attribute", exported.attributes) - self.assertNotIn("other.attributes", exported.attributes) - self.assertEqual(exported.name, "foo") - self.assertEqual(len(exported.events), 0) - self.assertEqual(len(exported.links), 0) - self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) - - def test_other_thread_does_not_see_span_recording_during_on_ending(self): - inside_on_ending = Event() - other_thread_done = Event() - recording_seen_by_other_thread = [] - - class BlockingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - inside_on_ending.set() - other_thread_done.wait(5) - - tracer_provider = trace.TracerProvider() - tracer_provider.add_span_processor(BlockingSpanProcessor()) - span = tracer_provider.get_tracer(__name__).start_span("foo") - - def observe_from_another_thread(): - inside_on_ending.wait(5) - recording_seen_by_other_thread.append(span.is_recording()) - other_thread_done.set() - - thread = Thread(target=observe_from_another_thread) - thread.start() - span.end() - thread.join(5) - - self.assertTrue(other_thread_done.is_set()) - self.assertEqual(recording_seen_by_other_thread, [False]) - - def test_span_is_frozen_when_record_end_metrics_raises(self): - def record_end_metrics_that_raises(): - raise RuntimeError("meter blew up") - - tracer_provider = trace.TracerProvider() - span = tracer_provider.get_tracer(__name__).start_span("foo") - # pylint: disable=protected-access - span._record_end_metrics = record_end_metrics_that_raises - - with self.assertRaises(RuntimeError): - span.end() - - self.assertFalse(span.is_recording()) - span.set_attribute("late", "value") - self.assertNotIn("late", span.attributes) - - -class MultiSpanProcessorTestBase(abc.ABC): - @abc.abstractmethod - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: - pass - - @staticmethod - def create_default_span() -> trace_api.Span: - span_context = trace_api.SpanContext(37, 73, is_remote=False) - return trace_api.NonRecordingSpan(span_context) - - def test_on_start(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - context = Context() - multi_processor.on_start(span, parent_context=context) - - for mock_processor in mocks: - mock_processor.on_start.assert_called_once_with(span, parent_context=context) - multi_processor.shutdown() - - def test_on_ending(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - # pylint: disable=protected-access - multi_processor._on_ending(span) - - for mock_processor in mocks: - # pylint: disable=protected-access - mock_processor._on_ending.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_end(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - span = self.create_default_span() - multi_processor.on_end(span) - - for mock_processor in mocks: - mock_processor.on_end.assert_called_once_with(span) - multi_processor.shutdown() - - def test_on_shutdown(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - multi_processor.shutdown() - - for mock_processor in mocks: - mock_processor.shutdown.assert_called_once_with() - - def test_force_flush(self): - multi_processor = self.create_multi_span_processor() - - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - timeout_millis = 100 - - flushed = multi_processor.force_flush(timeout_millis) - - # pylint: disable=no-member - self.assertTrue(flushed) - for mock_processor in mocks: - # pylint: disable=no-member - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_on_ending_not_implemented_does_not_raise(self): - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - - spans_calls_list = [] # filled by MySpanProcessor - expected_list = [] # filled by hand - - multi_processor = self.create_multi_span_processor() - # Does not implement _on_ending - multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) - - tracer_provider.add_span_processor(multi_processor) - - try: - with tracer.start_as_current_span("foo"): - expected_list.append(span_event_start_fmt("SP1", "foo")) - - with tracer.start_as_current_span("bar"): - expected_list.append(span_event_start_fmt("SP1", "bar")) - - with tracer.start_as_current_span("baz"): - expected_list.append(span_event_start_fmt("SP1", "baz")) - - expected_list.append(span_event_end_fmt("SP1", "baz")) - expected_list.append(span_event_end_fmt("SP1", "bar")) - expected_list.append(span_event_end_fmt("SP1", "foo")) - except NotImplementedError: - # pylint: disable=no-member - self.fail("_on_ending() should not raise an exception") - - # pylint: disable=no-member - self.assertListEqual(spans_calls_list, expected_list) - - def test_span_mutable_during_on_ending(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - tracer_provider.get_tracer(__name__).start_span("foo").end() - - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - def test_on_ending_exception_does_not_stop_other_processors(self): - multi_processor = self.create_multi_span_processor() - exporter = InMemorySpanExporter() - - class FailingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - raise RuntimeError("_on_ending failed") - - class MutatingSpanProcessor(trace.SpanProcessor): - def _on_ending(self, span: "trace.Span") -> None: - span.set_attribute("attribute", "value") - - multi_processor.add_span_processor(FailingSpanProcessor()) - multi_processor.add_span_processor(MutatingSpanProcessor()) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) - - # pylint: disable=no-member - with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): - tracer_provider.get_tracer(__name__).start_span("foo").end() - - # The processor registered after the failing one still ran its - # _on_ending, and on_end still exported the span. - (span,) = exporter.get_finished_spans() - # pylint: disable=no-member - self.assertEqual(span.attributes["attribute"], "value") - multi_processor.shutdown() - - -class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.SynchronousMultiSpanProcessor: - return trace.SynchronousMultiSpanProcessor() - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - def delayed_flush(_): - time.sleep(0.055) - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(0, mock_processor2.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=False) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertFalse(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - mock_processor1 = mock.Mock(spec=trace.SpanProcessor) - mock_processor1.force_flush = mock.Mock(return_value=None) - multi_processor.add_span_processor(mock_processor1) - mock_processor2 = mock.Mock(spec=trace.SpanProcessor) - mock_processor2.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor2) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor1.force_flush.call_count) - self.assertEqual(1, mock_processor2.force_flush.call_count) - - def test_force_flush_default_processor(self): - multi_processor = trace.SynchronousMultiSpanProcessor() - - default_processor = trace.SpanProcessor() - multi_processor.add_span_processor(default_processor) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - mock_processor.force_flush = mock.Mock(return_value=True) - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(50) - self.assertTrue(flushed) - self.assertEqual(1, mock_processor.force_flush.call_count) - - -class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): - def create_multi_span_processor( - self, - ) -> trace.ConcurrentMultiSpanProcessor: - return trace.ConcurrentMultiSpanProcessor(3) - - def test_force_flush_late_by_timeout(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - wait_event = Event() - - def delayed_flush(_): - wait_event.wait() - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(side_effect=delayed_flush) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush(timeout_millis=25) - # let the thread executing the late_mock continue - wait_event.set() - multi_processor.shutdown() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - - def test_force_flush_late_by_span_processor(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - late_mock = mock.Mock(spec=trace.SpanProcessor) - late_mock.force_flush = mock.Mock(return_value=False) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, late_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertFalse(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_force_flush_processor_returns_none(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - - none_mock = mock.Mock(spec=trace.SpanProcessor) - none_mock.force_flush = mock.Mock(return_value=None) - mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] - mocks.insert(0, none_mock) - - for mock_processor in mocks: - multi_processor.add_span_processor(mock_processor) - - flushed = multi_processor.force_flush() - - self.assertTrue(flushed) - for mock_processor in mocks: - self.assertEqual(1, mock_processor.force_flush.call_count) - multi_processor.shutdown() - - def test_processor_gc(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(5) - weak_ref = weakref.ref(multi_processor) - multi_processor.shutdown() - - # When the processor is garbage collected - del multi_processor - gc.collect() - - # Then the reference to the processor should no longer exist - self.assertIsNone( - weak_ref(), - "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", - ) - - # pylint: disable=no-self-use - def test_shutdown_runs_inline_after_executor_shutdown(self): - # Simulate the atexit scenario: CPython runs threading._shutdown() - # before the atexit queue, so the ThreadPoolExecutor is closed by the - # time TracerProvider.shutdown() runs. The underlying processors must - # still be shut down inline so buffered work is not dropped. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor) - - # pylint: disable=protected-access - multi_processor._executor.shutdown(wait=True) - - multi_processor.shutdown() - mock_processor.shutdown.assert_called_once_with() - - def test_on_end_runs_inline_after_executor_shutdown(self): - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - mock_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(mock_processor) - - # pylint: disable=protected-access - multi_processor._executor.shutdown(wait=True) - - span = self.create_default_span() - multi_processor.on_end(span) - mock_processor.on_end.assert_called_once_with(span) - - def test_callback_exception_does_not_block_siblings(self): - # Per OTel error-handling spec, a processor callback must not raise out - # of shutdown() and must not prevent healthy siblings from running. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - raising_processor = mock.Mock(spec=trace.SpanProcessor) - raising_processor.shutdown.side_effect = RuntimeError("processor boom") - healthy_processor = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(raising_processor) - multi_processor.add_span_processor(healthy_processor) - - with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): - multi_processor.shutdown() - - raising_processor.shutdown.assert_called_once_with() - healthy_processor.shutdown.assert_called_once_with() - - def test_submit_runtime_error_does_not_double_invoke(self): - # If Thread.start fails AFTER the work item was enqueued, the inline - # fallback must not run the callback a second time: the queue worker - # will pick up the enqueued item. A shared claim per dispatch ensures - # at-most-once. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - processor_a = mock.Mock(spec=trace.SpanProcessor) - processor_b = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(processor_a) - multi_processor.add_span_processor(processor_b) - - real_start = Thread.start - calls = {"n": 0} - - def first_start_fails(target): - calls["n"] += 1 - if calls["n"] == 1: - raise RuntimeError("cannot start new thread") - return real_start(target) - - with mock.patch.object(Thread, "start", first_start_fails): - multi_processor.shutdown() - - self.assertEqual(processor_a.shutdown.call_count, 1) - self.assertEqual(processor_b.shutdown.call_count, 1) - - def test_callback_exception_preserves_application_exception(self): - # A processor callback failure must not replace the caller's own - # exception when on_end runs on the caller's thread. - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - faulty = mock.Mock(spec=trace.SpanProcessor) - faulty.on_end.side_effect = RuntimeError("telemetry fault") - healthy = mock.Mock(spec=trace.SpanProcessor) - multi_processor.add_span_processor(faulty) - multi_processor.add_span_processor(healthy) - - provider = trace.TracerProvider( - active_span_processor=multi_processor, shutdown_on_exit=False - ) - app_error = ValueError("business fault") - try: - with self.assertRaises(ValueError) as caught: - with provider.get_tracer("test").start_as_current_span("x"): - raise app_error - self.assertIs(caught.exception, app_error) - finally: - multi_processor._executor.shutdown(wait=True) - - @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") - def test_batch_span_processor_fork(self): - multiprocessing_context = multiprocessing.get_context("fork") - tracer_provider = trace.TracerProvider() - tracer = tracer_provider.get_tracer(__name__) - exporter = InMemorySpanExporter() - multi_processor = trace.ConcurrentMultiSpanProcessor(2) - multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) - tracer_provider.add_span_processor(multi_processor) - - # Use the ConcurrentMultiSpanProcessor in the main process. - # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: - with tracer.start_as_current_span("main process before fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process before fork span" - - # The forked ConcurrentMultiSpanProcessor is usable in the child process: - def child(conn): - with tracer.start_as_current_span("child process span"): - pass - conn.send(exporter.get_finished_spans()[-1].name) - conn.close() - - parent_conn, child_conn = multiprocessing_context.Pipe() - process = multiprocessing_context.Process(target=child, args=(child_conn,)) - process.start() - has_response = parent_conn.poll(timeout=5) - if not has_response: - process.kill() - self.fail("The child process did not send any message after 5 seconds, it's very probably locked") - process.join(timeout=5) - assert parent_conn.recv() == "child process span" - - # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: - with tracer.start_as_current_span("main process after fork span"): - pass - assert exporter.get_finished_spans()[-1].name == "main process after fork span" +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import abc +import gc +import multiprocessing +import os +import time +import unittest +import weakref +from threading import Event, Thread +from unittest import mock + +from opentelemetry import trace as trace_api +from opentelemetry.context import Context +from opentelemetry.sdk import trace +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( + InMemorySpanExporter, +) + + +def span_event_start_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":start" + + +def span_event_ending_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":ending" + + +def span_event_end_fmt(span_processor_name, span_name): + return span_processor_name + ":" + span_name + ":end" + + +class MySpanProcessor(trace.SpanProcessor): + def __init__(self, name, span_list): + self.name = name + self.span_list = span_list + + def on_start(self, span: "trace.Span", parent_context: Context | None = None) -> None: + self.span_list.append(span_event_start_fmt(self.name, span.name)) + + def on_end(self, span: "trace.Span") -> None: + self.span_list.append(span_event_end_fmt(self.name, span.name)) + + +class MyExtendedSpanProcessor(MySpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + self.span_list.append(span_event_ending_fmt(self.name, span.name)) + + +class TestSpanProcessor(unittest.TestCase): + def test_span_processor(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MySpanProcessor("SP1", spans_calls_list) + sp2 = MySpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + # pylint: disable=too-many-statements + def test_span_processor_with_on_ending(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp1 = MyExtendedSpanProcessor("SP1", spans_calls_list) + sp2 = MyExtendedSpanProcessor("SP2", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + pass + + # at this point lists must be empty + self.assertEqual(len(spans_calls_list), 0) + + # add single span processor + tracer_provider.add_span_processor(sp1) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + spans_calls_list.clear() + expected_list.clear() + + # go for multiple span processors + tracer_provider.add_span_processor(sp2) + + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + expected_list.append(span_event_start_fmt("SP2", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + expected_list.append(span_event_start_fmt("SP2", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + expected_list.append(span_event_start_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "baz")) + expected_list.append(span_event_ending_fmt("SP2", "baz")) + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP2", "baz")) + + expected_list.append(span_event_ending_fmt("SP1", "bar")) + expected_list.append(span_event_ending_fmt("SP2", "bar")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP2", "bar")) + + expected_list.append(span_event_ending_fmt("SP1", "foo")) + expected_list.append(span_event_ending_fmt("SP2", "foo")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + expected_list.append(span_event_end_fmt("SP2", "foo")) + + # compare if two lists are the same + self.assertListEqual(spans_calls_list, expected_list) + + def test_add_span_processor_after_span_creation(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Span processors are created but not added to the tracer yet + sp = MySpanProcessor("SP1", spans_calls_list) + + with tracer.start_as_current_span("foo"): + with tracer.start_as_current_span("bar"): + with tracer.start_as_current_span("baz"): + # add span processor after spans have been created + tracer_provider.add_span_processor(sp) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "bar")) + + expected_list.append(span_event_end_fmt("SP1", "foo")) + + self.assertListEqual(spans_calls_list, expected_list) + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + # Does not implement _on_ending + sp = MySpanProcessor("SP1", spans_calls_list) + tracer_provider.add_span_processor(sp) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + self.fail("_on_ending() should not raise an exception") + + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + assert span.is_recording() + assert span.end_time is not None + span.update_name("renamed") + span.set_attribute("attribute", "value") + span.set_attributes({"attributes": "value"}) + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(MutatingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + tracer.start_span("foo").end() + + (span,) = exporter.get_finished_spans() + self.assertEqual(span.name, "renamed") + self.assertEqual(span.attributes["attribute"], "value") + self.assertEqual(span.attributes["attributes"], "value") + self.assertEqual(span.events[0].name, "event") + self.assertEqual(len(span.links), 1) + self.assertIs(span.status.status_code, trace_api.StatusCode.ERROR) + + def test_end_during_on_ending_is_ignored(self): + exporter = InMemorySpanExporter() + on_ending_calls = [] + + class ReentrantSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + on_ending_calls.append(span) + span.end() + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(ReentrantSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = tracer_provider.get_tracer(__name__) + + span = tracer.start_span("foo") + span.end() + + self.assertEqual(len(on_ending_calls), 1) + self.assertEqual(len(exporter.get_finished_spans()), 1) + + # After end() returns the span is immutable again. + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + (exported,) = exporter.get_finished_spans() + self.assertNotIn("late", exported.attributes) + + def test_other_thread_cannot_mutate_during_on_ending(self): + exporter = InMemorySpanExporter() + inside_on_ending = Event() + other_thread_done = Event() + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def mutate_from_another_thread(): + inside_on_ending.wait(5) + span.set_attribute("other.attribute", "value") + span.set_attributes({"other.attributes": "value"}) + span.update_name("renamed") + span.add_event("event") + span.add_link( + trace_api.SpanContext( + trace_id=0x1, + span_id=0x2, + is_remote=False, + ) + ) + span.set_status(trace_api.StatusCode.ERROR) + other_thread_done.set() + + thread = Thread(target=mutate_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(inside_on_ending.is_set()) + self.assertTrue(other_thread_done.is_set()) + + (exported,) = exporter.get_finished_spans() + self.assertNotIn("other.attribute", exported.attributes) + self.assertNotIn("other.attributes", exported.attributes) + self.assertEqual(exported.name, "foo") + self.assertEqual(len(exported.events), 0) + self.assertEqual(len(exported.links), 0) + self.assertIs(exported.status.status_code, trace_api.StatusCode.UNSET) + + def test_other_thread_does_not_see_span_recording_during_on_ending(self): + inside_on_ending = Event() + other_thread_done = Event() + recording_seen_by_other_thread = [] + + class BlockingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + inside_on_ending.set() + other_thread_done.wait(5) + + tracer_provider = trace.TracerProvider() + tracer_provider.add_span_processor(BlockingSpanProcessor()) + span = tracer_provider.get_tracer(__name__).start_span("foo") + + def observe_from_another_thread(): + inside_on_ending.wait(5) + recording_seen_by_other_thread.append(span.is_recording()) + other_thread_done.set() + + thread = Thread(target=observe_from_another_thread) + thread.start() + span.end() + thread.join(5) + + self.assertTrue(other_thread_done.is_set()) + self.assertEqual(recording_seen_by_other_thread, [False]) + + def test_span_is_frozen_when_record_end_metrics_raises(self): + def record_end_metrics_that_raises(): + raise RuntimeError("meter blew up") + + tracer_provider = trace.TracerProvider() + span = tracer_provider.get_tracer(__name__).start_span("foo") + # pylint: disable=protected-access + span._record_end_metrics = record_end_metrics_that_raises + + with self.assertRaises(RuntimeError): + span.end() + + self.assertFalse(span.is_recording()) + span.set_attribute("late", "value") + self.assertNotIn("late", span.attributes) + + +class MultiSpanProcessorTestBase(abc.ABC): + @abc.abstractmethod + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor | trace.ConcurrentMultiSpanProcessor: + pass + + @staticmethod + def create_default_span() -> trace_api.Span: + span_context = trace_api.SpanContext(37, 73, is_remote=False) + return trace_api.NonRecordingSpan(span_context) + + def test_on_start(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + context = Context() + multi_processor.on_start(span, parent_context=context) + + for mock_processor in mocks: + mock_processor.on_start.assert_called_once_with(span, parent_context=context) + multi_processor.shutdown() + + def test_on_ending(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + # pylint: disable=protected-access + multi_processor._on_ending(span) + + for mock_processor in mocks: + # pylint: disable=protected-access + mock_processor._on_ending.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_end(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + span = self.create_default_span() + multi_processor.on_end(span) + + for mock_processor in mocks: + mock_processor.on_end.assert_called_once_with(span) + multi_processor.shutdown() + + def test_on_shutdown(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + multi_processor.shutdown() + + for mock_processor in mocks: + mock_processor.shutdown.assert_called_once_with() + + def test_force_flush(self): + multi_processor = self.create_multi_span_processor() + + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(5)] + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + timeout_millis = 100 + + flushed = multi_processor.force_flush(timeout_millis) + + # pylint: disable=no-member + self.assertTrue(flushed) + for mock_processor in mocks: + # pylint: disable=no-member + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_on_ending_not_implemented_does_not_raise(self): + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + + spans_calls_list = [] # filled by MySpanProcessor + expected_list = [] # filled by hand + + multi_processor = self.create_multi_span_processor() + # Does not implement _on_ending + multi_processor.add_span_processor(MySpanProcessor("SP1", spans_calls_list)) + + tracer_provider.add_span_processor(multi_processor) + + try: + with tracer.start_as_current_span("foo"): + expected_list.append(span_event_start_fmt("SP1", "foo")) + + with tracer.start_as_current_span("bar"): + expected_list.append(span_event_start_fmt("SP1", "bar")) + + with tracer.start_as_current_span("baz"): + expected_list.append(span_event_start_fmt("SP1", "baz")) + + expected_list.append(span_event_end_fmt("SP1", "baz")) + expected_list.append(span_event_end_fmt("SP1", "bar")) + expected_list.append(span_event_end_fmt("SP1", "foo")) + except NotImplementedError: + # pylint: disable=no-member + self.fail("_on_ending() should not raise an exception") + + # pylint: disable=no-member + self.assertListEqual(spans_calls_list, expected_list) + + def test_span_mutable_during_on_ending(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + tracer_provider.get_tracer(__name__).start_span("foo").end() + + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + def test_on_ending_exception_does_not_stop_other_processors(self): + multi_processor = self.create_multi_span_processor() + exporter = InMemorySpanExporter() + + class FailingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + raise RuntimeError("_on_ending failed") + + class MutatingSpanProcessor(trace.SpanProcessor): + def _on_ending(self, span: "trace.Span") -> None: + span.set_attribute("attribute", "value") + + multi_processor.add_span_processor(FailingSpanProcessor()) + multi_processor.add_span_processor(MutatingSpanProcessor()) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider = trace.TracerProvider(active_span_processor=multi_processor) + + # pylint: disable=no-member + with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): + tracer_provider.get_tracer(__name__).start_span("foo").end() + + # The processor registered after the failing one still ran its + # _on_ending, and on_end still exported the span. + (span,) = exporter.get_finished_spans() + # pylint: disable=no-member + self.assertEqual(span.attributes["attribute"], "value") + multi_processor.shutdown() + + +class TestSynchronousMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.SynchronousMultiSpanProcessor: + return trace.SynchronousMultiSpanProcessor() + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + def delayed_flush(_): + time.sleep(0.055) + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(side_effect=delayed_flush) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(0, mock_processor2.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=False) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertFalse(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + mock_processor1 = mock.Mock(spec=trace.SpanProcessor) + mock_processor1.force_flush = mock.Mock(return_value=None) + multi_processor.add_span_processor(mock_processor1) + mock_processor2 = mock.Mock(spec=trace.SpanProcessor) + mock_processor2.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor2) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor1.force_flush.call_count) + self.assertEqual(1, mock_processor2.force_flush.call_count) + + def test_force_flush_default_processor(self): + multi_processor = trace.SynchronousMultiSpanProcessor() + + default_processor = trace.SpanProcessor() + multi_processor.add_span_processor(default_processor) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + mock_processor.force_flush = mock.Mock(return_value=True) + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(50) + self.assertTrue(flushed) + self.assertEqual(1, mock_processor.force_flush.call_count) + + +class TestConcurrentMultiSpanProcessor(MultiSpanProcessorTestBase, unittest.TestCase): + def create_multi_span_processor( + self, + ) -> trace.ConcurrentMultiSpanProcessor: + return trace.ConcurrentMultiSpanProcessor(3) + + def test_force_flush_late_by_timeout(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + wait_event = Event() + + def delayed_flush(_): + wait_event.wait() + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(side_effect=delayed_flush) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush(timeout_millis=25) + # let the thread executing the late_mock continue + wait_event.set() + multi_processor.shutdown() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + + def test_force_flush_late_by_span_processor(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + late_mock = mock.Mock(spec=trace.SpanProcessor) + late_mock.force_flush = mock.Mock(return_value=False) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, late_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertFalse(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_force_flush_processor_returns_none(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + + none_mock = mock.Mock(spec=trace.SpanProcessor) + none_mock.force_flush = mock.Mock(return_value=None) + mocks = [mock.Mock(spec=trace.SpanProcessor) for _ in range(4)] + mocks.insert(0, none_mock) + + for mock_processor in mocks: + multi_processor.add_span_processor(mock_processor) + + flushed = multi_processor.force_flush() + + self.assertTrue(flushed) + for mock_processor in mocks: + self.assertEqual(1, mock_processor.force_flush.call_count) + multi_processor.shutdown() + + def test_processor_gc(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(5) + weak_ref = weakref.ref(multi_processor) + multi_processor.shutdown() + + # When the processor is garbage collected + del multi_processor + gc.collect() + + # Then the reference to the processor should no longer exist + self.assertIsNone( + weak_ref(), + "The ConcurrentMultiSpanProcessor object created by this test wasn't garbage collected", + ) + + # pylint: disable=no-self-use + def test_shutdown_runs_inline_after_executor_shutdown(self): + # Simulate the atexit scenario: CPython runs threading._shutdown() + # before the atexit queue, so the ThreadPoolExecutor is closed by the + # time TracerProvider.shutdown() runs. The underlying processors must + # still be shut down inline so buffered work is not dropped. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + multi_processor.shutdown() + mock_processor.shutdown.assert_called_once_with() + + def test_on_end_runs_inline_after_executor_shutdown(self): + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + mock_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(mock_processor) + + # pylint: disable=protected-access + multi_processor._executor.shutdown(wait=True) + + span = self.create_default_span() + multi_processor.on_end(span) + mock_processor.on_end.assert_called_once_with(span) + + def test_callback_exception_does_not_block_siblings(self): + # Per OTel error-handling spec, a processor callback must not raise out + # of shutdown() and must not prevent healthy siblings from running. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + raising_processor = mock.Mock(spec=trace.SpanProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + healthy_processor = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(raising_processor) + multi_processor.add_span_processor(healthy_processor) + + with self.assertLogs("opentelemetry.sdk.trace", level="ERROR"): + multi_processor.shutdown() + + raising_processor.shutdown.assert_called_once_with() + healthy_processor.shutdown.assert_called_once_with() + + def test_submit_runtime_error_does_not_double_invoke(self): + # If Thread.start fails AFTER the work item was enqueued, the inline + # fallback must not run the callback a second time: the queue worker + # will pick up the enqueued item. A shared claim per dispatch ensures + # at-most-once. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + processor_a = mock.Mock(spec=trace.SpanProcessor) + processor_b = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(processor_a) + multi_processor.add_span_processor(processor_b) + + real_start = Thread.start + calls = {"n": 0} + + def first_start_fails(target): + calls["n"] += 1 + if calls["n"] == 1: + raise RuntimeError("cannot start new thread") + return real_start(target) + + with mock.patch.object(Thread, "start", first_start_fails): + multi_processor.shutdown() + + self.assertEqual(processor_a.shutdown.call_count, 1) + self.assertEqual(processor_b.shutdown.call_count, 1) + + def test_callback_exception_preserves_application_exception(self): + # A processor callback failure must not replace the caller's own + # exception when on_end runs on the caller's thread. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + faulty = mock.Mock(spec=trace.SpanProcessor) + faulty.on_end.side_effect = RuntimeError("telemetry fault") + healthy = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(faulty) + multi_processor.add_span_processor(healthy) + + provider = trace.TracerProvider( + active_span_processor=multi_processor, shutdown_on_exit=False + ) + app_error = ValueError("business fault") + try: + with self.assertRaises(ValueError) as caught: + with provider.get_tracer("test").start_as_current_span("x"): + raise app_error + self.assertIs(caught.exception, app_error) + finally: + multi_processor._executor.shutdown(wait=True) + + + def test_callback_logger_handler_raise_does_not_crash(self): + # A logging handler attached to the OTel logger that raises must not + # escape out of shutdown() nor deadlock; the reentry guard routes + # diagnostics to stderr and healthy siblings still run. + import logging + + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + raising_processor = mock.Mock(spec=trace.SpanProcessor) + raising_processor.shutdown.side_effect = RuntimeError("processor boom") + healthy = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(raising_processor) + multi_processor.add_span_processor(healthy) + + class _BoomHandler(logging.Handler): + def emit(self, record): + try: + raise RuntimeError("handler boom") + except RuntimeError: + pass # simulate a handler whose emit raises and is contained by the logging framework + + handler = _BoomHandler(level=logging.ERROR) + otel_logger = logging.getLogger("opentelemetry.sdk.trace") + otel_logger.addHandler(handler) + old_raise = logging.raiseExceptions + logging.raiseExceptions = False + try: + # Must not raise, must not recurse, must not deadlock. + multi_processor.shutdown() + finally: + logging.raiseExceptions = old_raise + otel_logger.removeHandler(handler) + multi_processor._executor.shutdown(wait=True) + + healthy.shutdown.assert_called_once_with() + + @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") + def test_batch_span_processor_fork(self): + multiprocessing_context = multiprocessing.get_context("fork") + tracer_provider = trace.TracerProvider() + tracer = tracer_provider.get_tracer(__name__) + exporter = InMemorySpanExporter() + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + multi_processor.add_span_processor(SimpleSpanProcessor(exporter)) + tracer_provider.add_span_processor(multi_processor) + + # Use the ConcurrentMultiSpanProcessor in the main process. + # This is necessary in this test to start using the underlying ThreadPoolExecutor and avoid false positive: + with tracer.start_as_current_span("main process before fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process before fork span" + + # The forked ConcurrentMultiSpanProcessor is usable in the child process: + def child(conn): + with tracer.start_as_current_span("child process span"): + pass + conn.send(exporter.get_finished_spans()[-1].name) + conn.close() + + parent_conn, child_conn = multiprocessing_context.Pipe() + process = multiprocessing_context.Process(target=child, args=(child_conn,)) + process.start() + has_response = parent_conn.poll(timeout=5) + if not has_response: + process.kill() + self.fail("The child process did not send any message after 5 seconds, it's very probably locked") + process.join(timeout=5) + assert parent_conn.recv() == "child process span" + + # The ConcurrentMultiSpanProcessor is still usable in the main process after the child process termination: + with tracer.start_as_current_span("main process after fork span"): + pass + assert exporter.get_finished_spans()[-1].name == "main process after fork span" From d87a6a0f9d5136f55cb496809a88fc098ce92b0c Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:04:36 +0800 Subject: [PATCH 09/26] fix(sdk): scope reentry guard to diagnostic dispatch; contain handler errors --- .../sdk/_logs/_internal/__init__.py | 33 +++++++++++++++++-- .../src/opentelemetry/sdk/trace/__init__.py | 33 +++++++++++++++++-- 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index c2f605c4db8..83b9130497c 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -90,6 +90,32 @@ _callback_log_reentry = threading.local() +def _safe_log_exception(logger, msg): + """Log an exception safely against recursive logging handlers. + + If we are already inside a diagnostic log dispatch on this thread, write + the record to stderr directly instead of going through the logging module + to prevent infinite feedback loops. Any exception raised by a user logging + handler during the outer dispatch is contained here. + """ + if getattr(_callback_log_reentry, "in_diag", False): + import sys as _sys + import traceback as _tb + print(msg, file=_sys.stderr) + _tb.print_exc(file=_sys.stderr) + return + _callback_log_reentry.in_diag = True + try: + logger.exception(msg) + except Exception: # pylint: disable=broad-exception-caught + import sys as _sys + import traceback as _tb + print(msg, file=_sys.stderr) + _tb.print_exc(file=_sys.stderr) + finally: + _callback_log_reentry.in_diag = False + + class BytesEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, bytes): @@ -508,7 +534,7 @@ def run(self) -> None: try: self._callback(*self._args, **self._kwargs) except Exception: # pylint: disable=broad-exception-caught - _logger.exception("Exception while calling LogRecordProcessor callback.") + _safe_log_exception(_logger, "Exception while calling LogRecordProcessor callback.") finally: _callback_log_reentry.active = reentry self._finished.set() @@ -561,8 +587,9 @@ def _submit_and_wait( callback = func(lp) callback(*args, **kwargs) except Exception: # pylint: disable=broad-exception-caught - _logger.exception( - "Exception while re-entering LogRecordProcessor callback inline." + _safe_log_exception( + _logger, + "Exception while re-entering LogRecordProcessor callback inline.", ) return attempts = [] diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 15fe678d0d3..20407b9f50c 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -78,6 +78,32 @@ _callback_log_reentry = threading.local() +def _safe_log_exception(logger, msg): + """Log an exception safely against recursive logging handlers. + + If we are already inside a diagnostic log dispatch on this thread, write + the record to stderr directly instead of going through the logging module + to prevent infinite feedback loops. Any exception raised by a user logging + handler during the outer dispatch is contained here. + """ + if getattr(_callback_log_reentry, "in_diag", False): + import sys as _sys + import traceback as _tb + print(msg, file=_sys.stderr) + _tb.print_exc(file=_sys.stderr) + return + _callback_log_reentry.in_diag = True + try: + logger.exception(msg) + except Exception: # pylint: disable=broad-exception-caught + import sys as _sys + import traceback as _tb + print(msg, file=_sys.stderr) + _tb.print_exc(file=_sys.stderr) + finally: + _callback_log_reentry.in_diag = False + + _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 128 _DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT = 128 @@ -265,7 +291,7 @@ def run(self) -> None: try: self._callback(*self._args, **self._kwargs) except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while calling SpanProcessor callback.") + _safe_log_exception(logger, "Exception while calling SpanProcessor callback.") finally: _callback_log_reentry.active = reentry self._finished.set() @@ -339,8 +365,9 @@ def _submit_and_await( callback = func(sp) callback(*args, **kwargs) except Exception: # pylint: disable=broad-exception-caught - logger.exception( - "Exception while re-entering SpanProcessor callback inline." + _safe_log_exception( + logger, + "Exception while re-entering SpanProcessor callback inline.", ) return attempts = [] From dda89e6345aa8b3154570c4202755f733e1a5674 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:13:37 +0800 Subject: [PATCH 10/26] style(sdk): move sys/traceback imports to top-level (fix pylint/precommit) --- .../opentelemetry/sdk/_logs/_internal/__init__.py | 13 +++++-------- .../src/opentelemetry/sdk/trace/__init__.py | 13 +++++-------- .../tests/logs/test_multi_log_processor.py | 3 --- .../tests/trace/test_span_processor.py | 3 +-- 4 files changed, 11 insertions(+), 21 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 83b9130497c..3ca33c67c42 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -8,6 +8,7 @@ import concurrent.futures import json import logging +import sys import os import threading import traceback @@ -99,19 +100,15 @@ def _safe_log_exception(logger, msg): handler during the outer dispatch is contained here. """ if getattr(_callback_log_reentry, "in_diag", False): - import sys as _sys - import traceback as _tb - print(msg, file=_sys.stderr) - _tb.print_exc(file=_sys.stderr) + print(msg, file=sys.stderr) + traceback.print_exc(file=sys.stderr) return _callback_log_reentry.in_diag = True try: logger.exception(msg) except Exception: # pylint: disable=broad-exception-caught - import sys as _sys - import traceback as _tb - print(msg, file=_sys.stderr) - _tb.print_exc(file=_sys.stderr) + print(msg, file=sys.stderr) + traceback.print_exc(file=sys.stderr) finally: _callback_log_reentry.in_diag = False diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 20407b9f50c..06390d70f86 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -7,6 +7,7 @@ import concurrent.futures import json import logging +import sys import os import threading import traceback @@ -87,19 +88,15 @@ def _safe_log_exception(logger, msg): handler during the outer dispatch is contained here. """ if getattr(_callback_log_reentry, "in_diag", False): - import sys as _sys - import traceback as _tb - print(msg, file=_sys.stderr) - _tb.print_exc(file=_sys.stderr) + print(msg, file=sys.stderr) + traceback.print_exc(file=sys.stderr) return _callback_log_reentry.in_diag = True try: logger.exception(msg) except Exception: # pylint: disable=broad-exception-caught - import sys as _sys - import traceback as _tb - print(msg, file=_sys.stderr) - _tb.print_exc(file=_sys.stderr) + print(msg, file=sys.stderr) + traceback.print_exc(file=sys.stderr) finally: _callback_log_reentry.in_diag = False diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 6e697885e4b..8ccc25736b4 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -312,8 +312,6 @@ def first_start_fails(target): def test_callback_logger_handler_raise_does_not_crash(self): # A logging handler attached to the OTel logger that raises must not # escape out of shutdown() nor deadlock; reentry guard routes to stderr. - import logging - multi = ConcurrentMultiLogRecordProcessor(2) raising = Mock(spec=LogRecordProcessor) raising.shutdown.side_effect = RuntimeError("processor boom") @@ -341,4 +339,3 @@ def emit(self, record): multi._executor.shutdown(wait=True) healthy.shutdown.assert_called_once_with() - diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 4b04254b605..78cbacea630 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -6,6 +6,7 @@ import multiprocessing import os import time +import logging import unittest import weakref from threading import Event, Thread @@ -815,8 +816,6 @@ def test_callback_logger_handler_raise_does_not_crash(self): # A logging handler attached to the OTel logger that raises must not # escape out of shutdown() nor deadlock; the reentry guard routes # diagnostics to stderr and healthy siblings still run. - import logging - multi_processor = trace.ConcurrentMultiSpanProcessor(2) raising_processor = mock.Mock(spec=trace.SpanProcessor) raising_processor.shutdown.side_effect = RuntimeError("processor boom") From cba9b5ac5aa1c5de7b8f708e3183a3b7cc7c3df0 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:27:57 +0800 Subject: [PATCH 11/26] style(sdk): sort imports (fix pre-commit I001) --- .../src/opentelemetry/sdk/_logs/_internal/__init__.py | 2 +- opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 3ca33c67c42..c53f2af3f3d 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -8,8 +8,8 @@ import concurrent.futures import json import logging -import sys import os +import sys import threading import traceback import warnings diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 06390d70f86..3b4e311e8d6 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -7,8 +7,8 @@ import concurrent.futures import json import logging -import sys import os +import sys import threading import traceback import typing From 9a0221e06d9bf13f09aff098273a7d1d73a2192d Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Mon, 26 Oct 2026 23:40:42 +0800 Subject: [PATCH 12/26] style(sdk): fix pylint W0621/W0212 and ruff I001 in test imports --- .../src/opentelemetry/sdk/_logs/_internal/__init__.py | 2 +- opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py | 4 ++-- opentelemetry-sdk/tests/trace/test_span_processor.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index c53f2af3f3d..0c145ff3700 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -91,7 +91,7 @@ _callback_log_reentry = threading.local() -def _safe_log_exception(logger, msg): +def _safe_log_exception(logger, msg): # pylint: disable=redefined-outer-name """Log an exception safely against recursive logging handlers. If we are already inside a diagnostic log dispatch on this thread, write diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index 3b4e311e8d6..bd75ae82b03 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -79,7 +79,7 @@ _callback_log_reentry = threading.local() -def _safe_log_exception(logger, msg): +def _safe_log_exception(logger, msg): # pylint: disable=redefined-outer-name """Log an exception safely against recursive logging handlers. If we are already inside a diagnostic log dispatch on this thread, write @@ -93,7 +93,7 @@ def _safe_log_exception(logger, msg): return _callback_log_reentry.in_diag = True try: - logger.exception(msg) + logger.exception(msg) # noqa: SHADOW except Exception: # pylint: disable=broad-exception-caught print(msg, file=sys.stderr) traceback.print_exc(file=sys.stderr) diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 78cbacea630..b26b360940c 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -3,10 +3,10 @@ import abc import gc +import logging import multiprocessing import os import time -import logging import unittest import weakref from threading import Event, Thread From d16c4768e816179ff757f23777f7606623d77556 Mon Sep 17 00:00:00 2001 From: RichardoMu <44485717+RichardoMrMu@users.noreply.github.com> Date: Tue, 27 Oct 2026 00:00:14 +0800 Subject: [PATCH 13/26] style(sdk): ruff format + pylint W0212 disable on _executor test access --- opentelemetry-sdk/tests/trace/test_span_processor.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index b26b360940c..57eadbccc61 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -799,9 +799,7 @@ def test_callback_exception_preserves_application_exception(self): multi_processor.add_span_processor(faulty) multi_processor.add_span_processor(healthy) - provider = trace.TracerProvider( - active_span_processor=multi_processor, shutdown_on_exit=False - ) + provider = trace.TracerProvider(active_span_processor=multi_processor, shutdown_on_exit=False) app_error = ValueError("business fault") try: with self.assertRaises(ValueError) as caught: @@ -811,7 +809,6 @@ def test_callback_exception_preserves_application_exception(self): finally: multi_processor._executor.shutdown(wait=True) - def test_callback_logger_handler_raise_does_not_crash(self): # A logging handler attached to the OTel logger that raises must not # escape out of shutdown() nor deadlock; the reentry guard routes From 094b2aab57da81539483f4c139e0509177240448 Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 03:02:41 +0800 Subject: [PATCH 14/26] style(sdk): use public shutdown() contract in span/log processor tests Remove protected _executor access in test teardown (pylint W0212) by shutting the concurrent processors down through their public shutdown() method, and collapse a double blank line flagged by ruff-format. No production behavior change; the callback containment, at-most-once dispatch and logging reentry isolation are already covered by the behavior tests. --- opentelemetry-sdk/tests/logs/test_multi_log_processor.py | 1 - opentelemetry-sdk/tests/trace/test_span_processor.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 8ccc25736b4..0a40be122f8 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -308,7 +308,6 @@ def first_start_fails(target): self.assertEqual(processor_a.shutdown.call_count, 1) self.assertEqual(processor_b.shutdown.call_count, 1) - def test_callback_logger_handler_raise_does_not_crash(self): # A logging handler attached to the OTel logger that raises must not # escape out of shutdown() nor deadlock; reentry guard routes to stderr. diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 57eadbccc61..bc8ab0b58cd 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -807,7 +807,9 @@ def test_callback_exception_preserves_application_exception(self): raise app_error self.assertIs(caught.exception, app_error) finally: - multi_processor._executor.shutdown(wait=True) + # Tear down through the public contract instead of reaching into + # the executor's protected member; the processor was never shut down. + multi_processor.shutdown() def test_callback_logger_handler_raise_does_not_crash(self): # A logging handler attached to the OTel logger that raises must not @@ -838,7 +840,6 @@ def emit(self, record): finally: logging.raiseExceptions = old_raise otel_logger.removeHandler(handler) - multi_processor._executor.shutdown(wait=True) healthy.shutdown.assert_called_once_with() From 885b576eb2e8ebe639c9d5cec8d4bbd078d11f12 Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 03:27:26 +0800 Subject: [PATCH 15/26] fix(logs): inline concurrent dispatch from real LoggingHandler bridge to avoid handler-lock deadlock A real logging handler holds its own lock across emit(). When ConcurrentMultiLogRecordProcessor.on_emit dispatches to its executor and then blocks waiting on the futures, a worker that re-enters logging through the same LoggingHandler deadlocks on the handler lock the dispatching thread still holds: the dispatcher waits for the worker, the worker waits for the lock. Mark the bridge boundary in LoggingHandler.emit with a thread-local and extend the existing reentry guard so the multi-processor runs callbacks inline whenever it is dispatched from inside that bridge, matching the callback reentry path. Add regression tests using the real LoggingHandler bridge (not a synthetic handler) for reentry inline dispatch and handler-raise isolation, asserting business errors are contained and healthy siblings still receive records. --- .../sdk/_logs/_internal/__init__.py | 26 +++++-- .../tests/logs/test_multi_log_processor.py | 68 +++++++++++++++++++ 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 0c145ff3700..995c3603506 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -573,12 +573,14 @@ def _submit_and_wait( *args: Any, **kwargs: Any, ): - # Reentry guard: if we are already running a callback on this thread - # (e.g. a LoggingHandler synchronously called back into on_emit), run - # the remaining callbacks inline instead of through the executor. - # Submitting would queue work no worker can pick up (workers blocked - # inside the outer callback), and future.result() would deadlock. - if getattr(_callback_log_reentry, "active", False): + # Reentry guard: if we are already running a callback on this thread, or + # we were dispatched from inside a real logging handler that holds its own + # lock across emit(), run the callbacks inline instead of through the + # executor. Submitting would either queue work no worker can pick up + # (workers blocked inside the outer callback) or block a re-entrant + # worker on the logging handler lock the dispatching thread still holds, + # and future.result() would deadlock in both cases. + if getattr(_callback_log_reentry, "active", False) or getattr(_callback_log_reentry, "in_bridge", False): for lp in self._log_record_processors: try: callback = func(lp) @@ -819,7 +821,17 @@ def emit(self, record: logging.LogRecord) -> None: """ logger = get_logger(record.name, logger_provider=self._logger_provider) if not isinstance(logger, NoOpLogger): - logger.emit(self._translate(record)) + # A real logging handler holds its own lock across emit(). If the + # concurrent multi-processor submits work to its executor and then + # blocks waiting on it, a worker that re-enters logging through this + # same handler would deadlock on that lock. Mark the bridge boundary + # so the multi-processor dispatches inline while we hold the handler + # lock, matching the re-entry guard for callbacks. + _callback_log_reentry.in_bridge = True + try: + logger.emit(self._translate(record)) + finally: + _callback_log_reentry.in_bridge = False def flush(self) -> None: """ diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 0a40be122f8..41c051991c7 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -338,3 +338,71 @@ def emit(self, record): multi._executor.shutdown(wait=True) healthy.shutdown.assert_called_once_with() + + def test_real_logging_handler_reentry_runs_inline(self): + # Wire the REAL LoggingHandler bridge into a ConcurrentMultiLogRecordProcessor. + # When a child processor emits a Python log on a worker thread, the bridge + # re-enters on_emit on that same thread. The ctive reentry guard must run + # the re-entered callbacks inline instead of submitting to the (busy) executor, + # so the call returns without deadlock and a healthy sibling still receives + # both the outer and the re-entered record. + multi = ConcurrentMultiLogRecordProcessor(2) + bridge_logger = logging.getLogger("otel.bridge.reentry") + bridge_logger.propagate = False + state = {"fired": False} + + class _Reenter(LogRecordProcessor): + def on_emit(self, log_record): + if not state["fired"]: + state["fired"] = True + bridge_logger.error("reentered") + + def shutdown(self): + pass + + def force_flush(self, timeout_millis=30000): + return True + + healthy = Mock(spec=LogRecordProcessor) + multi.add_log_record_processor(_Reenter()) + multi.add_log_record_processor(healthy) + provider = LoggerProvider(multi_log_record_processor=multi, shutdown_on_exit=False) + bridge = LoggingHandler(logger_provider=provider) + bridge_logger.addHandler(bridge) + try: + # Must return promptly: no executor deadlock, no unbounded recursion. + bridge_logger.error("outer") + finally: + bridge_logger.removeHandler(bridge) + multi._executor.shutdown(wait=True) + + # healthy sibling saw the outer record and the inline re-entered record. + self.assertEqual(healthy.on_emit.call_count, 2) + + def test_real_logging_handler_raise_is_isolated(self): + # Attach the REAL LoggingHandler bridge to the OTel internal logger. A + # raising child processor must not let the handler error escape shutdown(), + # must not deadlock, and must not prevent a healthy sibling from shutting + # down. The Python logging framework contains the handler error; the SDK's + # own diagnostic fallback keeps the record off the recursive path. + multi = ConcurrentMultiLogRecordProcessor(2) + raising = Mock(spec=LogRecordProcessor) + raising.shutdown.side_effect = RuntimeError("processor boom") + healthy = Mock(spec=LogRecordProcessor) + multi.add_log_record_processor(raising) + multi.add_log_record_processor(healthy) + provider = LoggerProvider(multi_log_record_processor=multi, shutdown_on_exit=False) + bridge = LoggingHandler(logger_provider=provider) + otel_logger = logging.getLogger("opentelemetry.sdk._logs._internal") + otel_logger.addHandler(bridge) + old_raise = logging.raiseExceptions + logging.raiseExceptions = False + try: + # Must not raise and must not deadlock. + multi.shutdown() + finally: + logging.raiseExceptions = old_raise + otel_logger.removeHandler(bridge) + multi._executor.shutdown(wait=True) + + healthy.shutdown.assert_called_once_with() From cd4e2ed9a3adf2f389731f98dd471bf858f274cf Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 03:44:52 +0800 Subject: [PATCH 16/26] test(logs): fix comment typo and remove stray control char in real-bridge reentry test --- opentelemetry-sdk/tests/logs/test_multi_log_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 41c051991c7..6fa579870d5 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -342,7 +342,7 @@ def emit(self, record): def test_real_logging_handler_reentry_runs_inline(self): # Wire the REAL LoggingHandler bridge into a ConcurrentMultiLogRecordProcessor. # When a child processor emits a Python log on a worker thread, the bridge - # re-enters on_emit on that same thread. The ctive reentry guard must run + # re-enters on_emit on that same thread. The active reentry guard must run # the re-entered callbacks inline instead of submitting to the (busy) executor, # so the call returns without deadlock and a healthy sibling still receives # both the outer and the re-entered record. From 8b7e96080b8257ed99ffe8fd67347ad47408007c Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 04:04:33 +0800 Subject: [PATCH 17/26] ci: retrigger public-symbols-check (previous run flaky/infrastructure; no code change) From 7954da92592089a2dc0008ea855a9e4f6cf72f4e Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 04:48:52 +0800 Subject: [PATCH 18/26] fix(sdk): isolate logging-handler errors from inline callback dispatch Wrap the executor-shutdown DEBUG diagnostic and the resolve/submit/future exception logs in safe wrappers so that a raising user logging handler cannot abort the inline fallback or starve healthy siblings. The DEBUG record and business exception traceback are still emitted (via stderr fallback if a handler raises); no log level is raised or diagnostic dropped. --- .../sdk/_logs/_internal/__init__.py | 29 ++++++++++++++++--- .../src/opentelemetry/sdk/trace/__init__.py | 29 ++++++++++++++++--- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py index 995c3603506..21ff4428755 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_logs/_internal/__init__.py @@ -113,6 +113,26 @@ def _safe_log_exception(logger, msg): # pylint: disable=redefined-outer-name _callback_log_reentry.in_diag = False +def _safe_log_debug(logger, msg): # pylint: disable=redefined-outer-name + """Emit a DEBUG diagnostic without letting a user logging handler error escape. + + The inline fallback path must run even when an attached logging handler + raises; the DEBUG record is still attempted through the standard handler, + and any handler error falls back to stderr instead of aborting the + callback dispatch. + """ + if getattr(_callback_log_reentry, "in_diag", False): + print(msg, file=sys.stderr) + return + _callback_log_reentry.in_diag = True + try: + logger.debug(msg) # noqa: SHADOW + except Exception: # pylint: disable=broad-exception-caught + print(msg, file=sys.stderr) + finally: + _callback_log_reentry.in_diag = False + + class BytesEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, bytes): @@ -597,7 +617,7 @@ def _submit_and_wait( try: callback = func(lp) except Exception: # pylint: disable=broad-exception-caught - _logger.exception("Exception while resolving LogRecordProcessor callback.") + _safe_log_exception(_logger, "Exception while resolving LogRecordProcessor callback.") continue attempt = _CallbackAttempt(callback, args, kwargs) attempts.append(attempt) @@ -610,17 +630,18 @@ def _submit_and_wait( # inline through the SAME attempt; the shared lock prevents # double invocation even if the work item was already enqueued # before Thread.start failed. - _logger.debug( + _safe_log_debug( + _logger, "Executor is shut down; running LogRecordProcessor callback inline.", ) attempt.run() except Exception: # pylint: disable=broad-exception-caught - _logger.exception("Exception while submitting LogRecordProcessor callback.") + _safe_log_exception(_logger, "Exception while submitting LogRecordProcessor callback.") for future in futures: try: future.result() except Exception: # pylint: disable=broad-exception-caught - _logger.exception("Exception in LogRecordProcessor worker.") + _safe_log_exception(_logger, "Exception in LogRecordProcessor worker.") for attempt in attempts: attempt.wait_if_started() diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py index bd75ae82b03..d5520424278 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/trace/__init__.py @@ -101,6 +101,26 @@ def _safe_log_exception(logger, msg): # pylint: disable=redefined-outer-name _callback_log_reentry.in_diag = False +def _safe_log_debug(logger, msg): # pylint: disable=redefined-outer-name + """Emit a DEBUG diagnostic without letting a user logging handler error escape. + + The inline fallback path must run even when an attached logging handler + raises; the DEBUG record is still attempted through the standard handler, + and any handler error falls back to stderr instead of aborting the + callback dispatch. + """ + if getattr(_callback_log_reentry, "in_diag", False): + print(msg, file=sys.stderr) + return + _callback_log_reentry.in_diag = True + try: + logger.debug(msg) # noqa: SHADOW + except Exception: # pylint: disable=broad-exception-caught + print(msg, file=sys.stderr) + finally: + _callback_log_reentry.in_diag = False + + _DEFAULT_OTEL_ATTRIBUTE_COUNT_LIMIT = 128 _DEFAULT_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = 128 _DEFAULT_OTEL_EVENT_ATTRIBUTE_COUNT_LIMIT = 128 @@ -373,7 +393,7 @@ def _submit_and_await( try: callback = func(sp) except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while resolving SpanProcessor callback.") + _safe_log_exception(logger, "Exception while resolving SpanProcessor callback.") continue attempt = _CallbackAttempt(callback, args, kwargs) attempts.append(attempt) @@ -386,17 +406,18 @@ def _submit_and_await( # TracerProvider.shutdown() runs. Run inline through the SAME # attempt; the shared lock prevents double invocation even if # the work item was already enqueued before Thread.start failed. - logger.debug( + _safe_log_debug( + logger, "Executor is shut down; running SpanProcessor callback inline.", ) attempt.run() except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception while submitting SpanProcessor callback.") + _safe_log_exception(logger, "Exception while submitting SpanProcessor callback.") for future in futures: try: future.result() except Exception: # pylint: disable=broad-exception-caught - logger.exception("Exception in SpanProcessor worker.") + _safe_log_exception(logger, "Exception in SpanProcessor worker.") for attempt in attempts: attempt.wait_if_started() From c4616bfc4f4b4f4859a96003be36fb5e0a553041 Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 04:52:29 +0800 Subject: [PATCH 19/26] test(sdk): outward-raising handler regression for inline fallback Add a handler that raises outward (no self-catch, DEBUG level, executor pre-shutdown) to prove the executor-shutdown diagnostic and per-callback exception logs no longer abort dispatch. Fails on 8b7e960, passes on the safe-log wrapper fix. --- .../tests/logs/test_multi_log_processor.py | 34 ++++++++++++++++++ .../tests/trace/test_span_processor.py | 36 +++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 6fa579870d5..47b006bcf4b 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -339,6 +339,40 @@ def emit(self, record): healthy.shutdown.assert_called_once_with() + + def test_logging_handler_outward_raise_inline_fallback_is_isolated(self): + # A user logging handler that raises OUTWARD (no self-catch, with + # raiseExceptions left at its default) must not abort the inline + # fallback when the executor is already shut down. The executor-shutdown + # DEBUG diagnostic and per-callback exception log are wrapped so a + # handler error falls back to stderr; both faulty and healthy shut down. + multi = ConcurrentMultiLogRecordProcessor(2) + faulty = Mock(spec=LogRecordProcessor) + faulty.shutdown.side_effect = RuntimeError("processor fault") + healthy = Mock(spec=LogRecordProcessor) + multi.add_log_record_processor(faulty) + multi.add_log_record_processor(healthy) + + class _OutwardBoom(logging.Handler): + def emit(self, record): + raise RuntimeError("handler boom outward") + + otel_logger = logging.getLogger("opentelemetry.sdk._logs._internal") + old_level = otel_logger.level + handler = _OutwardBoom(level=logging.DEBUG) + otel_logger.setLevel(logging.DEBUG) + otel_logger.addHandler(handler) + multi._executor.shutdown(wait=True) + try: + multi.shutdown() + finally: + otel_logger.removeHandler(handler) + otel_logger.setLevel(old_level) + multi._executor.shutdown(wait=True) + + healthy.shutdown.assert_called_once_with() + faulty.shutdown.assert_called_once_with() + def test_real_logging_handler_reentry_runs_inline(self): # Wire the REAL LoggingHandler bridge into a ConcurrentMultiLogRecordProcessor. # When a child processor emits a Python log on a worker thread, the bridge diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index bc8ab0b58cd..1dd11fd59a9 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -843,6 +843,42 @@ def emit(self, record): healthy.shutdown.assert_called_once_with() + + def test_logging_handler_outward_raise_inline_fallback_is_isolated(self): + # A user logging handler that raises OUTWARD (no self-catch, with + # raiseExceptions left at its default) must not abort the inline + # fallback when the executor is already shut down (the atexit path). + # The executor-shutdown DEBUG diagnostic and the per-callback exception + # log are wrapped so a handler error falls back to stderr instead of + # escaping; both the faulty and the healthy processor still shut down. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + faulty = mock.Mock(spec=trace.SpanProcessor) + faulty.shutdown.side_effect = RuntimeError("processor fault") + healthy = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(faulty) + multi_processor.add_span_processor(healthy) + + class _OutwardBoom(logging.Handler): + def emit(self, record): + raise RuntimeError("handler boom outward") + + otel_logger = logging.getLogger("opentelemetry.sdk.trace") + old_level = otel_logger.level + handler = _OutwardBoom(level=logging.DEBUG) + otel_logger.setLevel(logging.DEBUG) + otel_logger.addHandler(handler) + multi_processor._executor.shutdown(wait=True) + try: + # Must not raise despite the outward handler raise on the + # "Executor is shut down" DEBUG diagnostic. + multi_processor.shutdown() + finally: + otel_logger.removeHandler(handler) + otel_logger.setLevel(old_level) + + healthy.shutdown.assert_called_once_with() + faulty.shutdown.assert_called_once_with() + @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") def test_batch_span_processor_fork(self): multiprocessing_context = multiprocessing.get_context("fork") From 5fa884509a832b26bbcad7553d4cae982b77392d Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 04:57:45 +0800 Subject: [PATCH 20/26] test(sdk): outward-raising handler for resolve and future paths Replace the hollow _BoomHandler (try/except pass) with a handler that raises outward, and add a regression test for the callback-resolution diagnostic path. Covers resolve/submit/future/executor-shutdown-debug under a truly raising user handler. --- .../tests/logs/test_multi_log_processor.py | 44 +++++++++++++++--- .../tests/trace/test_span_processor.py | 45 +++++++++++++++---- 2 files changed, 74 insertions(+), 15 deletions(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 47b006bcf4b..cd20482630d 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -320,21 +320,51 @@ def test_callback_logger_handler_raise_does_not_crash(self): class _BoomHandler(logging.Handler): def emit(self, record): - try: - raise RuntimeError("handler boom") - except RuntimeError: - pass # simulate a handler whose emit raises and is contained by the logging framework + # Raises OUTWARD (no self-catch, raiseExceptions left default). + raise RuntimeError("handler boom") handler = _BoomHandler(level=logging.ERROR) otel_logger = logging.getLogger("opentelemetry.sdk._logs._internal") otel_logger.addHandler(handler) - old_raise = logging.raiseExceptions - logging.raiseExceptions = False + try: + # Normal-executor path: faulty callback raises -> future.result() -> + # safe_log_exception -> outward handler raise must not escape. + multi.shutdown() + finally: + otel_logger.removeHandler(handler) + multi._executor.shutdown(wait=True) + + healthy.shutdown.assert_called_once_with() + + def test_logging_handler_raise_during_callback_resolution_is_isolated(self): + # If resolving a callback (attribute lookup) raises AND an attached + # handler raises on the resulting exception log, dispatch continues to + # healthy siblings. + multi = ConcurrentMultiLogRecordProcessor(2) + + class _ResolveBoom: + @property + def shutdown(self): + raise RuntimeError("resolve boom") + + healthy = Mock(spec=LogRecordProcessor) + multi.add_log_record_processor(_ResolveBoom()) + multi.add_log_record_processor(healthy) + + class _OutwardBoom(logging.Handler): + def emit(self, record): + raise RuntimeError("handler boom outward") + + otel_logger = logging.getLogger("opentelemetry.sdk._logs._internal") + old_level = otel_logger.level + otel_logger.setLevel(logging.ERROR) + handler = _OutwardBoom(level=logging.ERROR) + otel_logger.addHandler(handler) try: multi.shutdown() finally: - logging.raiseExceptions = old_raise otel_logger.removeHandler(handler) + otel_logger.setLevel(old_level) multi._executor.shutdown(wait=True) healthy.shutdown.assert_called_once_with() diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 1dd11fd59a9..8b3fe599ca4 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -824,25 +824,54 @@ def test_callback_logger_handler_raise_does_not_crash(self): class _BoomHandler(logging.Handler): def emit(self, record): - try: - raise RuntimeError("handler boom") - except RuntimeError: - pass # simulate a handler whose emit raises and is contained by the logging framework + # Raises OUTWARD (no self-catch, raiseExceptions left default). + # The safe diagnostic wrapper must contain it and fall back to stderr. + raise RuntimeError("handler boom") handler = _BoomHandler(level=logging.ERROR) otel_logger = logging.getLogger("opentelemetry.sdk.trace") otel_logger.addHandler(handler) - old_raise = logging.raiseExceptions - logging.raiseExceptions = False try: - # Must not raise, must not recurse, must not deadlock. + # Normal-executor path: faulty callback raises -> future.result() -> + # safe_log_exception -> outward handler raise must not escape. multi_processor.shutdown() finally: - logging.raiseExceptions = old_raise otel_logger.removeHandler(handler) healthy.shutdown.assert_called_once_with() + def test_logging_handler_raise_during_callback_resolution_is_isolated(self): + # If resolving a callback (attribute lookup) raises AND an attached + # handler raises on the resulting exception log, dispatch must continue + # to healthy siblings. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + + class _ResolveBoom: + @property + def shutdown(self): + raise RuntimeError("resolve boom") + + healthy = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(_ResolveBoom()) + multi_processor.add_span_processor(healthy) + + class _OutwardBoom(logging.Handler): + def emit(self, record): + raise RuntimeError("handler boom outward") + + otel_logger = logging.getLogger("opentelemetry.sdk.trace") + old_level = otel_logger.level + otel_logger.setLevel(logging.ERROR) + handler = _OutwardBoom(level=logging.ERROR) + otel_logger.addHandler(handler) + try: + multi_processor.shutdown() + finally: + otel_logger.removeHandler(handler) + otel_logger.setLevel(old_level) + + healthy.shutdown.assert_called_once_with() + def test_logging_handler_outward_raise_inline_fallback_is_isolated(self): # A user logging handler that raises OUTWARD (no self-catch, with From e47c4e8017a5d79a2c9fa5a3b693a500bdeec038 Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 04:59:34 +0800 Subject: [PATCH 21/26] test(sdk): ruff format and scoped protected-access disable Address precommit lint: ruff format both test files and add # pylint: disable=protected-access where tests intentionally shut down the internal executor to exercise the atexit inline path. --- opentelemetry-sdk/tests/logs/test_multi_log_processor.py | 2 +- opentelemetry-sdk/tests/trace/test_span_processor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index cd20482630d..02745a98c3e 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -369,7 +369,6 @@ def emit(self, record): healthy.shutdown.assert_called_once_with() - def test_logging_handler_outward_raise_inline_fallback_is_isolated(self): # A user logging handler that raises OUTWARD (no self-catch, with # raiseExceptions left at its default) must not abort the inline @@ -392,6 +391,7 @@ def emit(self, record): handler = _OutwardBoom(level=logging.DEBUG) otel_logger.setLevel(logging.DEBUG) otel_logger.addHandler(handler) + # pylint: disable=protected-access multi._executor.shutdown(wait=True) try: multi.shutdown() diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 8b3fe599ca4..23648f096fd 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -872,7 +872,6 @@ def emit(self, record): healthy.shutdown.assert_called_once_with() - def test_logging_handler_outward_raise_inline_fallback_is_isolated(self): # A user logging handler that raises OUTWARD (no self-catch, with # raiseExceptions left at its default) must not abort the inline @@ -896,6 +895,7 @@ def emit(self, record): handler = _OutwardBoom(level=logging.DEBUG) otel_logger.setLevel(logging.DEBUG) otel_logger.addHandler(handler) + # pylint: disable=protected-access multi_processor._executor.shutdown(wait=True) try: # Must not raise despite the outward handler raise on the From c150213c6c5e4ebf00a30ff932c6c19d8e6d7c0c Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 05:09:29 +0800 Subject: [PATCH 22/26] test(sdk): queued submit RuntimeError + outward handler combo Patch executor.submit to enqueue via original then raise RuntimeError, with an outward-raising DEBUG handler. Asserts at-most-once dispatch (healthy/faulty each once), 2 enqueues, no escape. --- .../tests/logs/test_multi_log_processor.py | 44 +++++++++++++++++++ .../tests/trace/test_span_processor.py | 43 ++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 02745a98c3e..ea22f391d2f 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -403,6 +403,50 @@ def emit(self, record): healthy.shutdown.assert_called_once_with() faulty.shutdown.assert_called_once_with() + def test_submit_runtime_error_after_enqueue_does_not_double_invoke(self): + # When executor.submit raises RuntimeError AFTER the work item was + # actually enqueued, the inline fallback must not run the callback a + # second time. The shared _CallbackAttempt lock guarantees at-most-once, + # and an outward-raising user handler on the diagnostic log must not + # escape either. + multi = ConcurrentMultiLogRecordProcessor(2) + faulty = Mock(spec=LogRecordProcessor) + faulty.shutdown.side_effect = RuntimeError("processor fault") + healthy = Mock(spec=LogRecordProcessor) + multi.add_log_record_processor(faulty) + multi.add_log_record_processor(healthy) + + class _OutwardBoom(logging.Handler): + def emit(self, record): + raise RuntimeError("handler boom outward") + + otel_logger = logging.getLogger("opentelemetry.sdk._logs._internal") + old_level = otel_logger.level + otel_logger.setLevel(logging.DEBUG) + handler = _OutwardBoom(level=logging.DEBUG) + otel_logger.addHandler(handler) + + # pylint: disable=protected-access + orig_submit = multi._executor.submit + submit_calls = [] + + def patched(fn, *a, **kw): + fut = orig_submit(fn, *a, **kw) # actually enqueue + submit_calls.append(fut) + raise RuntimeError("cannot start new thread") + + multi._executor.submit = patched + try: + multi.shutdown() + finally: + otel_logger.removeHandler(handler) + otel_logger.setLevel(old_level) + multi._executor.shutdown(wait=True) + + assert len(submit_calls) == 2 + healthy.shutdown.assert_called_once_with() + faulty.shutdown.assert_called_once_with() + def test_real_logging_handler_reentry_runs_inline(self): # Wire the REAL LoggingHandler bridge into a ConcurrentMultiLogRecordProcessor. # When a child processor emits a Python log on a worker thread, the bridge diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 23648f096fd..6bfeaa991ec 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -908,6 +908,49 @@ def emit(self, record): healthy.shutdown.assert_called_once_with() faulty.shutdown.assert_called_once_with() + def test_submit_runtime_error_after_enqueue_does_not_double_invoke(self): + # When executor.submit raises RuntimeError AFTER the work item was + # actually enqueued (e.g. Thread.start failure), the inline fallback + # must not run the callback a second time. The shared _CallbackAttempt + # lock guarantees at-most-once, and an outward-raising user handler on + # the diagnostic log must not escape either. + multi_processor = trace.ConcurrentMultiSpanProcessor(2) + faulty = mock.Mock(spec=trace.SpanProcessor) + faulty.shutdown.side_effect = RuntimeError("processor fault") + healthy = mock.Mock(spec=trace.SpanProcessor) + multi_processor.add_span_processor(faulty) + multi_processor.add_span_processor(healthy) + + class _OutwardBoom(logging.Handler): + def emit(self, record): + raise RuntimeError("handler boom outward") + + otel_logger = logging.getLogger("opentelemetry.sdk.trace") + old_level = otel_logger.level + otel_logger.setLevel(logging.DEBUG) + handler = _OutwardBoom(level=logging.DEBUG) + otel_logger.addHandler(handler) + + # pylint: disable=protected-access + orig_submit = multi_processor._executor.submit + submit_calls = [] + + def patched(fn, *a, **kw): + fut = orig_submit(fn, *a, **kw) # actually enqueue + submit_calls.append(fut) + raise RuntimeError("cannot start new thread") # then raise + + multi_processor._executor.submit = patched + try: + multi_processor.shutdown() + finally: + otel_logger.removeHandler(handler) + otel_logger.setLevel(old_level) + + assert len(submit_calls) == 2 # one per processor, both enqueued + healthy.shutdown.assert_called_once_with() + faulty.shutdown.assert_called_once_with() + @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") def test_batch_span_processor_fork(self): multiprocessing_context = multiprocessing.get_context("fork") From 5c67262d36615ea5d5fa677e3de369e72189868f Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 05:11:36 +0800 Subject: [PATCH 23/26] test(sdk): combo test waits enqueued futures and asserts stderr fallback Wait on enqueued futures before asserting, then executor.shutdown(wait=True), and assert stderr contains the diagnostic fallback record. --- .../tests/logs/test_multi_log_processor.py | 18 +++++++++++---- .../tests/trace/test_span_processor.py | 22 +++++++++++++++---- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index ea22f391d2f..f5237f53a91 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -408,7 +408,10 @@ def test_submit_runtime_error_after_enqueue_does_not_double_invoke(self): # actually enqueued, the inline fallback must not run the callback a # second time. The shared _CallbackAttempt lock guarantees at-most-once, # and an outward-raising user handler on the diagnostic log must not - # escape either. + # escape. Stderr must still contain the diagnostic record. + import io + import sys as _sys + multi = ConcurrentMultiLogRecordProcessor(2) faulty = Mock(spec=LogRecordProcessor) faulty.shutdown.side_effect = RuntimeError("processor fault") @@ -428,24 +431,31 @@ def emit(self, record): # pylint: disable=protected-access orig_submit = multi._executor.submit - submit_calls = [] + enqueued_futures = [] def patched(fn, *a, **kw): fut = orig_submit(fn, *a, **kw) # actually enqueue - submit_calls.append(fut) + enqueued_futures.append(fut) raise RuntimeError("cannot start new thread") multi._executor.submit = patched + stderr_buf = io.StringIO() + old_stderr = _sys.stderr + _sys.stderr = stderr_buf try: multi.shutdown() + for f in enqueued_futures: + f.result() finally: + _sys.stderr = old_stderr otel_logger.removeHandler(handler) otel_logger.setLevel(old_level) multi._executor.shutdown(wait=True) - assert len(submit_calls) == 2 + assert len(enqueued_futures) == 2 healthy.shutdown.assert_called_once_with() faulty.shutdown.assert_called_once_with() + assert "Executor is shut down" in stderr_buf.getvalue() def test_real_logging_handler_reentry_runs_inline(self): # Wire the REAL LoggingHandler bridge into a ConcurrentMultiLogRecordProcessor. diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 6bfeaa991ec..c6bf6eb5414 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -913,7 +913,11 @@ def test_submit_runtime_error_after_enqueue_does_not_double_invoke(self): # actually enqueued (e.g. Thread.start failure), the inline fallback # must not run the callback a second time. The shared _CallbackAttempt # lock guarantees at-most-once, and an outward-raising user handler on - # the diagnostic log must not escape either. + # the diagnostic log must not escape. Stderr must still contain the + # diagnostic record (safe fallback). + import io + import sys as _sys + multi_processor = trace.ConcurrentMultiSpanProcessor(2) faulty = mock.Mock(spec=trace.SpanProcessor) faulty.shutdown.side_effect = RuntimeError("processor fault") @@ -933,23 +937,33 @@ def emit(self, record): # pylint: disable=protected-access orig_submit = multi_processor._executor.submit - submit_calls = [] + enqueued_futures = [] def patched(fn, *a, **kw): fut = orig_submit(fn, *a, **kw) # actually enqueue - submit_calls.append(fut) + enqueued_futures.append(fut) raise RuntimeError("cannot start new thread") # then raise multi_processor._executor.submit = patched + stderr_buf = io.StringIO() + old_stderr = _sys.stderr + _sys.stderr = stderr_buf try: multi_processor.shutdown() + # Wait for all enqueued workers to finish before asserting. + for f in enqueued_futures: + f.result() finally: + _sys.stderr = old_stderr otel_logger.removeHandler(handler) otel_logger.setLevel(old_level) + multi_processor._executor.shutdown(wait=True) - assert len(submit_calls) == 2 # one per processor, both enqueued + assert len(enqueued_futures) == 2 # one per processor, both enqueued healthy.shutdown.assert_called_once_with() faulty.shutdown.assert_called_once_with() + # Diagnostic stderr fallback must have recorded the shutdown message. + assert "Executor is shut down" in stderr_buf.getvalue() @unittest.skipUnless(hasattr(os, "fork"), "needs *nix") def test_batch_span_processor_fork(self): From cf60050f07a94a97f61ac272cd543bd36ef2c0a1 Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 05:12:53 +0800 Subject: [PATCH 24/26] test(sdk): move io/sys imports to top-level for lint compliance --- .../tests/logs/test_multi_log_processor.py | 10 +++++----- opentelemetry-sdk/tests/trace/test_span_processor.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index f5237f53a91..3693e11a043 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -3,7 +3,9 @@ # pylint:disable=protected-access,no-self-use,no-member +import io import logging +import sys import threading import time import unittest @@ -409,8 +411,6 @@ def test_submit_runtime_error_after_enqueue_does_not_double_invoke(self): # second time. The shared _CallbackAttempt lock guarantees at-most-once, # and an outward-raising user handler on the diagnostic log must not # escape. Stderr must still contain the diagnostic record. - import io - import sys as _sys multi = ConcurrentMultiLogRecordProcessor(2) faulty = Mock(spec=LogRecordProcessor) @@ -440,14 +440,14 @@ def patched(fn, *a, **kw): multi._executor.submit = patched stderr_buf = io.StringIO() - old_stderr = _sys.stderr - _sys.stderr = stderr_buf + old_stderr = sys.stderr + sys.stderr = stderr_buf try: multi.shutdown() for f in enqueued_futures: f.result() finally: - _sys.stderr = old_stderr + sys.stderr = old_stderr otel_logger.removeHandler(handler) otel_logger.setLevel(old_level) multi._executor.shutdown(wait=True) diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index c6bf6eb5414..8430a17c1ea 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -3,9 +3,11 @@ import abc import gc +import io import logging import multiprocessing import os +import sys import time import unittest import weakref @@ -915,8 +917,6 @@ def test_submit_runtime_error_after_enqueue_does_not_double_invoke(self): # lock guarantees at-most-once, and an outward-raising user handler on # the diagnostic log must not escape. Stderr must still contain the # diagnostic record (safe fallback). - import io - import sys as _sys multi_processor = trace.ConcurrentMultiSpanProcessor(2) faulty = mock.Mock(spec=trace.SpanProcessor) @@ -946,15 +946,15 @@ def patched(fn, *a, **kw): multi_processor._executor.submit = patched stderr_buf = io.StringIO() - old_stderr = _sys.stderr - _sys.stderr = stderr_buf + old_stderr = sys.stderr + sys.stderr = stderr_buf try: multi_processor.shutdown() # Wait for all enqueued workers to finish before asserting. for f in enqueued_futures: f.result() finally: - _sys.stderr = old_stderr + sys.stderr = old_stderr otel_logger.removeHandler(handler) otel_logger.setLevel(old_level) multi_processor._executor.shutdown(wait=True) From 14adf756d7d60380cc51b7801bbcca581703d80f Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 05:20:53 +0800 Subject: [PATCH 25/26] test(sdk): rename loop variable f to fut for pylint C0103 --- opentelemetry-sdk/tests/logs/test_multi_log_processor.py | 4 ++-- opentelemetry-sdk/tests/trace/test_span_processor.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py index 3693e11a043..6a5d22b961c 100644 --- a/opentelemetry-sdk/tests/logs/test_multi_log_processor.py +++ b/opentelemetry-sdk/tests/logs/test_multi_log_processor.py @@ -444,8 +444,8 @@ def patched(fn, *a, **kw): sys.stderr = stderr_buf try: multi.shutdown() - for f in enqueued_futures: - f.result() + for fut in enqueued_futures: + fut.result() finally: sys.stderr = old_stderr otel_logger.removeHandler(handler) diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 8430a17c1ea..4563d66035e 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -951,8 +951,8 @@ def patched(fn, *a, **kw): try: multi_processor.shutdown() # Wait for all enqueued workers to finish before asserting. - for f in enqueued_futures: - f.result() + for fut in enqueued_futures: + fut.result() finally: sys.stderr = old_stderr otel_logger.removeHandler(handler) From 7b99521804ed365099f6f5546ece97c98fd134ce Mon Sep 17 00:00:00 2001 From: RichardoMrMu <947676438@qq.com> Date: Tue, 27 Oct 2026 05:22:25 +0800 Subject: [PATCH 26/26] test(sdk): trim combo test docstring to satisfy pylint C0302 --- opentelemetry-sdk/tests/trace/test_span_processor.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/opentelemetry-sdk/tests/trace/test_span_processor.py b/opentelemetry-sdk/tests/trace/test_span_processor.py index 4563d66035e..89c50fc3a39 100644 --- a/opentelemetry-sdk/tests/trace/test_span_processor.py +++ b/opentelemetry-sdk/tests/trace/test_span_processor.py @@ -911,13 +911,9 @@ def emit(self, record): faulty.shutdown.assert_called_once_with() def test_submit_runtime_error_after_enqueue_does_not_double_invoke(self): - # When executor.submit raises RuntimeError AFTER the work item was - # actually enqueued (e.g. Thread.start failure), the inline fallback - # must not run the callback a second time. The shared _CallbackAttempt - # lock guarantees at-most-once, and an outward-raising user handler on - # the diagnostic log must not escape. Stderr must still contain the - # diagnostic record (safe fallback). - + # submit raises RuntimeError after enqueue: inline fallback must not + # double-invoke; outward DEBUG handler must not escape; stderr fallback + # must still record the diagnostic. multi_processor = trace.ConcurrentMultiSpanProcessor(2) faulty = mock.Mock(spec=trace.SpanProcessor) faulty.shutdown.side_effect = RuntimeError("processor fault")