diff --git a/.generator/schemas/v1/openapi.yaml b/.generator/schemas/v1/openapi.yaml index 47f01897fd..c8467da80b 100644 --- a/.generator/schemas/v1/openapi.yaml +++ b/.generator/schemas/v1/openapi.yaml @@ -9793,30 +9793,53 @@ components: - data_source - query type: object + MonitorFormulaAndFunctionDataJobsMonitorOptions: + description: Monitor configuration options for data jobs queries. + properties: + alert_on_every_failure: + description: |- + Alert separately for each failed run instead of on the count of trailing failures. + Only supported for `failed_runs` monitors with the `aastra` query dialect. It requires + a `> 0` threshold with no warning threshold, an evaluation window of at most two days, + and renotification disabled. + example: true + type: boolean + type: object MonitorFormulaAndFunctionDataJobsQueryDefinition: description: A formula and functions data jobs query. properties: job_type: description: |- The type of job being monitored. Valid values include: - `databricks.job`, `spark.application`, `airflow.dag`, - `dbt.job`, `dbt.model`, `dbt.test`, `glue.job`. + `databricks.job`, `spark.application`, `airflow.dag`, `dbt.job`, `glue.job`, `adf.pipeline`. + The `aastra` dialect also accepts `airflow.job`, `dbt.model`, and `dbt.test`. Custom job types are supported with the `custom.ol.` prefix. example: "databricks.job" type: string jobs_query: - description: Filter expression used to select the jobs to monitor. + description: |- + Filter expression used to select the jobs to monitor. Its syntax depends on `query_dialect`: + a metric tag filter such as `job_name:smoke*` for `metric`, or an Aastra entity search + such as ``search for databricks_job where `@team:data-eng` `` for `aastra`. example: "job_name:smoke*" type: string + monitor_options: + $ref: "#/components/schemas/MonitorFormulaAndFunctionDataJobsMonitorOptions" name: description: Name of the query for use in formulas. Must be `run_query`. example: "run_query" type: string query_dialect: description: |- - Query dialect for data jobs queries. Currently only `metric` is supported. + Query dialect for data jobs queries. Valid values are `metric` and `aastra`. example: "metric" type: string + run_filter: + description: |- + Optional filter on individual runs, using metrics filter syntax. + Only supported with the `aastra` query dialect. + example: "env:prod" + type: string required: - name - jobs_query @@ -9831,6 +9854,18 @@ components: type: string x-enum-varnames: - DATA_QUALITY_METRICS + MonitorFormulaAndFunctionDataQualityDetectionType: + description: |- + How a `data-jobs alert` job duration monitor detects a breach. Must match the + `.detection()` call in the monitor query. Required for job duration monitors. + enum: + - static + - anomaly + example: "anomaly" + type: string + x-enum-varnames: + - STATIC + - ANOMALY MonitorFormulaAndFunctionDataQualityDiffFunction: description: |- Function applied to the measure before it is compared against the predicted bounds. @@ -9891,7 +9926,8 @@ components: The data quality measure to query. Common values include: `bytes`, `cardinality`, `custom`, `freshness`, `max`, `mean`, `min`, `nullness`, `percent_negative`, `percent_zero`, `row_count`, `stddev`, - `sum`, `uniqueness`. Additional values may be supported. + `sum`, `uniqueness`. `data-jobs alert` job duration monitors use `duration`. + Additional values may be supported. example: "row_count" type: string MonitorFormulaAndFunctionDataQualityModelBoundsOverride: @@ -9961,6 +9997,14 @@ components: MonitorFormulaAndFunctionDataQualityMonitorOptions: description: Monitor configuration options for data quality queries. properties: + consecutive_runs: + description: |- + Number of consecutive runs that must breach before the monitor alerts. + Only supported by `data-jobs alert` job duration monitors. Defaults to `1`. + example: 1 + format: int64 + minimum: 1 + type: integer crontab_override: description: Crontab expression to override the default schedule. example: "* * * 10" @@ -9973,6 +10017,8 @@ components: description: Custom WHERE clause for the query. example: "USER_ID = 123" type: string + detection_type: + $ref: "#/components/schemas/MonitorFormulaAndFunctionDataQualityDetectionType" group_by_columns: description: Columns to group results by. example: ["col1", "col2"] @@ -10020,6 +10066,12 @@ components: description: Name of the query for use in formulas. example: "query1" type: string + run_filter: + description: |- + Optional filter on individual runs, using metrics filter syntax. + Only supported by `data-jobs alert` job duration monitors. + example: "env:prod" + type: string schema_version: description: Schema version for the data quality query. example: "0.0.1" @@ -37275,6 +37327,22 @@ paths: - `time_window` #m (between 1 and 2880), #h (between 1 and 48). - `operator` `<`, `<=`, `>`, `>=`, `==`, or `!=`. - `#` an integer or decimal number used to set the threshold. + + **Data Jobs Alert Query** + + Job status example: `formula("failed_runs(run_query)").by(group_keys).last(time_window) > #` + + Job duration examples: `formula("job_duration(run_query)").detection(static).by(group_keys) > duration` + and `formula("job_duration(run_query)").detection(anomaly).by(group_keys) > 0` + + - `run_query`: The name of the single query variable in `options.variables`. Job status monitors + use a data jobs query; job duration monitors use a data quality query with the `duration` measure. + - `group_keys`: The tags to group by, such as `entity_id,name,workspace_name`. + - `time_window`: Number of minutes (#m), hours (#h), or days (#d), up to `10d`. + - `#`: The number of trailing failed runs to alert above, as a non-negative integer. + - `duration`: The duration to alert above, such as `2h`, `30m`, or a number of seconds. + - `options.thresholds.critical`: Must equal the query threshold, in seconds for job duration. + Anomaly job duration monitors omit `options.thresholds`. operationId: CreateMonitor requestBody: content: diff --git a/examples/v1/monitors/CreateMonitor_1267012748.py b/examples/v1/monitors/CreateMonitor_1267012748.py new file mode 100644 index 0000000000..cbffcecde0 --- /dev/null +++ b/examples/v1/monitors/CreateMonitor_1267012748.py @@ -0,0 +1,62 @@ +""" +Create a Data Jobs duration monitor returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v1.api.monitors_api import MonitorsApi +from datadog_api_client.v1.model.monitor import Monitor +from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_data_source import ( + MonitorFormulaAndFunctionDataQualityDataSource, +) +from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_detection_type import ( + MonitorFormulaAndFunctionDataQualityDetectionType, +) +from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_monitor_options import ( + MonitorFormulaAndFunctionDataQualityMonitorOptions, +) +from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_query_definition import ( + MonitorFormulaAndFunctionDataQualityQueryDefinition, +) +from datadog_api_client.v1.model.monitor_options import MonitorOptions +from datadog_api_client.v1.model.monitor_type import MonitorType + +body = Monitor( + name="Example-Monitor", + type=MonitorType.DATA_JOBS_ALERT, + query='formula("job_duration(run_query)").detection(anomaly).by(name,workspace_name,entity_id) > 0', + message="Data jobs duration alert triggered", + tags=[ + "test:examplemonitor", + "env:ci", + ], + options=MonitorOptions( + variables=[ + MonitorFormulaAndFunctionDataQualityQueryDefinition( + name="run_query", + data_source=MonitorFormulaAndFunctionDataQualityDataSource.DATA_QUALITY_METRICS, + schema_version="0.0.1", + measure="duration", + filter="search for databricks_job", + run_filter="env:prod", + group_by=[ + "name", + "workspace_name", + "entity_id", + ], + monitor_options=MonitorFormulaAndFunctionDataQualityMonitorOptions( + detection_type=MonitorFormulaAndFunctionDataQualityDetectionType.ANOMALY, + consecutive_runs=2, + ), + ), + ], + ), +) + +configuration = Configuration() +configuration.access_token = environ["DD_BEARER_TOKEN"] +with ApiClient(configuration) as api_client: + api_instance = MonitorsApi(api_client) + response = api_instance.create_monitor(body=body) + + print(response) diff --git a/examples/v1/monitors/CreateMonitor_1302731819.py b/examples/v1/monitors/CreateMonitor_1302731819.py new file mode 100644 index 0000000000..da9748f655 --- /dev/null +++ b/examples/v1/monitors/CreateMonitor_1302731819.py @@ -0,0 +1,53 @@ +""" +Create a Data Jobs monitor with the aastra query dialect returns "OK" response +""" + +from os import environ +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v1.api.monitors_api import MonitorsApi +from datadog_api_client.v1.model.monitor import Monitor +from datadog_api_client.v1.model.monitor_formula_and_function_data_jobs_monitor_options import ( + MonitorFormulaAndFunctionDataJobsMonitorOptions, +) +from datadog_api_client.v1.model.monitor_formula_and_function_data_jobs_query_definition import ( + MonitorFormulaAndFunctionDataJobsQueryDefinition, +) +from datadog_api_client.v1.model.monitor_options import MonitorOptions +from datadog_api_client.v1.model.monitor_thresholds import MonitorThresholds +from datadog_api_client.v1.model.monitor_type import MonitorType + +body = Monitor( + name="Example-Monitor", + type=MonitorType.DATA_JOBS_ALERT, + query='formula("failed_runs(run_query)").by(name,workspace_name,entity_id).last(1d) > 0', + message="Data jobs alert triggered", + tags=[ + "test:examplemonitor", + "env:ci", + ], + options=MonitorOptions( + thresholds=MonitorThresholds( + critical=0.0, + ), + variables=[ + MonitorFormulaAndFunctionDataJobsQueryDefinition( + name="run_query", + jobs_query="search for databricks_job", + job_type="databricks.job", + query_dialect="aastra", + run_filter="env:prod", + monitor_options=MonitorFormulaAndFunctionDataJobsMonitorOptions( + alert_on_every_failure=True, + ), + ), + ], + ), +) + +configuration = Configuration() +configuration.access_token = environ["DD_BEARER_TOKEN"] +with ApiClient(configuration) as api_client: + api_instance = MonitorsApi(api_client) + response = api_instance.create_monitor(body=body) + + print(response) diff --git a/src/datadog_api_client/v1/api/monitors_api.py b/src/datadog_api_client/v1/api/monitors_api.py index 47bbf4cded..4e94af9f88 100644 --- a/src/datadog_api_client/v1/api/monitors_api.py +++ b/src/datadog_api_client/v1/api/monitors_api.py @@ -623,6 +623,22 @@ def create_monitor( * ``operator`` ``<`` , ``<=`` , ``>`` , ``>=`` , ``==`` , or ``!=``. * ``#`` an integer or decimal number used to set the threshold. + **Data Jobs Alert Query** + + Job status example: ``formula("failed_runs(run_query)").by(group_keys).last(time_window) > #`` + + Job duration examples: ``formula("job_duration(run_query)").detection(static).by(group_keys) > duration`` + and ``formula("job_duration(run_query)").detection(anomaly).by(group_keys) > 0`` + + * ``run_query`` : The name of the single query variable in ``options.variables``. Job status monitors + use a data jobs query; job duration monitors use a data quality query with the ``duration`` measure. + * ``group_keys`` : The tags to group by, such as ``entity_id,name,workspace_name``. + * ``time_window`` : Number of minutes (#m), hours (#h), or days (#d), up to ``10d``. + * ``#`` : The number of trailing failed runs to alert above, as a non-negative integer. + * ``duration`` : The duration to alert above, such as ``2h`` , ``30m`` , or a number of seconds. + * ``options.thresholds.critical`` : Must equal the query threshold, in seconds for job duration. + Anomaly job duration monitors omit ``options.thresholds``. + :param body: Create a monitor request body. :type body: Monitor :rtype: Monitor diff --git a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_jobs_monitor_options.py b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_jobs_monitor_options.py new file mode 100644 index 0000000000..44194f5a82 --- /dev/null +++ b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_jobs_monitor_options.py @@ -0,0 +1,39 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + +from typing import Union + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, + unset, + UnsetType, +) + + +class MonitorFormulaAndFunctionDataJobsMonitorOptions(ModelNormal): + @cached_property + def openapi_types(_): + return { + "alert_on_every_failure": (bool,), + } + + attribute_map = { + "alert_on_every_failure": "alert_on_every_failure", + } + + def __init__(self_, alert_on_every_failure: Union[bool, UnsetType] = unset, **kwargs): + """ + Monitor configuration options for data jobs queries. + + :param alert_on_every_failure: Alert separately for each failed run instead of on the count of trailing failures. + Only supported for ``failed_runs`` monitors with the ``aastra`` query dialect. It requires + a ``> 0`` threshold with no warning threshold, an evaluation window of at most two days, + and renotification disabled. + :type alert_on_every_failure: bool, optional + """ + if alert_on_every_failure is not unset: + kwargs["alert_on_every_failure"] = alert_on_every_failure + super().__init__(kwargs) diff --git a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_jobs_query_definition.py b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_jobs_query_definition.py index cad58de6f0..8b0c95eb60 100644 --- a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_jobs_query_definition.py +++ b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_jobs_query_definition.py @@ -3,49 +3,88 @@ # Copyright 2019-Present Datadog, Inc. from __future__ import annotations +from typing import Union, TYPE_CHECKING from datadog_api_client.model_utils import ( ModelNormal, cached_property, + unset, + UnsetType, ) +if TYPE_CHECKING: + from datadog_api_client.v1.model.monitor_formula_and_function_data_jobs_monitor_options import ( + MonitorFormulaAndFunctionDataJobsMonitorOptions, + ) + + class MonitorFormulaAndFunctionDataJobsQueryDefinition(ModelNormal): @cached_property def openapi_types(_): + from datadog_api_client.v1.model.monitor_formula_and_function_data_jobs_monitor_options import ( + MonitorFormulaAndFunctionDataJobsMonitorOptions, + ) + return { "job_type": (str,), "jobs_query": (str,), + "monitor_options": (MonitorFormulaAndFunctionDataJobsMonitorOptions,), "name": (str,), "query_dialect": (str,), + "run_filter": (str,), } attribute_map = { "job_type": "job_type", "jobs_query": "jobs_query", + "monitor_options": "monitor_options", "name": "name", "query_dialect": "query_dialect", + "run_filter": "run_filter", } - def __init__(self_, job_type: str, jobs_query: str, name: str, query_dialect: str, **kwargs): + def __init__( + self_, + job_type: str, + jobs_query: str, + name: str, + query_dialect: str, + monitor_options: Union[MonitorFormulaAndFunctionDataJobsMonitorOptions, UnsetType] = unset, + run_filter: Union[str, UnsetType] = unset, + **kwargs, + ): """ A formula and functions data jobs query. :param job_type: The type of job being monitored. Valid values include: - ``databricks.job`` , ``spark.application`` , ``airflow.dag`` , - ``dbt.job`` , ``dbt.model`` , ``dbt.test`` , ``glue.job``. + ``databricks.job`` , ``spark.application`` , ``airflow.dag`` , ``dbt.job`` , ``glue.job`` , ``adf.pipeline``. + The ``aastra`` dialect also accepts ``airflow.job`` , ``dbt.model`` , and ``dbt.test``. Custom job types are supported with the ``custom.ol.`` prefix. :type job_type: str - :param jobs_query: Filter expression used to select the jobs to monitor. + :param jobs_query: Filter expression used to select the jobs to monitor. Its syntax depends on ``query_dialect`` : + a metric tag filter such as ``job_name:smoke*`` for ``metric`` , or an Aastra entity search + such as ``search for databricks_job where `@team:data-eng` `` for ``aastra``. :type jobs_query: str + :param monitor_options: Monitor configuration options for data jobs queries. + :type monitor_options: MonitorFormulaAndFunctionDataJobsMonitorOptions, optional + :param name: Name of the query for use in formulas. Must be ``run_query``. :type name: str - :param query_dialect: Query dialect for data jobs queries. Currently only ``metric`` is supported. + :param query_dialect: Query dialect for data jobs queries. Valid values are ``metric`` and ``aastra``. :type query_dialect: str + + :param run_filter: Optional filter on individual runs, using metrics filter syntax. + Only supported with the ``aastra`` query dialect. + :type run_filter: str, optional """ + if monitor_options is not unset: + kwargs["monitor_options"] = monitor_options + if run_filter is not unset: + kwargs["run_filter"] = run_filter super().__init__(kwargs) self_.job_type = job_type diff --git a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_detection_type.py b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_detection_type.py new file mode 100644 index 0000000000..bd3bb5e5d1 --- /dev/null +++ b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_detection_type.py @@ -0,0 +1,39 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelSimple, + cached_property, +) + +from typing import ClassVar + + +class MonitorFormulaAndFunctionDataQualityDetectionType(ModelSimple): + """ + How a `data-jobs alert` job duration monitor detects a breach. Must match the + `.detection()` call in the monitor query. Required for job duration monitors. + + :param value: Must be one of ["static", "anomaly"]. + :type value: str + """ + + allowed_values = { + "static", + "anomaly", + } + STATIC: ClassVar["MonitorFormulaAndFunctionDataQualityDetectionType"] + ANOMALY: ClassVar["MonitorFormulaAndFunctionDataQualityDetectionType"] + + @cached_property + def openapi_types(_): + return { + "value": (str,), + } + + +MonitorFormulaAndFunctionDataQualityDetectionType.STATIC = MonitorFormulaAndFunctionDataQualityDetectionType("static") +MonitorFormulaAndFunctionDataQualityDetectionType.ANOMALY = MonitorFormulaAndFunctionDataQualityDetectionType("anomaly") diff --git a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_monitor_options.py b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_monitor_options.py index e457b56c12..b5a7d592c8 100644 --- a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_monitor_options.py +++ b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_monitor_options.py @@ -14,6 +14,9 @@ if TYPE_CHECKING: + from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_detection_type import ( + MonitorFormulaAndFunctionDataQualityDetectionType, + ) from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_model_configuration import ( MonitorFormulaAndFunctionDataQualityModelConfiguration, ) @@ -26,8 +29,17 @@ class MonitorFormulaAndFunctionDataQualityMonitorOptions(ModelNormal): + validations = { + "consecutive_runs": { + "inclusive_minimum": 1, + }, + } + @cached_property def openapi_types(_): + from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_detection_type import ( + MonitorFormulaAndFunctionDataQualityDetectionType, + ) from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_model_configuration import ( MonitorFormulaAndFunctionDataQualityModelConfiguration, ) @@ -39,9 +51,11 @@ def openapi_types(_): ) return { + "consecutive_runs": (int,), "crontab_override": (str,), "custom_sql": (str,), "custom_where": (str,), + "detection_type": (MonitorFormulaAndFunctionDataQualityDetectionType,), "group_by_columns": ([str],), "model_configuration": (MonitorFormulaAndFunctionDataQualityModelConfiguration,), "model_type_override": (MonitorFormulaAndFunctionDataQualityModelTypeOverride,), @@ -50,9 +64,11 @@ def openapi_types(_): } attribute_map = { + "consecutive_runs": "consecutive_runs", "crontab_override": "crontab_override", "custom_sql": "custom_sql", "custom_where": "custom_where", + "detection_type": "detection_type", "group_by_columns": "group_by_columns", "model_configuration": "model_configuration", "model_type_override": "model_type_override", @@ -62,9 +78,11 @@ def openapi_types(_): def __init__( self_, + consecutive_runs: Union[int, UnsetType] = unset, crontab_override: Union[str, UnsetType] = unset, custom_sql: Union[str, UnsetType] = unset, custom_where: Union[str, UnsetType] = unset, + detection_type: Union[MonitorFormulaAndFunctionDataQualityDetectionType, UnsetType] = unset, group_by_columns: Union[List[str], UnsetType] = unset, model_configuration: Union[MonitorFormulaAndFunctionDataQualityModelConfiguration, UnsetType] = unset, model_type_override: Union[MonitorFormulaAndFunctionDataQualityModelTypeOverride, UnsetType] = unset, @@ -75,6 +93,10 @@ def __init__( """ Monitor configuration options for data quality queries. + :param consecutive_runs: Number of consecutive runs that must breach before the monitor alerts. + Only supported by ``data-jobs alert`` job duration monitors. Defaults to ``1``. + :type consecutive_runs: int, optional + :param crontab_override: Crontab expression to override the default schedule. :type crontab_override: str, optional @@ -84,6 +106,10 @@ def __init__( :param custom_where: Custom WHERE clause for the query. :type custom_where: str, optional + :param detection_type: How a ``data-jobs alert`` job duration monitor detects a breach. Must match the + ``.detection()`` call in the monitor query. Required for job duration monitors. + :type detection_type: MonitorFormulaAndFunctionDataQualityDetectionType, optional + :param group_by_columns: Columns to group results by. :type group_by_columns: [str], optional @@ -102,12 +128,16 @@ def __init__( across two data entities and alerts on the difference between them. :type source_to_target_config: MonitorFormulaAndFunctionDataQualitySourceToTargetConfig, optional """ + if consecutive_runs is not unset: + kwargs["consecutive_runs"] = consecutive_runs if crontab_override is not unset: kwargs["crontab_override"] = crontab_override if custom_sql is not unset: kwargs["custom_sql"] = custom_sql if custom_where is not unset: kwargs["custom_where"] = custom_where + if detection_type is not unset: + kwargs["detection_type"] = detection_type if group_by_columns is not unset: kwargs["group_by_columns"] = group_by_columns if model_configuration is not unset: diff --git a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_query_definition.py b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_query_definition.py index ef57341eaa..7958de3ca6 100644 --- a/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_query_definition.py +++ b/src/datadog_api_client/v1/model/monitor_formula_and_function_data_quality_query_definition.py @@ -39,6 +39,7 @@ def openapi_types(_): "measure": (str,), "monitor_options": (MonitorFormulaAndFunctionDataQualityMonitorOptions,), "name": (str,), + "run_filter": (str,), "schema_version": (str,), "scope": (str,), } @@ -50,6 +51,7 @@ def openapi_types(_): "measure": "measure", "monitor_options": "monitor_options", "name": "name", + "run_filter": "run_filter", "schema_version": "schema_version", "scope": "scope", } @@ -62,6 +64,7 @@ def __init__( name: str, group_by: Union[List[str], UnsetType] = unset, monitor_options: Union[MonitorFormulaAndFunctionDataQualityMonitorOptions, UnsetType] = unset, + run_filter: Union[str, UnsetType] = unset, schema_version: Union[str, UnsetType] = unset, scope: Union[str, UnsetType] = unset, **kwargs, @@ -81,7 +84,8 @@ def __init__( :param measure: The data quality measure to query. Common values include: ``bytes`` , ``cardinality`` , ``custom`` , ``freshness`` , ``max`` , ``mean`` , ``min`` , ``nullness`` , ``percent_negative`` , ``percent_zero`` , ``row_count`` , ``stddev`` , - ``sum`` , ``uniqueness``. Additional values may be supported. + ``sum`` , ``uniqueness``. ``data-jobs alert`` job duration monitors use ``duration``. + Additional values may be supported. :type measure: str :param monitor_options: Monitor configuration options for data quality queries. @@ -90,6 +94,10 @@ def __init__( :param name: Name of the query for use in formulas. :type name: str + :param run_filter: Optional filter on individual runs, using metrics filter syntax. + Only supported by ``data-jobs alert`` job duration monitors. + :type run_filter: str, optional + :param schema_version: Schema version for the data quality query. :type schema_version: str, optional @@ -101,6 +109,8 @@ def __init__( kwargs["group_by"] = group_by if monitor_options is not unset: kwargs["monitor_options"] = monitor_options + if run_filter is not unset: + kwargs["run_filter"] = run_filter if schema_version is not unset: kwargs["schema_version"] = schema_version if scope is not unset: diff --git a/src/datadog_api_client/v1/model/monitor_formula_and_function_query_definition.py b/src/datadog_api_client/v1/model/monitor_formula_and_function_query_definition.py index ce4c3bbda6..07eb35e8a3 100644 --- a/src/datadog_api_client/v1/model/monitor_formula_and_function_query_definition.py +++ b/src/datadog_api_client/v1/model/monitor_formula_and_function_query_definition.py @@ -45,12 +45,17 @@ def __init__(self, **kwargs): :param measure: The data quality measure to query. Common values include: `bytes`, `cardinality`, `custom`, `freshness`, `max`, `mean`, `min`, `nullness`, `percent_negative`, `percent_zero`, `row_count`, `stddev`, - `sum`, `uniqueness`. Additional values may be supported. + `sum`, `uniqueness`. `data-jobs alert` job duration monitors use `duration`. + Additional values may be supported. :type measure: str :param monitor_options: Monitor configuration options for data quality queries. :type monitor_options: MonitorFormulaAndFunctionDataQualityMonitorOptions, optional + :param run_filter: Optional filter on individual runs, using metrics filter syntax. + Only supported by `data-jobs alert` job duration monitors. + :type run_filter: str, optional + :param schema_version: Schema version for the data quality query. :type schema_version: str, optional @@ -59,15 +64,17 @@ def __init__(self, **kwargs): :type scope: str, optional :param job_type: The type of job being monitored. Valid values include: - `databricks.job`, `spark.application`, `airflow.dag`, - `dbt.job`, `dbt.model`, `dbt.test`, `glue.job`. + `databricks.job`, `spark.application`, `airflow.dag`, `dbt.job`, `glue.job`, `adf.pipeline`. + The `aastra` dialect also accepts `airflow.job`, `dbt.model`, and `dbt.test`. Custom job types are supported with the `custom.ol.` prefix. :type job_type: str - :param jobs_query: Filter expression used to select the jobs to monitor. + :param jobs_query: Filter expression used to select the jobs to monitor. Its syntax depends on `query_dialect`: + a metric tag filter such as `job_name:smoke*` for `metric`, or an Aastra entity search + such as ``search for databricks_job where `@team:data-eng` `` for `aastra`. :type jobs_query: str - :param query_dialect: Query dialect for data jobs queries. Currently only `metric` is supported. + :param query_dialect: Query dialect for data jobs queries. Valid values are `metric` and `aastra`. :type query_dialect: str :param augment_query: Augment query for aggregate augmented queries. Can be an events query or a reference table query. diff --git a/src/datadog_api_client/v1/models/__init__.py b/src/datadog_api_client/v1/models/__init__.py index 135ebf6b1f..85954dfc40 100644 --- a/src/datadog_api_client/v1/models/__init__.py +++ b/src/datadog_api_client/v1/models/__init__.py @@ -558,12 +558,18 @@ from datadog_api_client.v1.model.monitor_formula_and_function_cost_query_definition import ( MonitorFormulaAndFunctionCostQueryDefinition, ) +from datadog_api_client.v1.model.monitor_formula_and_function_data_jobs_monitor_options import ( + MonitorFormulaAndFunctionDataJobsMonitorOptions, +) from datadog_api_client.v1.model.monitor_formula_and_function_data_jobs_query_definition import ( MonitorFormulaAndFunctionDataJobsQueryDefinition, ) from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_data_source import ( MonitorFormulaAndFunctionDataQualityDataSource, ) +from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_detection_type import ( + MonitorFormulaAndFunctionDataQualityDetectionType, +) from datadog_api_client.v1.model.monitor_formula_and_function_data_quality_diff_function import ( MonitorFormulaAndFunctionDataQualityDiffFunction, ) @@ -2017,8 +2023,10 @@ "MonitorFormulaAndFunctionCostAggregator", "MonitorFormulaAndFunctionCostDataSource", "MonitorFormulaAndFunctionCostQueryDefinition", + "MonitorFormulaAndFunctionDataJobsMonitorOptions", "MonitorFormulaAndFunctionDataJobsQueryDefinition", "MonitorFormulaAndFunctionDataQualityDataSource", + "MonitorFormulaAndFunctionDataQualityDetectionType", "MonitorFormulaAndFunctionDataQualityDiffFunction", "MonitorFormulaAndFunctionDataQualityDiffType", "MonitorFormulaAndFunctionDataQualityEntityMetricConfig", diff --git a/tests/generated-test/test-runner-data/manifest.json b/tests/generated-test/test-runner-data/manifest.json index 4f0bdf135b..387a953725 100644 --- a/tests/generated-test/test-runner-data/manifest.json +++ b/tests/generated-test/test-runner-data/manifest.json @@ -1428,6 +1428,13 @@ "scenario": "Create a Cost Monitor returns \"OK\" response", "version": "v1" }, + { + "feature": "Monitors", + "feature_file": "../../v1/features/monitors.feature", + "file": "v1/monitors/create-a-data-jobs-duration-monitor-returns-ok-response.json", + "scenario": "Create a Data Jobs duration monitor returns \"OK\" response", + "version": "v1" + }, { "feature": "Monitors", "feature_file": "../../v1/features/monitors.feature", @@ -1435,6 +1442,13 @@ "scenario": "Create a Data Jobs monitor returns \"OK\" response", "version": "v1" }, + { + "feature": "Monitors", + "feature_file": "../../v1/features/monitors.feature", + "file": "v1/monitors/create-a-data-jobs-monitor-with-the-aastra-query-dialect-returns-ok-response.json", + "scenario": "Create a Data Jobs monitor with the aastra query dialect returns \"OK\" response", + "version": "v1" + }, { "feature": "Monitors", "feature_file": "../../v1/features/monitors.feature", diff --git a/tests/generated-test/test-runner-data/v1/monitors/create-a-data-jobs-duration-monitor-returns-ok-response.json b/tests/generated-test/test-runner-data/v1/monitors/create-a-data-jobs-duration-monitor-returns-ok-response.json new file mode 100644 index 0000000000..97b89686c1 --- /dev/null +++ b/tests/generated-test/test-runner-data/v1/monitors/create-a-data-jobs-duration-monitor-returns-ok-response.json @@ -0,0 +1,56 @@ +{ + "api": "Monitors", + "expected_status": 200, + "feature": "Monitors", + "id": "v1/Monitors/Create a Data Jobs duration monitor returns \"OK\" response", + "operation_id": "CreateMonitor", + "request": { + "body": { + "schema": { + "format": null, + "ref": "Monitor", + "type": "object" + }, + "source": "inline", + "value": { + "message": "Data jobs duration alert triggered", + "name": "{{ unique }}", + "options": { + "variables": [ + { + "data_source": "data_quality_metrics", + "filter": "search for databricks_job", + "group_by": [ + "name", + "workspace_name", + "entity_id" + ], + "measure": "duration", + "monitor_options": { + "consecutive_runs": 2, + "detection_type": "anomaly" + }, + "name": "run_query", + "run_filter": "env:prod", + "schema_version": "0.0.1" + } + ] + }, + "query": "formula(\"job_duration(run_query)\").detection(anomaly).by(name,workspace_name,entity_id) > 0", + "tags": [ + "test:{{ unique_lower_alnum }}", + "env:ci" + ], + "type": "data-jobs alert" + } + }, + "content_type": "application/json", + "method": "POST", + "pagination": false, + "parameters": [], + "path": "/api/v1/monitor" + }, + "scenario": "Create a Data Jobs duration monitor returns \"OK\" response", + "schema_version": 1, + "version": "v1" +} diff --git a/tests/generated-test/test-runner-data/v1/monitors/create-a-data-jobs-monitor-with-the-aastra-query-dialect-returns-ok-response.json b/tests/generated-test/test-runner-data/v1/monitors/create-a-data-jobs-monitor-with-the-aastra-query-dialect-returns-ok-response.json new file mode 100644 index 0000000000..9f662996b9 --- /dev/null +++ b/tests/generated-test/test-runner-data/v1/monitors/create-a-data-jobs-monitor-with-the-aastra-query-dialect-returns-ok-response.json @@ -0,0 +1,52 @@ +{ + "api": "Monitors", + "expected_status": 200, + "feature": "Monitors", + "id": "v1/Monitors/Create a Data Jobs monitor with the aastra query dialect returns \"OK\" response", + "operation_id": "CreateMonitor", + "request": { + "body": { + "schema": { + "format": null, + "ref": "Monitor", + "type": "object" + }, + "source": "inline", + "value": { + "message": "Data jobs alert triggered", + "name": "{{ unique }}", + "options": { + "thresholds": { + "critical": 0 + }, + "variables": [ + { + "job_type": "databricks.job", + "jobs_query": "search for databricks_job", + "monitor_options": { + "alert_on_every_failure": true + }, + "name": "run_query", + "query_dialect": "aastra", + "run_filter": "env:prod" + } + ] + }, + "query": "formula(\"failed_runs(run_query)\").by(name,workspace_name,entity_id).last(1d) > 0", + "tags": [ + "test:{{ unique_lower_alnum }}", + "env:ci" + ], + "type": "data-jobs alert" + } + }, + "content_type": "application/json", + "method": "POST", + "pagination": false, + "parameters": [], + "path": "/api/v1/monitor" + }, + "scenario": "Create a Data Jobs monitor with the aastra query dialect returns \"OK\" response", + "schema_version": 1, + "version": "v1" +} diff --git a/tests/generated-test/test-server-data/v1/monitors.json b/tests/generated-test/test-server-data/v1/monitors.json index 1242fc1f2c..b1fd4a2ab7 100644 --- a/tests/generated-test/test-server-data/v1/monitors.json +++ b/tests/generated-test/test-server-data/v1/monitors.json @@ -192,6 +192,90 @@ "scenario": "Create a Cost Monitor returns \"OK\" response", "version": "v1" }, + { + "feature": "Monitors", + "frozen_at": "2026-11-06T16:42:00.361Z", + "interactions": [ + { + "request": { + "body": { + "type": "json", + "value": { + "message": "Data jobs duration alert triggered", + "name": "Test-Create_a_Data_Jobs_duration_monitor_returns_OK_response-1793983320", + "options": { + "variables": [ + { + "data_source": "data_quality_metrics", + "filter": "search for databricks_job", + "group_by": [ + "name", + "workspace_name", + "entity_id" + ], + "measure": "duration", + "monitor_options": { + "consecutive_runs": 2, + "detection_type": "anomaly" + }, + "name": "run_query", + "run_filter": "env:prod", + "schema_version": "0.0.1" + } + ] + }, + "query": "formula(\"job_duration(run_query)\").detection(anomaly).by(name,workspace_name,entity_id) > 0", + "tags": [ + "test:testcreateadatajobsdurationmonitorreturnsokresponse1793983320", + "env:ci" + ], + "type": "data-jobs alert" + } + }, + "content_type": "application/json", + "method": "POST", + "path": "/api/v1/monitor", + "query": [] + }, + "response": { + "body": { + "encoding": "text", + "value": "{\"id\":341751249,\"org_id\":321813,\"type\":\"data-jobs alert\",\"name\":\"Test-Create_a_Data_Jobs_duration_monitor_returns_OK_response-1793983320\",\"message\":\"Data jobs duration alert triggered\",\"tags\":[\"test:testcreateadatajobsdurationmonitorreturnsokresponse1793983320\",\"env:ci\"],\"query\":\"formula(\\\"job_duration(run_query)\\\").detection(anomaly).by(name,workspace_name,entity_id) > 0\",\"options\":{\"variables\":[{\"data_source\":\"data_quality_metrics\",\"filter\":\"search for databricks_job\",\"group_by\":[\"name\",\"workspace_name\",\"entity_id\"],\"measure\":\"duration\",\"monitor_options\":{\"consecutive_runs\":2,\"detection_type\":\"anomaly\"},\"name\":\"run_query\",\"run_filter\":\"env:prod\",\"schema_version\":\"0.0.1\"}],\"notify_no_data\":false,\"notify_audit\":false,\"new_host_delay\":300,\"include_tags\":true,\"silenced\":{}},\"multi\":true,\"created_at\":1793983320000,\"created\":\"2026-11-06T16:42:00.590725+00:00\",\"modified\":\"2026-11-06T16:42:00.590725+00:00\",\"deleted\":null,\"priority\":null,\"restricted_roles\":null,\"restriction_policy\":null,\"draft_status\":\"published\",\"assets\":[],\"overall_state_modified\":null,\"overall_state\":\"No Data\",\"creator\":{\"name\":\"C\\u00e9dric Van Rompay\",\"handle\":\"cedric.vanrompay@datadoghq.com\",\"email\":\"cedric.vanrompay@datadoghq.com\",\"id\":68397867}}\n" + }, + "headers": { + "content-type": "application/json" + }, + "reason": "OK", + "status": 200 + } + }, + { + "request": { + "body": { + "type": "empty", + "value": null + }, + "content_type": "", + "method": "DELETE", + "path": "/api/v1/monitor/341751249", + "query": [] + }, + "response": { + "body": { + "encoding": "text", + "value": "{\"deleted_monitor_id\":341751249}\n" + }, + "headers": { + "content-type": "application/json" + }, + "reason": "OK", + "status": 200 + } + } + ], + "scenario": "Create a Data Jobs duration monitor returns \"OK\" response", + "version": "v1" + }, { "feature": "Monitors", "frozen_at": "2026-05-14T10:58:54.153Z", @@ -268,6 +352,86 @@ "scenario": "Create a Data Jobs monitor returns \"OK\" response", "version": "v1" }, + { + "feature": "Monitors", + "frozen_at": "2026-11-06T16:42:01.032Z", + "interactions": [ + { + "request": { + "body": { + "type": "json", + "value": { + "message": "Data jobs alert triggered", + "name": "Test-Create_a_Data_Jobs_monitor_with_the_aastra_query_dialect_returns_OK_response-1793983321", + "options": { + "thresholds": { + "critical": 0 + }, + "variables": [ + { + "job_type": "databricks.job", + "jobs_query": "search for databricks_job", + "monitor_options": { + "alert_on_every_failure": true + }, + "name": "run_query", + "query_dialect": "aastra", + "run_filter": "env:prod" + } + ] + }, + "query": "formula(\"failed_runs(run_query)\").by(name,workspace_name,entity_id).last(1d) > 0", + "tags": [ + "test:testcreateadatajobsmonitorwiththeaastraquerydialectreturnsokresponse1793983321", + "env:ci" + ], + "type": "data-jobs alert" + } + }, + "content_type": "application/json", + "method": "POST", + "path": "/api/v1/monitor", + "query": [] + }, + "response": { + "body": { + "encoding": "text", + "value": "{\"id\":341751253,\"org_id\":321813,\"type\":\"data-jobs alert\",\"name\":\"Test-Create_a_Data_Jobs_monitor_with_the_aastra_query_dialect_returns_OK_response-1793983321\",\"message\":\"Data jobs alert triggered\",\"tags\":[\"test:testcreateadatajobsmonitorwiththeaastraquerydialectreturnsokresponse1793983321\",\"env:ci\"],\"query\":\"formula(\\\"failed_runs(run_query)\\\").by(name,workspace_name,entity_id).last(1d) > 0\",\"options\":{\"thresholds\":{\"critical\":0.0},\"variables\":[{\"job_type\":\"databricks.job\",\"jobs_query\":\"search for databricks_job\",\"monitor_options\":{\"alert_on_every_failure\":true},\"name\":\"run_query\",\"query_dialect\":\"aastra\",\"run_filter\":\"env:prod\"}],\"notify_no_data\":false,\"notify_audit\":false,\"new_host_delay\":300,\"include_tags\":true,\"silenced\":{}},\"multi\":true,\"created_at\":1793983321000,\"created\":\"2026-11-06T16:42:01.177378+00:00\",\"modified\":\"2026-11-06T16:42:01.177378+00:00\",\"deleted\":null,\"priority\":null,\"restricted_roles\":null,\"restriction_policy\":null,\"draft_status\":\"published\",\"assets\":[],\"overall_state_modified\":null,\"overall_state\":\"No Data\",\"creator\":{\"name\":\"C\\u00e9dric Van Rompay\",\"handle\":\"cedric.vanrompay@datadoghq.com\",\"email\":\"cedric.vanrompay@datadoghq.com\",\"id\":68397867}}\n" + }, + "headers": { + "content-type": "application/json" + }, + "reason": "OK", + "status": 200 + } + }, + { + "request": { + "body": { + "type": "empty", + "value": null + }, + "content_type": "", + "method": "DELETE", + "path": "/api/v1/monitor/341751253", + "query": [] + }, + "response": { + "body": { + "encoding": "text", + "value": "{\"deleted_monitor_id\":341751253}\n" + }, + "headers": { + "content-type": "application/json" + }, + "reason": "OK", + "status": 200 + } + } + ], + "scenario": "Create a Data Jobs monitor with the aastra query dialect returns \"OK\" response", + "version": "v1" + }, { "feature": "Monitors", "frozen_at": "2026-01-12T17:23:49.629Z", diff --git a/tests/v1/features/monitors.feature b/tests/v1/features/monitors.feature index 5584472ab2..ef8d3d387b 100644 --- a/tests/v1/features/monitors.feature +++ b/tests/v1/features/monitors.feature @@ -42,6 +42,18 @@ Feature: Monitors When the request is sent Then the response status is 200 OK + @team:ddoghq/monitor-app + Scenario: Create a Data Jobs duration monitor returns "OK" response + Given new "CreateMonitor" request + And body with value {"name": "{{ unique }}", "type": "data-jobs alert", "query": "formula(\"job_duration(run_query)\").detection(anomaly).by(name,workspace_name,entity_id) > 0", "message": "Data jobs duration alert triggered", "tags": ["test:{{ unique_lower_alnum }}", "env:ci"], "options": {"variables": [{"name": "run_query", "data_source": "data_quality_metrics", "schema_version": "0.0.1", "measure": "duration", "filter": "search for databricks_job", "run_filter": "env:prod", "group_by": ["name", "workspace_name", "entity_id"], "monitor_options": {"detection_type": "anomaly", "consecutive_runs": 2}}]}} + When the request is sent + Then the response status is 200 OK + And the response "name" is equal to "{{ unique }}" + And the response "type" is equal to "data-jobs alert" + And the response "options.variables[0].run_filter" is equal to "env:prod" + And the response "options.variables[0].monitor_options.detection_type" is equal to "anomaly" + And the response "options.variables[0].monitor_options.consecutive_runs" is equal to 2 + @team:ddoghq/monitor-app Scenario: Create a Data Jobs monitor returns "OK" response Given new "CreateMonitor" request @@ -51,6 +63,18 @@ Feature: Monitors And the response "name" is equal to "{{ unique }}" And the response "type" is equal to "data-jobs alert" + @team:ddoghq/monitor-app + Scenario: Create a Data Jobs monitor with the aastra query dialect returns "OK" response + Given new "CreateMonitor" request + And body with value {"name": "{{ unique }}", "type": "data-jobs alert", "query": "formula(\"failed_runs(run_query)\").by(name,workspace_name,entity_id).last(1d) > 0", "message": "Data jobs alert triggered", "tags": ["test:{{ unique_lower_alnum }}", "env:ci"], "options": {"thresholds": {"critical": 0}, "variables": [{"name": "run_query", "jobs_query": "search for databricks_job", "job_type": "databricks.job", "query_dialect": "aastra", "run_filter": "env:prod", "monitor_options": {"alert_on_every_failure": true}}]}} + When the request is sent + Then the response status is 200 OK + And the response "name" is equal to "{{ unique }}" + And the response "type" is equal to "data-jobs alert" + And the response "options.variables[0].query_dialect" is equal to "aastra" + And the response "options.variables[0].run_filter" is equal to "env:prod" + And the response "options.variables[0].monitor_options.alert_on_every_failure" is equal to true + @team:ddoghq/monitor-app Scenario: Create a Data Quality monitor returns "OK" response Given new "CreateMonitor" request