[FEATURE] Prometheus: per-query exemplars toggle (perses#3445)

docs/prometheus/README.md

@@ -134,7 +134,7 @@ See also technical docs related to this plugin:
 
 The Prometheus package supports displaying exemplars on the [Time Series Chart](../timeserieschart/README.md) panel:
 
-- When exemplars are enabled on the datasource (see below), the `PrometheusTimeSeriesQuery` plugin calls the Prometheus [`/api/v1/query_exemplars`](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars) endpoint in parallel with every range query, and attaches the returned exemplars to the series matching their labels.
+- When exemplars are enabled on the datasource (see below), the `PrometheusTimeSeriesQuery` plugin calls the Prometheus [`/api/v1/query_exemplars`](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-exemplars) endpoint in parallel with each range query that has not turned them off (see [per-query toggle](#per-query-toggle)), and attaches the returned exemplars to the series matching their labels.
 - On the chart, each exemplar is rendered as a diamond marker placed at its value and timestamp, using the color of the series it belongs to. Clicking a marker opens a dialog with its metadata: the labels of the series it belongs to, the exemplar labels (like `trace_id`), its value and its timestamp (e.g. to copy a trace ID and open the corresponding trace in your tracing UI).
 - A failed exemplars request never impacts the panel itself: the query results are displayed without exemplars.
 
@@ -157,6 +157,25 @@ Some requirements and caveats to be aware of:
 - Exemplars are only fetched for range queries: queries executed in instant mode never display exemplars.
 - When relying on the [proxy](https://perses.dev/perses/docs/concepts/proxy/) feature, make sure the `/api/v1/query_exemplars` endpoint is allowed so exemplars can be fetched through the Perses server.
 
+### Per-query toggle
+
+Each `PrometheusTimeSeriesQuery` has an **Exemplars** option in the query editor, with three modes:
+
+- **Auto** (default, `enableExemplars` omitted): follows the datasource setting.
+- **On** (`enableExemplars: true`): fetches exemplars for this query.
+- **Off** (`enableExemplars: false`): never fetches exemplars for this query, e.g. to save the extra request on a panel that doesn't need them.
+
+The toggle can only narrow the datasource setting, not widen it: **On** still requires exemplars to be enabled on the datasource, and a range query. When the datasource has exemplars disabled, or the query runs in instant mode, no exemplars are fetched whatever the toggle says.
+
+```yaml
+kind: "PrometheusTimeSeriesQuery"
+spec:
+  query: histogram_quantile(0.99, sum by (le) (rate(http_server_duration_bucket[5m])))
+  enableExemplars: false
+```
+
+When using the Dashboard-as-Code Go lib, use the `query.EnableExemplars(false)` option.
+
 ## Annotation (`PrometheusPromQLAnnotation`)
 
 The annotation plugin overlays contextual events on the panels of a dashboard, like deployments, incidents or maintenance windows. The provided PromQL expression is executed as a range query over the time range of the dashboard, and each series it returns becomes an annotation spanning from the timestamp of its first sample to the timestamp of its last one.

prometheus/schemas/prometheus-time-series-query/query.cue

@@ -27,4 +27,5 @@ spec: close({
 	minStep?:          =~ds.#durationRegex | =~common.#variableSyntaxRegex
 	resolution?:       number
 	instant?:          bool
+	enableExemplars?:  bool
 })

prometheus/sdk/go/query/options.go

@@ -65,3 +65,15 @@ func Instant(instant bool) Option {
 		return nil
 	}
 }
+
+// EnableExemplars sets the per-query exemplars toggle. Exemplars must still be
+// enabled on the Prometheus datasource (`exemplars.enable`): `true` cannot turn
+// them on for a datasource that has them disabled. When this option is omitted,
+// the query follows the datasource setting; `false` skips fetching exemplars for
+// this query. Exemplars are only fetched for range queries.
+func EnableExemplars(enableExemplars bool) Option {
+	return func(builder *Builder) error {
+		builder.EnableExemplars = &enableExemplars
+		return nil
+	}
+}

prometheus/sdk/go/query/query.go

@@ -29,6 +29,7 @@ type PluginSpec struct {
 	MinStep          common.Duration      `json:"minStep,omitempty" yaml:"minStep,omitempty"`
 	Resolution       int                  `json:"resolution,omitempty" yaml:"resolution,omitempty"`
 	Instant          *bool                `json:"instant,omitempty" yaml:"instant,omitempty"`
+	EnableExemplars  *bool                `json:"enableExemplars,omitempty" yaml:"enableExemplars,omitempty"`
 }
 
 type Option func(plugin *Builder) error

prometheus/src/plugins/prometheus-time-series-query/PrometheusTimeSeriesQueryEditor.tsx

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import { Button, ButtonGroup, FormControl, Stack, TextField } from '@mui/material';
+import { Button, ButtonGroup, FormControl, Stack, TextField, Typography } from '@mui/material';
 import { PanelEditorContext } from '@perses-dev/dashboards';
 import type { DatasourceSelectProps } from '@perses-dev/plugin-system';
 import {
@@ -42,7 +42,13 @@ import {
 import type { PrometheusDatasourceSpec } from '../types';
 import { DEFAULT_SCRAPE_INTERVAL } from '../types';
 import type { PrometheusTimeSeriesQueryEditorProps } from './query-editor-model';
-import { useQueryState, useFormatState, useMinStepState, useInstantState } from './query-editor-model';
+import {
+  useQueryState,
+  useFormatState,
+  useMinStepState,
+  useInstantState,
+  useEnableExemplarsState,
+} from './query-editor-model';
 /**
  * The options editor component for editing a PrometheusTimeSeriesQuery's spec.
  */
@@ -57,6 +63,7 @@ export function PrometheusTimeSeriesQueryEditor(props: PrometheusTimeSeriesQuery
   const datasourceSelectValue = datasource ?? DEFAULT_PROM;
 
   const datasourceSelectLabelID = useId(); // for panels with multiple queries, this component is rendered multiple times on the same page
+  const exemplarsLabelID = useId();
 
   const selectedDatasource = useDatasourceSelectValueToSelector(
     datasourceSelectValue,
@@ -71,6 +78,12 @@ export function PrometheusTimeSeriesQueryEditor(props: PrometheusTimeSeriesQuery
   const { format, handleFormatChange, handleFormatBlur } = useFormatState(props);
   const { minStep, handleMinStepChange, handleMinStepBlur } = useMinStepState(props);
   const { instant, handleInstantChange } = useInstantState(props);
+  const { enableExemplars, handleEnableExemplarsChange } = useEnableExemplarsState(props);
+  const datasourceSpec = datasourceResource?.plugin.spec as PrometheusDatasourceSpec | undefined;
+  // Undefined while the datasource resource is still loading: the control is disabled
+  // but no "disabled on the datasource" message is shown yet.
+  const exemplarsEnabledOnDatasource =
+    datasourceSpec === undefined ? undefined : datasourceSpec.exemplars?.enable === true;
   const minStepPlaceholder =
     minStep ??
     (datasourceResource && (datasourceResource.plugin.spec as PrometheusDatasourceSpec).scrapeInterval) ??
@@ -191,6 +204,43 @@ export function PrometheusTimeSeriesQueryEditor(props: PrometheusTimeSeriesQuery
           </Button>
         </ButtonGroup>
       </Stack>
+      <Stack direction="row" spacing={2} alignItems="center">
+        <ButtonGroup
+          aria-labelledby={exemplarsLabelID}
+          disabled={isReadonly || exemplarsEnabledOnDatasource !== true || instant === true}
+          sx={{ '& .MuiButton-root': { px: 2, py: 2 } }}
+        >
+          <Button
+            variant={enableExemplars === true ? 'contained' : 'outlined'}
+            aria-pressed={enableExemplars === true}
+            onClick={() => handleEnableExemplarsChange(true)}
+          >
+            On
+          </Button>
+          <Button
+            variant={enableExemplars === false ? 'contained' : 'outlined'}
+            aria-pressed={enableExemplars === false}
+            onClick={() => handleEnableExemplarsChange(false)}
+          >
+            Off
+          </Button>
+          <Button
+            variant={enableExemplars === undefined ? 'contained' : 'outlined'}
+            aria-pressed={enableExemplars === undefined}
+            onClick={() => handleEnableExemplarsChange(undefined)}
+          >
+            Auto
+          </Button>
+        </ButtonGroup>
+        <Typography id={exemplarsLabelID} variant="body2" color="textSecondary">
+          Exemplars
+          {exemplarsEnabledOnDatasource === false && ' (disabled on the datasource)'}
+          {exemplarsEnabledOnDatasource === true && instant === true && ' (not available for instant queries)'}
+          {exemplarsEnabledOnDatasource === true &&
+            instant === undefined &&
+            ' (skipped if the panel runs instant queries)'}
+        </Typography>
+      </Stack>
     </Stack>
   );
 }

prometheus/src/plugins/prometheus-time-series-query/get-time-series-data.ts

@@ -122,7 +122,11 @@ export const getTimeSeriesData: TimeSeriesQueryPlugin<PrometheusTimeSeriesQueryS
   const client: PrometheusClient = await context.datasourceStore.getDatasourceClient(selectedDatasource);
 
   // Make the request to Prom
-  const exemplarsEnabled = datasource.plugin.spec.exemplars?.enable === true && !isInstant;
+  // Exemplars are gated by the datasource-level feature toggle, can be overridden
+  // per query (`enableExemplars`: undefined = Auto), and are only fetched for range
+  // queries: the exemplars API is range-based and has no instant equivalent.
+  const exemplarsEnabled =
+    datasource.plugin.spec.exemplars?.enable === true && (spec.enableExemplars ?? true) && !isInstant;
 
   let response;
   let exemplarResponse: Awaited<ReturnType<PrometheusClient['queryExemplars']>> | undefined;

prometheus/src/plugins/prometheus-time-series-query/plugin.test.ts

@@ -296,6 +296,48 @@ describe('PrometheusTimeSeriesQuery', () => {
     ]);
   });
 
+  it('should not query exemplars when spec.enableExemplars is false, even if the datasource enables them', async () => {
+    const ctx = createStubContext();
+    getDatasource.mockImplementation((): DatasourceSpec<PrometheusDatasourceSpec> => {
+      return {
+        default: false,
+        plugin: {
+          kind: 'PrometheusDatasource',
+          spec: {
+            ...datasource,
+            exemplars: { enable: true },
+          },
+        },
+      };
+    });
+    (promStubClient.queryExemplars as Mock).mockClear();
+
+    const results = await PrometheusTimeSeriesQuery.getTimeSeriesData({ query: 'up', enableExemplars: false }, ctx);
+
+    expect(promStubClient.queryExemplars).not.toHaveBeenCalled();
+    expect(results.exemplars).toBeUndefined();
+  });
+
+  it('should not query exemplars when spec.enableExemplars is true but the datasource does not enable them', async () => {
+    const ctx = createStubContext();
+    getDatasource.mockImplementation((): DatasourceSpec<PrometheusDatasourceSpec> => {
+      return {
+        default: false,
+        plugin: {
+          kind: 'PrometheusDatasource',
+          // exemplars NOT enabled on the datasource: a per-query `true` must not override it
+          spec: datasource,
+        },
+      };
+    });
+    (promStubClient.queryExemplars as Mock).mockClear();
+
+    const results = await PrometheusTimeSeriesQuery.getTimeSeriesData({ query: 'up', enableExemplars: true }, ctx);
+
+    expect(promStubClient.queryExemplars).not.toHaveBeenCalled();
+    expect(results.exemplars).toBeUndefined();
+  });
+
   it('should keep the query working when the exemplar query fails', async () => {
     const ctx = createStubContext();
     getDatasource.mockImplementation((): DatasourceSpec<PrometheusDatasourceSpec> => {

prometheus/src/plugins/prometheus-time-series-query/query-editor-model.ts

@@ -157,3 +157,25 @@ export function useInstantState(props: PrometheusTimeSeriesQueryEditorProps): {
 
   return { instant, handleInstantChange };
 }
+
+/**
+ * Hook to manage `enableExemplars` state. The toggle commits immediately, like `instant`.
+ */
+export function useEnableExemplarsState(props: PrometheusTimeSeriesQueryEditorProps): {
+  enableExemplars: boolean | undefined;
+  handleEnableExemplarsChange: (enableExemplars: boolean | undefined) => void;
+} {
+  const { onChange, value } = props;
+
+  const enableExemplars = value.enableExemplars;
+
+  const handleEnableExemplarsChange = (next: boolean | undefined): void => {
+    onChange(
+      produce(value, (draft) => {
+        draft.enableExemplars = next;
+      }),
+    );
+  };
+
+  return { enableExemplars, handleEnableExemplarsChange };
+}

prometheus/src/plugins/prometheus-time-series-query/time-series-query-model.ts

@@ -24,5 +24,13 @@ export interface PrometheusTimeSeriesQuerySpec {
   minStep?: DurationString;
   resolution?: number;
   instant?: boolean;
+  /**
+   * Per-query toggle for the exemplars feature. Exemplars must be enabled on the
+   * selected datasource (`exemplars.enable`); this toggle cannot turn them on for
+   * a datasource that has them disabled. `undefined` (Auto) follows the datasource
+   * setting, `true` (On) fetches them when the datasource allows it, and `false`
+   * (Off) skips fetching for this query. Exemplars are only fetched for range queries.
+   */
+  enableExemplars?: boolean;
   datasource?: DatasourceSelectValue<PrometheusDatasourceSelector>;
 }