diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 951bc028..c1933a51 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -98,3 +98,13 @@ jobs: run: cargo test --workspace --locked env: RUSTC_WRAPPER: sccache + + - name: Verify MetricsQL frontend as a stable external dependency + run: bash tools/verify_metricsql_external_consumer.sh + env: + RUSTC_WRAPPER: sccache + + - name: Verify vendored MetricsQL parser baseline + run: python3 tools/verify_metricsql_vendored_baseline.py + env: + RUSTC_WRAPPER: sccache diff --git a/Cargo.lock b/Cargo.lock index 34f106da..1ee6f271 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,7 @@ dependencies = [ "const-random", "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -309,7 +310,7 @@ dependencies = [ "asap-types", "serde", "serde_json", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -326,6 +327,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "asap-frontend-metricsql" +version = "0.1.0" +dependencies = [ + "asap-types", + "metricsql_parser", + "thiserror 2.0.18", +] + [[package]] name = "asap-frontend-promql" version = "0.1.0" @@ -370,7 +380,7 @@ version = "0.1.0" dependencies = [ "serde", "serde_json", - "thiserror", + "thiserror 2.0.18", ] [[package]] @@ -423,6 +433,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "beef" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1" + [[package]] name = "bincode" version = "1.3.3" @@ -592,6 +608,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-targets", ] @@ -603,7 +620,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" dependencies = [ "chrono", - "phf", + "phf 0.12.1", ] [[package]] @@ -1174,6 +1191,15 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" +[[package]] +name = "enquote" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c36cb11dbde389f4096111698d8b567c0720e3452fd5ac3e6b4e47e1939932" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1777,6 +1803,39 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "logos" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7251356ef8cb7aec833ddf598c6cb24d17b689d20b993f9d11a3d764e34e6458" +dependencies = [ + "logos-derive", +] + +[[package]] +name = "logos-codegen" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f80069600c0d66734f5ff52cc42f2dabd6b29d205f333d61fd7832e9e9963f" +dependencies = [ + "beef", + "fnv", + "lazy_static", + "proc-macro2", + "quote", + "regex-syntax", + "syn", +] + +[[package]] +name = "logos-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fb722b06a9dc12adb0963ed585f19fc61dc5413e6a9be9422ef92c091e731d" +dependencies = [ + "logos-codegen", +] + [[package]] name = "lrlex" version = "0.13.10" @@ -1867,6 +1926,35 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "metricsql_common" +version = "0.1.0" +dependencies = [ + "chrono", +] + +[[package]] +name = "metricsql_parser" +version = "0.1.0" +dependencies = [ + "ahash", + "chrono", + "enquote", + "logos", + "logos-derive", + "metricsql_common", + "num-traits", + "phf 0.11.3", + "regex", + "scopeguard", + "serde", + "strum", + "strum_macros", + "thiserror 1.0.69", + "tinyvec", + "xxhash-rust", +] + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2103,13 +2191,55 @@ dependencies = [ "indexmap", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared 0.11.3", +] + [[package]] name = "phf" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" dependencies = [ - "phf_shared", + "phf_shared 0.12.1", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", ] [[package]] @@ -2572,13 +2702,33 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + [[package]] name = "thiserror" version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ - "thiserror-impl", + "thiserror-impl 2.0.18", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -2655,6 +2805,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.52.3" @@ -3193,6 +3358,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + [[package]] name = "xz2" version = "0.1.7" diff --git a/Cargo.toml b/Cargo.toml index 907f3ac2..d83f227d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,8 +4,11 @@ members = [ "crates/sql-function-catalog", "crates/asap-aware-mapping", "crates/frontend-promql", + "crates/frontend-metricsql", + "crates/metricsql-common-parser-support", "crates/frontend-sql", "crates/devtools", "crates/integration-tests", ] +exclude = ["crates/metricsql-parser-vendored"] resolver = "2" diff --git a/crates/frontend-metricsql/Cargo.toml b/crates/frontend-metricsql/Cargo.toml new file mode 100644 index 00000000..8fa341b8 --- /dev/null +++ b/crates/frontend-metricsql/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "asap-frontend-metricsql" +version = "0.1.0" +edition = "2021" + +[dependencies] +asap-types = { path = "../types" } +metricsql_parser = { path = "../metricsql-parser-vendored" } +thiserror = "2" diff --git a/crates/frontend-metricsql/src/lib.rs b/crates/frontend-metricsql/src/lib.rs new file mode 100644 index 00000000..adc68f4e --- /dev/null +++ b/crates/frontend-metricsql/src/lib.rs @@ -0,0 +1,325 @@ +//! MetricsQL AST to canonical `QueryExpr` frontend. + +use std::{rc::Rc, time::Duration}; + +use asap_types::pre_asap::{ + resolve_root, AggIntent, ArithmeticOpKind, BinaryOpKind, ColumnRef, CompareOpKind, GroupKeys, + Predicate, PromQLVectorSetOpKind, QueryExpr, Reduction, ScalarValue, Source, + UnresolvedQueryExpr as U, +}; +use asap_types::types::AccuracyTarget; +use metricsql_parser::ast::{AggregateModifier, DurationExpr, Expr, MetricExpr, RollupExpr}; +use metricsql_parser::functions::{AggregateFunction, BuiltinFunction, RollupFunction}; +use metricsql_parser::label::{LabelFilter, LabelFilterOp, NAME_LABEL}; +use thiserror::Error; + +pub use metricsql_parser::ast::Expr as MetricsqlExpr; + +#[derive(Debug, Error)] +pub enum MetricsqlError { + #[error("MetricsQL parse error: {0}")] + Parse(String), + #[error("unsupported MetricsQL feature: {0}")] + UnsupportedFeature(String), + #[error("MetricsQL column resolution failed: {0}")] + Resolve(String), +} + +pub fn parse_metricsql(query: &str) -> Result { + metricsql_parser::parser::parse(query).map_err(|e| MetricsqlError::Parse(e.to_string())) +} + +pub fn canonical_metricsql(query: &str) -> Result { + Ok(parse_metricsql(query)?.to_string()) +} + +pub fn lower_metricsql(query: &str, accuracy: AccuracyTarget) -> Result { + let ast = parse_metricsql(query)?; + let unresolved = Lowerer { accuracy }.lower(&ast)?; + resolve_root(&unresolved).map_err(|e| MetricsqlError::Resolve(e.to_string())) +} + +struct Lowerer { + accuracy: AccuracyTarget, +} + +impl Lowerer { + fn lower(&self, expr: &Expr) -> Result { + match expr { + Expr::MetricExpression(e) => self.metric(e), + Expr::Rollup(e) => self.rollup(e), + Expr::Function(e) => self.function(e), + Expr::Aggregation(e) => self.aggregate(e), + Expr::NumberLiteral(e) => Ok(U::promql_scalar(e.value)), + Expr::UnaryOperator(e) => Ok(U::BinaryOp { + op: BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + lhs: Rc::new(self.lower(&e.expr)?), + rhs: Rc::new(U::promql_scalar(-1.0)), + vector_match: None, + }), + Expr::BinaryOperator(e) => self.binary(e), + Expr::Parens(e) if e.expressions.len() == 1 => self.lower(&e.expressions[0]), + Expr::With(e) => self.lower(&e.expr), + other => Err(unsupported(format!("AST node `{other}`"))), + } + } + + fn metric(&self, metric: &MetricExpr) -> Result { + if metric.has_or_matchers() { + return Err(unsupported("or-delimited selector matchers")); + } + let name = metric + .metric_name() + .ok_or_else(|| unsupported("selector without one exact metric name"))?; + let mut filters: Vec<_> = metric + .matchers + .filter_iter() + .filter(|f| f.label != NAME_LABEL) + .collect(); + filters.sort_by(|a, b| a.label.cmp(&b.label).then(a.value.cmp(&b.value))); + Ok(U::Scan { + source: Source::TimeSeries { + metric: name.to_owned(), + }, + predicates: filters + .into_iter() + .map(|f| Predicate(Rc::new(matcher(f)))) + .collect(), + schema: None, + }) + } + + fn rollup(&self, rollup: &RollupExpr) -> Result { + if rollup.offset.is_some() || rollup.at.is_some() { + return Err(unsupported("offset and @ modifiers")); + } + if rollup.for_subquery() { + return Err(unsupported("subquery step or inherited step")); + } + let child = self.lower(&rollup.expr)?; + match &rollup.window { + None => Ok(child), + Some(window) => Ok(U::TimeRange { + range: duration(window)?, + child: Rc::new(child), + }), + } + } + + fn function( + &self, + function: &metricsql_parser::ast::FunctionExpr, + ) -> Result { + if function.keep_metric_names { + return Err(unsupported( + "keep_metric_names requires metric-name lineage", + )); + } + let BuiltinFunction::Rollup(rollup) = function.function else { + return Err(unsupported(format!("function `{}`", function.name()))); + }; + let expected_args = if rollup == RollupFunction::QuantileOverTime { + 2 + } else { + 1 + }; + require_arity(function.name(), function.args.len(), expected_args)?; + let child_index = usize::from(rollup == RollupFunction::QuantileOverTime); + let child = function + .args + .get(child_index) + .ok_or_else(|| unsupported(format!("missing argument for `{}`", function.name())))?; + let intent = match rollup { + RollupFunction::DefaultRollup | RollupFunction::LastOverTime => AggIntent::LastOverTime, + RollupFunction::FirstOverTime => AggIntent::FirstOverTime, + RollupFunction::AvgOverTime => AggIntent::Avg { col: None }, + RollupFunction::MinOverTime => AggIntent::Min { col: None }, + RollupFunction::MaxOverTime => AggIntent::Max { col: None }, + RollupFunction::SumOverTime => AggIntent::Sum { col: None }, + RollupFunction::CountOverTime => AggIntent::Count { + accuracy: self.accuracy.clone(), + }, + RollupFunction::StddevOverTime => AggIntent::StdDev { + col: None, + population: true, + }, + RollupFunction::StdvarOverTime => AggIntent::Variance { + col: None, + population: true, + }, + RollupFunction::Rate => AggIntent::Rate, + RollupFunction::IRate => AggIntent::IRate, + RollupFunction::Increase => AggIntent::Increase, + RollupFunction::Changes => AggIntent::Changes, + RollupFunction::Delta => AggIntent::Delta, + RollupFunction::IDelta => AggIntent::IDelta, + RollupFunction::Deriv => AggIntent::Deriv, + RollupFunction::Resets => AggIntent::Resets, + RollupFunction::MadOverTime => AggIntent::MadOverTime, + RollupFunction::PresentOverTime => AggIntent::PresentOverTime, + RollupFunction::AbsentOverTime => AggIntent::AbsentOverTime, + RollupFunction::QuantileOverTime => AggIntent::Quantile { + col: None, + q: number_arg(&function.args, 0)?, + accuracy: self.accuracy.clone(), + }, + _ => { + return Err(unsupported(format!( + "rollup function `{}`", + function.name() + ))) + } + }; + let child = self.lower(child)?; + if rollup == RollupFunction::DefaultRollup && !matches!(child, U::TimeRange { .. }) { + return Err(unsupported( + "default_rollup without an explicit range requires an evaluation step", + )); + } + Ok(aggregate(Reduction::PerEntity, intent, child)) + } + + fn aggregate( + &self, + expr: &metricsql_parser::ast::AggregationExpr, + ) -> Result { + if expr.limit != 0 || expr.keep_metric_names { + return Err(unsupported("aggregate limit or keep_metric_names")); + } + let expected_args = if expr.function == AggregateFunction::Quantile { + 2 + } else { + 1 + }; + require_arity(expr.name(), expr.args.len(), expected_args)?; + let child_index = expr + .arg_idx_for_optimization() + .ok_or_else(|| unsupported(format!("aggregate `{}` arguments", expr.name())))?; + let intent = match expr.function { + AggregateFunction::Sum => AggIntent::Sum { col: None }, + AggregateFunction::Avg => AggIntent::Avg { col: None }, + AggregateFunction::Min => AggIntent::Min { col: None }, + AggregateFunction::Max => AggIntent::Max { col: None }, + AggregateFunction::Count => AggIntent::Cardinality { + col: None, + accuracy: self.accuracy.clone(), + }, + AggregateFunction::StdDev => AggIntent::StdDev { + col: None, + population: true, + }, + AggregateFunction::StdVar => AggIntent::Variance { + col: None, + population: true, + }, + AggregateFunction::Group => AggIntent::Group, + AggregateFunction::Quantile => AggIntent::Quantile { + col: None, + q: number_arg(&expr.args, 0)?, + accuracy: self.accuracy.clone(), + }, + _ => return Err(unsupported(format!("aggregate `{}`", expr.name()))), + }; + let reduction = match &expr.modifier { + None => Reduction::by(vec![]), + Some(AggregateModifier::By(v)) => Reduction::by(names(v)), + Some(AggregateModifier::Without(v)) => Reduction::Reduce(GroupKeys::without(names(v))), + }; + let child = expr + .args + .get(child_index) + .ok_or_else(|| unsupported("missing aggregate input"))?; + Ok(aggregate(reduction, intent, self.lower(child)?)) + } + + fn binary(&self, expr: &metricsql_parser::ast::BinaryExpr) -> Result { + if expr.modifier.is_some() { + return Err(unsupported("binary vector matching modifiers")); + } + use metricsql_parser::ast::Operator as O; + let op = match expr.op { + O::Add => BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), + O::Sub => BinaryOpKind::Arithmetic(ArithmeticOpKind::Sub), + O::Mul => BinaryOpKind::Arithmetic(ArithmeticOpKind::Mul), + O::Div => BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + O::Mod => BinaryOpKind::Arithmetic(ArithmeticOpKind::Mod), + O::Pow => BinaryOpKind::Arithmetic(ArithmeticOpKind::Pow), + O::Atan2 => BinaryOpKind::Arithmetic(ArithmeticOpKind::Atan2), + O::Eql => BinaryOpKind::Compare(CompareOpKind::Eq), + O::NotEq => BinaryOpKind::Compare(CompareOpKind::Ne), + O::Lt => BinaryOpKind::Compare(CompareOpKind::Lt), + O::Lte => BinaryOpKind::Compare(CompareOpKind::Le), + O::Gt => BinaryOpKind::Compare(CompareOpKind::Gt), + O::Gte => BinaryOpKind::Compare(CompareOpKind::Ge), + O::And => BinaryOpKind::Set(PromQLVectorSetOpKind::And), + O::Or => BinaryOpKind::Set(PromQLVectorSetOpKind::Or), + O::Unless => BinaryOpKind::Set(PromQLVectorSetOpKind::Unless), + O::If | O::IfNot | O::Default => { + return Err(unsupported(format!("MetricsQL operator `{}`", expr.op))) + } + }; + Ok(U::BinaryOp { + op, + lhs: Rc::new(self.lower(&expr.left)?), + rhs: Rc::new(self.lower(&expr.right)?), + vector_match: None, + }) + } +} + +fn names(values: &[String]) -> Vec { + values.iter().cloned().map(ColumnRef::Named).collect() +} + +fn aggregate(reduction: Reduction, intent: AggIntent, child: U) -> U { + U::Aggregate { + reduction, + measures: vec![intent], + output_names: vec![String::new()], + having: None, + child: Rc::new(child), + } +} + +fn matcher(filter: &LabelFilter) -> U { + let op = match filter.op { + LabelFilterOp::Equal => CompareOpKind::Eq, + LabelFilterOp::NotEqual => CompareOpKind::Ne, + LabelFilterOp::RegexEqual => CompareOpKind::Regex, + LabelFilterOp::RegexNotEqual => CompareOpKind::NotRegex, + }; + U::Compare { + left: Rc::new(U::Column(ColumnRef::Named(filter.label.clone()))), + op, + right: Rc::new(U::Literal(ScalarValue::Utf8(filter.value.clone()))), + } +} + +fn duration(value: &DurationExpr) -> Result { + match value { + DurationExpr::Millis(ms) if *ms >= 0 => Ok(Duration::from_millis(*ms as u64)), + DurationExpr::StepValue(_) => Err(unsupported("step-relative duration")), + DurationExpr::Millis(_) => Err(unsupported("negative duration")), + } +} + +fn number_arg(args: &[Expr], index: usize) -> Result { + match args.get(index) { + Some(Expr::NumberLiteral(v)) if v.value.is_finite() => Ok(v.value), + _ => Err(unsupported(format!("numeric argument #{index}"))), + } +} + +fn require_arity(name: &str, actual: usize, expected: usize) -> Result<(), MetricsqlError> { + if actual == expected { + Ok(()) + } else { + Err(unsupported(format!( + "`{name}` with {actual} arguments; canonical lowering requires exactly {expected}" + ))) + } +} + +fn unsupported(message: impl Into) -> MetricsqlError { + MetricsqlError::UnsupportedFeature(message.into()) +} diff --git a/crates/frontend-metricsql/tests/lowering.rs b/crates/frontend-metricsql/tests/lowering.rs new file mode 100644 index 00000000..133fc328 --- /dev/null +++ b/crates/frontend-metricsql/tests/lowering.rs @@ -0,0 +1,153 @@ +use std::time::Duration; + +use asap_frontend_metricsql::{ + canonical_metricsql, lower_metricsql, parse_metricsql, MetricsqlError, +}; +use asap_types::pre_asap::{AggIntent, QueryExpr, Reduction, Source}; +use asap_types::types::AccuracyTarget; + +fn lower(query: &str) -> QueryExpr { + lower_metricsql(query, AccuracyTarget::Epsilon(0.01)).unwrap() +} + +#[test] +fn selector_range_aggregate_and_call_share_the_canonical_shape() { + let query = r#"sum by (job) (rate(http_requests_total{status=~"5.."}[5m]))"#; + let tree = lower(query); + let QueryExpr::Aggregate { + reduction, + measures, + child, + .. + } = tree + else { + panic!("expected outer aggregate"); + }; + assert_eq!(reduction, Reduction::by(vec![2])); + assert!(matches!(measures.as_slice(), [AggIntent::Sum { .. }])); + let QueryExpr::Aggregate { + measures, child, .. + } = child.as_ref() + else { + panic!("expected rate aggregate"); + }; + assert!(matches!(measures.as_slice(), [AggIntent::Rate])); + let QueryExpr::TimeRange { range, child } = child.as_ref() else { + panic!("expected range"); + }; + assert_eq!(*range, Duration::from_secs(300)); + assert!( + matches!(child.as_ref(), QueryExpr::Scan { source: Source::TimeSeries { metric }, predicates, .. } if metric == "http_requests_total" && predicates.len() == 1) + ); +} + +#[test] +fn default_rollup_with_explicit_range_is_last_over_time() { + let tree = lower("default_rollup(cpu_usage[5m])"); + let QueryExpr::Aggregate { + reduction, + measures, + child, + .. + } = tree + else { + panic!("expected aggregate"); + }; + assert_eq!(reduction, Reduction::PerEntity); + assert!(matches!(measures.as_slice(), [AggIntent::LastOverTime])); + assert!( + matches!(child.as_ref(), QueryExpr::TimeRange { range, .. } if *range == Duration::from_secs(300)) + ); +} + +#[test] +fn implicit_default_rollup_requires_runtime_step() { + let error = lower_metricsql("default_rollup(cpu_usage)", AccuracyTarget::Exact).unwrap_err(); + assert!( + matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("evaluation step")) + ); +} + +#[test] +fn keep_metric_names_is_preserved_in_ast_and_rejected_without_lineage() { + let ast = parse_metricsql("rate(requests_total[5m]) keep_metric_names").unwrap(); + assert!(ast.keep_metric_names()); + let error = lower_metricsql( + "rate(requests_total[5m]) keep_metric_names", + AccuracyTarget::Exact, + ) + .unwrap_err(); + assert!( + matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("metric-name lineage")) + ); +} + +/// Representative MetricsQL-only forms from VictoriaMetrics' parser/docs +/// corpus. Source: app/vmselect/vmui/assets/MetricsQL-*.md and +/// app/vmselect/promql/exec_test.go in VictoriaMetrics/VictoriaMetrics. +#[test] +fn victoria_metrics_extension_corpus_parses_natively_and_fails_closed() { + let cases = [ + r#"rate({__name__=~"foo|bar"}[5m]) keep_metric_names"#, + "time() ifnot time() > 1400 default -time()", + "rate(foo[5i])", + "sum(foo) by (job) limit 10", + r#"foo{job="a" or job="b"}"#, + ]; + for query in cases { + let ast = parse_metricsql(query) + .unwrap_or_else(|error| panic!("native MetricsQL parser rejected {query:?}: {error}")); + assert!(!ast.to_string().is_empty()); + let result = lower_metricsql(query, AccuracyTarget::Exact); + assert!( + result.is_err(), + "extension semantics must be represented or routed to exact fallback: {query}" + ); + } + + assert_eq!( + lower(r#"WITH (prefix="http_") {__name__=prefix+"requests_total"}"#), + lower("http_requests_total"), + "a fully expanded WITH selector has ordinary selector semantics" + ); +} + +#[test] +fn canonical_identity_ignores_compatible_formatting() { + let compact = canonical_metricsql("sum by(job)(rate(requests_total[5m]))").unwrap(); + let spaced = canonical_metricsql(" sum by ( job ) ( rate( requests_total[5m] ) ) ").unwrap(); + assert_eq!(compact, spaced); +} + +#[test] +fn canonical_identity_recursively_formats_metricsql_extensions() { + let compact = + canonical_metricsql("default_rollup(requests_total[5m]) keep_metric_names").unwrap(); + let spaced = + canonical_metricsql(" default_rollup( requests_total[5m] ) keep_metric_names ").unwrap(); + assert_eq!(compact, spaced); + assert_eq!( + compact, + "default_rollup(requests_total[5m]) keep_metric_names" + ); +} + +#[test] +fn metricsql_multi_argument_aggregates_fail_closed() { + for query in ["sum(foo, bar)", "avg(foo, bar)", "count(foo, bar)"] { + parse_metricsql(query).expect("MetricsQL accepts multi-argument aggregates"); + let error = lower_metricsql(query, AccuracyTarget::Exact).unwrap_err(); + assert!( + matches!(error, MetricsqlError::UnsupportedFeature(message) if message.contains("requires exactly 1")), + "{query} must not silently discard an aggregate input" + ); + } +} + +#[test] +fn supported_parameterized_functions_require_their_exact_arity() { + let quantile = lower("quantile(0.9, requests_total)"); + assert!(matches!(quantile, QueryExpr::Aggregate { .. })); + let rollup = lower("quantile_over_time(0.9, requests_total[5m])"); + assert!(matches!(rollup, QueryExpr::Aggregate { .. })); +} diff --git a/crates/frontend-metricsql/tests/victoriametrics_go_golden.rs b/crates/frontend-metricsql/tests/victoriametrics_go_golden.rs new file mode 100644 index 00000000..0508d0ab --- /dev/null +++ b/crates/frontend-metricsql/tests/victoriametrics_go_golden.rs @@ -0,0 +1,128 @@ +use asap_frontend_metricsql::{canonical_metricsql, parse_metricsql}; +use metricsql_parser::ast::{Expr, Operator}; + +/// Direct canonical outputs generated by the official VictoriaMetrics Go +/// parser `github.com/VictoriaMetrics/metricsql` v0.84.4. The expected side is +/// deliberately not parsed by Rust, so one Rust parser bug cannot affect both +/// sides of the assertion. +const OFFICIAL_GO_GOLDEN: &[(&str, &str)] = &[ + (r#"foo{job="a" or job="b"}"#, r#"foo{job="a" or job="b"}"#), + ( + r#"WITH (prefix="http_") {__name__=prefix+"requests_total"}"#, + "http_requests_total", + ), + ( + "WITH (x = rate(foo[5m])) sum(x) by (job)", + "sum(rate(foo[5m])) by(job)", + ), + ( + r#"sum(rate(foo{code!="500",env=~"prod|staging"}[5m])) by (job) limit 10"#, + r#"sum(rate(foo{code!="500", env=~"prod|staging"}[5m])) by(job) limit 10"#, + ), + ( + "rate(foo[5i]) keep_metric_names", + "rate(foo[5i]) keep_metric_names", + ), +]; + +#[test] +fn official_victoriametrics_corpus_matches_direct_canonical_output() { + for &(query, official) in OFFICIAL_GO_GOLDEN { + let actual = canonical_metricsql(query).unwrap_or_else(|error| { + panic!("Rust parser rejected official input {query:?}: {error}") + }); + assert_eq!( + strip_formatting_whitespace(&actual), + strip_formatting_whitespace(official), + "query: {query}" + ); + } +} + +fn strip_formatting_whitespace(value: &str) -> String { + let mut quoted = false; + let mut escaped = false; + let compact: String = value + .chars() + .filter(|&ch| { + if quoted && ch == '\\' && !escaped { + escaped = true; + return true; + } + if ch == '"' && !escaped { + quoted = !quoted; + } + escaped = false; + quoted || !ch.is_whitespace() + }) + .collect(); + let mut normalized = String::with_capacity(compact.len()); + let mut rest = compact.as_str(); + while let Some(open) = rest.find('{') { + let (prefix, after_open) = rest.split_at(open + 1); + normalized.push_str(prefix); + let Some(close) = after_open.find('}') else { + normalized.push_str(after_open); + return normalized; + }; + let (matchers, suffix) = after_open.split_at(close); + if matchers.contains("or") { + normalized.push_str(matchers); + } else { + let mut fields: Vec<_> = matchers.split(',').collect(); + fields.sort_unstable(); + normalized.push_str(&fields.join(",")); + } + normalized.push('}'); + rest = &suffix[1..]; + } + normalized.push_str(rest); + normalized +} + +#[test] +fn official_extension_cases_preserve_typed_ast_semantics() { + let conditional = parse_metricsql("time() ifnot time() > 1400 default -time()").unwrap(); + assert!( + matches!(conditional, Expr::BinaryOperator(ref binary) if binary.op == Operator::Default) + ); + + let matched = parse_metricsql("foo + on(job) group_left(instance) bar").unwrap(); + assert!( + matches!(matched, Expr::BinaryOperator(ref binary) if binary.op == Operator::Add && binary.modifier.is_some()) + ); + + let offset = parse_metricsql("foo offset 1.5h").unwrap(); + assert!(matches!(offset, Expr::Rollup(ref rollup) if rollup.offset.is_some())); +} + +#[test] +fn official_victoriametrics_numeric_tokens_have_the_expected_value() { + let ast = parse_metricsql("1_000 + 0x10 + 2.5Mi") + .expect("Rust parser must accept VictoriaMetrics numeric tokens"); + assert_eq!(eval_numeric(&ast), Some(2_622_456.0)); +} + +fn eval_numeric(expr: &Expr) -> Option { + match expr { + Expr::NumberLiteral(number) => Some(number.value), + Expr::Parens(parens) if parens.expressions.len() == 1 => { + eval_numeric(&parens.expressions[0]) + } + Expr::UnaryOperator(unary) => eval_numeric(&unary.expr).map(|value| -value), + Expr::BinaryOperator(binary) => { + let left = eval_numeric(&binary.left)?; + let right = eval_numeric(&binary.right)?; + match binary.op { + Operator::Add => Some(left + right), + Operator::Sub => Some(left - right), + Operator::Mul => Some(left * right), + Operator::Div => Some(left / right), + Operator::Mod => Some(left % right), + Operator::Pow => Some(left.powf(right)), + _ => None, + } + } + _ => None, + } +} diff --git a/crates/metricsql-common-parser-support/Cargo.toml b/crates/metricsql-common-parser-support/Cargo.toml new file mode 100644 index 00000000..4ce647d3 --- /dev/null +++ b/crates/metricsql-common-parser-support/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "metricsql_common" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" + +[dependencies] +chrono = "0.4" diff --git a/crates/metricsql-common-parser-support/src/lib.rs b/crates/metricsql-common-parser-support/src/lib.rs new file mode 100644 index 00000000..85d0110e --- /dev/null +++ b/crates/metricsql-common-parser-support/src/lib.rs @@ -0,0 +1,152 @@ +pub mod duration { + use std::fmt::{self, Formatter}; + + pub fn fmt_duration_ms(f: &mut Formatter<'_>, value: i64) -> fmt::Result { + if value == 0 { + return write!(f, "0ms"); + } + let mut remainder = value; + for (unit, suffix) in [ + (31_536_000_000, "y"), + (86_400_000, "d"), + (3_600_000, "h"), + (60_000, "m"), + (1_000, "s"), + ] { + let part = remainder / unit; + if part != 0 { + write!(f, "{part}{suffix}")?; + remainder %= unit; + } + } + if remainder != 0 { + write!(f, "{remainder}ms")?; + } + Ok(()) + } +} + +pub mod hash { + pub type FastHashMap = std::collections::HashMap; + pub type FastHashSet = std::collections::HashSet; + pub trait HashSetExt {} + impl HashSetExt for std::collections::HashSet {} +} + +pub mod prelude { + pub use crate::time::{datetime_part, timestamp_secs_to_utc_datetime, DateTimePart}; +} + +mod time { + use chrono::{DateTime, Datelike, NaiveDate, TimeZone, Timelike, Utc}; + + #[derive(Clone, Copy)] + pub enum DateTimePart { + DayOfMonth, + DayOfWeek, + DayOfYear, + DaysInMonth, + Hour, + Minute, + Month, + Second, + Year, + } + + pub fn timestamp_secs_to_utc_datetime(secs: i64) -> Option> { + DateTime::from_timestamp(secs, 0).map(|value| Utc.from_utc_datetime(&value.naive_utc())) + } + + pub fn datetime_part(value: DateTime, part: DateTimePart) -> Option { + Some(match part { + DateTimePart::DayOfMonth => value.day(), + DateTimePart::DayOfWeek => value.weekday().num_days_from_sunday(), + DateTimePart::DayOfYear => value.ordinal(), + DateTimePart::DaysInMonth => { + let (year, month) = (value.year(), value.month()); + let next = if month == 12 { + NaiveDate::from_ymd_opt(year + 1, 1, 1)? + } else { + NaiveDate::from_ymd_opt(year, month + 1, 1)? + }; + next.signed_duration_since(NaiveDate::from_ymd_opt(year, month, 1)?) + .num_days() as u32 + } + DateTimePart::Hour => value.hour(), + DateTimePart::Minute => value.minute(), + DateTimePart::Month => value.month(), + DateTimePart::Second => value.second(), + DateTimePart::Year => u32::try_from(value.year()).ok()?, + }) + } +} + +#[cfg(test)] +mod tests { + use super::{duration::fmt_duration_ms, time}; + use chrono::{TimeZone, Utc}; + use std::fmt; + + struct Millis(i64); + + impl fmt::Display for Millis { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + super::duration::fmt_duration_ms(formatter, self.0) + } + } + + #[test] + fn duration_format_matches_upstream_parser_support_cases() { + let cases = [ + (0, "0ms"), + (1, "1ms"), + (1_001, "1s1ms"), + (90_061, "1m30s61ms"), + (31_626_061_001, "1y1d1h1m1s1ms"), + (-1_001, "-1s-1ms"), + ]; + for (value, expected) in cases { + assert_eq!(Millis(value).to_string(), expected); + } + } + + #[test] + fn datetime_helpers_match_upstream_parser_support_cases() { + let epoch = time::timestamp_secs_to_utc_datetime(0).unwrap(); + assert_eq!( + time::datetime_part(epoch, time::DateTimePart::Year), + Some(1970) + ); + let leap = Utc.with_ymd_and_hms(2024, 2, 29, 23, 58, 57).unwrap(); + assert_eq!( + time::datetime_part(leap, time::DateTimePart::DaysInMonth), + Some(29) + ); + assert_eq!( + time::datetime_part(leap, time::DateTimePart::DayOfWeek), + Some(4) + ); + let negative_year = Utc.with_ymd_and_hms(-1, 1, 1, 0, 0, 0).unwrap(); + assert_eq!( + time::datetime_part(negative_year, time::DateTimePart::Year), + None + ); + } + + #[test] + fn hash_aliases_preserve_map_and_set_behavior() { + let mut map = super::hash::FastHashMap::default(); + map.insert("a", 1); + assert_eq!(map.get("a"), Some(&1)); + let mut set = super::hash::FastHashSet::default(); + set.insert("a"); + assert!(set.contains("a")); + } + + #[allow(dead_code)] + fn formatter_signature_is_the_upstream_signature( + formatter: &mut fmt::Formatter<'_>, + ) -> fmt::Result { + fmt_duration_ms(formatter, 1) + } +} diff --git a/crates/metricsql-parser-vendored/.gitignore b/crates/metricsql-parser-vendored/.gitignore new file mode 100644 index 00000000..042776aa --- /dev/null +++ b/crates/metricsql-parser-vendored/.gitignore @@ -0,0 +1,2 @@ +/Cargo.lock +/target/ diff --git a/crates/metricsql-parser-vendored/Cargo.toml b/crates/metricsql-parser-vendored/Cargo.toml new file mode 100644 index 00000000..0deaf46d --- /dev/null +++ b/crates/metricsql-parser-vendored/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "metricsql_parser" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +description = "Vendored MetricsQL parser from ccollie/metricsql" + +[dependencies] +ahash = { version = "0.8", features = ["std", "serde"] } +chrono = { version = "0.4", features = ["serde"] } +enquote = "1.1" +logos = "0.14" +logos-derive = "0.14" +metricsql_common = { path = "../metricsql-common-parser-support" } +num-traits = "0.2" +phf = { version = "0.11", features = ["macros"] } +regex = "1.10" +scopeguard = "1.2" +serde = { version = "1", features = ["derive"] } +strum = { version = "0.26", features = ["derive"] } +strum_macros = "0.26" +thiserror = "1" +tinyvec = { version = "1.8", features = ["alloc", "rustc_1_61"] } +xxhash-rust = { version = "0.8", features = ["xxh3"] } + +[dev-dependencies] +pretty_assertions = "1.4" +test-case = "3" + +[lib] +name = "metricsql_parser" + +[lints.rust] +unused_imports = "allow" +mismatched_lifetime_syntaxes = "allow" +dead_code = "allow" + +[lints.clippy] +all = "allow" + +# Keep this vendored dependency self-contained when the repository lives in a +# nested Git worktree and Cargo searches ancestor directories for a workspace. +[workspace] diff --git a/crates/metricsql-parser-vendored/LICENSE b/crates/metricsql-parser-vendored/LICENSE new file mode 100644 index 00000000..16fe87b0 --- /dev/null +++ b/crates/metricsql-parser-vendored/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/crates/metricsql-parser-vendored/UPSTREAM.md b/crates/metricsql-parser-vendored/UPSTREAM.md new file mode 100644 index 00000000..7bc108b2 --- /dev/null +++ b/crates/metricsql-parser-vendored/UPSTREAM.md @@ -0,0 +1,22 @@ +# Upstream provenance + +This crate vendors the `parser` directory from +[`ccollie/metricsql`](https://github.com/ccollie/metricsql) commit +`3046709308e449a42c56bfbfd45f95af848e6768` (Apache-2.0). + +`Cargo.toml` expands the upstream workspace dependencies and points +`metricsql_common` to ASAPPlanner's stable parser-only support crate. Parser +changes beyond `cargo fmt` are limited to compatibility fixes covered by the +official VictoriaMetrics Go parser golden corpus in +`crates/frontend-metricsql/tests/victoriametrics_go_golden.rs`. + +At the pinned commit, the upstream crate does not compile without changing its +`HashSetExt` import from `metricsql_common` to `ahash`. With that compile-only +fix and upstream's nightly toolchain, its library baseline is 230 passed and 23 +failed; its doctest baseline is 2 passed and 3 failed. The failures cover stale +optimizer/parser expectations and lexer edge cases. Our numeric underscore fix +resolves two of those lexer failures. The remaining 21 library and 3 doctest +failures are named explicitly and run on every CI build by +`tools/verify_metricsql_vendored_baseline.py`; any added or removed failure +causes CI to fail. Update the pinned commit, reproduce the upstream baseline, +and update that list before changing the vendored parser again. diff --git a/crates/metricsql-parser-vendored/src/ast/adjust_comparison_ops.rs b/crates/metricsql-parser-vendored/src/ast/adjust_comparison_ops.rs new file mode 100644 index 00000000..5c72a743 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/adjust_comparison_ops.rs @@ -0,0 +1,39 @@ +use crate::ast::Expr; + +/// Convert `num cmpOp query` expression to `query reverseCmpOp num` expression +/// like Prometheus does. For instance, `0.5 < foo` must be converted to `foo > 0.5` +/// in order to return valid values for `foo` that are bigger than 0.5. +pub fn adjust_comparison_ops(expr: &mut Expr) { + match expr { + Expr::Aggregation(agg) => { + for arg in agg.args.iter_mut() { + adjust_comparison_ops(arg); + } + } + Expr::UnaryOperator(ue) => { + adjust_comparison_ops(&mut ue.expr); + } + Expr::BinaryOperator(be) => { + adjust_comparison_ops(&mut be.left); + adjust_comparison_ops(&mut be.right); + be.adjust_comparison_op(); + } + Expr::Function(fe) => { + for arg in fe.args.iter_mut() { + adjust_comparison_ops(arg); + } + } + Expr::Parens(pe) => { + for e in pe.expressions.iter_mut() { + adjust_comparison_ops(e); + } + } + Expr::Rollup(re) => { + adjust_comparison_ops(&mut re.expr); + if let Some(ref mut at) = re.at { + adjust_comparison_ops(at.as_mut()); + } + } + _ => {} + } +} diff --git a/crates/metricsql-parser-vendored/src/ast/check_ast.rs b/crates/metricsql-parser-vendored/src/ast/check_ast.rs new file mode 100644 index 00000000..5ebd5144 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/check_ast.rs @@ -0,0 +1,246 @@ +// integrate checks from +// https://github.com/prometheus/prometheus/blob/fa6e05903fd3ce52e374a6e1bf4eb98c9f1f45a7/promql/parser/parse.go#L436 + +// Original Source: https://github.com/GreptimeTeam/promql-parser/blob/main/src/parser/ast.rs +use crate::ast::{ + AggregationExpr, BExpression, BinModifier, BinaryExpr, Expr, FunctionExpr, + InterpolatedSelector, MetricExpr, NumberLiteral, ParensExpr, RollupExpr, StringExpr, UnaryExpr, + VectorMatchCardinality, WithExpr, +}; +use crate::common::{Value, ValueType}; +use crate::functions::BuiltinFunction; +use crate::label::NAME_LABEL; + +/// check_ast checks the validity of the provided AST. This includes type checking. +/// Recursively check correct typing for child nodes and raise errors in case of bad typing. +pub fn check_ast(expr: Expr) -> Result { + use Expr::*; + match expr { + UnaryOperator(ex) => { + let modified = check_ast(*ex.expr)?; + Ok(UnaryOperator(UnaryExpr { + expr: Box::new(modified), + })) + } + BinaryOperator(ex) => check_ast_for_binary_expr(ex), + Aggregation(ex) => check_ast_for_aggregate_expr(ex), + Function(ex) => check_ast_for_call(ex), + MetricExpression(ex) => check_ast_for_vector_selector(ex), + Rollup(ex) => check_ast_for_rollup(ex), + Parens(ex) => check_ast_for_parens(ex), + StringExpr(ex) => check_ast_for_string_expr(ex), + With(ex) => check_ast_for_with(ex), + StringLiteral(_) | NumberLiteral(_) | Duration(_) => Ok(expr), + WithSelector(ws) => check_ast_for_interpolated_vector_selector(ws), + } +} + +pub fn validate_func_args(func: &BuiltinFunction, args: &[Expr]) -> Result<(), String> { + func.validate_args(args).map_err(|e| e.to_string()) +} + +fn check_ast_for_aggregate_expr(ex: AggregationExpr) -> Result { + let func = BuiltinFunction::Aggregate(ex.function); + validate_func_args(&func, &ex.args)?; + Ok(Expr::Aggregation(ex)) +} + +fn check_ast_for_call(expr: FunctionExpr) -> Result { + validate_func_args(&expr.function, &expr.args)?; + Ok(Expr::Function(expr)) +} + +fn check_ast_for_parens(expr: ParensExpr) -> Result { + let mut expressions: Vec = Vec::with_capacity(expr.len()); + for expr in expr.expressions.into_iter() { + expressions.push(check_ast(expr)?); + } + Ok(Expr::Parens(ParensExpr::new(expressions))) +} + +// TODO +fn check_ast_for_with(expr: WithExpr) -> Result { + Ok(Expr::With(expr)) +} + +/// TODO +fn check_ast_for_string_expr(expr: StringExpr) -> Result { + Ok(Expr::StringExpr(expr)) +} + +/// the original logic is redundant in +/// prometheus, and the following coding blocks +/// have been optimized for readability, but all logic SHOULD be covered. +fn check_ast_for_binary_expr(mut ex: BinaryExpr) -> Result { + use ValueType::*; + + let operator = ex.op; + let is_comparison = operator.is_comparison(); + + if ex.returns_bool() && !is_comparison { + return Err("bool modifier can only be used on comparison operators".into()); + } + + let left_type = ex.left.value_type(); + let right_type = ex.right.value_type(); + + // we're more lenient than prometheus here + // if is_comparison { + // match (&left_type, &right_type, ex.returns_bool()) { + // (ValueType::Scalar, ValueType::Scalar, false) => { + // return Err("comparisons between scalars must use BOOL modifier".into()); + // } + // (ValueType::String, ValueType::String, false) => { + // return Err("comparisons between strings must use BOOL modifier".into()); + // } + // _ => {} + // } + // } + + // For `on` matching, a label can only appear in one of the lists. + // Every time series of the result vector must be uniquely identifiable. + if ex.is_matching_on() && ex.is_labels_joint() { + if let Some(labels) = ex.intersect_labels() { + if let Some(label) = labels.first() { + return Err(format!( + "label '{label}' must not occur in ON and GROUP clause at once", + )); + } + }; + } + + if operator.is_set_operator() { + if left_type == String && right_type == String { + return Err(format!( + "operator '{operator}' not allowed in string string operations" + )); + } + + if left_type == String || right_type == String { + return Err(format!( + "set operator '{operator}' not allowed in binary {left_type}/{right_type} expression", + )); + } + + if left_type == InstantVector && right_type == InstantVector { + if let Some(ref modifier) = ex.modifier { + if matches!(modifier.card, VectorMatchCardinality::OneToMany(_)) + || matches!(modifier.card, VectorMatchCardinality::ManyToOne(_)) + { + return Err(format!("no grouping allowed for '{operator}' operation")); + } + }; + } + + match &mut ex.modifier { + Some(modifier) => { + if modifier.card == VectorMatchCardinality::OneToOne { + modifier.card = VectorMatchCardinality::ManyToMany; + } + } + None => { + ex.modifier = + Some(BinModifier::default().with_card(VectorMatchCardinality::ManyToMany)); + } + } + } + + if left_type == String && right_type == String { + if !operator.is_valid_string_op() { + return Err(format!( + "operator '{operator}' not allowed in string string operations" + )); + } + return Ok(Expr::BinaryOperator(ex)); + } + + let valid_types = [Scalar, InstantVector, RangeVector]; + + if !valid_types.contains(&left_type) || !valid_types.contains(&right_type) { + return Err("mismatched operand types in binary expression".into()); + } + + if (left_type != InstantVector || right_type != InstantVector) + && ex.is_matching_labels_not_empty() + { + return Err("vector matching only allowed between instant vectors".into()); + } + + Ok(Expr::BinaryOperator(ex)) +} + +fn check_ast_for_rollup(mut ex: RollupExpr) -> Result { + ex.expr = BExpression::from(check_ast(*ex.expr)?); + let value_type = ex.expr.return_type(); + if value_type != ValueType::InstantVector { + return Err(format!( + "subquery is only allowed on instant vector, got {value_type} instead" + )); + } + if let Some(at) = &ex.at { + let at_type = at.return_type(); + if at_type != ValueType::Scalar && value_type != ValueType::InstantVector { + return Err(format!( + "subquery @ modifier must be a scalar or expression, got {at_type} instead", + )); + } + if let Expr::NumberLiteral(NumberLiteral { value, .. }) = at.as_ref() { + if value.is_infinite() + || value.is_nan() + || value >= &(i64::MAX as f64) + || value <= &(i64::MIN as f64) + { + return Err(format!("timestamp out of bounds for @ modifier: {value}")); + } + } + } + + Ok(Expr::Rollup(ex)) +} + +fn check_ast_for_vector_selector(ex: MetricExpr) -> Result { + // A metric name may occur once in every OR branch. It is only duplicated + // when the same conjunction contains multiple __name__ matchers. + for branch in ex.matchers.iter() { + let metric_names: Vec<_> = branch + .iter() + .filter(|matcher| matcher.label == NAME_LABEL) + .collect(); + if metric_names.len() >= 2 { + return Err(format!( + "metric name must not be set twice: '{}' or '{}'", + metric_names[0].label, metric_names[1].label + )); + } + } + + if ex.metric_name().is_none() && ex.is_empty_matchers() { + // When name is None, a vector selector must contain at least one non-empty matcher + // to prevent implicit selection of all metrics (e.g. by a typo). + return Err("vector selector must contain at least one non-empty matcher".into()); + } + + Ok(Expr::MetricExpression(ex)) +} + +fn check_ast_for_interpolated_vector_selector(ex: InterpolatedSelector) -> Result { + // A Vector selector must contain at least one non-empty matcher to prevent + // implicit selection of all metrics (e.g. by a typo). + if ex.is_empty_matchers() { + return Err("vector selector must contain at least one non-empty matcher".into()); + } + + let mut du = ex.find_matchers(NAME_LABEL); + if du.len() >= 2 { + // this is to ensure that the err information can be predicted with fixed order + du.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); + + return Err(format!( + "metric name must not be set twice: '{}' or '{}'", + du[0].name(), + du[1].name() + )); + } + + Ok(Expr::WithSelector(ex)) +} diff --git a/crates/metricsql-parser-vendored/src/ast/expr.rs b/crates/metricsql-parser-vendored/src/ast/expr.rs new file mode 100644 index 00000000..f15f1f79 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/expr.rs @@ -0,0 +1,2356 @@ +use std::cmp::Ordering; +use std::fmt::{Display, Formatter, Write}; +use std::hash::{Hash, Hasher}; +use std::ops::{Deref, Neg, Range}; +use std::str::FromStr; +use std::time::{Duration, SystemTime}; +use std::{fmt, iter, ops}; + +use enquote::enquote; +use serde::{Deserialize, Serialize}; + +use metricsql_common::duration::fmt_duration_ms; + +use crate::ast::utils::string_vecs_equal_unordered; +use crate::ast::{ + expr_equals, indent, prettify_args, Operator, Prettier, StringExpr, MAX_CHARACTERS_PER_LINE, +}; +use crate::common::{hash_f64, join_vector, write_comma_separated, write_number, Value, ValueType}; +use crate::functions::{AggregateFunction, BuiltinFunction, TransformFunction}; +use crate::label::{LabelFilter, LabelFilterOp, Labels, Matchers, NAME_LABEL}; +use crate::parser::{escape_ident, ParseError, ParseResult}; +use crate::prelude::{ + get_aggregate_arg_idx_for_optimization, BuiltinFunctionType, InterpolatedSelector, + RollupFunction, +}; + +pub type BExpr = Box; + +/// Matching Modifier, for VectorMatching of binary expr. +/// Label lists provided to matching keywords will determine how vectors are combined. +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum VectorMatchModifier { + On(Labels), + Ignoring(Labels), +} + +impl Display for VectorMatchModifier { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + use VectorMatchModifier::*; + match self { + On(labels) => write!(f, "on({:?})", labels)?, + Ignoring(labels) => write!(f, "ignoring({:?})", labels)?, + } + Ok(()) + } +} + +impl VectorMatchModifier { + pub fn new(labels: Vec, is_on: bool) -> Self { + let names = Labels::new_from_iter(labels); + if is_on { + VectorMatchModifier::On(names) + } else { + VectorMatchModifier::Ignoring(names) + } + } + + pub fn labels(&self) -> &Labels { + match self { + VectorMatchModifier::On(l) => l, + VectorMatchModifier::Ignoring(l) => l, + } + } + + pub fn is_on(&self) -> bool { + matches!(*self, VectorMatchModifier::On(_)) + } +} + +/// Binary Expr Modifier +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct BinModifier { + /// The matching behavior for the operation if both operands are Vectors. + /// If they are not this field is None. + pub card: VectorMatchCardinality, + + /// on/ignoring on labels. + /// like a + b, no match modifier is needed. + #[serde(default, skip_serializing_if = "is_default")] + pub matching: Option, + + /// If keep_metric_names is set to true, then the operation should keep metric names. + #[serde(default, skip_serializing_if = "is_default")] + pub keep_metric_names: bool, + + /// If a comparison operator, return 0/1 rather than filtering. + /// For example, `foo > bool bar`. + #[serde(default, skip_serializing_if = "is_default")] + pub return_bool: bool, +} + +impl Default for BinModifier { + fn default() -> Self { + Self { + card: VectorMatchCardinality::OneToOne, + matching: None, + keep_metric_names: false, + return_bool: false, + } + } +} + +impl BinModifier { + pub fn with_card(mut self, card: VectorMatchCardinality) -> Self { + self.card = card; + self + } + + pub fn with_matching(mut self, matching: Option) -> Self { + self.matching = matching; + if self.matching.is_none() { + self.card = VectorMatchCardinality::OneToOne; + } + self + } + + pub fn with_return_bool(mut self, return_bool: bool) -> Self { + self.return_bool = return_bool; + self + } + + pub fn with_keep_metric_names(mut self, keep_metric_names: bool) -> Self { + self.keep_metric_names = keep_metric_names; + self + } + + pub fn is_labels_joint(&self) -> bool { + matches!((self.card.labels(), &self.matching), + (Some(labels), Some(matching)) if !labels.is_joint(matching.labels())) + } + + pub fn intersect_labels(&self) -> Option> { + if let Some(labels) = self.card.labels() { + if let Some(matching) = &self.matching { + let res = labels.intersect(matching.labels()); + return Some(res.0); + } + }; + None + } + + pub fn is_matching_on(&self) -> bool { + matches!(&self.matching, Some(matching) if matching.is_on()) + } + + pub fn is_matching_labels_not_empty(&self) -> bool { + matches!(&self.matching, Some(matching) if !matching.labels().is_empty()) + } + + pub fn is_default(&self) -> bool { + self.card == VectorMatchCardinality::OneToOne + && self.matching.is_none() + && !self.keep_metric_names + && !self.return_bool + } +} + +impl Display for BinModifier { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + use VectorMatchCardinality::*; + if self.return_bool { + write!(f, "bool")?; + } + match &self.card { + ManyToOne(labels) => { + write!(f, " group_left")?; + write_comma_separated(labels.iter(), f, true)?; + } + OneToMany(labels) => { + write!(f, " group_right")?; + write_comma_separated(labels.iter(), f, true)?; + } + _ => {} + } + if let Some(matching) = &self.matching { + match matching { + VectorMatchModifier::On(labels) => { + write!(f, " on")?; + write_comma_separated(labels.iter(), f, true)?; + } + VectorMatchModifier::Ignoring(labels) => { + write!(f, " ignoring")?; + write_comma_separated(labels.iter(), f, true)?; + } + } + } + if self.keep_metric_names { + write!(f, " keep_metric_names")?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, Eq, Serialize, Deserialize)] +pub enum AggregateModifier { + // todo: use BtreeSet, since the runtime expects these to be sorted + By(Vec), + Without(Vec), +} + +impl AggregateModifier { + /// Creates a new AggregateModifier with the Left op + pub fn by() -> Self { + AggregateModifier::By(vec![]) + } + + /// Creates a new AggregateModifier with the Right op + pub fn without() -> Self { + AggregateModifier::Without(vec![]) + } + + /// Adds a label key to this AggregateModifier + pub fn arg>(&mut self, arg: S) { + match self { + AggregateModifier::By(ref mut args) => { + args.push(arg.into()); + args.sort(); + } + AggregateModifier::Without(ref mut args) => { + args.push(arg.into()); + args.sort(); + } + } + } + + pub fn get_args(&self) -> &Vec { + match self { + AggregateModifier::By(val) => val, + AggregateModifier::Without(val) => val, + } + } + + pub fn is_empty(&self) -> bool { + self.get_args().is_empty() + } +} + +impl Display for AggregateModifier { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + AggregateModifier::By(vec) => { + write!(f, "by ")?; + write_comma_separated(vec.iter(), f, true)?; + } + AggregateModifier::Without(vec) => { + write!(f, "without ")?; + write_comma_separated(vec.iter(), f, true)?; + } + } + Ok(()) + } +} + +impl PartialEq for AggregateModifier { + fn eq(&self, other: &AggregateModifier) -> bool { + match (self, other) { + (AggregateModifier::Without(left), AggregateModifier::Without(right)) => { + string_vecs_equal_unordered(left, right) + } + (AggregateModifier::By(left), AggregateModifier::By(right)) => { + string_vecs_equal_unordered(left, right) + } + _ => false, + } + } +} + +// See https://prometheus.io/docs/prometheus/latest/querying/operators/#vector-matching +#[derive(Debug, Clone, PartialEq, Eq, Copy, Hash, Serialize, Deserialize)] +pub enum GroupModifierOp { + On, + Ignoring, +} + +impl Display for GroupModifierOp { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + use GroupModifierOp::*; + match self { + On => write!(f, "on")?, + Ignoring => write!(f, "ignoring")?, + } + Ok(()) + } +} + +impl TryFrom<&str> for GroupModifierOp { + type Error = ParseError; + + fn try_from(op: &str) -> Result { + use GroupModifierOp::*; + + match op { + op if op.eq_ignore_ascii_case("on") => Ok(On), + op if op.eq_ignore_ascii_case("ignoring") => Ok(Ignoring), + _ => Err(ParseError::General(format!( + "Unknown group_modifier op: {op}", + ))), + } + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub enum GroupType { + GroupLeft, + GroupRight, +} + +#[derive(Debug, Clone, Eq, Hash, PartialEq, Serialize, Deserialize)] +pub enum VectorMatchCardinality { + OneToOne, + /// on(labels)/ignoring(labels) GROUP_LEFT + ManyToOne(Labels), + /// on(labels)/ignoring(labels) GROUP_RIGHT + OneToMany(Labels), + /// logical/set binary operators + ManyToMany, +} + +impl VectorMatchCardinality { + pub fn group_left(labels: Labels) -> Self { + VectorMatchCardinality::ManyToOne(labels) + } + + pub fn group_right(labels: Labels) -> Self { + VectorMatchCardinality::OneToMany(labels) + } + pub fn is_group_left(&self) -> bool { + matches!(self, VectorMatchCardinality::ManyToOne(_)) + } + + pub fn is_group_right(&self) -> bool { + matches!(self, VectorMatchCardinality::OneToMany(_)) + } + + pub fn is_grouping(&self) -> bool { + matches!( + self, + VectorMatchCardinality::ManyToOne(_) | VectorMatchCardinality::OneToMany(_) + ) + } + + pub fn labels(&self) -> Option<&Labels> { + match self { + VectorMatchCardinality::ManyToOne(labels) + | VectorMatchCardinality::OneToMany(labels) => Some(labels), + _ => None, + } + } + + pub fn group_type(&self) -> Option { + match self { + VectorMatchCardinality::ManyToOne(_) => Some(GroupType::GroupLeft), + VectorMatchCardinality::OneToMany(_) => Some(GroupType::GroupRight), + _ => None, + } + } +} + +impl Display for VectorMatchCardinality { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + use VectorMatchCardinality::*; + let str = match self { + OneToOne => "OneToOne".to_string(), + OneToMany(labels) => format!("group_right({:?})", labels), + ManyToOne(labels) => format!("group_left({:?})", labels), + ManyToMany => "ManyToMany".to_string(), + }; + write!(f, "{}", str)?; + Ok(()) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Copy, Hash, Serialize, Deserialize)] +pub enum JoinModifierOp { + GroupLeft, + GroupRight, +} + +impl Display for JoinModifierOp { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + use JoinModifierOp::*; + match self { + GroupLeft => write!(f, "group_left")?, + GroupRight => write!(f, "group_right")?, + } + Ok(()) + } +} + +impl TryFrom<&str> for JoinModifierOp { + type Error = ParseError; + + fn try_from(op: &str) -> Result { + use JoinModifierOp::*; + + match op { + op if op.eq_ignore_ascii_case("group_left") => Ok(GroupLeft), + op if op.eq_ignore_ascii_case("group_right") => Ok(GroupRight), + _ => { + let msg = format!("Unknown join_modifier op: {}", op); + Err(ParseError::General(msg)) + } + } + } +} + +/// Expression Trait. Useful for cases where match is not ergonomic +pub trait ExpressionNode { + fn cast(self) -> Expr; +} + +/// NumberExpr represents number expression. +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct NumberLiteral { + /// value is the parsed number, i.e. `1.23`, `-234`, etc. + pub value: f64, +} + +impl NumberLiteral { + pub fn new(v: f64) -> Self { + NumberLiteral { value: v } + } + + pub fn return_type(&self) -> ValueType { + ValueType::Scalar + } +} + +impl Value for NumberLiteral { + fn value_type(&self) -> ValueType { + ValueType::Scalar + } +} + +impl From for NumberLiteral { + fn from(value: f64) -> Self { + NumberLiteral::new(value) + } +} + +impl From for NumberLiteral { + fn from(value: i64) -> Self { + NumberLiteral::new(value as f64) + } +} + +impl From for NumberLiteral { + fn from(value: usize) -> Self { + NumberLiteral::new(value as f64) + } +} + +impl PartialEq for NumberLiteral { + fn eq(&self, other: &Self) -> bool { + are_floats_equal(self.value, other.value) + } +} + +impl PartialOrd for NumberLiteral { + fn partial_cmp(&self, other: &Self) -> Option { + self.value.partial_cmp(&other.value) + } +} + +impl Eq for NumberLiteral {} + +impl ExpressionNode for NumberLiteral { + fn cast(self) -> Expr { + Expr::NumberLiteral(self.clone()) + } +} + +impl Display for NumberLiteral { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write_number(f, self.value) + } +} + +impl Prettier for NumberLiteral { + fn needs_split(&self, _max: usize) -> bool { + false + } +} + +impl Hash for NumberLiteral { + fn hash(&self, state: &mut H) { + hash_f64(state, self.value); + } +} + +impl Neg for NumberLiteral { + type Output = Self; + + fn neg(self) -> Self::Output { + let value = -self.value; + NumberLiteral { value } + } +} + +impl Deref for NumberLiteral { + type Target = f64; + + fn deref(&self) -> &Self::Target { + &self.value + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StringLiteral(pub String); + +impl StringLiteral { + pub fn new>(s: S) -> Self { + StringLiteral(s.into()) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +impl Display for StringLiteral { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +impl Prettier for StringLiteral { + fn needs_split(&self, _max: usize) -> bool { + false + } +} + +impl Deref for StringLiteral { + type Target = String; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl From for StringLiteral { + fn from(s: String) -> Self { + StringLiteral(s) + } +} + +/// DurationExpr contains a duration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub enum DurationExpr { + /// duration value in milliseconds + Millis(i64), + /// a value that is multiplied at evaluation time by the step value + StepValue(f64), +} + +impl DurationExpr { + pub fn from_secs(secs: i64) -> Self { + Self::Millis(secs * 1000) + } + + pub fn new(millis: i64) -> Self { + Self::Millis(millis) + } + + pub fn new_step(value: f64) -> Self { + Self::StepValue(value) + } + + pub fn requires_step(&self) -> bool { + matches!(self, DurationExpr::StepValue(_)) + } + + /// Duration returns the duration from de in milliseconds. + pub fn value(&self, step: i64) -> i64 { + match self { + DurationExpr::Millis(v) => *v, + DurationExpr::StepValue(v) => (*v * step as f64) as i64, + } + } + + pub fn value_as_secs(&self, step: i64) -> i64 { + self.value(step) / 1000 + } + + pub fn non_negative_value(&self, step: i64) -> Result { + let v = self.value(step); + if v < 0 { + return Err(format!("unexpected negative duration {v}dms").to_string()); + } + Ok(v) + } + + pub fn return_type(&self) -> ValueType { + ValueType::Scalar + } +} + +impl Default for DurationExpr { + fn default() -> Self { + Self::Millis(0) + } +} + +impl Display for DurationExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + DurationExpr::Millis(v) => fmt_duration_ms(f, *v), + DurationExpr::StepValue(v) => write!(f, "{v}i"), + } + } +} + +impl TryFrom for DurationExpr { + type Error = String; + + fn try_from(num: NumberLiteral) -> Result { + DurationExpr::try_from(num.value) + } +} + +impl TryFrom for DurationExpr { + type Error = String; + + fn try_from(ex: Expr) -> Result { + match ex { + Expr::NumberLiteral(nl) => DurationExpr::try_from(nl), + _ => Err("invalid float value after @ modifier".into()), + } + } +} + +impl TryFrom for DurationExpr { + type Error = String; + + fn try_from(secs: f64) -> Result { + let err_info = format!("timestamp out of bounds for @ modifier: {secs}"); + + if secs.is_nan() || secs.is_infinite() || secs >= f64::MAX || secs <= f64::MIN { + return Err(err_info); + } + let milli = (secs * 1000f64).round().abs() as u64; + + let duration = Duration::from_millis(milli); + let mut st = Some(SystemTime::UNIX_EPOCH); + if secs.is_sign_positive() { + st = SystemTime::UNIX_EPOCH.checked_add(duration); + } + if secs.is_sign_negative() { + st = SystemTime::UNIX_EPOCH.checked_sub(duration); + } + if let Some(st) = st { + let millis = st + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + return Ok(DurationExpr::Millis(millis)); + } + Err(err_info) + } +} + +impl PartialEq for DurationExpr { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (DurationExpr::Millis(v1), DurationExpr::Millis(v2)) => v1 == v2, + (DurationExpr::StepValue(v1), DurationExpr::StepValue(v2)) => { + are_floats_equal(*v1, *v2) + } + _ => false, + } + } +} + +impl Eq for DurationExpr {} + +impl Prettier for DurationExpr { + fn needs_split(&self, _max: usize) -> bool { + false + } +} + +// todo: MetricExpr => Selector +/// MetricExpr represents MetricsQL metric with optional filters, i.e. `foo{...}`. +/// +/// Curly braces may contain or-delimited list of filters. For example: +/// +/// `x{job="foo",instance="bar" or job="x",instance="baz"}` +/// +/// In this case the filter returns all the series, which match at least one of the following filters: +/// +/// `x{job="foo",instance="bar"}` +/// `x{job="x",instance="baz"}` +/// +/// This allows using or-delimited list of filters inside rollup functions. For example, +/// the following query calculates rate per each matching series for the given or-delimited filters: +/// +/// `rate(x{job="foo",instance="bar" or job="x",instance="baz"}[5m])` +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MetricExpr { + /// name is the metric name. + pub name: Option, + /// matchers contains a list of label filters from curly braces. + pub matchers: Matchers, +} + +impl MetricExpr { + pub fn new>(name: S) -> MetricExpr { + let name_filter = LabelFilter { + op: LabelFilterOp::Equal, + label: NAME_LABEL.to_string(), + value: name.into(), + }; + MetricExpr { + // name: Some(name.into()), + name: None, + matchers: Matchers::default().append(name_filter), + } + } + + pub fn with_filters(filters: Vec) -> Self { + MetricExpr { + name: None, + matchers: Matchers::new(filters), + } + } + + pub fn with_or_filters(filters: Vec>) -> Self { + MetricExpr { + name: None, + matchers: Matchers::with_or_matchers(filters), + } + } + + pub fn is_empty(&self) -> bool { + self.matchers.is_empty() + } + + pub fn is_only_metric_name(&self) -> bool { + self.matchers.is_only_metric_name() + } + + pub fn metric_name(&self) -> Option<&str> { + self.matchers.metric_name() + } + + pub fn append(mut self, filter: LabelFilter) -> Self { + self.matchers = self.matchers.append(filter); + self + } + + pub fn append_or(mut self, filter: LabelFilter) -> Self { + self.matchers = self.matchers.append_or(filter); + self + } + + pub fn return_type(&self) -> ValueType { + ValueType::InstantVector + } + + pub fn is_empty_matchers(&self) -> bool { + self.matchers.is_empty_matchers() + } + + /// find all the matchers whose name equals the specified name. + pub fn find_matchers(&self, name: &str) -> Vec<&LabelFilter> { + self.matchers.find_matchers(name) + } + + pub fn sort_filters(&mut self) { + self.matchers.sort_filters(); + } + + pub fn has_or_matchers(&self) -> bool { + !self.matchers.or_matchers.is_empty() + } +} + +impl Value for MetricExpr { + fn value_type(&self) -> ValueType { + ValueType::InstantVector + } +} + +impl Display for MetricExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + if self.is_empty() { + write!(f, "{{}}")?; + return Ok(()); + } + + let mut offset = 0; + let metric_name = self.metric_name().unwrap_or(""); + if !metric_name.is_empty() { + write!(f, "{}", escape_ident(metric_name))?; + offset = 1; + } + + if self.is_only_metric_name() { + return Ok(()); + } + write!(f, "{{")?; + let mut count = 0; + for lfs in self.matchers.iter() { + if lfs.len() < offset { + continue; + } + if count > 0 { + write!(f, " or ")?; + } + let lfs_ = &lfs[offset..]; + write!(f, "{}", join_vector(lfs_, ", ", false))?; + count += 1; + } + write!(f, "}}")?; + Ok(()) + } +} + +/// directly create an instant vector with only METRIC_NAME matcher. +/// +/// # Examples +/// +/// Basic usage: +/// +/// ``` rust +/// use metricsql_parser::ast::MetricExpr; +/// use metricsql_parser::label::Matchers; +/// +/// let vs = MetricExpr { +/// name: Some(String::from("foo")), +/// matchers: Matchers::empty(), +/// }; +/// +/// assert_eq!(MetricExpr::from("foo"), vs); +/// ``` +impl From<&str> for MetricExpr { + fn from(name: &str) -> Self { + MetricExpr::from(name.to_string()) + } +} + +impl From for MetricExpr { + fn from(name: String) -> Self { + MetricExpr::new(name) + } +} + +impl Neg for MetricExpr { + type Output = UnaryExpr; + + fn neg(self) -> Self::Output { + let ex = Expr::MetricExpression(self); + UnaryExpr { expr: Box::new(ex) } + } +} + +impl ExpressionNode for MetricExpr { + fn cast(self) -> Expr { + Expr::MetricExpression(self) + } +} + +impl Prettier for MetricExpr { + fn needs_split(&self, _max: usize) -> bool { + false + } +} + +/// FuncExpr represents MetricsQL function such as `rate(...)` +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FunctionExpr { + pub function: BuiltinFunction, + + /// Args contains function args. + pub args: Vec, + + /// If keep_metric_names is set to true, then the function should keep metric names. + #[serde(default, skip_serializing_if = "is_default")] + pub keep_metric_names: bool, +} + +impl FunctionExpr { + pub fn new(name: &str, args: Vec) -> ParseResult { + let func_name = if name.is_empty() { "union" } else { name }; + let function = BuiltinFunction::new(func_name)?; + + Ok(Self { + args, + keep_metric_names: false, + function, + }) + } + + pub fn name(&self) -> &'static str { + self.function.name() + } + + pub fn return_type(&self) -> ValueType { + self.function + .return_type(&self.args) + .unwrap_or(ValueType::Scalar) + } + + pub fn function_type(&self) -> BuiltinFunctionType { + self.function.get_type() + } + + pub fn arg_idx_for_optimization(&self) -> Option { + match self.function { + BuiltinFunction::Aggregate(aggr_fn) => self.get_aggr_arg_idx_for_optimization(aggr_fn), + _ => self.function.get_arg_idx_for_optimization(self.args.len()), + } + } + + pub fn arg_for_optimization(&self) -> Option<&Expr> { + match self.arg_idx_for_optimization() { + None => None, + Some(idx) => self.args.get(idx), + } + } + + fn get_aggr_arg_idx_for_optimization(&self, func: AggregateFunction) -> Option { + let arg_count = self.args.len(); + use AggregateFunction::*; + // todo: just examine the signature and return the position containing a vector + match func { + Bottomk | BottomkAvg | BottomkMax | BottomkMedian | BottomkLast | BottomkMin + | Limitk | Outliersk | OutliersMAD | Quantile | Topk | TopkAvg | TopkMax + | TopkMedian | TopkLast | TopkMin => Some(1), + CountValues => None, + Quantiles => Some(arg_count - 1), + _ => { + for e in &self.args { + if let Expr::Aggregation(_) = e { + return None; + } + } + Some(0) + } + } + } + + pub fn default_rollup(arg: Expr) -> ParseResult { + Self::from_single_arg("default_rollup", arg) + } + + pub fn from_single_arg(name: &str, arg: Expr) -> ParseResult { + let args = vec![arg]; + Self::new(name, args) + } + + pub fn is_rollup(&self) -> bool { + self.function_type() == BuiltinFunctionType::Rollup + } + + pub fn is_rollup_function(&self, rf: RollupFunction) -> bool { + match self.function { + BuiltinFunction::Rollup(r) => r == rf, + _ => false, + } + } + + pub fn is_aggregate_function(&self, af: AggregateFunction) -> bool { + match self.function { + BuiltinFunction::Aggregate(a) => a == af, + _ => false, + } + } + + pub fn is_transform_function(&self, tf: TransformFunction) -> bool { + match self.function { + BuiltinFunction::Transform(f) => f == tf, + _ => false, + } + } +} + +impl Display for FunctionExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", self.function.name())?; + write_comma_separated(&mut self.args.iter(), f, true)?; + if self.keep_metric_names { + write!(f, " keep_metric_names")?; + } + Ok(()) + } +} + +impl Prettier for FunctionExpr { + fn format(&self, level: usize, max: usize) -> String { + let spaces = indent(level); + format!( + "{spaces}{}(\n{}\n{spaces}{})", + self.name(), + prettify_args(&self.args, level + 1, max), + if self.keep_metric_names { + " keep_metric_names" + } else { + "" + } + ) + } +} + +/// AggregationExpr represents aggregate function such as `sum(...) by (...)` +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct AggregationExpr { + /// function is the aggregation function. + pub function: AggregateFunction, + + /// function args. + pub args: Vec, + + /// optional modifier such as `by (...)` or `without (...)`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub modifier: Option, + + /// optional limit for the number of output time series. + /// This is an MetricsQL extension. + /// + /// Example: `sum(...) by (...) limit 10` would return maximum 10 time series. + #[serde(default, skip_serializing_if = "is_default")] + pub limit: usize, + + #[serde(default, skip_serializing_if = "is_default")] + pub keep_metric_names: bool, +} + +impl AggregationExpr { + pub fn new(function: AggregateFunction, args: Vec) -> AggregationExpr { + let mut ae = AggregationExpr { + args, + modifier: None, + limit: 0, + function, + keep_metric_names: false, + }; + + ae.set_keep_metric_names(); + ae + } + + /// the aggregation function name. + pub fn name(&self) -> &'static str { + self.function.name() + } + + pub fn from_name(name: &str) -> ParseResult { + let function = AggregateFunction::from_str(name)?; + Ok(Self::new(function, vec![])) + } + + pub fn with_modifier(mut self, modifier: AggregateModifier) -> Self { + self.modifier = Some(modifier); + self + } + + pub fn with_args(mut self, args: &[Expr]) -> Self { + self.args = args.to_vec(); + self.set_keep_metric_names(); + self + } + + fn set_keep_metric_names(&mut self) { + if self.args.len() != 1 { + self.keep_metric_names = false; + return; + } + match &self.args[0] { + Expr::Function(fe) => { + self.keep_metric_names = fe.keep_metric_names; + } + Expr::Aggregation(ae) => { + self.keep_metric_names = ae.keep_metric_names; + } + _ => self.keep_metric_names = false, + } + } + + pub fn return_type(&self) -> ValueType { + ValueType::InstantVector + } + + pub fn get_arg_for_optimization(&self) -> Option<&'_ Expr> { + match self.arg_idx_for_optimization() { + None => None, + Some(idx) => Some(&self.args[idx]), + } + } + + pub fn arg_idx_for_optimization(&self) -> Option { + get_aggregate_arg_idx_for_optimization(self.function, self.args.len()) + } + + /// Check if args[0] contains one of the following: + /// - metricExpr + /// - metricExpr[d] + /// - RollupFunc(metricExpr) + /// - RollupFunc(metricExpr[d]) + pub fn can_incrementally_eval(&self) -> bool { + if self.args.len() != 1 { + return false; + } + + fn validate(me: &MetricExpr, for_subquery: bool) -> bool { + if me.is_empty() || for_subquery { + return false; + } + + true + } + + fn validate_expr(expr: &Expr) -> bool { + match expr { + Expr::MetricExpression(me) => validate(me, false), + Expr::Rollup(re) => match &*re.expr { + Expr::MetricExpression(me) => validate(me, re.for_subquery()), + _ => false, + }, + _ => false, + } + } + + let first = &self.args[0]; + match first { + Expr::Function(fe) => match fe.function { + BuiltinFunction::Rollup(_) => { + if let Some(arg) = fe.arg_for_optimization() { + validate_expr(arg) + } else { + false + } + } + _ => false, + }, + _ => validate_expr(first), + } + } + + pub fn is_non_grouping(&self) -> bool { + match &self.modifier { + Some(modifier) => modifier.is_empty(), + _ => true, + } + } + + fn get_op_string(&self) -> String { + let mut s = self.function.to_string(); + + if let Some(modifier) = &self.modifier { + match modifier { + AggregateModifier::By(ls) if !ls.is_empty() => write!(s, " {modifier} ").unwrap(), + AggregateModifier::Without(_) => write!(s, " {modifier} ").unwrap(), + _ => (), + } + } + s + } +} + +impl Display for AggregationExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", self.function)?; + let args_len = self.args.len(); + if args_len > 0 { + write_comma_separated(&mut self.args.iter(), f, true)?; + } + if let Some(modifier) = &self.modifier { + write!(f, " {}", modifier)?; + } + if self.limit > 0 { + write!(f, " limit {}", self.limit)?; + } + Ok(()) + } +} + +impl Prettier for AggregationExpr { + fn format(&self, level: usize, max: usize) -> String { + let spaces = indent(level); + let mut s = format!("{spaces}{}(\n", self.get_op_string()); + let args = prettify_args(&self.args, level + 1, max); + if !args.is_empty() { + writeln!(s, "{}", args).unwrap(); + } + write!(s, "{spaces})").unwrap(); + if self.limit > 0 { + write!(s, " limit {}", self.limit).unwrap(); + } + if self.keep_metric_names { + write!(s, " keep_metric_names").unwrap(); + } + s + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +/// RollupExpr represents an MetricsQL expression which contains at least `offset` or `[...]` part. +pub struct RollupExpr { + /// The expression for the rollup. Usually it is MetricExpr, but may be arbitrary expr + /// if subquery is used. https://prometheus.io/blog/2019/01/28/subquery-support/ + pub expr: BExpression, + + /// window contains optional window value from square brackets. Equivalent to `range` in + /// prometheus terminology + /// + /// For example, `http_requests_total[5m]` will have window value `5m`. + #[serde(skip_serializing_if = "Option::is_none")] + pub window: Option, + + /// step contains optional step value from square brackets. Equivalent to `resolution` + /// in the prometheus docs + /// + /// For example, `foobar[1h:3m]` will have step value `3m`. + #[serde(skip_serializing_if = "Option::is_none")] + pub step: Option, + + /// offset contains optional value from `offset` part. + /// + /// For example, `foobar{baz="aa"} offset 5m` will have offset value `5m`. + #[serde(skip_serializing_if = "Option::is_none")] + pub offset: Option, + + /// if set to true, then `foo[1h:]` would print the same instead of `foo[1h]`. + #[serde(default, skip_serializing_if = "is_default")] + pub inherit_step: bool, + + /// at contains an optional expression after `@` modifier. + /// + /// For example, `foo @ end()` or `bar[5m] @ 12345` + /// See https://prometheus.io/docs/prometheus/latest/querying/basics/#modifier + #[serde(skip_serializing_if = "Option::is_none")] + pub at: Option, +} + +impl RollupExpr { + pub fn new(expr: Expr) -> Self { + RollupExpr { + expr: Box::new(expr), + window: None, + offset: None, + step: None, + inherit_step: false, + at: None, + } + } + + pub fn for_subquery(&self) -> bool { + self.step.is_some() || self.inherit_step + } + + pub fn set_at(mut self, expr: impl ExpressionNode) -> Self { + self.at = Some(Box::new(expr.cast())); + self + } + + pub fn set_offset(&mut self, expr: DurationExpr) { + self.offset = Some(expr); + } + + pub fn set_window(mut self, expr: DurationExpr) -> ParseResult { + self.window = Some(expr); + self.validate().map_err(ParseError::General)?; + Ok(self) + } + + pub fn set_expr(&mut self, expr: impl ExpressionNode) -> ParseResult<()> { + self.expr = Box::new(expr.cast()); + self.validate().map_err(ParseError::General) + } + + fn validate(&self) -> Result<(), String> { + // range + subquery is not allowed (however this is syntactically invalid) + // if self.window.is_some() && self.for_subquery() { + // return Err( + // "range and subquery are not allowed together in a rollup expression".to_string(), + // ); + // } + Ok(()) + } + + pub fn return_type(&self) -> ValueType { + // sub queries turn instant vectors into ranges + match (self.window.is_some(), self.for_subquery()) { + (false, false) => ValueType::InstantVector, + (false, true) => ValueType::RangeVector, + (true, false) => ValueType::RangeVector, + (true, true) => { + ValueType::RangeVector + // unreachable!("range and subquery are not allowed together in a rollup expression") + } + } + } + + pub fn wraps_metric_expr(&self) -> bool { + matches!(*self.expr, Expr::MetricExpression(_)) + } + + fn get_time_suffix_string(&self) -> Result { + let mut s = String::with_capacity(12); + if self.window.is_some() || self.inherit_step || self.step.is_some() { + s.push('['); + if let Some(win) = &self.window { + write!(s, "{}", win)?; + } + if let Some(step) = &self.step { + s.push(':'); + write!(s, "{}", step)?; + } else if self.inherit_step { + s.push(':'); + } + s.push(']'); + } + if let Some(offset) = &self.offset { + write!(s, " offset {}", offset)?; + } + if let Some(at) = &self.at { + let parens_needed = at.is_binary_op(); + s.push_str(" @ "); + if parens_needed { + s.push('('); + } + write!(s, "{}", at)?; + if parens_needed { + s.push(')'); + } + } + Ok(s) + } +} + +impl Display for RollupExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + let need_parens = match self.expr.as_ref() { + Expr::Rollup(_) => true, + Expr::BinaryOperator(_) => true, + Expr::Aggregation(ae) => ae.modifier.is_some(), + _ => false, + }; + if need_parens { + write!(f, "(")?; + } + write!(f, "{}", self.expr)?; + if need_parens { + write!(f, ")")?; + } + let suffix = self.get_time_suffix_string().map_err(|_| fmt::Error)?; + + write!(f, "{}", suffix)?; + Ok(()) + } +} + +impl Prettier for RollupExpr { + fn pretty(&self, level: usize, max: usize) -> String { + let suffix = self.get_time_suffix_string().unwrap_or("".to_string()); + format!("{}{}", self.expr.pretty(level, max), suffix) + } +} + +/// BinaryOpExpr represents a binary operation. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct BinaryExpr { + /// left contains left arg for the `left op right` expression. + pub left: BExpression, + + /// contains right arg for the `left op right` expression. + pub right: BExpression, + + /// Op is the operation itself, i.e. `+`, `-`, `*`, etc. + pub op: Operator, + + #[serde(skip_serializing_if = "Option::is_none")] + pub modifier: Option, +} + +impl BinaryExpr { + pub fn new(op: Operator, lhs: Expr, rhs: Expr) -> Self { + // operators can only have instant vectors or scalars + // TODO + BinaryExpr { + op, + left: Box::new(lhs), + right: Box::new(rhs), + modifier: None, + } + } + + pub fn is_matching_on(&self) -> bool { + matches!(&self.modifier, Some(modifier) if modifier.is_matching_on()) + } + + pub fn is_matching_labels_not_empty(&self) -> bool { + matches!(&self.modifier, Some(modifier) if modifier.is_matching_labels_not_empty()) + } + + /// check if labels of card and matching are joint + pub fn is_labels_joint(&self) -> bool { + matches!(&self.modifier, Some(modifier) if modifier.is_labels_joint()) + } + + /// intersect labels of card and matching + pub fn intersect_labels(&self) -> Option> { + self.modifier + .as_ref() + .and_then(|modifier| modifier.intersect_labels()) + } + + pub fn with_bool_modifier(mut self) -> Self { + if !self.op.is_comparison() { + panic!("bool modifier is only allowed for comparison operators"); + } + if let Some(modifier) = &mut self.modifier { + modifier.return_bool = true; + } else { + let modifier = BinModifier { + return_bool: true, + ..Default::default() + }; + self.modifier = Some(modifier); + } + self + } + + pub fn set_keep_metric_names(&mut self) { + if let Some(modifier) = &mut self.modifier { + modifier.keep_metric_names = true; + } else { + let modifier = BinModifier { + keep_metric_names: true, + ..Default::default() + }; + self.modifier = Some(modifier); + } + } + + /// Convert `num cmpOp query` expression to `query reverseCmpOp num` expression + /// like Prometheus does. For instance, `0.5 < foo` must be converted to `foo > 0.5` + /// in order to return valid values for `foo` that are bigger than 0.5. + pub fn adjust_comparison_op(&mut self) -> bool { + if self.should_adjust_comparison_op() { + self.op = self.op.get_reverse_cmp(); + std::mem::swap(&mut self.left, &mut self.right); + return true; + } + false + } + + pub fn should_adjust_comparison_op(&self) -> bool { + if !self.op.is_comparison() { + return false; + } + + if Expr::is_number(&self.right) || !Expr::is_scalar(&self.left) { + return false; + } + true + } + + pub fn should_reset_metric_group(&self) -> bool { + let op = self.op; + if op.is_comparison() && !self.returns_bool() { + // do not reset MetricGroup for non-boolean `compare` binary ops like Prometheus does. + return false; + } + !matches!(op, Operator::Default | Operator::If | Operator::IfNot) + } + + pub fn return_type(&self) -> ValueType { + let lhs_ret = self.left.return_type(); + let rhs_ret = self.right.return_type(); + + match (lhs_ret, rhs_ret) { + (ValueType::Scalar, ValueType::Scalar) => ValueType::Scalar, + (ValueType::RangeVector, ValueType::RangeVector) => ValueType::RangeVector, + (ValueType::InstantVector, ValueType::InstantVector) => ValueType::InstantVector, + (ValueType::InstantVector, ValueType::Scalar) => ValueType::InstantVector, + (ValueType::Scalar, ValueType::InstantVector) => ValueType::InstantVector, + (ValueType::String, ValueType::String) => { + if self.op.is_comparison() { + return ValueType::Scalar; + } + debug_assert!( + self.op == Operator::Add, + "Operator {} is not valid for (String, String)", + self.op + ); + ValueType::String + } + _ => ValueType::InstantVector, + } + } + + /// indicates whether `bool` modifier is present. + /// For example, `foo > bool bar`. + pub fn returns_bool(&self) -> bool { + matches!(&self.modifier, Some(modifier) if modifier.return_bool) + } + + /// Determines if the result of the operation should keep metric names. + pub fn keep_metric_names(&self) -> bool { + matches!(&self.modifier, Some(modifier) if modifier.keep_metric_names) + } + + pub fn vector_match_cardinality(&self) -> Option<&VectorMatchCardinality> { + if let Some(modifier) = self.modifier.as_ref() { + return Some(&modifier.card); + } + None + } + + fn fmt_no_keep_metric_name(&self, f: &mut Formatter) -> fmt::Result { + write!( + f, + "{} {} {}", + self.left, + self.get_op_matching_string(), + self.right + ) + } + + fn get_op_matching_string(&self) -> String { + match &self.modifier { + Some(modifier) => format!("{}{modifier}", self.op), + None => self.op.to_string(), + } + } +} + +impl Display for BinaryExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + if self.keep_metric_names() { + write!(f, "(")?; + self.fmt_no_keep_metric_name(f)?; + write!(f, ") keep_metric_names")?; + } else { + self.fmt_no_keep_metric_name(f)?; + } + Ok(()) + } +} + +impl Prettier for BinaryExpr { + fn format(&self, level: usize, max: usize) -> String { + format!( + "{}\n{}{}{}\n{}", + self.left.pretty(level + 1, max), + indent(level), + self.get_op_matching_string(), + if self.keep_metric_names() { + "\n keep_metric_names" + } else { + "" + }, + self.right.pretty(level + 1, max) + ) + } +} + +/// UnaryExpr will negate the expr +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct UnaryExpr { + pub expr: Box, +} + +impl UnaryExpr { + pub fn new(expr: Expr) -> Self { + UnaryExpr { + expr: Box::new(expr), + } + } +} + +impl Display for UnaryExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "-{}", self.expr) + } +} + +impl Prettier for UnaryExpr { + fn pretty(&self, level: usize, max: usize) -> String { + format!( + "{}-{}", + indent(level), + self.expr.pretty(level, max).trim_start() + ) + } +} + +// TODO: ParensExpr => GroupExpr +/// Expression(s) explicitly grouped in parens +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ParensExpr { + pub expressions: Vec, +} + +impl ParensExpr { + pub fn new(expressions: Vec) -> Self { + ParensExpr { expressions } + } + + pub fn len(&self) -> usize { + self.expressions.len() + } + + pub fn is_empty(&self) -> bool { + self.expressions.is_empty() + } + + pub fn return_type(&self) -> ValueType { + if let Some(inner) = self.innermost_expr() { + return inner.return_type(); + } + if self.len() == 1 { + return self.expressions[0].return_type(); + } + + // Treat as a function with empty name, i.e. union() + TransformFunction::Union.return_type() + } + + pub fn to_function(self) -> FunctionExpr { + // Treat parensExpr as a function with empty name, i.e. union() + // todo: how to avoid clone + let name = "union"; + let func = BuiltinFunction::from_str(name).unwrap(); // if union is not defined, we have a fatal issue + + FunctionExpr { + args: self.expressions, + keep_metric_names: false, + function: func, + } + } + + /// Return the innermost expression wrapped by a `ParensExpr` if the `ParensExpr` contains + /// exactly one expression. For example : (((x + y))) would return a reef to `x + y` + pub fn innermost_expr(&self) -> Option<&Expr> { + match self.len() { + 0 => None, + 1 => match &self.expressions[0] { + Expr::Parens(pe2) => pe2.innermost_expr(), + expr => Some(expr), + }, + _ => None, + } + } +} + +impl Value for ParensExpr { + fn value_type(&self) -> ValueType { + if let Some(inner) = self.innermost_expr() { + return inner.return_type(); + } + // Treat as a function with empty name, i.e. union() + TransformFunction::Union.return_type() + } +} + +impl Display for ParensExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write_comma_separated(&mut self.expressions.iter(), f, true)?; + Ok(()) + } +} + +impl Prettier for ParensExpr { + fn format(&self, level: usize, max: usize) -> String { + let mut s = String::with_capacity(64); + let sub_indent = indent(level + 1); + + for (i, expr) in self.expressions.iter().enumerate() { + if i > 0 { + s.push_str(",\n"); + } + s.push_str(&sub_indent); + s.push_str(&expr.pretty(level + 1, max)); + } + + format!("{}(\n{s}\n{})", indent(level), indent(level)) + } +} + +impl ExpressionNode for ParensExpr { + fn cast(self) -> Expr { + Expr::Parens(self) + } +} + +/// WithExpr represents `with (...)` extension from MetricsQL. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WithExpr { + pub was: Vec, + pub expr: BExpr, +} + +impl WithExpr { + pub fn new(expr: Expr, was: Vec) -> Self { + WithExpr { + expr: Box::new(expr), + was, + } + } + + pub fn return_type(&self) -> ValueType { + self.expr.return_type() + } +} + +impl Value for WithExpr { + fn value_type(&self) -> ValueType { + self.expr.value_type() + } +} + +impl Display for WithExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "WITH (")?; + for (i, was) in self.was.iter().enumerate() { + if (i + 1) < self.was.len() { + write!(f, ", ")?; + } + write!(f, "{}", was)?; + } + write!(f, ") ")?; + write!(f, "{}", self.expr)?; + Ok(()) + } +} + +impl Prettier for WithExpr { + fn format(&self, level: usize, max: usize) -> String { + let mut s = format!("{}WITH (\n", indent(level)); + for (i, was) in self.was.iter().enumerate() { + if i > 0 { + s.push_str(",\n"); + } + s.push_str(&was.pretty(level + 1, max)); + } + s.push_str(&format!("\n{})", indent(level))); + s + } +} + +impl ExpressionNode for WithExpr { + fn cast(self) -> Expr { + Expr::With(self) + } +} + +/// WithArgExpr represents a single entry from WITH expression. +#[derive(Debug, Clone, Eq, Serialize, Deserialize)] +pub struct WithArgExpr { + pub name: String, + pub args: Vec, + pub expr: Expr, + pub(crate) token_range: Range, +} + +impl PartialEq for WithArgExpr { + fn eq(&self, other: &Self) -> bool { + self.name == other.name && self.args == other.args && expr_equals(&self.expr, &other.expr) + } +} + +impl WithArgExpr { + pub fn new_function>(name: S, expr: Expr, args: Vec) -> Self { + WithArgExpr { + name: name.into(), + args, + expr, + token_range: Default::default(), + } + } + + pub fn new>(name: S, expr: Expr, args: Vec) -> Self { + WithArgExpr { + name: name.into(), + args, + expr, + token_range: Default::default(), + } + } + + pub fn new_number>(name: S, value: f64) -> Self { + Self::new(name, Expr::from(value), vec![]) + } + + pub fn new_string>(name: S, value: String) -> Self { + Self::new(name, Expr::from(value), vec![]) + } + + pub fn return_value(&self) -> ValueType { + self.expr.return_type() + } +} + +impl Value for WithArgExpr { + fn value_type(&self) -> ValueType { + self.expr.value_type() + } +} + +impl Display for WithArgExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}", escape_ident(&self.name))?; + write_comma_separated(self.args.iter(), f, !self.args.is_empty())?; + write!(f, " = {}", self.expr)?; + Ok(()) + } +} + +impl Prettier for WithArgExpr { + fn format(&self, level: usize, max: usize) -> String { + let mut s = format!("{}{} = ", indent(level), self.name); + if !self.args.is_empty() { + s.push_str(&self.args.join(", ")); + s.push_str(" = "); + } + s.push_str(&self.expr.pretty(level, max)); + s + } +} + +/// A root expression node. +/// +/// These are all valid root expression ast. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Expr { + /// A single scalar number. + NumberLiteral(NumberLiteral), + + Duration(DurationExpr), + + /// A single scalar string. + /// + /// Prometheus' docs claim strings aren't currently implemented, but they're + /// valid as function arguments. + StringLiteral(StringLiteral), + + /// A function call + Function(FunctionExpr), + + /// Aggregation represents aggregate functions such as `sum(...) by (...)` + Aggregation(AggregationExpr), + + /// A unary operator expression + UnaryOperator(UnaryExpr), + + /// A binary operator expression + BinaryOperator(BinaryExpr), + + /// RollupExpr represents an MetricsQL expression which contains at least `offset` or `[...]` part. + Rollup(RollupExpr), + + /// MetricExpr represents a MetricsQL metric with optional filters, i.e. `foo{...}`. + MetricExpression(MetricExpr), + + /// A grouped expression wrapped in parentheses + Parens(ParensExpr), + + /// String expression parsed in the context of a `with` statement. + /// + /// Prometheus' docs claim strings aren't currently implemented, but they're + /// valid as function arguments. + StringExpr(StringExpr), + + /// A MetricsQL specific WITH statement node. Transformed at parse time to one + /// of the other variants + With(WithExpr), + + /// An interpolated MetricsQL metric with optional filters, i.e. `foo{...}` parsed in the + /// context of a `WITH` statement. Transformed at parse time to a MetricExpr + WithSelector(InterpolatedSelector), +} + +pub type BExpression = Box; + +impl Expr { + pub fn is_scalar(expr: &Expr) -> bool { + match expr { + Expr::Duration(_) | Expr::NumberLiteral(_) => true, + Expr::Function(f) => f.function.is_scalar(), + _ => false, + } + } + + pub fn is_number(expr: &Expr) -> bool { + matches!(expr, Expr::NumberLiteral(_)) + } + + pub fn is_string(expr: &Expr) -> bool { + matches!(expr, Expr::StringLiteral(_)) + } + + pub fn is_primitive(expr: &Expr) -> bool { + matches!(expr, Expr::NumberLiteral(_) | Expr::StringLiteral(_)) + } + + pub fn is_duration(expr: &Expr) -> bool { + matches!(expr, Expr::Duration(_)) + } + + pub fn vectors(&self) -> Box + '_> { + match self { + Self::MetricExpression(v) => Box::new(v.matchers.filter_iter()), + Self::Rollup(re) => Box::new(re.expr.vectors().chain(if let Some(at) = &re.at { + at.vectors() + } else { + Box::new(iter::empty()) + })), + Self::UnaryOperator(u) => u.expr.vectors(), + Self::BinaryOperator(be) => Box::new(be.left.vectors().chain(be.right.vectors())), + Self::Aggregation(ae) => Box::new(ae.args.iter().flat_map(|node| node.vectors())), + Self::Function(fe) => Box::new(fe.args.iter().flat_map(|node| node.vectors())), + Self::Parens(pe) => Box::new(pe.expressions.iter().flat_map(|node| node.vectors())), + Self::NumberLiteral(_) + | Self::Duration(_) + | Self::StringLiteral(_) + | Self::StringExpr(_) + | Self::With(_) => Box::new(iter::empty()), + // this node type should not appear in the AST after parsing + Expr::WithSelector(_) => Box::new(iter::empty()), + } + } + + /** + Return an iterator of series names present in this node. + ``` rust + let query = r#" + sum(1 - something_used{env="production"} / something_total) by (instance) + and ignoring (instance) + sum(rate(some_queries{instance=~"localhost\\d+"} [5m])) > 100 + "#; + let ast = metricsql_parser::parser::parse(query).expect("valid query"); + let series: Vec = ast.series_names().collect(); + assert_eq!(series, vec![ + "something_used".to_string(), + "something_total".to_string(), + "some_queries".to_string(), + ], + ); + ``` + */ + pub fn series_names(&self) -> impl Iterator + '_ { + self.vectors().map(|x| { + if x.label == NAME_LABEL { + x.value.clone() + // String::from_utf8(x.value.clone()) + // .expect("series names should always be valid utf8") + } else { + x.label.clone() + } + }) + } + + pub fn contains_subquery(&self) -> bool { + use Expr::*; + match self { + Function(fe) => fe.args.iter().any(|e| e.contains_subquery()), + BinaryOperator(bo) => bo.left.contains_subquery() || bo.right.contains_subquery(), + Aggregation(aggr) => aggr.args.iter().any(|e| e.contains_subquery()), + Rollup(re) => re.for_subquery(), + _ => false, + } + } + + pub fn return_type(&self) -> ValueType { + match self { + Expr::NumberLiteral(_) => ValueType::Scalar, + Expr::Duration(dur) => dur.return_type(), + Expr::StringLiteral(_) | Expr::StringExpr(_) => ValueType::String, + Expr::Function(fe) => fe.return_type(), + Expr::Aggregation(ae) => ae.return_type(), + Expr::UnaryOperator(u) => u.expr.return_type(), + Expr::BinaryOperator(be) => be.return_type(), + Expr::Rollup(re) => re.return_type(), + Expr::Parens(me) => me.return_type(), + Expr::MetricExpression(me) => me.return_type(), + Expr::With(w) => w.return_type(), + Expr::WithSelector(_) => ValueType::InstantVector, + } + } + + pub fn variant_name(&self) -> &'static str { + match self { + Expr::NumberLiteral(_) => "Scalar", + Expr::Duration(_) => "Duration", + Expr::StringLiteral(_) | Expr::StringExpr(_) => "String", + Expr::Function(_) => "Function", + Expr::Aggregation(_) => "Aggregation", + Expr::UnaryOperator(_) => "UnaryOperator", + Expr::BinaryOperator(_) => "BinaryOperator", + Expr::Rollup(_) => "Rollup", + Expr::Parens(_) => "Parens", + Expr::MetricExpression(_) => "VectorSelector", + Expr::With(_) => "With", + Expr::WithSelector(_) => "WithSelector", + } + } + + pub fn cast(self) -> Expr { + // this code seems suspicious + match self { + Expr::Aggregation(a) => Expr::Aggregation(a), + Expr::UnaryOperator(u) => Expr::UnaryOperator(u), + Expr::BinaryOperator(b) => Expr::BinaryOperator(b), + Expr::Duration(d) => Expr::Duration(d), + Expr::Function(f) => Expr::Function(f), + Expr::NumberLiteral(n) => Expr::NumberLiteral(n), + Expr::MetricExpression(m) => Expr::MetricExpression(m), + Expr::Parens(m) => Expr::Parens(m), + Expr::Rollup(r) => Expr::Rollup(r), + Expr::StringLiteral(s) => Expr::StringLiteral(s), + Expr::StringExpr(s) => Expr::StringExpr(s), + Expr::With(w) => Expr::With(w), + Expr::WithSelector(ws) => Expr::WithSelector(ws), + } + } + + pub fn is_metric_expression(&self) -> bool { + matches!(self, Expr::MetricExpression(_)) + } + + pub fn is_binary_op(&self) -> bool { + matches!(self, Expr::BinaryOperator(_)) + } + + pub fn is_rollup(&self) -> bool { + matches!(self, Expr::Rollup(_)) + } + + /// returns a scalar expression + pub fn scalar(value: f64) -> Expr { + Expr::from(value) + } + + /// returns a string literal expression + pub fn string_literal(value: &str) -> Expr { + Expr::from(value) + } + + /// Return `self == other` + pub fn eq(self, other: Expr) -> Expr { + binary_expr(self, Operator::Eql, other) + } + + /// Return `self != other` + pub fn not_eq(self, other: Expr) -> Expr { + binary_expr(self, Operator::NotEq, other) + } + + /// Return `self > other` + pub fn gt(self, other: Expr) -> Expr { + binary_expr(self, Operator::Gt, other) + } + + /// Return `self >= other` + pub fn gt_eq(self, other: Expr) -> Expr { + binary_expr(self, Operator::Gte, other) + } + + /// Return `self < other` + pub fn lt(self, other: Expr) -> Expr { + binary_expr(self, Operator::Lt, other) + } + + /// Return `self <= other` + pub fn lt_eq(self, other: Expr) -> Expr { + binary_expr(self, Operator::Lte, other) + } + + /// Return `self AND other` + pub fn and(self, other: Expr) -> Expr { + binary_expr(self, Operator::And, other) + } + + /// Return `self OR other` + pub fn or(self, other: Expr) -> Expr { + binary_expr(self, Operator::Or, other) + } + + /// Calculate the modulus of two expressions. + /// Return `self % other` + pub fn modulus(self, other: Expr) -> Expr { + binary_expr(self, Operator::Mod, other) + } + + pub fn call(func: &str, args: Vec) -> ParseResult { + let expr = FunctionExpr::new(func, args)?; + Ok(Expr::Function(expr)) + } + + pub fn new_vector_selector>(name: Option, matchers: Matchers) -> Expr { + let mut matchers = matchers; + if let Some(name) = name { + let metric_name = LabelFilter::equal(NAME_LABEL, name.into().as_str()); + if matchers.or_matchers.is_empty() { + matchers.matchers.push(metric_name); + } else { + for branch in &mut matchers.or_matchers { + branch.push(metric_name.clone()); + } + } + matchers.sort_filters(); + } + let selector = MetricExpr { + name: None, + matchers, + }; + Expr::MetricExpression(selector) + } + + pub fn new_binary_expr( + lhs: Expr, + op: Operator, + modifier: Option, + rhs: Expr, + ) -> Result { + let ex = BinaryExpr { + left: Box::new(lhs), + right: Box::new(rhs), + op, + modifier, + }; + Ok(Expr::BinaryOperator(ex)) + } + + pub fn at_expr(self, at: Expr) -> Result { + let already_set_err = Err("@ may not be set multiple times".into()); + match self { + Expr::Rollup(mut s) => match s.at { + None => { + s.at = Some(Box::new(at)); + Ok(Expr::Rollup(s)) + } + Some(_) => already_set_err, + }, + _ => { + Err("@ modifier must be preceded by an vector selector or matrix selector or a subquery".into()) + } + } + } + + pub fn prettify(&self) -> String { + self.pretty(0, MAX_CHARACTERS_PER_LINE) + } + + pub fn keep_metric_names(&self) -> bool { + match self { + Expr::UnaryOperator(ue) => ue.expr.keep_metric_names(), + Expr::BinaryOperator(be) => be.keep_metric_names(), + Expr::Rollup(re) => re.wraps_metric_expr(), + Expr::Function(fe) => fe.keep_metric_names, + Expr::Aggregation(ae) => ae.keep_metric_names, + Expr::Parens(pe) => { + if let Some(expr) = pe.innermost_expr() { + return expr.keep_metric_names(); + } + false + } + _ => false, + } + } +} + +impl Display for Expr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + Expr::Aggregation(a) => write!(f, "{}", a)?, + Expr::UnaryOperator(ue) => write!(f, "{}", ue)?, + Expr::BinaryOperator(be) => write!(f, "{}", be)?, + Expr::Duration(d) => write!(f, "{}", d)?, + Expr::Function(func) => write!(f, "{}", func)?, + Expr::NumberLiteral(n) => write!(f, "{}", n)?, + Expr::MetricExpression(me) => write!(f, "{}", me)?, + Expr::Parens(p) => write!(f, "{}", p)?, + Expr::Rollup(re) => write!(f, "{}", re)?, + Expr::StringLiteral(s) => write!(f, "{}", enquote('"', s))?, + Expr::StringExpr(s) => write!(f, "{}", s)?, + Expr::With(w) => write!(f, "{}", w)?, + Expr::WithSelector(ws) => write!(f, "{}", ws)?, + } + Ok(()) + } +} + +impl Prettier for Expr { + fn pretty(&self, level: usize, max: usize) -> String { + match self { + Expr::Aggregation(ex) => ex.pretty(level, max), + Expr::UnaryOperator(ex) => ex.pretty(level, max), + Expr::BinaryOperator(ex) => ex.pretty(level, max), + Expr::Parens(ex) => ex.pretty(level, max), + Expr::Rollup(ex) => ex.pretty(level, max), + Expr::NumberLiteral(ex) => ex.pretty(level, max), + Expr::StringLiteral(ex) => ex.pretty(level, max), + Expr::MetricExpression(ex) => ex.pretty(level, max), + Expr::Function(ex) => ex.pretty(level, max), + Expr::Duration(d) => d.pretty(level, max), + Expr::StringExpr(se) => se.pretty(level, max), + Expr::With(we) => we.pretty(level, max), + Expr::WithSelector(ws) => ws.pretty(level, max), + } + } +} + +impl Value for Expr { + fn value_type(&self) -> ValueType { + match self { + Expr::Aggregation(a) => a.return_type(), + Expr::UnaryOperator(ue) => ue.expr.return_type(), + Expr::BinaryOperator(be) => be.return_type(), + Expr::Duration(d) => d.return_type(), + Expr::Function(func) => func.return_type(), + Expr::NumberLiteral(n) => n.return_type(), + Expr::MetricExpression(me) => me.value_type(), + Expr::Parens(p) => p.return_type(), + Expr::Rollup(re) => re.return_type(), + Expr::StringLiteral(_) => ValueType::String, + Expr::StringExpr(_) => ValueType::String, + Expr::With(w) => w.return_type(), + Expr::WithSelector(_) => ValueType::InstantVector, + } + } +} + +// crate private +impl Default for Expr { + fn default() -> Self { + Expr::from(1.0) + } +} + +impl From for Expr { + fn from(v: f64) -> Self { + Expr::NumberLiteral(NumberLiteral::new(v)) + } +} + +impl From for Expr { + fn from(v: i64) -> Self { + Self::from(v as f64) + } +} + +impl From for Expr { + fn from(s: String) -> Self { + Expr::StringLiteral(StringLiteral(s)) + } +} + +impl From<&str> for Expr { + fn from(s: &str) -> Self { + Expr::StringLiteral(StringLiteral(s.to_string())) + } +} + +impl From for Expr { + fn from(vs: MetricExpr) -> Self { + Expr::MetricExpression(vs) + } +} + +pub(crate) fn binary_expr(left: Expr, op: Operator, right: Expr) -> Expr { + let mut expr = BinaryExpr::new(op, left, right); + if op.is_comparison() { + expr = expr.with_bool_modifier(); + } + Expr::BinaryOperator(expr) +} + +impl ops::Add for Expr { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + binary_expr(self, Operator::Add, rhs) + } +} + +impl ops::Sub for Expr { + type Output = Self; + + fn sub(self, rhs: Self) -> Self { + binary_expr(self, Operator::Sub, rhs) + } +} + +impl ops::Mul for Expr { + type Output = Self; + + fn mul(self, rhs: Self) -> Self { + binary_expr(self, Operator::Mul, rhs) + } +} + +impl ops::Div for Expr { + type Output = Self; + + fn div(self, rhs: Self) -> Self { + binary_expr(self, Operator::Div, rhs) + } +} + +impl ops::Rem for Expr { + type Output = Self; + + fn rem(self, rhs: Self) -> Self { + binary_expr(self, Operator::Mod, rhs) + } +} + +impl ops::BitAnd for Expr { + type Output = Self; + + fn bitand(self, rhs: Self) -> Self { + binary_expr(self, Operator::And, rhs) + } +} + +impl ops::BitOr for Expr { + type Output = Self; + + fn bitor(self, rhs: Self) -> Self { + binary_expr(self, Operator::Or, rhs) + } +} + +impl ops::BitXor for Expr { + type Output = Self; + + fn bitxor(self, rhs: Self) -> Self { + binary_expr(self, Operator::Pow, rhs) + } +} + +impl Neg for Expr { + type Output = Self; + + fn neg(self) -> Self::Output { + match self { + Expr::NumberLiteral(nl) => Expr::NumberLiteral(-nl), + _ => Expr::UnaryOperator(UnaryExpr { + expr: Box::new(self), + }), + } + } +} + +fn are_floats_equal(left: f64, right: f64) -> bool { + // Special handling for nan == nan. + left == right || left.is_nan() && right.is_nan() +} + +fn is_default(t: &T) -> bool { + t == &T::default() +} diff --git a/crates/metricsql-parser-vendored/src/ast/expr_tree_node.rs b/crates/metricsql-parser-vendored/src/ast/expr_tree_node.rs new file mode 100644 index 00000000..1a9552ec --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/expr_tree_node.rs @@ -0,0 +1,198 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Tree node implementation for Ast expr + +use crate::ast::{ + AggregationExpr, BExpression, BinaryExpr, Expr, FunctionExpr, ParensExpr, RollupExpr, + UnaryExpr, WithArgExpr, WithExpr, +}; +use crate::common::{TreeNode, VisitRecursion}; +use crate::parser::ParseResult; + +pub type Result = ParseResult; + +impl TreeNode for Expr { + fn apply_children(&self, op: &mut F) -> Result + where + F: FnMut(&Self) -> Result, + { + let children = match self { + Expr::StringLiteral(_) + | Expr::StringExpr(_) + | Expr::NumberLiteral(_) + | Expr::MetricExpression(_) + | Expr::WithSelector(_) + | Expr::Duration(_) => vec![], + Expr::UnaryOperator(u) => vec![u.expr.as_ref().clone()], + Expr::BinaryOperator(BinaryExpr { left, right, .. }) => { + vec![left.as_ref().clone(), right.as_ref().clone()] + } + Expr::Aggregation(AggregationExpr { args, .. }) + | Expr::Function(FunctionExpr { args, .. }) => args.clone(), + Expr::Parens(p) => p.expressions.clone(), + Expr::Rollup(RollupExpr { expr, at, .. }) => { + let mut expr_vec = Vec::with_capacity(2); + expr_vec.push(expr.as_ref().clone()); + if let Some(at_expr) = at { + expr_vec.push(at_expr.as_ref().clone()); + } + // todo: window, step, offset + expr_vec + } + Expr::With(w) => { + let mut expr_vec = Vec::with_capacity(1 + w.was.len()); + expr_vec.push(w.expr.as_ref().clone()); + for wa in w.was.iter() { + expr_vec.push(wa.expr.clone()); + } + expr_vec + } + }; + + for child in children.iter() { + match op(child)? { + VisitRecursion::Continue => {} + VisitRecursion::Skip => return Ok(VisitRecursion::Continue), + VisitRecursion::Stop => return Ok(VisitRecursion::Stop), + } + } + + Ok(VisitRecursion::Continue) + } + + fn map_children(self, transform: F) -> Result + where + F: FnMut(Self) -> Result, + { + let mut transform = transform; + + // recurse into all sub expressions(and cover all expression types) + let expr = match self { + Expr::Aggregation(AggregationExpr { + function, + args, + modifier, + limit, + keep_metric_names, + }) => Expr::Aggregation(AggregationExpr { + function, + args: transform_vec(args, &mut transform)?, + modifier, + limit, + keep_metric_names, + }), + Expr::UnaryOperator(u) => Expr::UnaryOperator(UnaryExpr { + expr: transform_boxed(u.expr, &mut transform)?, + }), + Expr::BinaryOperator(BinaryExpr { + left, + op, + right, + modifier, + }) => Expr::BinaryOperator(BinaryExpr { + left: transform_boxed(left, &mut transform)?, + op, + right: transform_boxed(right, &mut transform)?, + modifier, + }), + Expr::Duration(_) => self.clone(), + Expr::Function(FunctionExpr { + args, + keep_metric_names, + function, + }) => Expr::Function(FunctionExpr { + args: transform_vec(args, &mut transform)?, + keep_metric_names, + function, + }), + Expr::NumberLiteral(_) => self.clone(), + Expr::Rollup(RollupExpr { + expr, + window, + step, + offset, + inherit_step, + at, + }) => Expr::Rollup(RollupExpr { + expr: transform_boxed(expr, &mut transform)?, + window, + step, + offset, + inherit_step, + at: transform_option_box(at, &mut transform)?, + }), + Expr::MetricExpression(_) => self.clone(), + Expr::Parens(ParensExpr { expressions }) => Expr::Parens(ParensExpr { + expressions: transform_vec(expressions, &mut transform)?, + }), + Expr::StringLiteral(_) | Expr::StringExpr(_) => self.clone(), + Expr::With(w) => { + let mut was: Vec = Vec::with_capacity(w.was.len()); + for wa in w.was.into_iter() { + let new_wa = WithArgExpr { + name: wa.name.clone(), + args: wa.args.clone(), + expr: transform(wa.expr)?, + token_range: wa.token_range, + }; + was.push(new_wa); + } + let with = WithExpr { + was, + expr: Box::new(transform(*w.expr)?), + }; + Expr::With(with) + } + Expr::WithSelector(_) => self.clone(), + }; + + Ok(expr) + } +} + +#[allow(clippy::boxed_local)] +fn transform_boxed(boxed_expr: Box, transform: &mut F) -> Result +where + F: FnMut(Expr) -> Result, +{ + // TODO: + // It might be possible to avoid an allocation (the Box::new) below by reusing the box. + let expr: Expr = *boxed_expr; + let rewritten_expr = transform(expr)?; + Ok(Box::new(rewritten_expr)) +} + +fn transform_option_box( + option_box: Option, + transform: &mut F, +) -> Result> +where + F: FnMut(Expr) -> Result, +{ + option_box + .map(|expr| transform_boxed(expr, transform)) + .transpose() +} + +/// &mut transform a `Vec` of `Expr`s +fn transform_vec(v: Vec, transform: &mut F) -> Result> +where + F: FnMut(Expr) -> Result, +{ + v.into_iter().map(transform).collect() +} diff --git a/crates/metricsql-parser-vendored/src/ast/interpolated_selector.rs b/crates/metricsql-parser-vendored/src/ast/interpolated_selector.rs new file mode 100644 index 00000000..0126108f --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/interpolated_selector.rs @@ -0,0 +1,159 @@ +use std::fmt; +use std::fmt::{Display, Formatter}; + +use serde::{Deserialize, Serialize}; + +use crate::ast::{Prettier, StringExpr}; +use crate::common::{Value, ValueType}; +use crate::label::{LabelFilterExpr, Matchers, NAME_LABEL}; +use crate::parser::ParseResult; + +/// InterpolatedSelector represents a Vector Selector in the context of a `WITH` expression. +#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InterpolatedSelector { + /// a list of label filter expressions from WITH clause. + /// This is transformed into label_filters during compilation. + pub(crate) matchers: Vec>, +} + +impl InterpolatedSelector { + pub fn new>(name: S) -> InterpolatedSelector { + let name_filter = LabelFilterExpr::equal(NAME_LABEL, StringExpr::new(name)).unwrap(); + InterpolatedSelector { + matchers: vec![vec![name_filter]], + } + } + + pub fn with_filters(filters: Vec) -> Self { + InterpolatedSelector { + matchers: vec![filters], + } + } + + pub fn with_or_filters(filters: Vec>) -> Self { + InterpolatedSelector { matchers: filters } + } + + pub fn is_empty(&self) -> bool { + self.matchers.is_empty() + } + + pub fn is_resolved(&self) -> bool { + self.matchers.is_empty() + || self + .matchers + .iter() + .all(|x| x.iter().all(|label| label.is_resolved())) + } + + pub fn metric_name(&self) -> Option<&str> { + let lfss = &self.matchers; + if lfss.is_empty() { + return None; + } + + fn get_name(lf: &LabelFilterExpr) -> Option<&str> { + if lf.is_metric_name_filter() { + if let Some(literal) = lf.value.get_literal().unwrap_or(None) { + return Some(literal.as_str()); + } + } + None + } + + if let Some((first, rest)) = lfss.split_first() { + if let Some(lf) = first.first() { + if let Some(literal) = get_name(lf) { + let metric_name = literal; + for lf in rest { + if let Some(head) = lf.first() { + if let Some(literal) = get_name(head) { + if literal != metric_name { + return None; + } + } + } else { + return None; + } + } + return Some(metric_name); + } + } + } + None + } + + pub fn return_type(&self) -> ValueType { + ValueType::InstantVector + } + + pub fn to_matchers(&self) -> ParseResult { + if !self.is_resolved() { + // todo: err + } + + let mut or_matchers = vec![]; + for m in &self.matchers { + let mut and_matchers = vec![]; + for l in m { + and_matchers.push(l.to_label_filter()?); + } + or_matchers.push(and_matchers); + } + + Ok(Matchers::with_or_matchers(or_matchers)) + } + + pub fn is_empty_matchers(&self) -> bool { + self.matchers.is_empty() + || self + .matchers + .iter() + .all(|x| x.iter().all(|y| y.is_empty_matcher())) + } + + /// find all the matchers whose name equals the specified name. + pub fn find_matchers(&self, name: &str) -> Vec<&LabelFilterExpr> { + self.matchers + .iter() + .flatten() + .filter(|m| m.label.eq_ignore_ascii_case(name)) + .collect() + } +} + +impl Value for InterpolatedSelector { + fn value_type(&self) -> ValueType { + ValueType::InstantVector + } +} + +impl Display for InterpolatedSelector { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{{")?; + + let exprs = &self.matchers[0..]; + + for (i, filter_list) in exprs.iter().enumerate() { + for (j, filter) in filter_list.iter().enumerate() { + write!(f, "{}", filter)?; + if j + 1 < filter_list.len() { + write!(f, ", ")?; + } + } + if i + 1 < exprs.len() { + write!(f, " or ")?; + } + } + + write!(f, "}}")?; + + Ok(()) + } +} + +impl Prettier for InterpolatedSelector { + fn needs_split(&self, _max: usize) -> bool { + false + } +} diff --git a/crates/metricsql-parser-vendored/src/ast/mod.rs b/crates/metricsql-parser-vendored/src/ast/mod.rs new file mode 100644 index 00000000..105ef8fc --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/mod.rs @@ -0,0 +1,108 @@ +pub use adjust_comparison_ops::*; +pub use check_ast::*; +pub use expr::*; +pub use expr_tree_node::*; +pub use interpolated_selector::*; +pub use operator::*; +pub use string_expr::*; +pub(crate) use utils::*; + +mod adjust_comparison_ops; +mod check_ast; +mod expr; +mod expr_tree_node; +mod interpolated_selector; +pub mod operator; +mod string_expr; +pub mod utils; +mod visitor; + +const INDENT_STR: &str = " "; +const MAX_CHARACTERS_PER_LINE: usize = 100; + +/// Approach +/// -------- +/// When a PromQL query is parsed, it is converted into PromQL AST, +/// which is a nested structure of nodes. Each node has a depth/level +/// (distance from the root), that is passed by its parent. +/// +/// While prettifying, a Node considers 2 things: +/// 1. Did the current Node's parent add a new line? +/// 2. Does the current Node needs to be prettified? +/// +/// The level of a Node determines if it should be indented or not. +/// The answer to the 1 is NO if the level passed is 0. This means, the +/// parent Node did not apply a new line, so the current Node must not +/// apply any indentation as prefix. +/// If level > 1, a new line is applied by the parent. So, the current Node +/// should prefix an indentation before writing any of its content. This indentation +/// will be ([level/depth of current Node] * " "). +/// +/// The answer to 2 is YES if the normalized length of the current Node exceeds +/// the [MAX_CHARACTERS_PER_LINE] limit. Hence, it applies the indentation equal to +/// its depth and increments the level by 1 before passing down the child. +/// If the answer is NO, the current Node returns the normalized string value of itself. +pub trait Prettier: std::fmt::Display { + /// max param is short for max_characters_per_line. + fn pretty(&self, level: usize, max: usize) -> String { + if self.needs_split(max) { + self.format(level, max) + } else { + format!("{}{self}", indent(level)) + } + } + + /// override format if expr needs to be split into multiple lines + fn format(&self, level: usize, _max: usize) -> String { + format!("{}{self}", indent(level)) + } + + /// override needs_split to return false, in order not to split multiple lines + fn needs_split(&self, max: usize) -> bool { + self.to_string().len() > max + } +} + +fn indent(n: usize) -> String { + INDENT_STR.repeat(n) +} + +pub(super) fn prettify_args(args: &[Expr], level: usize, max: usize) -> String { + if args.is_empty() { + return "".to_string(); + } + let mut v = Vec::with_capacity(args.len()); + for ex in args { + v.push(ex.pretty(level, max)); + } + v.join(",\n") +} + +#[cfg(test)] +mod tests { + use super::*; + + struct Pretty(String); + + impl std::fmt::Display for Pretty { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } + } + + impl Prettier for Pretty {} + + #[test] + fn test_prettier_trait() { + let max = 10; + let level = 1; + + let p = Pretty("demo".into()); + assert!(!p.needs_split(max)); + assert_eq!(p.format(level, max), p.pretty(level, max)); + + let p = Pretty("demo_again.".into()); + assert!(p.needs_split(max)); + assert_eq!(p.format(level, max), p.pretty(level, max)); + } +} diff --git a/crates/metricsql-parser-vendored/src/ast/operator.rs b/crates/metricsql-parser-vendored/src/ast/operator.rs new file mode 100644 index 00000000..0653bd0b --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/operator.rs @@ -0,0 +1,294 @@ +use std::fmt; +use std::str::FromStr; + +use phf::phf_map; +use serde::{Deserialize, Serialize}; +use strum_macros::EnumIter; + +use crate::parser::tokens::Token; +use crate::parser::ParseError; + +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq, Hash, EnumIter, Serialize, Deserialize)] +pub enum Operator { + Add, + And, + Atan2, + Default, + Div, + #[default] + Eql, + Mod, + Mul, + Pow, + Sub, + Gt, + Gte, + If, + IfNot, + Lt, + Lte, + NotEq, + Or, + Unless, +} + +pub static BINARY_OPS_MAP: phf::Map<&'static str, Operator> = phf_map! { + "+" => Operator::Add, + "-" => Operator::Sub, + "*" => Operator::Mul, + "/" => Operator::Div, + "%" => Operator::Mod, + "^" => Operator::Pow, + + // See https://github.com/prometheus/prometheus/pull/9248 + "atan2" => Operator::Atan2, + + // cmp ops + "==" => Operator::Eql, + "!=" => Operator::NotEq, + "<" => Operator::Lt, + ">" => Operator::Gt, + "<=" => Operator::Lte, + ">=" => Operator::Gte, + + // logic set ops + "and" => Operator::And, + "or" => Operator::Or, + "unless" => Operator::Unless, + + // New ops for MetricsQL + "if" => Operator::If, + "ifnot" => Operator::IfNot, + "default" => Operator::Default, +}; + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum BinaryOpKind { + Arithmetic, + Comparison, + Logical, +} + +pub type Precedence = usize; + +impl Operator { + #[inline] + pub const fn precedence(&self) -> Precedence { + use Operator::*; + + match self { + Default => 0, + If | IfNot => 1, + // See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence + Or => 10, + And | Unless => 20, + Eql | Gte | Gt | Lt | Lte | NotEq => 30, + Add | Sub => 40, + Mul | Div | Mod | Atan2 => 50, + Pow => 60, + } + } + + #[inline] + pub const fn kind(&self) -> BinaryOpKind { + use BinaryOpKind::*; + use Operator::*; + + match self { + Add | Sub | Mul | Div | Mod | Pow | Atan2 => Arithmetic, + Eql | Gte | Gt | Lt | Lte | NotEq => Comparison, + And | Unless | Or | If | IfNot | Default => Logical, + } + } + + // See https://prometheus.io/docs/prometheus/latest/querying/operators/#binary-operator-precedence + pub const fn is_right_associative(self) -> bool { + matches!(self, Operator::Pow) + } + + pub const fn is_arithmetic_op(&self) -> bool { + use Operator::*; + matches!(self, Add | Sub | Mul | Div | Mod | Pow | Atan2) + } + + pub const fn is_logical_op(&self) -> bool { + use Operator::*; + matches!(self, And | Or | Unless | If | IfNot | Default) + } + + pub const fn is_comparison(&self) -> bool { + use Operator::*; + matches!(self, Eql | Gte | Gt | Lt | Lte | NotEq) + } + + pub const fn is_valid_string_op(&self) -> bool { + use Operator::*; + matches!(self, Add | Eql | Gte | Gt | Lt | Lte | NotEq) + } + + pub const fn get_reverse_cmp(&self) -> Operator { + match self { + Operator::Gt => Operator::Lt, + Operator::Lt => Operator::Gt, + Operator::Gte => Operator::Lte, + Operator::Lte => Operator::Gte, + // there is no need in changing `==` and `!=`. + _ => *self, + } + } + + #[inline] + pub const fn is_set_operator(&self) -> bool { + use Operator::*; + matches!(self, And | Or | Unless) + } + + pub const fn as_str(&self) -> &'static str { + use Operator::*; + match self { + Add => "+", + And => "and", + Atan2 => "atan2", + Default => "default", + Div => "/", + Eql => "==", + Gt => ">", + Gte => ">=", + If => "if", + IfNot => "ifNot", + Mod => "%", + Mul => "*", + Lt => "<", + Lte => "<=", + NotEq => "!=", + Or => "or", + Pow => "^", + Sub => "-", + Unless => "unless", + } + } +} + +impl FromStr for Operator { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + Operator::try_from(s) + } +} + +impl TryFrom<&str> for Operator { + type Error = ParseError; + + fn try_from(op: &str) -> Result { + if let Some(ch) = op.chars().next() { + let value = if !ch.is_alphabetic() { + BINARY_OPS_MAP.get(op) + } else { + // slight optimization - don't lowercase if not needed (save allocation) + BINARY_OPS_MAP.get(op).or_else(|| { + let lower = op.to_ascii_lowercase(); + BINARY_OPS_MAP.get(&lower) + }) + }; + if let Some(operator) = value { + return Ok(*operator); + } + } + Err(ParseError::General(format!("Unknown binary op {}", op))) + } +} + +impl TryFrom for Operator { + type Error = ParseError; + + fn try_from(token: Token) -> Result { + match token { + Token::OpAnd => Ok(Operator::And), + Token::OpAtan2 => Ok(Operator::Atan2), + Token::OpDefault => Ok(Operator::Default), + Token::OpDiv => Ok(Operator::Div), + Token::OpEqual => Ok(Operator::Eql), + Token::OpGreaterThan => Ok(Operator::Gt), + Token::OpGreaterThanOrEqual => Ok(Operator::Gte), + Token::OpIf => Ok(Operator::If), + Token::OpIfNot => Ok(Operator::IfNot), + Token::OpMod => Ok(Operator::Mod), + Token::OpMul => Ok(Operator::Mul), + Token::OpMinus => Ok(Operator::Sub), + Token::OpLessThan => Ok(Operator::Lt), + Token::OpLessThanOrEqual => Ok(Operator::Lte), + Token::OpNotEqual => Ok(Operator::NotEq), + Token::OpOr => Ok(Operator::Or), + Token::OpPow => Ok(Operator::Pow), + Token::OpUnless => Ok(Operator::Unless), + Token::OpPlus => Ok(Operator::Add), + _ => Err(ParseError::General(format!( + "Unknown binary op {:?}", + token + ))), + } + } +} + +impl fmt::Display for Operator { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.as_str())?; + Ok(()) + } +} + +pub fn is_binary_op(op: &str) -> bool { + if let Some(ch) = op.chars().next() { + return if !ch.is_alphabetic() { + BINARY_OPS_MAP.contains_key(op) + } else { + BINARY_OPS_MAP.contains_key(op.to_lowercase().as_str()) + }; + } + false +} + +#[cfg(test)] +mod tests { + use crate::ast::is_binary_op; + + #[test] + fn test_is_binary_op_success() { + let f = |s: &str| assert!(is_binary_op(s), "expecting valid binaryOp: {}", s); + + f("and"); + f("AND"); + f("unless"); + f("unleSS"); + f("=="); + f("!="); + f(">="); + f("<="); + f("or"); + f("Or"); + f("+"); + f("-"); + f("*"); + f("/"); + f("%"); + f("atan2"); + f("^"); + f(">"); + f("<"); + } + + #[test] + fn test_is_binary_op_error() { + let f = |s: &str| { + assert!(!is_binary_op(s), "unexpected valid binaryOp: {}", s); + }; + + f("foobar"); + f("=~"); + f("!~"); + f("="); + f("<=="); + f("234"); + } +} diff --git a/crates/metricsql-parser-vendored/src/ast/string_expr.rs b/crates/metricsql-parser-vendored/src/ast/string_expr.rs new file mode 100644 index 00000000..e315be24 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/string_expr.rs @@ -0,0 +1,294 @@ +use std::cmp::Ordering; +use std::fmt::{Display, Formatter}; +use std::{fmt, ops}; + +use serde::{Deserialize, Serialize}; + +use crate::ast::Prettier; +use crate::common::ValueType; +use crate::parser::ParseError; +use crate::prelude::ParseResult; + +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum StringSegment { + Literal(String), + Ident(String), +} + +impl Default for StringSegment { + fn default() -> Self { + StringSegment::Literal(String::new()) + } +} + +impl Display for StringSegment { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + StringSegment::Literal(lit) => write!(f, "{}", enquote::enquote('"', lit))?, + StringSegment::Ident(ident) => write!(f, "{}", ident)?, + } + Ok(()) + } +} + +impl PartialOrd for StringSegment { + fn partial_cmp(&self, other: &Self) -> Option { + match (self, other) { + (StringSegment::Literal(s), StringSegment::Literal(o)) => s.partial_cmp(o), + (StringSegment::Ident(s), StringSegment::Ident(o)) => s.partial_cmp(o), + (StringSegment::Literal(s), StringSegment::Ident(o)) => s.partial_cmp(o), + (StringSegment::Ident(s), StringSegment::Literal(o)) => s.partial_cmp(o), + } + } +} + +/// StringExpr represents a string expression which may be composed of multiple segments. +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub struct StringExpr(Vec, bool); + +impl StringExpr { + pub fn new>(s: S) -> Self { + let segments = vec![StringSegment::Literal(s.into())]; + StringExpr(segments, true) + } + + pub fn new_identifier>(ident: S) -> Self { + let segments = vec![StringSegment::Ident(ident.into())]; + StringExpr(segments, false) + } + + pub fn with_segment_capacity(initial_capacity: usize) -> Self { + StringExpr(Vec::with_capacity(initial_capacity), true) + } + + pub fn set_from_string>(&mut self, s: S) { + if self.0.len() == 1 { + if let Some(StringSegment::Literal(elem)) = self.0.first_mut() { + elem.clear(); + elem.push_str(s.into().as_str()) + } + return; + } + self.0.clear(); + self.0.push(StringSegment::Literal(s.into())); + self.1 = true + } + + pub fn push_str(&mut self, tok: &str) { + if let Some(StringSegment::Literal(value)) = self.0.last_mut() { + value.push_str(tok); + self.1 = self.0.len() == 1; + } else { + self.0.push(StringSegment::Literal(tok.to_string())); + } + } + + pub fn push_ident(&mut self, tok: &str) { + self.0.push(StringSegment::Ident(tok.to_string())); + self.1 = false; + } + + pub fn push(&mut self, segment: &StringSegment) { + match segment { + StringSegment::Literal(s) => self.push_str(s), + StringSegment::Ident(ident) => self.push_ident(ident), + } + } + + pub fn clear(&mut self) { + self.0.clear(); + self.1 = true; + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn segment_count(&self) -> usize { + self.0.len() + } + + pub(crate) fn is_expanded(&self) -> bool { + self.is_literal_only() + } + + pub fn is_literal_only(&self) -> bool { + self.1 + } + + pub fn is_identifier(&self) -> bool { + if self.0.len() == 1 { + let first = self.0.first().unwrap(); + return matches!(first, StringSegment::Ident(_)); + } + false + } + + pub(crate) fn estimate_result_capacity(&self) -> usize { + self.0.iter().fold(0, |acc, s| { + let res = match s { + StringSegment::Literal(lit) => lit.len(), + StringSegment::Ident(_) => 4, // todo: proper named constant + }; + acc + res + }) + } + + pub fn resolve(&self, resolve_fn: F) -> ParseResult + where + F: Fn(&str) -> ParseResult>, + { + if self.is_literal_only() { + if let Some(first) = self.0.first() { + match first { + StringSegment::Literal(lit) => return Ok(lit.clone()), + _ => panic!("BUG: string segment should be all literal"), + } + } else { + return Ok("".to_string()); + } + } + let min_capacity = self.estimate_result_capacity(); + + let mut res = String::with_capacity(min_capacity); + for s in self.0.iter() { + match s { + StringSegment::Literal(lit) => res.push_str(lit), + StringSegment::Ident(ident) => { + if let Some(ident_value) = resolve_fn(ident)? { + res.push_str(&ident_value); + } else { + let msg = format!( + "unknown identifier {:?} in string expression of {:?}", + ident, self + ); + return Err(ParseError::WithExprExpansionError(msg)); + } + } + }; + } + + Ok(res) + } + + pub fn return_type(&self) -> ValueType { + ValueType::String + } + + pub fn iter(&self) -> impl Iterator + '_ { + self.0.iter() + } + + pub fn get_literal(&self) -> ParseResult> { + if self.is_literal_only() { + if let Some(first) = self.0.first() { + return match first { + StringSegment::Literal(lit) => Ok(Some(lit)), + _ => Err(ParseError::General( + "BUG: string segment should be all literal".to_string(), + )), + }; + } + } + Ok(None) + } + + pub fn as_identifier(&self) -> Option<&String> { + if self.is_identifier() { + if let Some(first) = self.0.first() { + return match first { + StringSegment::Ident(ident) => Some(ident), + _ => None, + }; + } + } + None + } +} + +impl PartialOrd for StringExpr { + fn partial_cmp(&self, other: &Self) -> Option { + for (left, right) in self.0.iter().zip(other.0.iter()) { + if let Some(cmp) = left.partial_cmp(right) { + if cmp != Ordering::Equal { + break; + } + } + } + Some(Ordering::Equal) + } +} + +impl Default for StringExpr { + fn default() -> Self { + StringExpr(vec![], true) + } +} + +impl From for StringExpr { + fn from(s: String) -> Self { + let segment = StringSegment::Literal(s); + StringExpr(vec![segment], true) + } +} + +impl From<&str> for StringExpr { + fn from(s: &str) -> Self { + Self::from(s.to_string()) + } +} + +impl Display for StringExpr { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + for (i, segment) in self.0.iter().enumerate() { + if i > 0 { + write!(f, " + ")?; + } + write!(f, "{}", segment)?; + } + Ok(()) + } +} + +impl Prettier for StringExpr { + fn format(&self, level: usize, _max: usize) -> String { + let mut s = String::with_capacity(32); + let spaces = " ".repeat(level * 2); + for (i, segment) in self.0.iter().enumerate() { + if i > 0 { + s.push_str(&spaces); + s.push_str(" + \n"); + } + s.push_str(&format!("{spaces}{segment}\n")); + } + s + } + + fn needs_split(&self, max: usize) -> bool { + if self.is_literal_only() { + return false; + } + let mut len = 0; + for s in self.iter() { + let segment_len = match s { + StringSegment::Literal(lit) => lit.len(), + StringSegment::Ident(id) => id.len(), + }; + len += segment_len; + } + len += self.segment_count() * 3; + len > max + } +} + +impl ops::Add for StringExpr { + type Output = Self; + + fn add(self, rhs: Self) -> Self { + let mut result = self.clone(); + for item in rhs.0.iter() { + result.0.push(item.clone()) + } + result + } +} diff --git a/crates/metricsql-parser-vendored/src/ast/utils.rs b/crates/metricsql-parser-vendored/src/ast/utils.rs new file mode 100644 index 00000000..beeaf2eb --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/utils.rs @@ -0,0 +1,319 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Utility functions for expression simplification + +use ahash::AHashSet; + +use crate::ast::visitor::{walk_expr, ExprVisitor}; +use crate::ast::{BinaryExpr, Expr, MetricExpr, NumberLiteral, Operator, ParensExpr}; +use crate::functions::{get_rollup_arg_idx, BuiltinFunction}; + +/// Create a selector expression based on a qualified or unqualified column name +/// +/// example: +/// ``` rust +/// use crate::metricsql_parser::ast::utils::*; +/// let c = selector("latency"); +/// ``` +pub fn selector(ident: impl Into) -> Expr { + Expr::MetricExpression(MetricExpr::new(ident.into())) +} + +pub fn lit(str: &str) -> Expr { + Expr::from(str) +} + +pub fn number(val: f64) -> Expr { + Expr::from(val) +} + +/// returns true if `needle` is found in a chain of search_op +/// expressions. Such as: (A AND B) AND C +pub fn expr_contains(expr: &Expr, needle: &Expr, search_op: Operator) -> bool { + match expr { + Expr::BinaryOperator(BinaryExpr { + left, op, right, .. + }) if *op == search_op => { + expr_contains(left, needle, search_op) || expr_contains(right, needle, search_op) + } + _ => expr == needle, + } +} + +pub fn is_number_value(s: &Expr, val: f64) -> bool { + match s { + Expr::NumberLiteral(NumberLiteral { value, .. }) => *value == val, + _ => false, + } +} + +pub fn is_zero(s: &Expr) -> bool { + is_number_value(s, 0.0) +} + +pub fn is_one(s: &Expr) -> bool { + is_number_value(s, 1.0) +} + +pub fn is_null(expr: &Expr) -> bool { + match expr { + Expr::NumberLiteral(NumberLiteral { value, .. }) => value.is_nan(), + _ => false, + } +} + +/// returns true if `haystack` looks like (needle OP X) or (X OP needle) +pub(crate) fn is_op_with(target_op: Operator, haystack: &Expr, needle: &Expr) -> bool { + matches!(haystack, Expr::BinaryOperator(BinaryExpr { left, op, right, .. }) + if op == &target_op && (needle == left.as_ref() || needle == right.as_ref())) +} + +/// Combines an array of filter expressions into a single filter +/// expression consisting of the input filter expressions joined with +/// logical AND. +/// +/// Returns None if the filters array is empty. +/// +/// # Example +/// ``` rust +/// use crate::metricsql_parser::ast::utils::{selector, number, conjunction}; +/// // a=1 AND b=2 +/// let expr = selector("a").eq(number(1.0)).and(selector("b").eq(number(2.0))); +/// +/// // [a=1, b=2] +/// let split = vec![ +/// selector("a").eq(number(1.0)), +/// selector("b").eq(number(2.0)), +/// ]; +/// +/// // use conjunction to join them together with `AND` +/// assert_eq!(conjunction(split), Some(expr)); +/// ``` +pub fn conjunction(filters: impl IntoIterator) -> Option { + filters.into_iter().reduce(|accum, expr| accum.and(expr)) +} + +/// Combines an array of filter expressions into a single filter +/// expression consisting of the input filter expressions joined with +/// logical OR. +/// +/// Returns None if the filters array is empty. +pub fn disjunction(filters: impl IntoIterator) -> Option { + filters.into_iter().reduce(|accum, expr| accum.or(expr)) +} + +// all this nonsense because f64 used in NumberExpr doesn't implement Eq +pub fn expr_equals(expr1: &Expr, expr2: &Expr) -> bool { + use Expr::*; + + fn compare_parens(parens: &ParensExpr, expr: &Expr) -> bool { + if let Some(other) = parens.innermost_expr() { + return expr == other; + } + match expr { + Parens(p) => parens == p, + _ => false, + } + } + + match (expr1, expr2) { + (Parens(p1), Parens(p2)) => { + // println!("p1: {:?}, p2: {:?}", p1, p2); + p1 == p2 + } + // special case: (x) == x. I don't know if I like this + (Parens(p), e) => p.len() == 1 && compare_parens(p, e), + (e, Parens(p)) => p.len() == 1 && compare_parens(p, e), + (a, b) => a == b, + } +} + +pub(super) fn string_vecs_equal_unordered(a: &[String], b: &[String]) -> bool { + if a.len() != b.len() { + return false; + } + let hash_a: AHashSet<_> = a.iter().collect(); + b.iter().all(|x| hash_a.contains(x)) +} + +struct InvalidExprVisitor { + has_implicit_conversion: bool, +} + +impl ExprVisitor for InvalidExprVisitor { + type Error = (); + + fn pre_visit(&mut self, expr: &Expr) -> Result { + if self.has_implicit_conversion { + return Ok(true); + } + if let Expr::Function(f) = expr { + if let BuiltinFunction::Rollup(rollup) = f.function { + let idx = get_rollup_arg_idx(&rollup, f.args.len()); + if idx < 0 { + return Ok(true); + } + let arg = &f.args[idx as usize]; + match arg { + Expr::Rollup(re) => { + if re.window.is_none() { + self.has_implicit_conversion = true; + } + } + Expr::MetricExpression(_) => {} + _ => { + self.has_implicit_conversion = true; + } + } + } + } + Ok(true) + } +} + +/// is_likely_invalid returns true if an expression contains tricky implicit conversions which is invalid most of the time. +/// +/// Examples of invalid expressions: +/// +/// * rate(sum(foo)) +/// * rate(abs(foo)) +/// * rate(foo + bar) +/// * rate(foo > 10) +/// +/// These expressions are implicitly converted into another expressions, which returns unexpected results most of the time: +/// +/// * rate(default_rollup(sum(foo))[1i:1i]) +/// * rate(default_rollup(abs(foo))[1i:1i]) +/// * rate(default_rollup(foo + bar)[1i:1i]) +/// * rate(default_rollup(foo > 10)[1i:1i]) +/// +/// See https://docs.victoriametrics.com/metricsql/#implicit-query-conversions +/// +/// Note that rate(foo) is valid expression, since it returns the expected results most of the time, e.g. rate(foo[1i]). +pub fn is_likely_invalid(e: &Expr) -> bool { + let mut visitor = InvalidExprVisitor { + has_implicit_conversion: false, + }; + // unwrap is fine since the visitor doesn't error + walk_expr(&mut visitor, e).unwrap(); + visitor.has_implicit_conversion +} + +#[cfg(test)] +pub mod tests { + use crate::ast::utils::{conjunction, disjunction, selector}; + use crate::parser::parse; + use crate::prelude::utils::is_likely_invalid; + + #[test] + fn test_conjunction_empty() { + assert_eq!(conjunction(vec![]), None); + } + + #[test] + fn test_conjunction() { + // `[A, B, C]` + let expr = conjunction(vec![selector("a"), selector("b"), selector("c")]); + + // --> `(A AND B) AND C` + assert_eq!( + expr, + Some(selector("a").and(selector("b")).and(selector("c"))) + ); + + // which is different from `A AND (B AND C)` + assert_ne!( + expr, + Some(selector("a").and(selector("b").and(selector("c")))) + ); + } + + #[test] + fn test_disjunction_empty() { + assert_eq!(disjunction(vec![]), None); + } + + #[test] + fn test_disjunction() { + // `[A, B, C]` + let expr = disjunction(vec![selector("a"), selector("b"), selector("c")]); + + // --> `(A OR B) OR C` + assert_eq!( + expr, + Some(selector("a").or(selector("b")).or(selector("c"))) + ); + + // which is different from `A OR (B OR C)` + assert_ne!( + expr, + Some(selector("a").or(selector("b").or(selector("c")))) + ); + } + + #[test] + fn test_is_likely_invalid() { + fn f(q: &str, result_expected: bool) { + let expr = parse(q).unwrap(); + let result = is_likely_invalid(&expr); + assert_eq!( + result, result_expected, + "unexpected result for is_likely_invalid({}); got {}; want {}", + q, result, result_expected + ) + } + + f("1", false); + f(r#"foo{bar="baz"}"#, false); + + // This should be OK, since it is easy to reason about + f("rate(foo)", false); + f("foo[5m]", false); + f("1 + foo[5m]", false); + + f("rate(foo[5s])", false); + f(r#"rate(foo{bar=~"baz"}[5s])"#, false); + f(r#"rate(foo{bar=~"baz"}[5s] offset 1h)"#, false); + + // Explicit subqueries are allowed + f("sum_over_time((up > 0)[5m:1s])", false); + f("rate(sum(foo)[5m])", false); + f("rate(sum(foo)[5m:3s])", false); + + // Implicit step in the subquery is OK + f("sum_over_time((up > 0)[5m])", false); + + // This is OK, since it is supported by Prometheus + f(r#"rate(foo{bar=~"baz"}[5m:1s])"#, false); + f(r#"rate(foo{bar=~"baz"}[5m:1s] offset 1h)"#, false); + + f("sum(foo)", false); + f("sum(rate(foo))", false); + f("abs(foo)", false); + f("sum(abs(foo))", false); + + // This isn't OK, since these queries work unexpectedly most of the time + f("rate(sum(foo))", true); + f("rate(abs(foo))", true); + f("rate(1)", true); + f("rate(foo + bar)", true); + f("rate(rate(foo))", true); + f("rate(sum(foo) offset 5m)", true); + f(r#"1 + rate(label_set(foo, "bar", "baz"))"#, true); + } +} diff --git a/crates/metricsql-parser-vendored/src/ast/visitor.rs b/crates/metricsql-parser-vendored/src/ast/visitor.rs new file mode 100644 index 00000000..220ea0b5 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/ast/visitor.rs @@ -0,0 +1,222 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::ast::{AggregationExpr, BinaryExpr, Expr, FunctionExpr, ParensExpr, RollupExpr}; + +/// Trait that implements the [Visitor pattern](https://en.wikipedia.org/wiki/Visitor_pattern) +/// for a depth first walk on [Expr] AST. [`pre_visit`](ExprVisitor::pre_visit) is called +/// before any children are visited, and then [`post_visit`](ExprVisitor::post_visit) is called +/// after all children have been visited. Only [`pre_visit`](ExprVisitor::pre_visit) is required. +pub trait ExprVisitor { + type Error; + + /// Called before any children are visited. Return `Ok(false)` to cut short the recursion + /// (skip traversing and return). + fn pre_visit(&mut self, plan: &Expr) -> Result; + + /// Called after all children are visited. Return `Ok(false)` to cut short the recursion + /// (skip traversing and return). + fn post_visit(&mut self, _plan: &Expr) -> Result { + Ok(true) + } +} + +/// A util function that traverses an AST [Expr] in depth-first order. Returns +/// `Ok(true)` if all nodes were visited, and `Ok(false)` if any call to +/// [`pre_visit`](ExprVisitor::pre_visit) or [`post_visit`](ExprVisitor::post_visit) +/// returned `Ok(false)` and may have cut short the recursion. +pub fn walk_expr(visitor: &mut V, expr: &Expr) -> Result { + if !visitor.pre_visit(expr)? { + return Ok(false); + } + + let recurse = match expr { + Expr::UnaryOperator(u) => walk_expr(visitor, &u.expr)?, + Expr::BinaryOperator(BinaryExpr { left, right, .. }) => { + walk_expr(visitor, left)? && walk_expr(visitor, right)? + } + Expr::Parens(ParensExpr { expressions }) => { + for expr in expressions { + if !walk_expr(visitor, expr)? { + return Ok(false); + } + } + true + } + Expr::Rollup(RollupExpr { expr, at, .. }) => { + if !walk_expr(visitor, expr)? { + return Ok(false); + } + if let Some(at_expr) = at { + if !walk_expr(visitor, at_expr)? { + return Ok(false); + } + } + true + } + Expr::Aggregation(AggregationExpr { args, .. }) + | Expr::Function(FunctionExpr { args, .. }) => { + for func_argument_expr in args { + if !walk_expr(visitor, func_argument_expr)? { + return Ok(false); + } + } + true + } + Expr::StringLiteral(_) + | Expr::StringExpr(_) + | Expr::NumberLiteral(_) + | Expr::MetricExpression(_) + | Expr::With(_) + | Expr::WithSelector(_) + | Expr::Duration(_) => true, + }; + + if !recurse { + return Ok(false); + } + + if !visitor.post_visit(expr)? { + return Ok(false); + } + + Ok(true) +} + +#[cfg(test)] +mod tests { + use crate::ast::MetricExpr; + use crate::label::LabelFilterOp; + use crate::parser; + + use super::*; + + struct NamespaceVisitor { + namespace: String, + } + + fn vector_selector_includes_namespace(namespace: &str, vector_selector: &MetricExpr) -> bool { + vector_selector + .matchers + .find_matchers("namespace") + .iter() + .any(|x| x.op == LabelFilterOp::Equal && x.value.eq(namespace)) + } + + impl ExprVisitor for NamespaceVisitor { + type Error = &'static str; + + fn pre_visit(&mut self, expr: &Expr) -> Result { + match expr { + Expr::MetricExpression(matrix_selector) => { + let included = vector_selector_includes_namespace( + self.namespace.as_str(), + matrix_selector, + ); + return Ok(included); + } + Expr::NumberLiteral(_) | Expr::StringLiteral(_) => return Ok(false), + _ => (), + } + Ok(true) + } + } + + #[test] + fn test_check_for_namespace_basic_query() { + let expr = "pg_stat_activity_count{namespace=\"sample\"}"; + let ast = parser::parse(expr).unwrap(); + let mut visitor = NamespaceVisitor { + namespace: "sample".to_string(), + }; + assert!(walk_expr(&mut visitor, &ast).unwrap()); + } + + #[test] + fn test_check_for_namespace_label_present() { + let expr = "(sum by (namespace) (max_over_time(pg_stat_activity_count{namespace=\"sample\"}[1h])))"; + let ast = parser::parse(expr).unwrap(); + let mut visitor = NamespaceVisitor { + namespace: "sample".to_string(), + }; + assert!(walk_expr(&mut visitor, &ast).unwrap()); + } + + #[test] + fn test_check_for_namespace_label_wrong_namespace() { + let expr = "(sum by (namespace) (max_over_time(pg_stat_activity_count{namespace=\"sample\"}[1h])))"; + let ast = parser::parse(expr).unwrap(); + let mut visitor = NamespaceVisitor { + namespace: "foobar".to_string(), + }; + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + } + + #[test] + fn test_check_for_namespace_label_missing_namespace() { + let expr = "(sum by (namespace) (max_over_time(pg_stat_activity_count{}[1h])))"; + let ast = parser::parse(expr).unwrap(); + let mut visitor = NamespaceVisitor { + namespace: "sample".to_string(), + }; + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + } + + #[test] + fn test_literal_expr() { + let mut visitor = NamespaceVisitor { + namespace: "sample".to_string(), + }; + + let ast = parser::parse("1").unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + + let ast = parser::parse("1 + 1").unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + + let ast = parser::parse(r#""1""#).unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + } + + #[test] + fn test_binary_expr() { + let mut visitor = NamespaceVisitor { + namespace: "sample".to_string(), + }; + + let ast = parser::parse( + "pg_stat_activity_count{namespace=\"sample\"} + pg_stat_activity_count{}", + ) + .unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + + let ast = parser::parse( + "pg_stat_activity_count{} - pg_stat_activity_count{namespace=\"sample\"}", + ) + .unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + + let ast = parser::parse("pg_stat_activity_count{} * pg_stat_activity_count{}").unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + + let ast = parser::parse("pg_stat_activity_count{namespace=\"sample\"} / 1").unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + + let ast = parser::parse("1 % pg_stat_activity_count{namespace=\"sample\"}").unwrap(); + assert!(!walk_expr(&mut visitor, &ast).unwrap()); + + let ast = parser::parse("pg_stat_activity_count{namespace=\"sample\"} ^ pg_stat_activity_count{namespace=\"sample\"}").unwrap(); + assert!(walk_expr(&mut visitor, &ast).unwrap()); + } +} diff --git a/crates/metricsql-parser-vendored/src/binaryop/mod.rs b/crates/metricsql-parser-vendored/src/binaryop/mod.rs new file mode 100644 index 00000000..a8879898 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/binaryop/mod.rs @@ -0,0 +1,330 @@ +use crate::ast::Operator; +use crate::parser::{ParseError, ParseResult}; + +pub type BinopFunc = fn(left: f64, right: f64) -> f64; + +/// eq returns true of left == right. +#[inline] +fn op_eq(left: f64, right: f64) -> bool { + // Special handling for nan == nan. + if left.is_nan() { + return right.is_nan(); + } + left == right +} + +/// neq returns true of left != right. +#[inline] +fn op_neq(left: f64, right: f64) -> bool { + // Special handling for comparison with nan. + if left.is_nan() { + return !right.is_nan(); + } + if right.is_nan() { + return true; + } + left != right +} + +fn op_and(left: f64, right: f64) -> f64 { + if left.is_nan() || right.is_nan() { + f64::NAN + } else { + left + } +} + +// return the first non-NaN item. If both left and right are NaN, it returns NaN. +fn op_or(left: f64, right: f64) -> f64 { + if !left.is_nan() { + return left; + } else if !right.is_nan() { + return right; + } + f64::NAN +} + +/// gt returns true of left > right +#[inline] +fn op_gt(left: f64, right: f64) -> bool { + left > right +} + +/// lt returns true if left < right +#[inline] +fn op_lt(left: f64, right: f64) -> bool { + left < right +} + +/// Gte returns true if left >= right +#[inline] +fn op_gte(left: f64, right: f64) -> bool { + left >= right +} + +/// Lte returns true if left <= right +#[inline] +fn op_lte(left: f64, right: f64) -> bool { + left <= right +} + +/// Plus returns left + right +#[inline] +fn op_plus(left: f64, right: f64) -> f64 { + left + right +} + +/// Minus returns left - right +#[inline] +fn op_minus(left: f64, right: f64) -> f64 { + left - right +} + +/// Mul returns left * right +#[inline] +fn op_mul(left: f64, right: f64) -> f64 { + left * right +} + +/// Div returns left / right +/// Todo: protect against div by zero +#[inline] +fn op_div(left: f64, right: f64) -> f64 { + left / right +} + +/// returns left % right +#[inline] +fn op_mod(left: f64, right: f64) -> f64 { + left % right +} + +/// pow returns pow(left, right) +#[inline] +fn op_pow(left: f64, right: f64) -> f64 { + left.powf(right) +} + +/// returns atan2(left, right) +#[inline] +fn op_atan2(left: f64, right: f64) -> f64 { + left.atan2(right) +} + +/// returns left or right if left is NaN. +#[inline] +fn op_default(left: f64, right: f64) -> f64 { + if left.is_nan() { + return right; + } + left +} + +/// If returns left if right is not NaN. Otherwise, NaN is returned. +#[inline] +fn op_if(left: f64, right: f64) -> f64 { + if right.is_nan() { + return f64::NAN; + } + left +} + +/// if_not returns left if right is NaN. Otherwise, NaN is returned. +#[inline] +pub fn op_if_not(left: f64, right: f64) -> f64 { + if right.is_nan() { + return left; + } + f64::NAN +} + +#[inline] +pub fn op_unless(left: f64, right: f64) -> f64 { + if right != left { + return f64::NAN; + } + left +} + +/// convert true to x, false to NaN. +#[inline] +pub const fn to_comparison_value(b: bool, x: f64) -> f64 { + if b { + x + } else { + f64::NAN + } +} + +macro_rules! make_comparison_func { + ($name: ident, $func: expr) => { + pub fn $name(left: f64, right: f64) -> f64 { + to_comparison_value($func(left, right), left) + } + }; +} + +macro_rules! make_comparison_func_bool { + ($name: ident, $func: expr) => { + pub fn $name(left: f64, right: f64) -> f64 { + if left.is_nan() { + return f64::NAN; + } + if $func(left, right) { + 1_f64 + } else { + 0_f64 + } + } + }; +} + +make_comparison_func!(compare_eq, op_eq); +make_comparison_func!(compare_neq, op_neq); +make_comparison_func!(compare_gt, op_gt); +make_comparison_func!(compare_lt, op_lt); +make_comparison_func!(compare_gte, op_gte); +make_comparison_func!(compare_lte, op_lte); + +make_comparison_func_bool!(compare_eq_bool, op_eq); +make_comparison_func_bool!(compare_neq_bool, op_neq); +make_comparison_func_bool!(compare_gt_bool, op_gt); +make_comparison_func_bool!(compare_lt_bool, op_lt); +make_comparison_func_bool!(compare_gte_bool, op_gte); +make_comparison_func_bool!(compare_lte_bool, op_lte); + +pub const fn get_scalar_comparison_handler(op: Operator, is_bool: bool) -> BinopFunc { + if is_bool { + match op { + Operator::Eql => compare_eq_bool, + Operator::NotEq => compare_neq_bool, + Operator::Gt => compare_gt_bool, + Operator::Lt => compare_lt_bool, + Operator::Gte => compare_gte_bool, + Operator::Lte => compare_lte_bool, + _ => unreachable!(), + } + } else { + match op { + Operator::Eql => compare_eq, + Operator::NotEq => compare_neq, + Operator::Gt => compare_gt, + Operator::Lt => compare_lt, + Operator::Gte => compare_gte, + Operator::Lte => compare_lte, + _ => unreachable!(), + } + } +} + +pub const fn get_scalar_binop_handler(op: Operator, is_bool: bool) -> BinopFunc { + match op { + Operator::Add => op_plus, + Operator::Atan2 => op_atan2, + Operator::Default => op_default, + Operator::Div => op_div, + Operator::Mod => op_mod, + Operator::Mul => op_mul, + Operator::Pow => op_pow, + Operator::Sub => op_minus, + Operator::If => op_if, + Operator::IfNot => op_if_not, + Operator::Unless => op_unless, + Operator::And => op_and, + Operator::Or => op_or, + Operator::Eql => get_scalar_comparison_handler(Operator::Eql, is_bool), + Operator::NotEq => get_scalar_comparison_handler(Operator::NotEq, is_bool), + Operator::Gt => get_scalar_comparison_handler(Operator::Gt, is_bool), + Operator::Lt => get_scalar_comparison_handler(Operator::Lt, is_bool), + Operator::Gte => get_scalar_comparison_handler(Operator::Gte, is_bool), + Operator::Lte => get_scalar_comparison_handler(Operator::Lte, is_bool), + } +} + +pub fn eval_binary_op(left: f64, right: f64, op: Operator, is_bool: bool) -> f64 { + let handler = get_scalar_binop_handler(op, is_bool); + handler(left, right) +} + +pub fn string_compare(a: &str, b: &str, op: Operator, is_bool: bool) -> ParseResult { + let res = match op { + Operator::Eql => a == b, + Operator::NotEq => a != b, + Operator::Lt => a < b, + Operator::Gt => a > b, + Operator::Lte => a <= b, + Operator::Gte => a >= b, + _ => { + return Err(ParseError::Unsupported(format!( + "unexpected operator {op} in string comparison" + ))) + } + }; + Ok(if res { + 1_f64 + } else if is_bool { + 0_f64 + } else { + f64::NAN + }) +} + +/// Supported operation between two float type values. +/// For one-off operations. This differs from the `get_scalar_binop_handler` in that it +/// is optimized for a single operation. The `get_scalar_binop_handler` is optimized for +/// a single operation that is applied to many values (it minimizes the number of branches). +pub fn scalar_binary_operation( + lhs: f64, + rhs: f64, + op: Operator, + return_bool: bool, +) -> ParseResult { + use Operator::*; + + let value = if op.is_comparison() { + let val = match op { + Eql => op_eq(lhs, rhs), + NotEq => op_neq(lhs, rhs), + Gt => lhs > rhs, + Lt => lhs < rhs, + Gte => lhs >= rhs, + Lte => lhs <= rhs, + _ => { + unreachable!("Unsupported scalar comparison operation: {lhs} {op} {rhs}",) + } + }; + if return_bool { + val as u32 as f64 + } else { + // if the return value was true, that means our element + // satisfies the comparison, hence return it + if val { + lhs + } else { + f64::NAN + } + } + } else { + match op { + Add => lhs + rhs, + Sub => lhs - rhs, + Mul => lhs * rhs, + Div => lhs / rhs, + Pow => lhs.powf(rhs), + Mod => lhs % rhs, + Atan2 => lhs.atan2(rhs), + Default => op_default(lhs, rhs), + If => op_if(lhs, rhs), + IfNot => op_if_not(lhs, rhs), + And => op_and(lhs, rhs), + Or => op_or(lhs, rhs), + Unless => f64::NAN, + _ => { + return Err(ParseError::Unsupported(format!( + "Unsupported scalar operation: {lhs} {op} {rhs}", + ))) + } + } + }; + Ok(value) +} diff --git a/crates/metricsql-parser-vendored/src/common/mod.rs b/crates/metricsql-parser-vendored/src/common/mod.rs new file mode 100644 index 00000000..4a44bc61 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/common/mod.rs @@ -0,0 +1,7 @@ +pub use tree_node::*; +pub(crate) use utils::{hash_f64, join_vector, write_comma_separated, write_number}; +pub use value::*; + +mod tree_node; +mod utils; +mod value; diff --git a/crates/metricsql-parser-vendored/src/common/tree_node.rs b/crates/metricsql-parser-vendored/src/common/tree_node.rs new file mode 100644 index 00000000..7e8f6282 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/common/tree_node.rs @@ -0,0 +1,338 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! This module provides common traits for visiting or rewriting tree +//! data structures easily. + +use std::sync::Arc; + +use crate::parser::ParseResult; + +/// Defines a visitable and rewriteable a tree node. This trait is +/// implemented for expression trees ([`Expr`]) in MetricSQL +pub trait TreeNode: Sized { + /// Use preorder to iterate the node on the tree so that we can + /// stop fast for some cases. + /// + /// The `op` closure can be used to collect some info from the + /// tree node or do some checking for the tree node. + fn apply(&self, op: &mut F) -> ParseResult + where + F: FnMut(&Self) -> ParseResult, + { + match op(self)? { + VisitRecursion::Continue => {} + // If the recursion should skip, do not apply to its children. And let the recursion continue + VisitRecursion::Skip => return Ok(VisitRecursion::Continue), + // If the recursion should stop, do not apply to its children + VisitRecursion::Stop => return Ok(VisitRecursion::Stop), + }; + + self.apply_children(&mut |node| node.apply(op)) + } + + /// Visit the tree node using the given [TreeNodeVisitor] + /// It performs a depth first walk of a node and its children. + /// + /// For a node tree such as + /// ```text + /// ParentNode + /// left: ChildNode1 + /// right: ChildNode2 + /// ``` + /// + /// The nodes are visited using the following order + /// ```text + /// pre_visit(ParentNode) + /// pre_visit(ChildNode1) + /// post_visit(ChildNode1) + /// pre_visit(ChildNode2) + /// post_visit(ChildNode2) + /// post_visit(ParentNode) + /// ``` + /// + /// If an Err result is returned, recursion is stopped immediately + /// + /// If [`VisitRecursion::Stop`] is returned on a call to pre_visit, no + /// children of that node will be visited, nor is post_visit + /// called on that node. Details see [`TreeNodeVisitor`] + /// + /// If using the default [`TreeNodeVisitor::post_visit`] that does + /// nothing, [`Self::apply`] should be preferred. + fn visit>(&self, visitor: &mut V) -> ParseResult { + match visitor.pre_visit(self)? { + VisitRecursion::Continue => {} + // If the recursion should skip, do not apply to its children. And let the recursion continue + VisitRecursion::Skip => return Ok(VisitRecursion::Continue), + // If the recursion should stop, do not apply to its children + VisitRecursion::Stop => return Ok(VisitRecursion::Stop), + }; + + match self.apply_children(&mut |node| node.visit(visitor))? { + VisitRecursion::Continue => {} + // If the recursion should skip, do not apply to its children. And let the recursion continue + VisitRecursion::Skip => return Ok(VisitRecursion::Continue), + // If the recursion should stop, do not apply to its children + VisitRecursion::Stop => return Ok(VisitRecursion::Stop), + } + + visitor.post_visit(self) + } + + /// Convenience utils for writing optimizers rule: recursively apply the given `op` to the node tree. + /// When `op` does not apply to a given node, it is left unchanged. + /// The default tree traversal direction is transform_up(Postorder Traversal). + fn transform(self, op: &F) -> ParseResult + where + F: Fn(Self) -> ParseResult>, + { + self.transform_up(op) + } + + /// Convenience utils for writing optimizers rule: recursively apply the given 'op' to the node and all of its + /// children(Preorder Traversal). + /// When the `op` does not apply to a given node, it is left unchanged. + fn transform_down(self, op: &F) -> ParseResult + where + F: Fn(Self) -> ParseResult>, + { + let after_op = op(self)?.into(); + after_op.map_children(|node| node.transform_down(op)) + } + + /// Convenience utils for writing optimizers rule: recursively apply the given 'op' first to all of its + /// children and then itself(Postorder Traversal). + /// When the `op` does not apply to a given node, it is left unchanged. + fn transform_up(self, op: &F) -> ParseResult + where + F: Fn(Self) -> ParseResult>, + { + let after_op_children = self.map_children(|node| node.transform_up(op))?; + + let new_node = op(after_op_children)?.into(); + Ok(new_node) + } + + /// Transform the tree node using the given [TreeNodeRewriter] + /// It performs a depth first walk of a node and its children. + /// + /// For a node tree such as + /// ```text + /// ParentNode + /// left: ChildNode1 + /// right: ChildNode2 + /// ``` + /// + /// The nodes are visited using the following order + /// ```text + /// pre_visit(ParentNode) + /// pre_visit(ChildNode1) + /// mutate(ChildNode1) + /// pre_visit(ChildNode2) + /// mutate(ChildNode2) + /// mutate(ParentNode) + /// ``` + /// + /// If an Err result is returned, recursion is stopped immediately + /// + /// If [`false`] is returned on a call to pre_visit, no + /// children of that node will be visited, nor is mutate + /// called on that node + /// + /// If using the default [`TreeNodeRewriter::pre_visit`] which + /// returns `true`, [`Self::transform`] should be preferred. + fn rewrite>(self, rewriter: &mut R) -> ParseResult { + let need_mutate = match rewriter.pre_visit(&self)? { + RewriteRecursion::Mutate => return rewriter.mutate(self), + RewriteRecursion::Stop => return Ok(self), + RewriteRecursion::Continue => true, + RewriteRecursion::Skip => false, + }; + + let after_op_children = self.map_children(|node| node.rewrite(rewriter))?; + + // now rewrite this node itself + if need_mutate { + rewriter.mutate(after_op_children) + } else { + Ok(after_op_children) + } + } + + /// Apply the closure `F` to the node's children + fn apply_children(&self, op: &mut F) -> ParseResult + where + F: FnMut(&Self) -> ParseResult; + + /// Apply transform `F` to the node's children, the transform `F` might have a direction(Preorder or Postorder) + fn map_children(self, transform: F) -> ParseResult + where + F: FnMut(Self) -> ParseResult; +} + +/// Implements the [visitor +/// pattern](https://en.wikipedia.org/wiki/Visitor_pattern) for recursively walking [`TreeNode`]s. +/// +/// [`TreeNodeVisitor`] allows keeping the algorithms +/// separate from the code to traverse the structure of the `TreeNode` +/// tree and makes it easier to add new types of tree node and +/// algorithms. +/// +/// When passed to[`TreeNode::visit`], [`TreeNodeVisitor::pre_visit`] +/// and [`TreeNodeVisitor::post_visit`] are invoked recursively +/// on an node tree. +/// +/// If an [`Err`] result is returned, recursion is stopped +/// immediately. +/// +/// If [`VisitRecursion::Stop`] is returned on a call to pre_visit, no +/// children of that tree node are visited, nor is post_visit +/// called on that tree node +/// +/// If [`VisitRecursion::Stop`] is returned on a call to post_visit, no +/// siblings of that tree node are visited, nor is post_visit +/// called on its parent tree node +/// +/// If [`VisitRecursion::Skip`] is returned on a call to pre_visit, no +/// children of that tree node are visited. +pub trait TreeNodeVisitor: Sized { + /// The node type which is visitable. + type N: TreeNode; + + /// Invoked before any children of `node` are visited. + fn pre_visit(&mut self, node: &Self::N) -> ParseResult; + + /// Invoked after all children of `node` are visited. Default + /// implementation does nothing. + fn post_visit(&mut self, _node: &Self::N) -> ParseResult { + Ok(VisitRecursion::Continue) + } +} + +/// Trait for potentially recursively transform an [`TreeNode`] node +/// tree. When passed to `TreeNode::rewrite`, `TreeNodeRewriter::mutate` is +/// invoked recursively on all nodes of a tree. +pub trait TreeNodeRewriter: Sized { + /// The node type which is rewritable. + type N: TreeNode; + + /// Invoked before (Preorder) any children of `node` are rewritten / + /// visited. Default implementation returns `Ok(Recursion::Continue)` + fn pre_visit(&mut self, _node: &Self::N) -> ParseResult { + Ok(RewriteRecursion::Continue) + } + + /// Invoked after (Postorder) all children of `node` have been mutated and + /// returns a potentially modified node. + fn mutate(&mut self, node: Self::N) -> ParseResult; +} + +/// Controls how the [`TreeNode`] recursion should proceed for [`TreeNode::rewrite`]. +#[derive(Debug)] +pub enum RewriteRecursion { + /// Continue rewrite this node tree. + Continue, + /// Call 'op' immediately and return. + Mutate, + /// Do not rewrite the children of this node. + Stop, + /// Keep recursive but skip apply op on this node + Skip, +} + +/// Controls how the [`TreeNode`] recursion should proceed for [`TreeNode::visit`]. +#[derive(Debug)] +pub enum VisitRecursion { + /// Continue the visit to this node tree. + Continue, + /// Keep recursive but skip applying op on the children + Skip, + /// Stop the visit to this node tree. + Stop, +} + +pub enum Transformed { + /// The item was transformed / rewritten somehow + Yes(T), + /// The item was not transformed + No(T), +} + +impl Transformed { + pub fn into(self) -> T { + match self { + Transformed::Yes(t) => t, + Transformed::No(t) => t, + } + } + + pub fn into_pair(self) -> (T, bool) { + match self { + Transformed::Yes(t) => (t, true), + Transformed::No(t) => (t, false), + } + } +} + +/// Helper trait for implementing [`TreeNode`] that have children stored as Arc's +/// +/// If some trait object, such as `dyn T`, implements this trait, +/// its related `Arc` will automatically implement [`TreeNode`] +pub trait DynTreeNode { + /// Returns all children of the specified TreeNode + fn arc_children(&self) -> Vec>; + + /// construct a new self with the specified children + fn with_new_arc_children( + &self, + arc_self: Arc, + new_children: Vec>, + ) -> ParseResult>; +} + +/// Blanket implementation for Arc for any type that implements +/// [`DynTreeNode`] +impl TreeNode for Arc { + fn apply_children(&self, op: &mut F) -> ParseResult + where + F: FnMut(&Self) -> ParseResult, + { + for child in self.arc_children() { + match op(&child)? { + VisitRecursion::Continue => {} + VisitRecursion::Skip => return Ok(VisitRecursion::Continue), + VisitRecursion::Stop => return Ok(VisitRecursion::Stop), + } + } + + Ok(VisitRecursion::Continue) + } + + fn map_children(self, transform: F) -> ParseResult + where + F: FnMut(Self) -> ParseResult, + { + let children = self.arc_children(); + if !children.is_empty() { + let new_children: ParseResult> = children.into_iter().map(transform).collect(); + let arc_self = Arc::clone(&self); + self.with_new_arc_children(arc_self, new_children?) + } else { + Ok(self) + } + } +} diff --git a/crates/metricsql-parser-vendored/src/common/utils.rs b/crates/metricsql-parser-vendored/src/common/utils.rs new file mode 100644 index 00000000..1dd6dde3 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/common/utils.rs @@ -0,0 +1,65 @@ +use std::fmt; +use std::fmt::{Display, Formatter}; +use std::hash::Hasher; + +/// Returns the mantissa, exponent and sign as integers. +pub fn integer_decode(val: f64) -> (u64, i16, i8) { + let bits: u64 = val.to_bits(); + let sign: i8 = if bits >> 63 == 0 { 1 } else { -1 }; + let mut exponent: i16 = ((bits >> 52) & 0x7ff) as i16; + let mantissa = if exponent == 0 { + (bits & 0xfffffffffffff) << 1 + } else { + (bits & 0xfffffffffffff) | 0x10000000000000 + }; + + exponent -= 1023 + 52; + (mantissa, exponent, sign) +} + +pub fn hash_f64(state: &mut H, value: f64) { + let (mantissa, exponent, sign) = integer_decode(value); + state.write_u64(mantissa); + state.write_i16(exponent); + state.write_i8(sign) +} + +pub fn write_number(f: &mut Formatter<'_>, value: f64) -> fmt::Result { + if value.is_finite() { + write!(f, "{}", value) + } else if value.is_nan() { + write!(f, "NaN") + } else if value.is_sign_positive() { + write!(f, "+Inf") + } else { + write!(f, "-Inf") + } +} + +pub(crate) fn write_comma_separated( + values: impl Iterator, + f: &mut Formatter, + use_parens: bool, +) -> Result<(), fmt::Error> { + if use_parens { + write!(f, "(")?; + } + for (i, arg) in values.enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{}", arg)?; + } + if use_parens { + write!(f, ")")?; + } + Ok(()) +} + +pub fn join_vector(v: &[T], sep: &str, sort: bool) -> String { + let mut vs = v.iter().map(|x| x.to_string()).collect::>(); + if sort { + vs.sort(); + } + vs.join(sep) +} diff --git a/crates/metricsql-parser-vendored/src/common/value.rs b/crates/metricsql-parser-vendored/src/common/value.rs new file mode 100644 index 00000000..a2cbd763 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/common/value.rs @@ -0,0 +1,64 @@ +use std::fmt::{Display, Formatter}; + +use serde::{Deserialize, Serialize}; + +/// A query value type +#[derive(Debug, Default, Copy, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum ValueType { + /// A 64-bit floating point number. + Scalar, + /// An owned String + String, + #[default] + InstantVector, + RangeVector, +} + +impl ValueType { + /// Returns true if this `ValueType` is a valid sub-expression of an + /// operator, false if not. + pub const fn is_operator_valid(&self) -> bool { + match self { + ValueType::Scalar | + ValueType::String | + ValueType::RangeVector | // ??????? + ValueType::InstantVector => true, + } + } + + pub const fn as_str(&self) -> &'static str { + match self { + ValueType::Scalar => "Scalar", + ValueType::String => "String", + ValueType::InstantVector => "InstantVector", + ValueType::RangeVector => "RangeVector", + } + } + + pub const fn is_scalar(&self) -> bool { + matches!(self, ValueType::Scalar) + } +} + +impl Display for ValueType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +pub trait Value { + fn value_type(&self) -> ValueType; +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_value_type() { + assert_eq!(ValueType::Scalar.to_string(), "Scalar"); + assert_eq!(ValueType::String.to_string(), "String"); + assert_eq!(ValueType::InstantVector.to_string(), "InstantVector"); + assert_eq!(ValueType::RangeVector.to_string(), "RangeVector"); + } +} diff --git a/crates/metricsql-parser-vendored/src/functions/aggregate.rs b/crates/metricsql-parser-vendored/src/functions/aggregate.rs new file mode 100644 index 00000000..b1a78c47 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/functions/aggregate.rs @@ -0,0 +1,250 @@ +//! AggregateFunction module contains enum for available aggregation AggregateFunctions. + +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use strum_macros::EnumIter; + +use crate::common::ValueType; +use crate::functions::signature::{Signature, Volatility}; +use crate::functions::{BuiltinFunction, FunctionMeta, MAX_ARG_COUNT}; +use crate::parser::ParseError; + +/// Aggregation AggregateFunctions +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Hash, EnumIter, Serialize, Deserialize)] +pub enum AggregateFunction { + /// calculate the average over dimensions + Avg, + /// smallest k elements by sample value + Bottomk, + /// count the number of elements in the vector + Count, + /// count the number of elements with the same value + CountValues, + /// calculate maximum over dimensions + Max, + /// calculate minimum over dimensions + Min, + /// calculate population standard deviation over dimensions + StdDev, + /// calculate population standard variance over dimensions + StdVar, + /// largest k elements by sample value + Topk, + Group, + /// calculate φ-quantile (0 ≤ φ ≤ 1) over dimensions + Quantile, + Quantiles, + /// calculate sum over dimensions + Sum, + // PromQL extension functions + /// any(q) by (group_labels) returns a single series per group_labels out of time series returned by q. + /// See also group. + Any, + BottomkMin, + BottomkMax, + BottomkAvg, + BottomkLast, + BottomkMedian, + Distinct, + GeoMean, + Histogram, + Limitk, + MAD, + Median, + Mode, + OutliersIQR, + Outliersk, + OutliersMAD, + Sum2, + TopkMin, + TopkMax, + TopkAvg, + TopkLast, + TopkMedian, + Share, + ZScore, +} + +impl AggregateFunction { + pub fn signature(&self) -> Signature { + aggregate_function_signature(self) + } + + pub const fn may_sort_results(&self) -> bool { + use AggregateFunction::*; + matches!( + self, + Topk | Bottomk + | Outliersk + | TopkMax + | TopkMin + | TopkAvg + | TopkMedian + | TopkLast + | BottomkMax + | BottomkMin + | BottomkAvg + | BottomkMedian + | BottomkLast + ) + } + + pub const fn name(&self) -> &'static str { + use AggregateFunction::*; + + match self { + Any => "any", + Avg => "avg", + Bottomk => "bottomk", + BottomkAvg => "bottomk_avg", + BottomkLast => "bottomk_last", + BottomkMax => "bottomk_max", + BottomkMedian => "bottomk_median", + BottomkMin => "bottomk_min", + Count => "count", + CountValues => "count_values", + Distinct => "distinct", + GeoMean => "geomean", + Group => "group", + Histogram => "histogram", + Limitk => "limitk", + MAD => "mad", + Max => "max", + Median => "median", + Min => "min", + Mode => "mode", + OutliersIQR => "outliers_iqr", + Outliersk => "outliersk", + OutliersMAD => "outliers_mad", + Quantile => "quantile", + Quantiles => "quantiles", + Share => "share", + StdDev => "stddev", + StdVar => "stdvar", + Sum => "sum", + Sum2 => "sum2", + Topk => "topk", + TopkMin => "topk_min", + TopkMax => "topk_max", + TopkAvg => "topk_avg", + TopkLast => "topk_last", + TopkMedian => "topk_median", + ZScore => "score", + } + } + + pub fn return_type(&self) -> ValueType { + ValueType::InstantVector + } + + pub fn can_accept_multiple_args(&self) -> bool { + can_accept_multiple_args_for_aggr_func(*self) + } +} + +impl Display for AggregateFunction { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } +} + +impl FromStr for AggregateFunction { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + if let Some(meta) = FunctionMeta::lookup(s) { + if let BuiltinFunction::Aggregate(ag) = &meta.function { + return Ok(*ag); + } + } + Err(ParseError::InvalidFunction(s.to_string())) + } +} + +/// the signatures supported by the function `fun`. +pub fn aggregate_function_signature(fun: &AggregateFunction) -> Signature { + use AggregateFunction::*; + match fun { + CountValues => Signature::exact( + vec![ValueType::String, ValueType::InstantVector], + Volatility::Stable, + ), + Topk | Limitk | Outliersk => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Stable, + ), + OutliersMAD => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Stable, + ), + TopkMin | TopkMax | TopkAvg | TopkMedian | BottomkMin | BottomkMax | BottomkAvg + | BottomkLast | BottomkMedian => Signature::exact_with_min_args( + vec![ + ValueType::Scalar, + ValueType::InstantVector, + ValueType::String, + ], + 2, + Volatility::Stable, + ), + Quantile => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Stable, + ), + Quantiles => { + // todo: + let mut quantile_types: Vec = vec![ValueType::Scalar; MAX_ARG_COUNT]; + quantile_types.insert(0, ValueType::String); + quantile_types.push(ValueType::InstantVector); + Signature::variadic_min(quantile_types, 3, Volatility::Volatile) + } + _ => Signature::variadic_min( + vec![ValueType::InstantVector, ValueType::Scalar], + 1, + Volatility::Stable, + ), + } +} + +pub const fn get_aggregate_arg_idx_for_optimization( + func: AggregateFunction, + arg_count: usize, +) -> Option { + use AggregateFunction::*; + // todo: just examine the signature and return the position containing a vector + match func { + Bottomk | BottomkAvg | BottomkMax | BottomkMedian | BottomkLast | BottomkMin | Limitk + | Outliersk | OutliersMAD | Quantile | Topk | TopkAvg | TopkMax | TopkMedian | TopkLast + | TopkMin => Some(1), + CountValues => None, + Quantiles => Some(arg_count - 1), + _ => Some(0), + } +} + +// todo: use signature +pub(crate) const fn can_accept_multiple_args_for_aggr_func(func: AggregateFunction) -> bool { + use AggregateFunction::*; + matches!( + func, + Any | Avg + | Count + | Distinct + | GeoMean + | Group + | Histogram + | MAD + | Max + | Median + | Min + | Mode + | Share + | StdDev + | StdVar + | Sum + | Sum2 + | ZScore + ) +} diff --git a/crates/metricsql-parser-vendored/src/functions/mod.rs b/crates/metricsql-parser-vendored/src/functions/mod.rs new file mode 100644 index 00000000..b02c982b --- /dev/null +++ b/crates/metricsql-parser-vendored/src/functions/mod.rs @@ -0,0 +1,405 @@ +pub use aggregate::*; +use metricsql_common::hash::FastHashMap; +pub use rollup::*; +use serde::{Deserialize, Serialize}; +pub use signature::*; +use std::fmt; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; +use std::sync::OnceLock; +use strum::IntoEnumIterator; +pub use transform::*; + +use crate::ast::Expr; +use crate::common::ValueType; +use crate::parser::{validate_function_args, ParseError, ParseResult}; + +mod aggregate; +mod rollup; +mod signature; +mod transform; + +/// Maximum number of arguments permitted in a rollup function. This really only applies +/// to variadic functions like `aggr_over_time` and `quantiles_over_time` +const MAX_ARG_COUNT: usize = 32; + +#[derive(Debug, Clone, Hash, PartialEq, Eq, Serialize, Deserialize)] +pub enum BuiltinFunction { + Aggregate(AggregateFunction), + Rollup(RollupFunction), + Transform(TransformFunction), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum BuiltinFunctionType { + Aggregate, + Rollup, + Transform, +} + +impl BuiltinFunctionType { + pub const fn to_str(&self) -> &'static str { + match self { + BuiltinFunctionType::Aggregate => "aggregate", + BuiltinFunctionType::Rollup => "rollup", + BuiltinFunctionType::Transform => "transform", + } + } +} + +impl FromStr for BuiltinFunctionType { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + match s { + s if s.eq_ignore_ascii_case("aggregate") => Ok(BuiltinFunctionType::Aggregate), + s if s.eq_ignore_ascii_case("rollup") => Ok(BuiltinFunctionType::Rollup), + s if s.eq_ignore_ascii_case("transform") => Ok(BuiltinFunctionType::Transform), + _ => Err(ParseError::InvalidFunction(s.to_string())), + } + } +} + +impl Display for BuiltinFunctionType { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.to_str())?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct FunctionMeta { + pub name: &'static str, + pub function: BuiltinFunction, + pub signature: Signature, +} + +impl FunctionMeta { + pub fn lookup(name: &str) -> Option<&'static FunctionMeta> { + get_registry() + .get(name) + .or_else(|| get_registry().get(name.to_lowercase().as_str())) + } + + pub fn get_rollup_function(name: &str) -> ParseResult<&'static FunctionMeta> { + if let Some(meta) = FunctionMeta::lookup(name) { + if let BuiltinFunction::Rollup(_) = &meta.function { + return Ok(meta); + } + } + Err(ParseError::InvalidFunction(format!("rollup::{name}"))) + } + + pub fn get_aggregate_function(name: &str) -> ParseResult<&'static FunctionMeta> { + if let Some(meta) = FunctionMeta::lookup(name) { + if let BuiltinFunction::Aggregate(_) = &meta.function { + return Ok(meta); + } + } + Err(ParseError::InvalidFunction(format!("aggregate::{name}"))) + } + + pub fn get_type(&self) -> BuiltinFunctionType { + self.function.get_type() + } + + pub fn validate_arg_count(&self, name: &str, arg_len: usize) -> ParseResult<()> { + self.signature.validate_arg_count(name, arg_len) + } + + pub fn validate_args(&self, args: &[Expr]) -> ParseResult<()> { + validate_function_args(&self.function, args) + } + + pub fn is_aggregation(&self) -> bool { + matches!(self.function, BuiltinFunction::Aggregate(_)) + } + + pub fn is_scalar(&self) -> bool { + match self.function { + BuiltinFunction::Transform(func) => func.return_type() == ValueType::Scalar, + _ => false, + } + } + + pub fn is_rollup_function(&self, func: RollupFunction) -> bool { + match &self.function { + BuiltinFunction::Rollup(rf) => rf == &func, + _ => false, + } + } + + pub fn is_variadic(&self) -> bool { + self.signature.is_variadic() + } + + pub fn args_iter(&self) -> TypeIterator<'_> { + self.signature.args_iter() + } +} + +impl Display for FunctionMeta { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "{}(", self.name)?; + let mut min = self.signature.min_args(); + if min == 0 { + min = MAX_ARG_COUNT; + } + let is_variadic = self.is_variadic(); + let mut bracket_written = false; + for (i, arg) in self.args_iter().enumerate() { + if i > 0 { + if i == min { + if is_variadic { + write!(f, ", ...")?; + break; + } + if !bracket_written { + bracket_written = true; + write!(f, "[")?; + } + } + write!(f, ", ")?; + } + write!(f, "{}", arg)?; + } + + if bracket_written { + write!(f, "]")?; + } + + write!(f, ")") + } +} +type FunctionRegistry = FastHashMap<&'static str, FunctionMeta>; +static REGISTRY: OnceLock = OnceLock::new(); + +pub fn get_registry() -> &'static FunctionRegistry { + REGISTRY.get_or_init(init_registry) +} + +fn init_registry() -> FunctionRegistry { + let mut registry = FunctionRegistry::default(); + + for af in AggregateFunction::iter() { + let name = af.name(); + let function = BuiltinFunction::Aggregate(af); + let signature = af.signature(); + registry.insert( + name, + FunctionMeta { + name, + function, + signature, + }, + ); + } + + for rf in RollupFunction::iter() { + let name = rf.name(); + let function = BuiltinFunction::Rollup(rf); + let signature = rf.signature(); + registry.insert( + name, + FunctionMeta { + name, + function, + signature, + }, + ); + } + + for tf in TransformFunction::iter() { + let name = tf.name(); + let function = BuiltinFunction::Transform(tf); + let signature = tf.signature(); + registry.insert( + name, + FunctionMeta { + name, + function, + signature, + }, + ); + } + + registry +} + +impl BuiltinFunction { + pub fn new(name: &str) -> ParseResult { + if let Some(meta) = FunctionMeta::lookup(name) { + return Ok(meta.function.clone()); + } + Err(ParseError::InvalidFunction(format!("built-in::{name}"))) + } + + pub fn is_supported(name: &str) -> bool { + Self::new(name).is_ok() + } + + pub fn name(&self) -> &'static str { + use BuiltinFunction::*; + match self { + Aggregate(af) => af.name(), + Rollup(rf) => rf.name(), + Transform(tf) => tf.name(), + } + } + + pub fn signature(&self) -> Signature { + use BuiltinFunction::*; + match self { + Aggregate(af) => af.signature(), + Rollup(rf) => rf.signature(), + Transform(tf) => tf.signature(), + } + } + + pub fn validate_args(&self, args: &[Expr]) -> ParseResult<()> { + validate_function_args(self, args) + } + + pub fn type_name(&self) -> &'static str { + self.get_type().to_str() + } + + pub fn get_type(&self) -> BuiltinFunctionType { + use BuiltinFunction::*; + match self { + Aggregate(_) => BuiltinFunctionType::Aggregate, + Rollup(_) => BuiltinFunctionType::Rollup, + Transform(_) => BuiltinFunctionType::Transform, + } + } + + pub fn is_type(&self, other: BuiltinFunction) -> bool { + self.type_name() == other.type_name() + } + + pub fn is_aggregate_func(name: &str) -> bool { + if let Some(meta) = FunctionMeta::lookup(name) { + return meta.is_aggregation(); + } + false + } + + pub fn is_aggregation(&self) -> bool { + matches!(self, BuiltinFunction::Aggregate(_)) + } + + pub fn is_scalar(&self) -> bool { + match self { + BuiltinFunction::Transform(func) => func.return_type() == ValueType::Scalar, + _ => false, + } + } + + pub fn is_rollup_function(&self, func: RollupFunction) -> bool { + match self { + BuiltinFunction::Rollup(rf) => rf == &func, + _ => false, + } + } + + pub fn may_sort_results(&self) -> bool { + use BuiltinFunction::*; + match self { + Aggregate(af) => af.may_sort_results(), + Rollup(_) => false, // todo + Transform(tf) => tf.may_sort_results(), + } + } + + pub fn get_arg_for_optimization<'a>(&'a self, args: &'a [Expr]) -> Option<&Expr> { + match self.get_arg_idx_for_optimization(args.len()) { + Some(idx) => args.get(idx), + None => None, + } + } + + pub fn get_arg_idx_for_optimization(&self, args_len: usize) -> Option { + match self { + BuiltinFunction::Aggregate(af) => get_aggregate_arg_idx_for_optimization(*af, args_len), + BuiltinFunction::Rollup(rf) => get_rollup_arg_idx_for_optimization(*rf, args_len), + BuiltinFunction::Transform(tf) => get_transform_arg_idx_for_optimization(*tf, args_len), + } + } + + pub fn return_type(&self, args: &[Expr]) -> ParseResult { + if self.is_scalar() { + return Ok(ValueType::Scalar); + } + + // determine the arg to pass through + let arg = self.get_arg_for_optimization(args); + + let kind = if let Some(exp) = arg { + exp.return_type() + } else { + // todo: does this depend on the function type (rollup, transform, aggregation) + ValueType::InstantVector + }; + + match self { + BuiltinFunction::Rollup(rf) => { + if is_rollup_aggregation_over_time(*rf) { + match kind { + ValueType::RangeVector => Ok(ValueType::InstantVector), + ValueType::Scalar | ValueType::InstantVector => { + Ok(ValueType::InstantVector) + } + _ => { + // invalid arg + Err(ParseError::General(format!( + "aggregation over time is not valid with Expr returning {:?}", + kind + ))) + } + } + } else { + Ok(kind) + } + } + _ => Ok(kind), + } + } +} + +impl Display for BuiltinFunction { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + use BuiltinFunction::*; + match self { + Aggregate(af) => write!(f, "{}", af), + Rollup(rf) => write!(f, "{}", rf), + Transform(tf) => write!(f, "{}", tf), + } + } +} + +impl FromStr for BuiltinFunction { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + Self::new(s) + } +} + +impl TryFrom<&str> for BuiltinFunction { + type Error = ParseError; + + fn try_from(value: &str) -> Result { + Self::new(value) + } +} + +#[cfg(test)] +mod tests { + #[test] + fn test_is_supported() { + assert!(super::BuiltinFunction::is_supported("geomean")); + assert!(super::BuiltinFunction::is_supported("Predict_Linear")); + assert!(super::BuiltinFunction::is_supported("minute")); + assert!(!super::BuiltinFunction::is_supported("foo")); + } +} diff --git a/crates/metricsql-parser-vendored/src/functions/rollup.rs b/crates/metricsql-parser-vendored/src/functions/rollup.rs new file mode 100644 index 00000000..3b886a35 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/functions/rollup.rs @@ -0,0 +1,496 @@ +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use strum_macros::EnumIter; + +use crate::common::ValueType; +use crate::functions::signature::{Signature, Volatility}; +use crate::functions::{BuiltinFunction, FunctionMeta, MAX_ARG_COUNT}; +use crate::parser::ParseError; + +/// Built-in Rollup Functions +#[derive( + Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Hash, Default, EnumIter, Serialize, Deserialize, +)] +pub enum RollupFunction { + AbsentOverTime, + AggrOverTime, + AscentOverTime, + AvgOverTime, + Changes, + ChangesPrometheus, + CountEqOverTime, + CountGtOverTime, + CountLeOverTime, + CountNeOverTime, + CountOverTime, + CountValuesOverTime, + DecreasesOverTime, + #[default] + DefaultRollup, + Delta, + DeltaPrometheus, + Deriv, + DerivFast, + DescentOverTime, + DistinctOverTime, + DurationOverTime, + FirstOverTime, + GeomeanOverTime, + HistogramOverTime, + HoeffdingBoundLower, + HoeffdingBoundUpper, + HoltWinters, + IDelta, + IDeriv, + Increase, + IncreasePrometheus, + IncreasePure, + IncreasesOverTime, + Integrate, + IQROverTime, + IRate, // + rollupFuncsRemoveCounterResets + Lag, + LastOverTime, + Lifetime, + MadOverTime, + MaxOverTime, + MedianOverTime, + MinOverTime, + ModeOverTime, + OutlierIQROverTime, + PredictLinear, + PresentOverTime, + QuantileOverTime, + QuantilesOverTime, + RangeOverTime, + Rate, + RateOverSum, + Resets, + Rollup, + RollupCandlestick, + RollupDelta, + RollupDeriv, + RollupIncrease, + RollupRate, + RollupScrapeInterval, + ScrapeInterval, + ShareEqOverTime, + ShareGtOverTime, + ShareLeOverTime, + StaleSamplesOverTime, + StddevOverTime, + StdvarOverTime, + SumEqOverTime, + SumGtOverTime, + SumLeOverTime, + SumOverTime, + Sum2OverTime, + TFirstOverTime, + Timestamp, + TimestampWithName, + TLastChangeOverTime, + TLastOverTime, + TMaxOverTime, + TMinOverTime, + ZScoreOverTime, +} + +impl Display for RollupFunction { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } +} + +impl RollupFunction { + pub const fn name(&self) -> &'static str { + use RollupFunction::*; + + match self { + AbsentOverTime => "absent_over_time", + AggrOverTime => "aggr_over_time", + AscentOverTime => "ascent_over_time", + AvgOverTime => "avg_over_time", + Changes => "changes", + ChangesPrometheus => "changes_prometheus", + CountEqOverTime => "count_eq_over_time", + CountGtOverTime => "count_gt_over_time", + CountLeOverTime => "count_le_over_time", + CountNeOverTime => "count_ne_over_time", + CountOverTime => "count_over_time", + CountValuesOverTime => "count_values_over_time", + DecreasesOverTime => "decreases_over_time", + DefaultRollup => "default_rollup", + Delta => "delta", + DeltaPrometheus => "delta_prometheus", + Deriv => "deriv", + DerivFast => "deriv_fast", + DescentOverTime => "descent_over_time", + DistinctOverTime => "distinct_over_time", + DurationOverTime => "duration_over_time", + FirstOverTime => "first_over_time", + GeomeanOverTime => "geomean_over_time", + HistogramOverTime => "histogram_over_time", + HoeffdingBoundLower => "hoeffding_bound_lower", + HoeffdingBoundUpper => "hoeffding_bound_upper", + HoltWinters => "holt_winters", + IDelta => "idelta", + IDeriv => "ideriv", + Increase => "increase", + IncreasePrometheus => "increase_prometheus", + IncreasePure => "increase_pure", + IncreasesOverTime => "increases_over_time", + Integrate => "integrate", + IQROverTime => "iqr_over_time", + IRate => "irate", + Lag => "lag", + LastOverTime => "last_over_time", + Lifetime => "lifetime", + MadOverTime => "mad_over_time", + MaxOverTime => "max_over_time", + MedianOverTime => "median_over_time", + MinOverTime => "min_over_time", + ModeOverTime => "mode_over_time", + OutlierIQROverTime => "outlier_iqr_over_time", + PredictLinear => "predict_linear", + PresentOverTime => "present_over_time", + QuantileOverTime => "quantile_over_time", + QuantilesOverTime => "quantiles_over_time", + RangeOverTime => "range_over_time", + Rate => "rate", + RateOverSum => "rate_over_sum", + Resets => "resets", + Rollup => "rollup", + RollupCandlestick => "rollup_candlestick", + RollupDelta => "rollup_delta", + RollupDeriv => "rollup_deriv", + RollupIncrease => "rollup_increase", + RollupRate => "rollup_rate", + RollupScrapeInterval => "rollup_scrape_interval", + ScrapeInterval => "scrape_interval", + ShareEqOverTime => "share_eq_over_time", + ShareGtOverTime => "share_gt_over_time", + ShareLeOverTime => "share_le_over_time", + StaleSamplesOverTime => "stale_samples_over_time", + StddevOverTime => "stddev_over_time", + StdvarOverTime => "stdvar_over_time", + SumEqOverTime => "sum_eq_over_time", + SumGtOverTime => "sum_gt_over_time", + SumLeOverTime => "sum_le_over_time", + SumOverTime => "sum_over_time", + Sum2OverTime => "sum2_over_time", + TFirstOverTime => "tfirst_over_time", + Timestamp => "timestamp", + TimestampWithName => "timestamp_with_name", + TLastChangeOverTime => "tlast_change_over_time", + TLastOverTime => "tlast_over_time", + TMaxOverTime => "tmax_over_time", + TMinOverTime => "tmin_over_time", + ZScoreOverTime => "zscore_over_time", + } + } + + /// the signatures supported by the function `fun`. + pub fn signature(&self) -> Signature { + use RollupFunction::*; + use ValueType::*; + + // note: the physical expression must accept the type returned by this function or the execution panics. + match self { + CountEqOverTime | CountLeOverTime | CountNeOverTime | CountGtOverTime + | DurationOverTime | PredictLinear | ShareEqOverTime | ShareGtOverTime + | ShareLeOverTime | SumEqOverTime | SumGtOverTime | SumLeOverTime | TFirstOverTime + | TLastChangeOverTime | TLastOverTime => { + Signature::exact(vec![RangeVector, Scalar], Volatility::Immutable) + } + CountValuesOverTime => { + Signature::exact(vec![String, RangeVector], Volatility::Immutable) + } + HoeffdingBoundLower | HoeffdingBoundUpper => { + Signature::exact(vec![Scalar, RangeVector], Volatility::Immutable) + } + HoltWinters => { + Signature::exact(vec![RangeVector, Scalar, Scalar], Volatility::Immutable) + } + AggrOverTime => { + let mut quantile_types: Vec = vec![String; MAX_ARG_COUNT]; + quantile_types.insert(0, RangeVector); + Signature::variadic_min(quantile_types, 2, Volatility::Volatile) + } + QuantilesOverTime => { + let mut quantile_types: Vec = vec![RangeVector; MAX_ARG_COUNT]; + quantile_types.insert(0, RangeVector); + Signature::variadic_min(quantile_types, 3, Volatility::Volatile) + } + Rollup | RollupDelta | RollupDeriv | RollupIncrease | RollupRate + | RollupScrapeInterval | RollupCandlestick => { + Signature::variadic_min(vec![RangeVector, String], 1, Volatility::Volatile) + } + _ => { + // default + Signature::uniform(1, RangeVector, Volatility::Immutable) + } + } + } + + /// These functions don't change physical meaning of input time series, + /// so they don't drop metric name + pub const fn keep_metric_name(&self) -> bool { + use RollupFunction::*; + matches!( + self, + AvgOverTime + | DefaultRollup + | FirstOverTime + | GeomeanOverTime + | HoeffdingBoundLower + | HoeffdingBoundUpper + | HoltWinters + | LastOverTime + | MaxOverTime + | MinOverTime + | ModeOverTime + | IQROverTime + | PredictLinear + | QuantileOverTime + | QuantilesOverTime + | Rollup + | RollupCandlestick + | TimestampWithName + ) + } + + pub const fn should_remove_counter_resets(&self) -> bool { + use RollupFunction::*; + matches!( + self, + Increase + | IncreasePrometheus + | IncreasePure + | IRate + | Rate + | RollupIncrease + | RollupRate + ) + } + + pub const fn is_aggregate_function(&self) -> bool { + use RollupFunction::*; + matches!( + self, + AbsentOverTime + | AscentOverTime + | AvgOverTime + | Changes + | CountOverTime + | DecreasesOverTime + | DefaultRollup + | Delta + | Deriv + | DerivFast + | DescentOverTime + | DistinctOverTime + | FirstOverTime + | GeomeanOverTime + | IDelta + | IDeriv + | Increase + | IncreasePure + | IncreasesOverTime + | Integrate + | IRate + | Lag + | LastOverTime + | Lifetime + | MaxOverTime + | MinOverTime + | MedianOverTime + | ModeOverTime + | PresentOverTime + | RangeOverTime + | Rate + | RateOverSum + | Resets + | ScrapeInterval + | StaleSamplesOverTime + | StddevOverTime + | StdvarOverTime + | SumOverTime + | Sum2OverTime + | TFirstOverTime + | Timestamp + | TimestampWithName + | TLastChangeOverTime + | TLastOverTime + | TMaxOverTime + | TMinOverTime + | ZScoreOverTime + ) + } + + // All rollup the functions which do not rely on the previous sample + // before the lookbehind window (aka prev_value), do not need silence interval. + pub const fn need_silence_interval(&self) -> bool { + use RollupFunction::*; + !matches!( + self, + AscentOverTime + | Changes + | DecreasesOverTime + // The default_rollup implicitly relies on the previous samples in order to fill gaps. + // See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/5388 + | DefaultRollup + | Delta + | DerivFast + | DescentOverTime + | IDelta + | IDeriv + | Increase + | IncreasePure + | IncreasesOverTime + | Integrate + | IRate + | Lag + | Lifetime + | Rate + | Resets + | Rollup + | RollupCandlestick + | RollupDelta + | RollupDeriv + | RollupIncrease + | RollupRate + | RollupScrapeInterval + | ScrapeInterval + | TLastChangeOverTime + ) + } +} + +/// We can extend lookbehind window for these functions in order to make sure it contains enough +/// points for returning non-empty results. +/// +/// This is needed for returning the expected non-empty graphs when zooming in the graph in Grafana, +/// which is built with `func_name(metric)` query. +pub const fn can_adjust_window(func: RollupFunction) -> bool { + use RollupFunction::*; + matches!( + func, + DefaultRollup + | Deriv + | DerivFast + | IDeriv + | IRate + | Rate + | RateOverSum + | Rollup + | RollupCandlestick + | RollupDeriv + | RollupRate + | RollupScrapeInterval + | ScrapeInterval + | Timestamp + ) +} + +impl FromStr for RollupFunction { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + if let Some(meta) = FunctionMeta::lookup(s) { + if let BuiltinFunction::Rollup(rf) = &meta.function { + return Ok(*rf); + } + } + Err(ParseError::InvalidFunction(s.to_string())) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Hash, EnumIter, Serialize, Deserialize)] +pub enum RollupTag { + Min, + Max, + Avg, +} + +impl Display for RollupTag { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + RollupTag::Min => write!(f, "min"), + RollupTag::Max => write!(f, "max"), + RollupTag::Avg => write!(f, "avg"), + } + } +} + +impl FromStr for RollupTag { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + match s { + s if s.eq_ignore_ascii_case("min") => Ok(RollupTag::Min), + s if s.eq_ignore_ascii_case("max") => Ok(RollupTag::Max), + s if s.eq_ignore_ascii_case("avg") => Ok(RollupTag::Avg), + _ => Err(ParseError::InvalidFunction(format!( + "invalid rollup tag::{s}", + ))), + } + } +} + +/// get_rollup_arg_idx returns the argument index for the given fe, which accepts the rollup argument. +/// +/// -1 is returned if fe isn't a rollup function. +pub const fn get_rollup_arg_idx(fe: &RollupFunction, arg_count: usize) -> i32 { + use RollupFunction::*; + match fe { + QuantileOverTime | HoeffdingBoundLower | HoeffdingBoundUpper => 1, + QuantilesOverTime => (arg_count - 1) as i32, + _ => 0, + } +} + +pub const fn get_rollup_arg_idx_for_optimization( + func: RollupFunction, + arg_count: usize, +) -> Option { + // This must be kept in sync with GetRollupArgIdx() + use RollupFunction::*; + match func { + AbsentOverTime => None, + QuantileOverTime | HoeffdingBoundLower | HoeffdingBoundUpper => Some(1), + QuantilesOverTime => Some(arg_count - 1), + _ => Some(0), + } +} + +/// Determines if a given rollup function converts a range vector to an instant vector +/// +/// Note that `_over_time` functions do not affect labels, unlike their regular +/// counterparts +pub fn is_rollup_aggregation_over_time(func: RollupFunction) -> bool { + use RollupFunction::*; + let name = func.name(); + + if name.ends_with("over_time") { + return true; + } + + matches!(func, |Delta| DeltaPrometheus + | Deriv + | DerivFast + | IDelta + | IDeriv + | Increase + | IncreasePure + | IncreasePrometheus + | IRate + | PredictLinear + | Rate + | Resets + | RollupDeriv + | RollupDelta + | RollupIncrease + | RollupRate) +} diff --git a/crates/metricsql-parser-vendored/src/functions/signature.rs b/crates/metricsql-parser-vendored/src/functions/signature.rs new file mode 100644 index 00000000..5947b4b6 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/functions/signature.rs @@ -0,0 +1,333 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Signature module contains foundational types that are used to represent signatures, types, +//! and return types of functions. + +use serde::{Deserialize, Serialize}; +use std::fmt::Display; + +use crate::common::ValueType; +use crate::functions::MAX_ARG_COUNT; +use crate::parser::{ParseError, ParseResult}; + +///A function's volatility, which defines the functions eligibility for certain optimizations +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)] +pub enum Volatility { + /// Immutable - An immutable function will always return the same output when given the same + /// input. An example of this is [super::BuiltinScalarFunction::Cos]. + Immutable, + /// Stable - A stable function may return different values given the same input across different + /// queries but must return the same value for a given input within a query. An example of + /// this is [super::BuiltinScalarFunction::Now]. + Stable, + /// Volatile - A volatile function may change the return value from evaluation to evaluation. + /// Multiple invocations of a volatile function may return different results when used in the + /// same query. An example of this is [super::BuiltinScalarFunction::Random]. + Volatile, +} + +/// A function's type signature, which defines the function's supported argument types. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum TypeSignature { + /// arbitrary number of arguments of any common type out of a list of valid types + /// second element is the min number of arguments + /// A function such as `concat` is `Variadic(vec![ValueType::String, ValueType::Float], 1)` + Variadic(Vec, usize), + /// arbitrary number of arguments of an arbitrary but equal type, with possible minimum + /// A function such as `array` is `VariadicEqual` + VariadicEqual(ValueType, usize), + /// arbitrary number of arguments of any type, with possible minimum + VariadicAny(usize), + /// fixed number of arguments of an arbitrary type out of a list of valid types + /// A function of one argument of f64 is `Uniform(1, ValueType::Scalar)` + Uniform(ValueType, usize), + /// arguments of an exact type with an optional minimum + Exact(Vec, Option), + /// fixed number of arguments of arbitrary types + Any(usize), +} + +impl TypeSignature { + /// Validate argument counts matches the `signature`. + pub fn validate_arg_count(&self, name: &str, arg_len: usize) -> ParseResult<()> { + fn expect_arg_count(name: &str, arg_len: usize, expected: usize) -> ParseResult<()> { + if arg_len != expected { + return Err(ParseError::ArgumentError(format!( + "The function {name}() expected {expected} arguments but received {arg_len}", + ))); + } + Ok(()) + } + + fn expect_min_args(name: &str, args_len: usize, min: usize) -> ParseResult<()> { + if args_len < min { + return Err(ParseError::ArgumentError(format!( + "The function {name}() expected a minimum of {min} arguments but received {args_len}", + ))); + } + Ok(()) + } + + match self { + TypeSignature::Exact(valid_types, min) => { + let max = valid_types.len(); + if let Some(min) = min { + if !(*min..=max).contains(&arg_len) { + return Err(ParseError::ArgumentError(format!( + "The function {name}() expected between {min} and {max} arguments but received {arg_len}", + ))); + } + return Ok(()); + } + expect_arg_count(name, arg_len, max) + } + TypeSignature::Any(min) + | TypeSignature::Uniform(_, min) + | TypeSignature::VariadicEqual(_, min) + | TypeSignature::Variadic(_, min) + | TypeSignature::VariadicAny(min) => expect_min_args(name, arg_len, *min), + } + } + + pub fn is_variadic(&self) -> bool { + matches!( + self, + TypeSignature::Variadic(_, _) + | TypeSignature::VariadicEqual(_, _) + | TypeSignature::VariadicAny(_) + ) + } +} + +/// The Signature of a function defines its supported input types as well as its volatility. +#[derive(Debug, Clone, PartialEq, Hash)] +pub struct Signature { + /// type_signature - The types that the function accepts. See [TypeSignature] for more information. + pub type_signature: TypeSignature, + /// volatility - The volatility of the function. See [Volatility] for more information. + pub volatility: Volatility, + + pub return_type: ValueType, +} + +impl Signature { + /// new - Creates a new Signature from any type signature and the volatility. + pub fn new(type_signature: TypeSignature, volatility: Volatility) -> Self { + Signature { + type_signature, + volatility, + return_type: ValueType::InstantVector, + } + } + + /// variadic - Creates a variadic signature that represents an arbitrary number of arguments all from a type in common_types. + pub fn variadic(common_types: Vec, volatility: Volatility) -> Self { + Self { + type_signature: TypeSignature::Variadic(common_types, 0), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// variadic - Creates a variadic signature that represents an arbitrary number of arguments all from a type in common_types. + pub fn variadic_min(common_types: Vec, min: usize, volatility: Volatility) -> Self { + Self { + type_signature: TypeSignature::Variadic(common_types, min), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// variadic_equal - Creates a variadic signature that represents an arbitrary number of arguments of the same type. + pub fn variadic_equal(valid_type: ValueType, min: usize, volatility: Volatility) -> Self { + Self { + type_signature: TypeSignature::VariadicEqual(valid_type, min), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// uniform - Creates a signature with a fixed number of arguments of the same type, which must be from valid_types. + pub fn uniform(arg_count: usize, valid_type: ValueType, volatility: Volatility) -> Self { + Self { + type_signature: TypeSignature::Uniform(valid_type, arg_count), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// exact - Creates a signature which must match the types in exact_types in order. + pub fn exact(exact_types: Vec, volatility: Volatility) -> Self { + Signature { + type_signature: TypeSignature::Exact(exact_types, None), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// exact - Creates a signature which must match the types in exact_types in order, but with + /// a minimum number of args. + pub fn exact_with_min_args( + exact_types: Vec, + min: usize, + volatility: Volatility, + ) -> Self { + // todo: panic if out of range + let min_arg = min.clamp(0, exact_types.len()); + Signature { + type_signature: TypeSignature::Exact(exact_types, Some(min_arg)), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// any - Creates a signature which can be made of any type but of a fixed number + pub fn any(arg_count: usize, volatility: Volatility) -> Self { + Signature { + type_signature: TypeSignature::Any(arg_count), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// variadic_any - creates a variadic signature that represents an arbitrary number of arguments of any type. + pub fn variadic_any(min_arg_count: usize, volatility: Volatility) -> Self { + Signature { + type_signature: TypeSignature::VariadicAny(min_arg_count), + volatility, + return_type: ValueType::InstantVector, + } + } + + /// Validate argument counts matches the `signature`. + pub fn validate_arg_count(&self, name: &str, arg_len: usize) -> Result<(), ParseError> { + self.type_signature.validate_arg_count(name, arg_len) + } + + pub fn expand_types(&self) -> (Vec, usize) { + // todo: also return min count + match &self.type_signature { + TypeSignature::Variadic(types, min) => (types.clone(), *min), + TypeSignature::VariadicEqual(data_type, min) => (vec![*data_type; MAX_ARG_COUNT], *min), + TypeSignature::Uniform(data_type, count) => (vec![*data_type; MAX_ARG_COUNT], *count), + TypeSignature::Exact(types, min) => (types.clone(), min.unwrap_or(types.len())), + TypeSignature::VariadicAny(min) => { + (vec![ValueType::InstantVector; MAX_ARG_COUNT], *min) + } + TypeSignature::Any(count) => { + (vec![ValueType::InstantVector; *count], *count) // TODO:: !!!! have a ValueType::Any + } + } + } + + pub fn types(&self) -> TypeIterator<'_> { + TypeIterator::new(self) + } + + pub fn min_args(&self) -> usize { + match &self.type_signature { + TypeSignature::Exact(_, min) => min.unwrap_or(0), + TypeSignature::Any(count) => *count, + TypeSignature::Uniform(_, count) => *count, + TypeSignature::Variadic(_, min) => *min, + TypeSignature::VariadicEqual(_, min) => *min, + TypeSignature::VariadicAny(min) => *min, + } + } + + pub fn is_variadic(&self) -> bool { + self.type_signature.is_variadic() + } + + pub fn args_iter(&self) -> TypeIterator<'_> { + TypeIterator::new(self) + } +} + +impl Display for Signature { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let (types, min) = self.expand_types(); + write!(f, "Signature({:?}, min: {})", types, min) + } +} + +pub struct TypeIterator<'a> { + signature: &'a Signature, + arg_index: usize, +} + +impl<'a> Iterator for TypeIterator<'a> { + type Item = ValueType; + + fn next(&mut self) -> Option { + use TypeSignature::*; + + match &self.signature.type_signature { + Variadic(types, _min) => { + if self.arg_index < types.len() { + self.arg_index += 1; + Some(types[self.arg_index - 1]) + } else { + Some(types[types.len() - 1]) + } + } + VariadicEqual(data_type, _) => { + if self.arg_index < MAX_ARG_COUNT { + self.arg_index += 1; + Some(*data_type) + } else { + None + } + } + Uniform(data_type, count) => { + if self.arg_index < *count { + self.arg_index += 1; + Some(*data_type) + } else { + None + } + } + Exact(types, _) => { + if self.arg_index < types.len() { + self.arg_index += 1; + Some(types[self.arg_index - 1]) + } else { + None + } + } + VariadicAny(count) | Any(count) => { + if self.arg_index < *count { + self.arg_index += 1; + // ?? TODO: !!!! have a ValueType::Any + Some(ValueType::InstantVector) + } else { + None + } + } + } + } +} + +impl<'a> TypeIterator<'a> { + pub fn new(signature: &'a Signature) -> Self { + Self { + signature, + arg_index: 0, + } + } +} diff --git a/crates/metricsql-parser-vendored/src/functions/transform.rs b/crates/metricsql-parser-vendored/src/functions/transform.rs new file mode 100644 index 00000000..bf9a789f --- /dev/null +++ b/crates/metricsql-parser-vendored/src/functions/transform.rs @@ -0,0 +1,538 @@ +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +use serde::{Deserialize, Serialize}; +use strum_macros::EnumIter; + +use crate::common::ValueType; +use crate::functions::signature::{Signature, Volatility}; +use crate::functions::{BuiltinFunction, FunctionMeta, MAX_ARG_COUNT}; +use crate::parser::ParseError; + +// TODO: ttf + +/// Transform functions calculate transformations over rollup results. +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Hash, EnumIter, Serialize, Deserialize)] +pub enum TransformFunction { + Abs, + Absent, + Acos, + Acosh, + Alias, + Asin, + Asinh, + Atan, + Atanh, + BitmapAnd, + BitmapOr, + BitmapXor, + BucketsLimit, + Ceil, + Clamp, + ClampMax, + ClampMin, + Cos, + Cosh, + DayOfMonth, + DayOfWeek, + DayOfYear, + DaysInMonth, + Deg, + DropCommonLabels, + DropEmptySeries, + End, + Exp, + Floor, + HistogramAvg, + HistogramQuantile, + HistogramQuantiles, + HistogramShare, + HistogramStddev, + HistogramStdvar, + Hour, + Interpolate, + KeepLastValue, + KeepNextValue, + LabelCopy, + LabelDel, + LabelGraphiteGroup, + LabelJoin, + LabelKeep, + LabelLowercase, + LabelMap, + LabelMatch, + LabelMismatch, + LabelMove, + LabelReplace, + LabelSet, + LabelTransform, + LabelUppercase, + LabelValue, + LabelsEqual, + LimitOffset, + Ln, + Log2, + Log10, + Minute, + Month, + Now, + Pi, + PrometheusBuckets, + Rad, + Random, + RandExponential, + RandNormal, + RangeAvg, + RangeFirst, + RangeLast, + RangeLinearRegression, + RangeMax, + RangeMedian, + RangeMin, + RangeNormalize, + RangeQuantile, + RangeStdDev, + RangeStdVar, + RangeSum, + RangeTrimOutliers, + RangeTrimSpikes, + RangeTrimZScore, + RangeZScore, + RemoveResets, + Round, + Ru, + RunningAvg, + RunningMax, + RunningMin, + RunningSum, + Scalar, + Sgn, + Sin, + Sinh, + SmoothExponential, + Sort, + SortByLabel, + SortByLabelDesc, + SortByLabelNumeric, + SortByLabelNumericDesc, + SortDesc, + Sqrt, + Start, + Step, + Tan, + Tanh, + Time, + // "timestamp" has been moved to rollup funcs. See https://github.com/VictoriaMetrics/VictoriaMetrics/issues/415 + TimezoneOffset, + Union, + Vector, + Year, +} + +impl Display for TransformFunction { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.name()) + } +} + +impl FromStr for TransformFunction { + type Err = ParseError; + + fn from_str(s: &str) -> Result { + if let Some(meta) = FunctionMeta::lookup(s) { + if let BuiltinFunction::Transform(tf) = &meta.function { + return Ok(*tf); + } + } + Err(ParseError::InvalidFunction(s.to_string())) + } +} + +impl TransformFunction { + pub fn name(&self) -> &'static str { + use TransformFunction::*; + match self { + Abs => "abs", + Absent => "absent", + Acos => "acos", + Acosh => "acosh", + Alias => "alias", + Asin => "asin", + Asinh => "asinh", + Atan => "atan", + Atanh => "atanh", + BitmapAnd => "bitmap_and", + BitmapOr => "bitmap_or", + BitmapXor => "bitmap_xor", + BucketsLimit => "buckets_limit", + Ceil => "ceil", + Clamp => "clamp", + ClampMax => "clamp_max", + ClampMin => "clamp_min", + Cos => "cos", + Cosh => "cosh", + DayOfMonth => "day_of_month", + DayOfWeek => "day_of_week", + DayOfYear => "day_of_year", + DaysInMonth => "days_in_month", + Deg => "deg", + DropCommonLabels => "drop_common_labels", + DropEmptySeries => "drop_empty_series", + End => "end", + Exp => "exp", + Floor => "floor", + HistogramAvg => "histogram_avg", + HistogramQuantile => "histogram_quantile", + HistogramQuantiles => "histogram_quantiles", + HistogramShare => "histogram_share", + HistogramStddev => "histogram_stddev", + HistogramStdvar => "histogram_stdvar", + Hour => "hour", + Interpolate => "interpolate", + KeepLastValue => "keep_last_value", + KeepNextValue => "keep_next_value", + LabelCopy => "label_copy", + LabelDel => "label_del", + LabelGraphiteGroup => "label_graphite_group", + LabelJoin => "label_join", + LabelKeep => "label_keep", + LabelLowercase => "label_lowercase", + LabelMap => "label_map", + LabelMatch => "label_match", + LabelMismatch => "label_mismatch", + LabelMove => "label_move", + LabelReplace => "label_replace", + LabelSet => "label_set", + LabelTransform => "label_transform", + LabelUppercase => "label_uppercase", + LabelValue => "label_value", + LabelsEqual => "labels_equal", + LimitOffset => "limit_offset", + Ln => "ln", + Log2 => "log2", + Log10 => "log10", + Minute => "minute", + Month => "month", + Now => "now", + Pi => "pi", + PrometheusBuckets => "prometheus_buckets", + Rad => "rad", + Random => "rand", + RandExponential => "rand_exponential", + RandNormal => "rand_normal", + RangeAvg => "range_avg", + RangeFirst => "range_first", + RangeLast => "range_last", + RangeLinearRegression => "range_linear_regression", + RangeMax => "range_max", + RangeMedian => "range_median", + RangeMin => "range_min", + RangeNormalize => "range_normalize", + RangeQuantile => "range_quantile", + RangeStdDev => "range_stddev", + RangeStdVar => "range_stdvar", + RangeSum => "range_sum", + RangeTrimSpikes => "range_trim_spikes", + RangeTrimOutliers => "range_trim_outliers", + RangeTrimZScore => "range_trim_zscore", + RangeZScore => "range_zscore", + RemoveResets => "remove_resets", + Round => "round", + Ru => "ru", + RunningAvg => "running_avg", + RunningMax => "running_max", + RunningMin => "running_min", + RunningSum => "running_sum", + Scalar => "scalar", + Sgn => "sgn", + Sin => "sin", + Sinh => "sinh", + SmoothExponential => "smooth_exponential", + Sort => "sort", + SortByLabel => "sort_by_label", + SortByLabelDesc => "sort_by_label_desc", + SortByLabelNumeric => "sort_by_label_numeric", + SortByLabelNumericDesc => "sort_by_label_numeric_desc", + SortDesc => "sort_desc", + Sqrt => "sqrt", + Start => "start", + Step => "step", + Tan => "tan", + Tanh => "tanh", + Time => "time", + TimezoneOffset => "timezone_offset", + Union => "union", + Vector => "vector", + Year => "year", + } + } + + /// These functions don't change physical meaning of input time series, + /// so they don't drop metric name + pub const fn keep_metric_name(&self) -> bool { + use TransformFunction::*; + matches!( + self, + Ceil | Clamp + | ClampMax + | ClampMin + | Floor + | Interpolate + | KeepLastValue + | KeepNextValue + | RangeAvg + | RangeFirst + | RangeLast + | RangeLinearRegression + | RangeMax + | RangeMedian + | RangeMin + | RangeNormalize + | RangeQuantile + | RangeStdDev + | RangeStdVar + | Round + | Ru + | RunningAvg + | RunningMax + | RunningMin + | SmoothExponential + ) + } + + pub const fn may_sort_results(&self) -> bool { + use TransformFunction::*; + matches!( + &self, + Sort | SortDesc + | SortByLabel + | SortByLabelDesc + | SortByLabelNumeric + | SortByLabelNumericDesc + ) + } + + pub const fn manipulates_labels(&self) -> bool { + use TransformFunction::*; + matches!( + self, + Alias + | DropCommonLabels + | DropEmptySeries + | LabelCopy + | LabelDel + | LabelGraphiteGroup + | LabelJoin + | LabelKeep + | LabelLowercase + | LabelMap + | LabelMove + | LabelReplace + | LabelSet + | LabelTransform + | LabelUppercase + ) + } + + pub fn signature(&self) -> Signature { + use TransformFunction::*; + + // note: the expression must accept the type returned by this function or the execution panics. + match self { + Alias => Signature::exact( + vec![ValueType::InstantVector, ValueType::String], + Volatility::Stable, + ), + BitmapAnd | BitmapOr | BitmapXor => Signature::exact( + vec![ValueType::InstantVector, ValueType::Scalar], + Volatility::Immutable, + ), + BucketsLimit => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Immutable, + ), + Clamp => Signature::exact( + vec![ + ValueType::InstantVector, + ValueType::Scalar, + ValueType::Scalar, + ], + Volatility::Volatile, + ), + ClampMax | ClampMin => Signature::exact( + vec![ValueType::InstantVector, ValueType::Scalar], + Volatility::Immutable, + ), + Start | End => Signature::exact(vec![], Volatility::Stable), + DropCommonLabels => { + Signature::variadic_equal(ValueType::InstantVector, 1, Volatility::Stable) + } + HistogramQuantile => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Stable, + ), + HistogramQuantiles => { + // histogram_quantiles("phiLabel", phi1, ..., phiN, buckets) + // todo: need a better way to handle variadic args with specific types + Signature::variadic_any(3, Volatility::Stable) + } + // histogram_share(le, buckets) + HistogramShare => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Stable, + ), + LabelCopy | LabelMove | LabelSet => { + let mut types = vec![ValueType::String; MAX_ARG_COUNT]; + types.insert(0, ValueType::InstantVector); + Signature::exact_with_min_args(types, 3, Volatility::Stable) + } + LabelDel | LabelKeep | LabelLowercase | LabelUppercase => { + let mut types = vec![ValueType::String; MAX_ARG_COUNT]; + types.insert(0, ValueType::InstantVector); + Signature::exact_with_min_args(types, 2, Volatility::Stable) + } + LabelJoin => { + let mut types = vec![ValueType::String; MAX_ARG_COUNT]; + types.insert(0, ValueType::InstantVector); + Signature::exact_with_min_args(types, 4, Volatility::Stable) + } + LabelMap => { + let mut types = vec![ValueType::String; MAX_ARG_COUNT]; + types.insert(0, ValueType::InstantVector); + Signature::exact_with_min_args(types, 4, Volatility::Stable) + } + LabelMatch | LabelMismatch => { + let types = vec![ + ValueType::InstantVector, + ValueType::String, + ValueType::String, + ]; + Signature::exact_with_min_args(types, 3, Volatility::Stable) + } + LabelGraphiteGroup => { + // label_graphite_group(q, groupNum1, ... groupNumN) + let mut types = vec![ValueType::Scalar; MAX_ARG_COUNT]; + types.insert(0, ValueType::InstantVector); + Signature::exact(types, Volatility::Stable) + } + LabelReplace => { + // label_replace(q, "dst_label", "replacement", "src_label", "regex") + Signature::exact( + vec![ + ValueType::InstantVector, + ValueType::String, + ValueType::String, + ValueType::String, + ValueType::String, + ], + Volatility::Stable, + ) + } + LabelTransform => { + // label_transform(q, "label", "regexp", "replacement") + Signature::exact( + vec![ + ValueType::InstantVector, + ValueType::String, + ValueType::String, + ValueType::String, + ], + Volatility::Stable, + ) + } + LabelValue => Signature::exact( + vec![ValueType::InstantVector, ValueType::String], + Volatility::Stable, + ), + LimitOffset => Signature::exact( + vec![ + ValueType::Scalar, + ValueType::Scalar, + ValueType::InstantVector, + ], + Volatility::Stable, + ), + Now => Signature::exact(vec![], Volatility::Stable), + Pi => Signature::exact(vec![], Volatility::Immutable), + Random | RandExponential | RandNormal => { + Signature::exact_with_min_args(vec![ValueType::Scalar], 0, Volatility::Volatile) + } + RangeNormalize => { + Signature::variadic_min(vec![ValueType::InstantVector], 1, Volatility::Stable) + } + RangeTrimOutliers | RangeTrimSpikes | RangeTrimZScore => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Stable, + ), + RangeQuantile => Signature::exact( + vec![ValueType::Scalar, ValueType::InstantVector], + Volatility::Stable, + ), + Round => Signature::exact_with_min_args( + vec![ValueType::InstantVector, ValueType::Scalar], + 1, + Volatility::Stable, + ), + Ru => Signature::exact( + vec![ValueType::RangeVector, ValueType::RangeVector], + Volatility::Stable, + ), + Scalar => Signature::any(1, Volatility::Stable), + SmoothExponential => Signature::exact( + vec![ValueType::InstantVector, ValueType::Scalar], + Volatility::Stable, + ), + Sort => Signature::exact(vec![ValueType::RangeVector], Volatility::Stable), + SortByLabel | SortByLabelDesc | SortByLabelNumeric | SortByLabelNumericDesc => { + let mut types = vec![ValueType::String; MAX_ARG_COUNT]; + types.insert(0, ValueType::RangeVector); + Signature::exact_with_min_args(types, 2, Volatility::Stable) + } + Step => Signature::exact(vec![], Volatility::Stable), + Time => Signature::exact(vec![], Volatility::Stable), + TimezoneOffset => Signature::exact(vec![ValueType::String], Volatility::Stable), + Union => { + let types = vec![ValueType::InstantVector; MAX_ARG_COUNT]; + Signature::exact_with_min_args(types, 1, Volatility::Stable) + } + Vector => Signature::exact(vec![ValueType::InstantVector], Volatility::Stable), + // DateTime functions + DayOfMonth | DayOfWeek | DayOfYear | DaysInMonth | Hour | Minute | Month | Year => { + Signature::exact_with_min_args( + vec![ValueType::InstantVector], + 0, + Volatility::Immutable, + ) + } + _ => { + // by default we take a single arg containing series + Signature::exact(vec![ValueType::InstantVector], Volatility::Immutable) + } + } + } + + pub fn return_type(&self) -> ValueType { + ValueType::InstantVector + } +} + +pub const fn get_transform_arg_idx_for_optimization( + func: TransformFunction, + arg_count: usize, +) -> Option { + if func.manipulates_labels() { + return None; + } + + use TransformFunction::*; + match func { + Absent | Scalar | Union | Vector => None, + End | Now | Pi | Start | Step | Time => None, // todo Ru + LimitOffset => Some(2), + BucketsLimit | HistogramQuantile | HistogramShare | RangeQuantile | RangeTrimSpikes => { + Some(1) + } + HistogramQuantiles => Some(arg_count - 1), + _ => Some(0), + } +} diff --git a/crates/metricsql-parser-vendored/src/label/label_filter.rs b/crates/metricsql-parser-vendored/src/label/label_filter.rs new file mode 100644 index 00000000..ef539968 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/label/label_filter.rs @@ -0,0 +1,821 @@ +use std::cmp::Ordering; +use std::fmt; +use std::hash::{Hash, Hasher}; + +use ahash::AHashMap; +use regex::Regex; +use serde::{Deserialize, Serialize}; +use xxhash_rust::xxh3::Xxh3; + +use crate::common::join_vector; +use crate::parser::{compile_regexp, escape_ident, is_empty_regex, quote, ParseError}; + +pub const NAME_LABEL: &str = "__name__"; +pub type LabelName = String; + +pub type LabelValue = String; + +// NOTE: https://github.com/rust-lang/regex/issues/668 +#[derive(Debug, Default, Clone)] +pub enum MatchOp { + #[default] + Equal, + NotEqual, + Re(Regex), + NotRe(Regex), +} + +impl MatchOp { + pub fn is_negative(&self) -> bool { + matches!(self, MatchOp::NotEqual | MatchOp::NotRe(_)) + } + + pub fn is_regex(&self) -> bool { + matches!(self, Self::NotRe(_) | Self::Re(_)) + } +} + +impl fmt::Display for MatchOp { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MatchOp::Equal => write!(f, "="), + MatchOp::NotEqual => write!(f, "!="), + MatchOp::Re(reg) => write!(f, "=~{reg}"), + MatchOp::NotRe(reg) => write!(f, "!~{reg}"), + } + } +} + +impl PartialEq for MatchOp { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (MatchOp::Equal, MatchOp::Equal) => true, + (MatchOp::NotEqual, MatchOp::NotEqual) => true, + (MatchOp::Re(s), MatchOp::Re(o)) => s.as_str().eq(o.as_str()), + (MatchOp::NotRe(s), MatchOp::NotRe(o)) => s.as_str().eq(o.as_str()), + _ => false, + } + } +} + +impl Eq for MatchOp {} + +impl Hash for MatchOp { + fn hash(&self, state: &mut H) { + match self { + MatchOp::Equal => "eq".hash(state), + MatchOp::NotEqual => "ne".hash(state), + MatchOp::Re(s) => format!("re:{}", s.as_str()).hash(state), + MatchOp::NotRe(s) => format!("nre:{}", s.as_str()).hash(state), + } + } +} + +#[derive( + Default, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Copy, Hash, Serialize, Deserialize, +)] +pub enum LabelFilterOp { + #[default] + Equal, + NotEqual, + RegexEqual, + RegexNotEqual, +} + +impl LabelFilterOp { + pub fn is_negative(&self) -> bool { + matches!(self, LabelFilterOp::NotEqual | LabelFilterOp::RegexNotEqual) + } + + pub fn is_regex(&self) -> bool { + matches!( + self, + LabelFilterOp::RegexEqual | LabelFilterOp::RegexNotEqual + ) + } + + pub fn as_str(&self) -> &'static str { + match self { + LabelFilterOp::Equal => "=", + LabelFilterOp::NotEqual => "!=", + LabelFilterOp::RegexEqual => "=~", + LabelFilterOp::RegexNotEqual => "!~", + } + } +} + +impl TryFrom<&str> for LabelFilterOp { + type Error = ParseError; + + fn try_from(op: &str) -> Result { + match op { + "=" => Ok(LabelFilterOp::Equal), + "!=" => Ok(LabelFilterOp::NotEqual), + "=~" => Ok(LabelFilterOp::RegexEqual), + "!~" => Ok(LabelFilterOp::RegexNotEqual), + _ => Err(ParseError::General(format!( + "Unexpected match op literal: {op}" + ))), + } + } +} + +impl fmt::Display for LabelFilterOp { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.as_str()) + } +} + +/// LabelFilter represents MetricsQL label filter like `foo="bar"`. +#[derive(Default, Debug, Clone, Serialize, Deserialize)] +pub struct LabelFilter { + pub op: LabelFilterOp, + + /// label contains label name for the filter. + pub label: String, + + /// value contains unquoted value for the filter. + pub value: String, +} + +impl LabelFilter { + pub fn new(match_op: LabelFilterOp, label: N, value: V) -> Result + where + N: Into, + V: Into, + { + let label = label.into(); + + assert!(!label.is_empty()); + let value = match match_op { + LabelFilterOp::RegexEqual | LabelFilterOp::RegexNotEqual => { + let label_value = value.into(); + let converted = try_escape_for_repeat_re(&label_value); + let re_anchored = format!("^(?:{})$", converted); + if compile_regexp(&re_anchored).is_err() { + return Err(ParseError::InvalidRegex(label_value)); + } + converted + } + _ => value.into(), + }; + + Ok(Self { + label, + op: match_op, + value, + }) + } + + pub fn equal>(key: S, value: S) -> Self { + LabelFilter { + op: LabelFilterOp::Equal, + label: key.into(), + value: value.into(), + } + } + + pub fn not_equal>(key: S, value: S) -> Self { + LabelFilter { + op: LabelFilterOp::NotEqual, + label: key.into(), + value: value.into(), + } + } + + pub fn regex_equal>(key: S, value: S) -> Result { + LabelFilter::new(LabelFilterOp::RegexEqual, key, value) + } + + pub fn regex_notequal>(key: S, value: S) -> Result { + LabelFilter::new(LabelFilterOp::RegexNotEqual, key, value) + } + + /// is_regexp represents whether the filter is regexp, i.e. `=~` or `!~`. + pub fn is_regexp(&self) -> bool { + self.op.is_regex() + } + + /// is_negative represents whether the filter is negative, i.e. '!=' or '!~'. + pub fn is_negative(&self) -> bool { + self.op.is_negative() + } + + pub fn is_metric_name_filter(&self) -> bool { + self.label == NAME_LABEL && self.op == LabelFilterOp::Equal + } + + pub fn is_name_label(&self) -> bool { + self.label == NAME_LABEL && self.op == LabelFilterOp::Equal + } + + /// Vector selectors must either specify a name or at least one label + /// matcher that does not match the empty string. + /// + /// The following expression is illegal: + /// {job=~".*"} # Bad! + pub fn is_empty_matcher(&self) -> bool { + use LabelFilterOp::*; + // if we're matching against __name__, a negative comparison against the empty + // string is valid + let is_name_label = self.label == NAME_LABEL; + match self.op { + Equal => self.value.is_empty(), + NotEqual => !self.value.is_empty() && !is_name_label, + RegexEqual => is_empty_regex(&self.value), + RegexNotEqual => is_empty_regex(&self.value) && !is_name_label, + } + } + + pub fn is_match(&self, str: &str) -> bool { + match self.op { + LabelFilterOp::Equal => self.value.eq(str), + LabelFilterOp::NotEqual => self.value.ne(str), + LabelFilterOp::RegexEqual => { + // slight optimization for frequent case + if str.is_empty() { + return is_empty_regex(&self.value); + } + if let Ok(re) = compile_regexp(&self.value) { + re.is_match(str) + } else { + false + } + } + LabelFilterOp::RegexNotEqual => { + if let Ok(re) = compile_regexp(&self.value) { + !re.is_match(str) + } else { + false + } + } + } + } + + pub fn as_string(&self) -> String { + format!( + "{}{}{}", + escape_ident(&self.label), + self.op, + quote(&self.value) + ) + } + + pub fn name(&self) -> String { + if self.label == NAME_LABEL { + return self.value.to_string(); + } + self.label.clone() + } +} + +impl PartialEq for LabelFilter { + fn eq(&self, other: &Self) -> bool { + self.op.eq(&other.op) && self.label.eq(&other.label) && self.value.eq(&other.value) + } +} + +impl PartialOrd for LabelFilter { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Eq for LabelFilter {} + +impl Ord for LabelFilter { + fn cmp(&self, other: &Self) -> Ordering { + let mut cmp = self.label.cmp(&other.label); + if cmp == Ordering::Equal { + cmp = self.value.cmp(&other.value); + if cmp == Ordering::Equal { + cmp = self.op.cmp(&other.op); + } + } + cmp + } +} + +impl fmt::Display for LabelFilter { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "{}{}{}", + escape_ident(&self.label), + self.op, + quote(&self.value) + )?; + Ok(()) + } +} + +impl Hash for LabelFilter { + fn hash(&self, state: &mut H) { + self.op.hash(state); + self.label.hash(state); + self.value.hash(state); + } +} + +pub type Matcher = LabelFilter; + +#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct Matchers { + pub matchers: Vec, + pub or_matchers: Vec>, +} + +impl Matchers { + pub fn new(filters: Vec) -> Self { + Matchers { + matchers: filters, + or_matchers: vec![], + } + } + + pub fn empty() -> Self { + Self { + matchers: vec![], + or_matchers: vec![], + } + } + + pub fn is_empty(&self) -> bool { + self.matchers.is_empty() && self.or_matchers.is_empty() + } + + pub fn with_or_matchers(or_matchers: Vec>) -> Self { + Matchers { + matchers: vec![], + or_matchers, + } + } + + pub fn append(mut self, matcher: LabelFilter) -> Self { + // Check the latest or_matcher group. If it is not empty, + // we need to add the current matcher to this group. + if let Some(last_or_matcher) = self.or_matchers.last_mut() { + last_or_matcher.push(matcher); + } else { + self.matchers.push(matcher); + } + self + } + + pub fn append_or(mut self, matcher: LabelFilter) -> Self { + if !self.matchers.is_empty() { + // Be careful not to move ownership here, because it + // will be used by the subsequent append method. + let last_matchers = std::mem::take(&mut self.matchers); + self.or_matchers.push(last_matchers); + } + let new_or_matchers = vec![matcher]; + self.or_matchers.push(new_or_matchers); + self + } + + pub fn merge(mut self, other: Matchers) -> Self { + if !other.or_matchers.is_empty() { + if !self.matchers.is_empty() { + self.or_matchers.push(std::mem::take(&mut self.matchers)); + } + self.or_matchers.extend(other.or_matchers); + } else { + self.matchers.extend(other.matchers); + } + self + } + + /// Vector selectors must either specify a name or at least one label + /// matcher that does not match the empty string. + /// + /// The following expression is illegal: + /// {job=~".*"} -- Bad! + pub fn is_empty_matchers(&self) -> bool { + (self.matchers.is_empty() && self.or_matchers.is_empty()) + || self + .matchers + .iter() + .chain(self.or_matchers.iter().flatten()) + .all(|m| m.is_match("")) + } + + /// find the matcher's value whose name equals the specified name. This function + /// is designed to prepare error message of invalid promql expression. + #[allow(dead_code)] + pub(crate) fn find_matcher_value(&self, name: &str) -> Option<&String> { + self.matchers + .iter() + .chain(self.or_matchers.iter().flatten()) + .find(|m| m.label.eq(name)) + .map(|m| &m.value) + } + + /// find matchers whose name equals the specified name + pub fn find_matchers(&self, name: &str) -> Vec<&LabelFilter> { + self.matchers + .iter() + .chain(self.or_matchers.iter().flatten()) + .filter(|m| m.label.eq(name)) + .collect() + } + + pub fn sort_filters(&mut self) { + if !self.matchers.is_empty() { + self.matchers.sort(); + } + if !self.or_matchers.is_empty() { + for filter_list in self.or_matchers.iter_mut() { + filter_list.sort(); + } + } + } + + pub fn is_only_metric_name(&self) -> bool { + if !self.matchers.is_empty() { + return self.matchers.len() == 1 && self.matchers[0].is_metric_name_filter(); + } + if !self.or_matchers.is_empty() { + if self.metric_name().is_none() { + return false; + } + return self.or_matchers.iter().all(|lfs| lfs.len() <= 1); + } + true + } + + pub fn metric_name(&self) -> Option<&str> { + if !self.matchers.is_empty() { + let found = self + .matchers + .iter() + .find(|m| m.is_metric_name_filter()) + .map(|m| m.value.as_str()); + + // todo: make sure only 1 is specified + return found; + } + if !self.or_matchers.is_empty() { + let lfs = self.or_matchers.first().unwrap(); + if lfs.is_empty() { + return None; + } + let head = &lfs[0]; + if !head.is_metric_name_filter() { + return None; + } + let metric_name = head.value.as_str(); + for or_matchers in &self.or_matchers[1..] { + if or_matchers.is_empty() { + return None; + } + let first = &or_matchers[0]; + if !first.is_metric_name_filter() || first.value.as_str() != metric_name { + return None; + } + } + return Some(metric_name); + } + None + } + + pub fn dedup(&mut self) { + if !self.matchers.is_empty() { + remove_duplicate_label_filters(&mut self.matchers); + } + if !self.or_matchers.is_empty() { + for or_matchers in &mut self.or_matchers { + remove_duplicate_label_filters(or_matchers); + } + } + } + + pub fn iter(&self) -> impl Iterator> { + OrIter::new(&self.matchers, &self.or_matchers) + } + + pub fn filter_iter(&self) -> impl Iterator { + self.matchers + .iter() + .chain(self.or_matchers.iter().flatten()) + } +} + +fn hash_filters(hasher: &mut impl Hasher, filters: &[LabelFilter]) { + for filter in filters { + filter.hash(hasher); + } +} + +impl Hash for Matchers { + fn hash(&self, state: &mut H) { + if !self.matchers.is_empty() { + hash_filters(state, &self.matchers); + } + for filters in &self.or_matchers { + hash_filters(state, filters); + } + } +} + +impl fmt::Display for Matchers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let simple_matchers = &self.matchers; + let or_matchers = &self.or_matchers; + if or_matchers.is_empty() { + write!(f, "{}", join_vector(simple_matchers, ",", true)) + } else { + let or_matchers_string = + self.or_matchers + .iter() + .fold(String::new(), |or_matchers_str, pair| { + format!("{} or {}", or_matchers_str, join_vector(pair, ", ", false)) + }); + let or_matchers_string = or_matchers_string.trim_start_matches(" or").trim(); + write!(f, "{}", or_matchers_string) + } + } +} + +struct OrIter<'a> { + matchers: &'a Vec, + or_matchers: &'a Vec>, + index: usize, + first: bool, +} + +impl<'a> OrIter<'a> { + fn new(matchers: &'a Vec, or_matchers: &'a Vec>) -> Self { + Self { + matchers, + or_matchers, + index: 0, + first: true, + } + } +} +impl<'a> Iterator for OrIter<'a> { + type Item = &'a Vec; + + fn next(&mut self) -> Option { + if self.first { + self.first = false; + if !self.matchers.is_empty() { + return Some(self.matchers); + } + } + if self.index < self.or_matchers.len() { + let index = self.index; + self.index += 1; + return Some(&self.or_matchers[index]); + } + None + } +} + +pub(crate) fn remove_duplicate_label_filters(filters: &mut Vec) { + fn get_hash(hasher: &mut Xxh3, filter: &LabelFilter) -> u64 { + hasher.reset(); + hasher.write(filter.label.as_bytes()); + hasher.write(filter.op.as_str().as_bytes()); + hasher.write(filter.value.as_bytes()); + hasher.finish() + } + + let mut hasher = Xxh3::new(); + let mut hash_map: AHashMap = AHashMap::with_capacity(filters.len()); + + for i in (0..filters.len()).rev() { + let hash = get_hash(&mut hasher, &filters[i]); + if let std::collections::hash_map::Entry::Vacant(e) = hash_map.entry(hash) { + e.insert(true); + } else { + filters.remove(i); + } + } +} + +/// Go and Rust handle the repeat pattern differently +/// in Go the following is valid: `aaa{bbb}ccc` +/// in Rust {bbb} is seen as an invalid repeat and must be escaped \{bbb} +/// This escapes the opening "{" if it's not followed by valid repeat pattern (e.g. 4,6). +pub fn try_escape_for_repeat_re(re: &str) -> String { + fn is_repeat(chars: &mut std::str::Chars<'_>) -> (bool, String) { + let mut buf = String::new(); + let mut comma_seen = false; + for c in chars.by_ref() { + buf.push(c); + match c { + ',' if comma_seen => { + return (false, buf); // ,, is invalid + } + ',' if buf == "," => { + return (false, buf); // {, is invalid + } + ',' if !comma_seen => comma_seen = true, + '}' if buf == "}" => { + return (false, buf); // {} is invalid + } + '}' => { + return (true, buf); + } + _ if c.is_ascii_digit() => continue, + _ => { + return (false, buf); // false if visit non-digit char + } + } + } + (false, buf) // not ended with "}" + } + + let mut result = String::with_capacity(re.len() + 1); + let mut chars = re.chars(); + + while let Some(c) = chars.next() { + match c { + '\\' => { + if let Some(cc) = chars.next() { + result.push(c); + result.push(cc); + } + } + '{' => { + let (is, s) = is_repeat(&mut chars); + if !is { + result.push('\\'); + } + result.push(c); + result.push_str(&s); + } + _ => result.push(c), + } + } + result +} + +#[cfg(test)] +mod tests { + use crate::label::{LabelFilter, LabelFilterOp, Matchers}; + + use super::try_escape_for_repeat_re; + + #[test] + fn test_matcher_eq_ne() { + let op = LabelFilterOp::Equal; + let matcher = LabelFilter::new(op, "name", "up").unwrap(); + assert!(matcher.is_match("up")); + assert!(!matcher.is_match("down")); + + let op = LabelFilterOp::NotEqual; + let matcher = LabelFilter::new(op, "name", "up").unwrap(); + assert!(matcher.is_match("foo")); + assert!(matcher.is_match("bar")); + assert!(!matcher.is_match("up")); + } + + #[test] + fn test_matcher_re() { + let value = "api/v1/.*"; + let matcher = LabelFilter::new(LabelFilterOp::RegexEqual, "name", value).unwrap(); + assert!(matcher.is_match("api/v1/query")); + assert!(matcher.is_match("api/v1/range_query")); + assert!(!matcher.is_match("api/v2")); + } + + #[test] + fn test_eq_matcher_equality() { + assert_eq!( + LabelFilter::equal("code", "200"), + LabelFilter::equal("code", "200") + ); + + assert_ne!( + LabelFilter::equal("code", "200"), + LabelFilter::equal("code", "201") + ); + + assert_ne!( + LabelFilter::equal("code", "200"), + LabelFilter::not_equal("code", "200") + ); + } + + #[test] + fn test_ne_matcher_equality() { + assert_eq!( + LabelFilter::not_equal("code", "200"), + LabelFilter::not_equal("code", "200") + ); + + assert_ne!( + LabelFilter::not_equal("code", "200"), + LabelFilter::not_equal("code", "201") + ); + + assert_ne!( + LabelFilter::not_equal("code", "200"), + LabelFilter::equal("code", "200") + ); + } + + #[test] + fn test_re_matcher_equality() { + assert_eq!( + LabelFilter::regex_equal("code", "2??"), + LabelFilter::regex_equal("code", "2??") + ); + + assert_ne!( + LabelFilter::regex_equal("code", "2??",), + LabelFilter::regex_equal("code", "2*?",) + ); + + assert_ne!( + LabelFilter::new(LabelFilterOp::RegexEqual, "code", "2??",), + LabelFilter::new(LabelFilterOp::Equal, "code", "2??") + ); + } + + #[test] + fn test_not_re_matcher_equality() { + assert_eq!( + LabelFilter::regex_notequal("code", "2??",), + LabelFilter::regex_notequal("code", "2??",) + ); + + assert_ne!( + LabelFilter::regex_notequal("code", "2??"), + LabelFilter::regex_notequal("code", "2*?",) + ); + + assert_ne!( + LabelFilter::regex_equal("code", "2??").unwrap(), + LabelFilter::equal("code", "2??") + ); + } + + #[test] + fn test_matchers_equality() { + assert_eq!( + Matchers::empty() + .append(LabelFilter::equal("name1", "val1")) + .append(LabelFilter::equal("name2", "val2")), + Matchers::empty() + .append(LabelFilter::equal("name1", "val1")) + .append(LabelFilter::equal("name2", "val2")) + ); + + assert_ne!( + Matchers::empty().append(LabelFilter::equal("name1", "val1")), + Matchers::empty().append(LabelFilter::equal("name2", "val2")) + ); + + assert_ne!( + Matchers::empty().append(LabelFilter::equal("name1", "val1")), + Matchers::empty().append(LabelFilter::not_equal("name1", "val1")) + ); + + assert_eq!( + Matchers::empty() + .append(LabelFilter::equal("name1", "val1")) + .append(LabelFilter::not_equal("name2", "val2")) + .append(LabelFilter::regex_equal("name2", "\\d+").unwrap()) + .append(LabelFilter::regex_notequal("name2", "\\d+").unwrap()), + Matchers::empty() + .append(LabelFilter::equal("name1", "val1")) + .append(LabelFilter::not_equal("name2", "val2")) + .append(LabelFilter::regex_equal("name2", "\\d+").unwrap()) + .append(LabelFilter::regex_notequal("name2", "\\d+").unwrap()) + ); + } + + #[test] + fn test_find_matchers() { + let matchers = Matchers::empty() + .append(LabelFilter::equal("foo", "bar")) + .append(LabelFilter::not_equal("foo", "bar")) + .append(LabelFilter::equal("FOO", "bar")) + .append(LabelFilter::not_equal("bar", "bar")); + + let ms = matchers.find_matchers("foo"); + assert_eq!(4, ms.len()); + } + + #[test] + fn test_convert_re() { + assert_eq!(try_escape_for_repeat_re("abc{}"), r"abc\{}"); + assert_eq!(try_escape_for_repeat_re("abc{def}"), r"abc\{def}"); + assert_eq!(try_escape_for_repeat_re("abc{def"), r"abc\{def"); + assert_eq!(try_escape_for_repeat_re("abc{1}"), "abc{1}"); + assert_eq!(try_escape_for_repeat_re("abc{1,}"), "abc{1,}"); + assert_eq!(try_escape_for_repeat_re("abc{1,2}"), "abc{1,2}"); + assert_eq!(try_escape_for_repeat_re("abc{,2}"), r"abc\{,2}"); + assert_eq!(try_escape_for_repeat_re("abc{{1,2}}"), r"abc\{{1,2}}"); + assert_eq!(try_escape_for_repeat_re(r"abc\{abc"), r"abc\{abc"); + assert_eq!(try_escape_for_repeat_re("abc{1a}"), r"abc\{1a}"); + assert_eq!(try_escape_for_repeat_re("abc{1,a}"), r"abc\{1,a}"); + assert_eq!(try_escape_for_repeat_re("abc{1,2a}"), r"abc\{1,2a}"); + assert_eq!(try_escape_for_repeat_re("abc{1,2,3}"), r"abc\{1,2,3}"); + assert_eq!(try_escape_for_repeat_re("abc{1,,2}"), r"abc\{1,,2}"); + } +} diff --git a/crates/metricsql-parser-vendored/src/label/label_filter_expr.rs b/crates/metricsql-parser-vendored/src/label/label_filter_expr.rs new file mode 100644 index 00000000..9bc6a5495 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/label/label_filter_expr.rs @@ -0,0 +1,215 @@ +use serde::{Deserialize, Serialize}; +use std::fmt; + +use crate::ast::StringExpr; +use crate::label::{LabelFilter, LabelFilterOp, LabelName, NAME_LABEL}; +use crate::parser::{compile_regexp, escape_ident, is_empty_regex, ParseError, ParseResult}; + +/// LabelFilterExpr represents `foo ident + "bar"` expression, where is `=`, `!=`, `=~` or `!~`. +/// For internal use only, in the context of WITH expressions +#[derive(Default, Debug, Clone, Eq, Serialize, Deserialize)] +pub struct LabelFilterExpr { + pub op: LabelFilterOp, + + /// Label contains label name for the filter. + pub label: String, + + /// Value contains unquoted value for the filter. + pub value: StringExpr, + + is_variable: bool, +} + +impl LabelFilterExpr { + pub fn new(label: N, match_op: LabelFilterOp, value: StringExpr) -> ParseResult + where + N: Into, + { + let label = label.into(); + + assert!(!label.is_empty()); + + if (match_op == LabelFilterOp::RegexEqual || match_op == LabelFilterOp::RegexNotEqual) + && value.is_expanded() + { + let resolved_value = value.to_string(); + if compile_regexp(&resolved_value).is_err() { + return Err(ParseError::InvalidRegex(resolved_value)); + } + } + + Ok(Self { + label, + op: match_op, + value, + is_variable: false, + }) + } + + pub fn named(name: &str) -> Self { + Self { + label: NAME_LABEL.to_string(), + op: LabelFilterOp::Equal, + value: StringExpr::from(name), + is_variable: false, + } + } + + pub(crate) fn variable(name: &str) -> Self { + Self { + label: NAME_LABEL.to_string(), + op: LabelFilterOp::Equal, + value: StringExpr::from(name), + is_variable: true, + } + } + + pub fn equal>(key: S, value: StringExpr) -> ParseResult { + LabelFilterExpr::new(key, LabelFilterOp::Equal, value) + } + + pub fn not_equal>(key: S, value: StringExpr) -> ParseResult { + LabelFilterExpr::new(key, LabelFilterOp::NotEqual, value) + } + + /// is_negative represents whether the filter is negative, i.e. '!=' or '!~'. + pub fn is_negative(&self) -> bool { + self.op.is_negative() + } + + /// is_regexp represents whether the filter is regexp, i.e. `=~` or `!~`. + pub fn is_regexp(&self) -> bool { + self.op.is_regex() + } + + pub fn is_metric_name_filter(&self) -> bool { + self.label == NAME_LABEL && self.op == LabelFilterOp::Equal + } + + pub fn is_name_label(&self) -> bool { + self.label == NAME_LABEL && self.op == LabelFilterOp::Equal + } + + pub fn set_value>(&mut self, value: S) { + self.value = StringExpr::from(value.into()) + } + + pub fn as_string(&self) -> String { + format!("{}{}{}", escape_ident(&self.label), self.op, self.value) + } + + pub fn is_resolved(&self) -> bool { + self.value.is_literal_only() && !self.is_variable + } + + pub fn is_variable(&self) -> bool { + self.is_variable + } + + pub fn name(&self) -> String { + if self.label == NAME_LABEL { + if self.value.is_literal_only() { + // todo: better error handling + if let Some(value) = self.value.get_literal().unwrap() { + return value.to_string(); + } + } + if let Some(ident) = self.value.as_identifier() { + return ident.to_string(); + } + // todo: panic + return self.value.to_string(); + } + self.label.clone() + } + + /// Vector selectors must either specify a name or at least one label + /// matcher that does not match the empty string. + /// + /// The following expression is illegal: + /// {job=~".*"} # Bad! + pub fn is_empty_matcher(&self) -> bool { + use LabelFilterOp::*; + match self.op { + Equal => self.value.is_empty(), + NotEqual => !self.value.is_empty(), + RegexEqual => { + let str = self.value.to_string(); + is_empty_regex(&str) + } + RegexNotEqual => { + let str = self.value.to_string(); + is_empty_regex(&str) + } + } + } + + pub fn is_match(&self, str: &str) -> bool { + use LabelFilterOp::*; + let haystack = self.value.to_string(); + match self.op { + Equal => haystack.eq(str), + NotEqual => haystack.ne(str), + RegexEqual => compile_regexp(&haystack) + .map(|re| re.is_match(str)) + .unwrap_or(false), + RegexNotEqual => { + let str = self.value.to_string(); + compile_regexp(&haystack) + .map(|re| !re.is_match(&str)) + .unwrap_or(false) + } + } + } + + pub fn to_label_filter(&self) -> ParseResult { + let empty_str = "".to_string(); + let value = self.value.get_literal()?.unwrap_or(&empty_str).to_string(); + LabelFilter::new(self.op, &self.label, value) + } + + // + pub fn is_raw_ident(&self) -> bool { + self.value.is_empty() + && self.op == LabelFilterOp::Equal + && !self.label.is_empty() + && self.label != NAME_LABEL + } +} + +impl PartialOrd for LabelFilterExpr { + fn partial_cmp(&self, other: &Self) -> Option { + let cmp = self.label.cmp(&other.label); + if cmp != std::cmp::Ordering::Equal { + return Some(cmp); + } + if let Some(cmp) = self.value.partial_cmp(&other.value) { + if cmp != std::cmp::Ordering::Equal { + return Some(cmp); + } + } + Some(std::cmp::Ordering::Equal) + } +} + +impl PartialEq for LabelFilterExpr { + fn eq(&self, other: &Self) -> bool { + self.label == other.label && self.op == other.op && self.value == other.value + } +} + +impl fmt::Display for LabelFilterExpr { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + // todo: this is a mess. refactor + if self.is_variable || self.is_metric_name_filter() { + write!(f, "{}", escape_ident(&self.value.to_string()))?; + return Ok(()); + } + if self.is_raw_ident() { + write!(f, "{}", escape_ident(&self.label))?; + return Ok(()); + } + write!(f, "{}{}{}", escape_ident(&self.label), self.op, &self.value)?; + Ok(()) + } +} diff --git a/crates/metricsql-parser-vendored/src/label/labels.rs b/crates/metricsql-parser-vendored/src/label/labels.rs new file mode 100644 index 00000000..a9289546 --- /dev/null +++ b/crates/metricsql-parser-vendored/src/label/labels.rs @@ -0,0 +1,196 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Label matchers and Well-known label names used by Prometheus components. + +use std::collections::BTreeSet; +use std::fmt; +use std::hash::Hash; +use std::str::FromStr; + +use ahash::AHashSet; +use serde::{Deserialize, Serialize}; + +/// "__name__" +pub const METRIC_NAME: &str = "__name__"; +/// "alertname" +pub const ALERT_NAME: &str = "alertname"; +/// "le" +pub const BUCKET_LABEL: &str = "le"; +/// "instance" +pub const INSTANCE_NAME: &str = "instance"; + +pub type Label = String; + +#[derive(Debug, Clone, Default, Serialize, Deserialize, Eq)] +pub struct Labels(pub(crate) Vec