diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bd9ef73..9eb736ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `tree`: `with_seed` on `DecisionTreeRegressorParameters` and `DecisionTreeClassifierParameters`. - `ensemble`: `ExtraTreesRegressorParameters` gets a `bootstrap` field and a `with_bootstrap` method. The default is `false`, so each tree still uses the whole dataset. - `ensemble`: `ExtraTreesClassifier` added with an API very similar to `ExtraTreesRegressor` +- `ensemble`: `MaxFeatures` enum (`Sqrt`, `Fraction`, `Count`, `All`) sets the number of features to test at each split in the forest estimators. ### Fixed - `ensemble`: `ExtraTreesRegressor::predict_oob` returns an error when the forest was fitted without bootstrap. Before this change, it returned `NaN` for a float target and panicked for an integer target (#520). @@ -31,6 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `ensemble`: each forest uses one RNG for bootstrap sampling and for the tree seeds (before: `seed + tree_idx`). Results for a given seed differ from earlier versions. - **Breaking change:** `ensemble`: `ExtraTreesRegressorParameters` gets a public `bootstrap` field. Struct literals without `..Default::default()` must add `bootstrap: false` to keep the earlier behavior. Serialized parameters without this field still deserialize, with `bootstrap = false` (#520). - `ensemble`: Extra trees do not sort the feature columns. Trees are not bit-identical to the old ones because of different summation order. +- **Breaking change:** `ensemble`: `m` on the four forest parameter structs is now `MaxFeatures` (was `Option`). `with_m(3)` still works. `fit` returns an error when `m` is not valid for the data. Parameters serialized with an earlier version that contain `m` no longer deserialize. +- **Breaking change:** `ensemble`: `m` on `RandomForestClassifierSearchParameters` and `RandomForestRegressorSearchParameters` is now `Vec` (was `Vec>`). The default grid is `[MaxFeatures::Sqrt]`. Change `vec![None, Some(3)]` to `vec![MaxFeatures::Sqrt, MaxFeatures::Count(3)]`. ## [0.7.0] - 2026-10-26 ### Added diff --git a/src/ensemble/base_forest.rs b/src/ensemble/base_forest.rs index 41dd4450..69b50659 100644 --- a/src/ensemble/base_forest.rs +++ b/src/ensemble/base_forest.rs @@ -4,6 +4,7 @@ use std::fmt::Debug; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +use crate::ensemble::max_features::MaxFeatures; use crate::error::{Failed, FailedError}; use crate::linalg::basic::arrays::{Array1, Array2}; use crate::linalg::basic::arrays::{ArrayView1, MutArrayView1}; @@ -33,8 +34,8 @@ pub struct BaseForestParameters { /// The number of trees in the forest. pub n_trees: usize, #[cfg_attr(feature = "serde", serde(default))] - /// Number of random sample of predictors to use as split candidates. - pub m: Option, + /// Number of features to test at each split. See [`MaxFeatures`]. + pub m: MaxFeatures, #[cfg_attr(feature = "serde", serde(default))] /// Whether to keep samples used for tree generation. This is required for OOB prediction. pub keep_samples: bool, @@ -127,9 +128,7 @@ impl< )); } - let mtry = parameters - .m - .unwrap_or((num_attributes as f64).sqrt().floor() as usize); + let mtry = parameters.m.resolve(num_attributes)?; let mut rng = get_rng_impl(parameters.seed); let n_trees = parameters.n_trees; @@ -436,7 +435,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 5, - m: None, + m: MaxFeatures::Sqrt, keep_samples: true, seed: Some(42), bootstrap: true, @@ -465,7 +464,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 5, - m: None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(0), bootstrap: true, @@ -494,7 +493,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 5, - m: None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(0), bootstrap: true, @@ -520,7 +519,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 5, - m: None, + m: MaxFeatures::Sqrt, keep_samples: true, seed: Some(42), bootstrap: false, @@ -579,7 +578,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: None, + m: MaxFeatures::Sqrt, keep_samples: true, // keep samples used for each tree, so we can check that they are different seed: Some(42), bootstrap: true, // Use bootstrapping @@ -625,7 +624,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Some(1), + m: MaxFeatures::Count(1), keep_samples: false, seed: Some(42), bootstrap: false, @@ -654,7 +653,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: None, + m: MaxFeatures::Sqrt, keep_samples: true, seed: Some(42), bootstrap: false, // No bootstrapping @@ -697,7 +696,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 500, // Use more trees than before to smooth out randomness - m: None, + m: MaxFeatures::Sqrt, keep_samples: true, // keep samples used for each tree, so we can check that they are different seed: Some(42), bootstrap: true, // Use bootstrapping @@ -757,7 +756,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Some(1), + m: MaxFeatures::Count(1), keep_samples: true, seed: Some(42), bootstrap: true, @@ -817,7 +816,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Some(1), + m: MaxFeatures::Count(1), keep_samples: false, seed: Some(42), bootstrap, @@ -890,7 +889,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Some(1), + m: MaxFeatures::Count(1), keep_samples: false, seed, bootstrap: true, @@ -940,4 +939,164 @@ mod tests { "two fits with seed None give the same forest" ); } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn fit_returns_error_for_invalid_max_features() { + let n_rows = 10; + let n_features = 4; + let x: DenseMatrix = DenseMatrix::from_iterator( + (0..n_features * n_rows).map(|k| ((k * 7919) % 101) as f64), + n_rows, + n_features, + 0, + ); + let y: Vec = (0..n_rows).map(|i| i as f64).collect(); + + let invalid = [ + MaxFeatures::Count(0), + MaxFeatures::Count(n_features + 1), + MaxFeatures::Fraction(0.0), + MaxFeatures::Fraction(-0.5), + MaxFeatures::Fraction(1.5), + MaxFeatures::Fraction(f64::NAN), + MaxFeatures::Fraction(f64::INFINITY), + ]; + + for splitter in [Splitter::Best, Splitter::Random] { + for m in invalid { + let params = BaseForestParameters { + max_depth: None, + min_samples_leaf: 1, + min_samples_split: 2, + n_trees: 3, + m, + keep_samples: false, + seed: Some(42), + bootstrap: true, + splitter, + }; + let result = BaseForest::fit(&x, &y, None, &MSECriterion, params); + let err = result.err().unwrap_or_else(|| { + panic!("fit must fail for m = {m:?} (splitter: {splitter:?})") + }); + assert_eq!( + err.error(), + FailedError::ParametersError, + "m = {m:?} (splitter: {splitter:?})" + ); + } + } + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn fit_accepts_boundary_max_features() { + let n_rows = 10; + let n_features = 4; + let x: DenseMatrix = DenseMatrix::from_iterator( + (0..n_features * n_rows).map(|k| ((k * 7919) % 101) as f64), + n_rows, + n_features, + 0, + ); + let y: Vec = (0..n_rows).map(|i| i as f64).collect(); + + let valid = [ + MaxFeatures::Sqrt, + MaxFeatures::All, + MaxFeatures::Count(1), + MaxFeatures::Count(n_features), + MaxFeatures::Fraction(1.0), + MaxFeatures::Fraction(0.01), + ]; + + for m in valid { + let params = BaseForestParameters { + max_depth: None, + min_samples_leaf: 1, + min_samples_split: 2, + n_trees: 3, + m, + keep_samples: false, + seed: Some(42), + bootstrap: true, + splitter: Splitter::Best, + }; + let forest = BaseForest::fit(&x, &y, None, &MSECriterion, params) + .unwrap_or_else(|e| panic!("fit must succeed for m = {m:?}: {e}")); + assert_eq!(forest.trees.unwrap().len(), 3, "m = {m:?}"); + } + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn equivalent_max_features_give_same_forest() { + let n_rows = 10; + let n_features = 4; + let x: DenseMatrix = DenseMatrix::from_iterator( + (0..n_features * n_rows).map(|k| ((k * 7919) % 101) as f64), + n_rows, + n_features, + 0, + ); + let y: Vec = (0..n_rows).map(|i| i as f64).collect(); + let mut params = BaseForestParameters { + max_depth: None, + min_samples_leaf: 1, + min_samples_split: 2, + n_trees: 3, + m: MaxFeatures::All, + keep_samples: false, + seed: Some(42), + bootstrap: true, + splitter: Splitter::Best, + }; + + let forest_all = + BaseForest::fit(&x, &y, None, &MSECriterion, params.clone()).expect("Fit should work"); + + params.m = MaxFeatures::Fraction(1.0); + + let forest_fraction = + BaseForest::fit(&x, &y, None, &MSECriterion, params.clone()).expect("Fit should work"); + + params.m = MaxFeatures::Count(4); + + let forest_count = + BaseForest::fit(&x, &y, None, &MSECriterion, params.clone()).expect("Fit should work"); + + assert_eq!( + forest_all, forest_count, + "Forest fit by 'all' and 'count' is different" + ); + assert_eq!( + forest_all, forest_fraction, + "Forest fit by 'all' and 'fraction' is different" + ); + + params.m = MaxFeatures::Sqrt; + + let forest_sqrt = + BaseForest::fit(&x, &y, None, &MSECriterion, params.clone()).expect("Fit should work"); + + params.m = MaxFeatures::Count(2); + + let forest_count_sqrt = + BaseForest::fit(&x, &y, None, &MSECriterion, params.clone()).expect("Fit should work"); + + assert_eq!( + forest_sqrt, forest_count_sqrt, + "Forest fit by 'sqrt' and 'count' is different" + ); + } } diff --git a/src/ensemble/extra_trees_classifier.rs b/src/ensemble/extra_trees_classifier.rs index 288e8ef9..ca1f3350 100644 --- a/src/ensemble/extra_trees_classifier.rs +++ b/src/ensemble/extra_trees_classifier.rs @@ -81,6 +81,7 @@ use crate::api::{Predictor, SupervisedEstimator}; use crate::ensemble::base_forest::{ BaseForest, BaseForestParameters, predict_mode, predict_mode_oob, }; +use crate::ensemble::max_features::MaxFeatures; use crate::error::Failed; use crate::linalg::basic::arrays::{Array1, Array2}; use crate::numbers::basenum::Number; @@ -112,8 +113,8 @@ pub struct ExtraTreesClassifierParameters { /// The number of trees in the forest. pub n_trees: usize, #[cfg_attr(feature = "serde", serde(default))] - /// Number of random sample of predictors to use as split candidates. - pub m: Option, + /// Number of features to test at each split. See [`MaxFeatures`]. + pub m: MaxFeatures, #[cfg_attr(feature = "serde", serde(default))] /// Whether to keep samples used for tree generation. This is required for OOB prediction. pub keep_samples: bool, @@ -168,9 +169,9 @@ impl ExtraTreesClassifierParameters { self.n_trees = n_trees; self } - /// Number of random sample of predictors to use as split candidates. - pub fn with_m(mut self, m: usize) -> Self { - self.m = Some(m); + /// Number of features to test at each split. See [`MaxFeatures`]. + pub fn with_m(mut self, m: impl Into) -> Self { + self.m = m.into(); self } /// Seed used for feature selection and for the random split points of each tree. @@ -209,7 +210,7 @@ impl Default for ExtraTreesClassifierParameters { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Option::None, bootstrap: false, diff --git a/src/ensemble/extra_trees_regressor.rs b/src/ensemble/extra_trees_regressor.rs index b75ee096..666c6020 100644 --- a/src/ensemble/extra_trees_regressor.rs +++ b/src/ensemble/extra_trees_regressor.rs @@ -63,6 +63,7 @@ use crate::api::{Predictor, SupervisedEstimator}; use crate::ensemble::base_forest::{ BaseForest, BaseForestParameters, predict_mean, predict_mean_oob, }; +use crate::ensemble::max_features::MaxFeatures; use crate::error::Failed; use crate::linalg::basic::arrays::{Array1, Array2}; use crate::numbers::basenum::Number; @@ -89,8 +90,8 @@ pub struct ExtraTreesRegressorParameters { /// The number of trees in the forest. pub n_trees: usize, #[cfg_attr(feature = "serde", serde(default))] - /// Number of random sample of predictors to use as split candidates. - pub m: Option, + /// Number of features to test at each split. See [`MaxFeatures`]. + pub m: MaxFeatures, #[cfg_attr(feature = "serde", serde(default))] /// Whether to keep samples used for tree generation. This is required for OOB prediction. pub keep_samples: bool, @@ -138,9 +139,9 @@ impl ExtraTreesRegressorParameters { self.n_trees = n_trees; self } - /// Number of random sample of predictors to use as split candidates. - pub fn with_m(mut self, m: usize) -> Self { - self.m = Some(m); + /// Number of features to test at each split. See [`MaxFeatures`]. + pub fn with_m(mut self, m: impl Into) -> Self { + self.m = m.into(); self } @@ -170,7 +171,7 @@ impl Default for ExtraTreesRegressorParameters { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Option::None, bootstrap: false, @@ -594,7 +595,7 @@ mod tests { #[test] fn serde_without_bootstrap_field_uses_default() { // Parameters serialized before the `bootstrap` field was added. - let json = r#"{"max_depth":null,"min_samples_leaf":1,"min_samples_split":2,"n_trees":10,"m":null,"keep_samples":true,"seed":42}"#; + let json = r#"{"max_depth":null,"min_samples_leaf":1,"min_samples_split":2,"m":"Sqrt","n_trees":10,"keep_samples":true,"seed":42}"#; let parameters: ExtraTreesRegressorParameters = serde_json::from_str(json).unwrap(); assert!(!parameters.bootstrap); @@ -610,6 +611,41 @@ mod tests { assert_eq!(forest.predict_oob(&x).err(), Some(Failed::predict(msg))); } + #[cfg(feature = "serde")] + #[test] + fn serde_without_m_field_uses_default() { + let json = r#"{"max_depth":null,"min_samples_leaf":1,"min_samples_split":2,"n_trees":10,"keep_samples":true,"seed":42}"#; + let parameters: ExtraTreesRegressorParameters = serde_json::from_str(json).unwrap(); + assert_eq!(parameters.m, MaxFeatures::Sqrt); + } + + #[cfg(feature = "serde")] + #[test] + fn serde_with_old_m_format_fails() { + // Before 0.8.0, `m` was `Option`. Serde always wrote it, as `null` or as a number. + // `m` is now `MaxFeatures`, so these old values no longer deserialize (see CHANGELOG). + let old_formats = [ + r#"{"max_depth":null,"min_samples_leaf":1,"min_samples_split":2,"m":null,"n_trees":10,"keep_samples":true,"seed":42}"#, + r#"{"max_depth":null,"min_samples_leaf":1,"min_samples_split":2,"m":3,"n_trees":10,"keep_samples":true,"seed":42}"#, + ]; + for json in old_formats { + let result: Result = serde_json::from_str(json); + assert!(result.is_err(), "old format must not deserialize: {json}"); + } + + // The new format deserializes. + let new_formats = [ + (r#"{"m":"Sqrt"}"#, MaxFeatures::Sqrt), + (r#"{"m":"All"}"#, MaxFeatures::All), + (r#"{"m":{"Count":3}}"#, MaxFeatures::Count(3)), + (r#"{"m":{"Fraction":0.5}}"#, MaxFeatures::Fraction(0.5)), + ]; + for (json, expected) in new_formats { + let parameters: ExtraTreesRegressorParameters = serde_json::from_str(json).unwrap(); + assert_eq!(parameters.m, expected, "json = {json}"); + } + } + #[cfg_attr( all(target_arch = "wasm32", not(target_os = "wasi")), wasm_bindgen_test::wasm_bindgen_test diff --git a/src/ensemble/max_features.rs b/src/ensemble/max_features.rs new file mode 100644 index 00000000..7298dee8 --- /dev/null +++ b/src/ensemble/max_features.rs @@ -0,0 +1,248 @@ +//! # Max Features +//! At each node, a forest of decision trees tests only a random subset of the features. +//! [`MaxFeatures`] sets the size of this subset. The forest calculates the size from _p_, +//! the number of features in the training data. +//! +//! | Variant | Number of features | Condition | +//! |------------------|-------------------------------|-------------------| +//! | `Sqrt` (default) | \\(\max(1, \lfloor\sqrt{p}\rfloor)\\) | | +//! | `Fraction(f)` | \\(\max(1, \lfloor f p \rfloor)\\) | \\(0 < f \le 1\\) | +//! | `Count(n)` | \\(n\\) | \\(1 \le n \le p\\) | +//! | `All` | \\(p\\) | | +//! +//! A small subset makes the trees less correlated, so the forest has a lower variance. +//! A large subset makes each tree stronger. \\(\sqrt{p}\\) is the usual value for classification. +//! With `All`, a random forest is the same as bagging. +//! +//! If the value is not valid for the data (for example, `Count(n)` with \\(n > p\\)), `fit` returns an error. +//! +//! The four forest estimators use this type: [random forest classifier](../random_forest_classifier/index.html), +//! [random forest regressor](../random_forest_regressor/index.html), +//! [extra trees classifier](../extra_trees_classifier/index.html) and +//! [extra trees regressor](../extra_trees_regressor/index.html). +//! +//! ## References: +//! +//! * ["Random Forests", Breiman L., Machine Learning 45(1), 5-32, 2001](https://doi.org/10.1023/A:1010933404324) +//! * ["Extremely randomized trees", Geurts P., Ernst D., Wehenkel L., Machine Learning 63(1), 3-42, 2006](https://doi.org/10.1007/s10994-006-6226-1) +//! * ["An Introduction to Statistical Learning", James G., Witten D., Hastie T., Tibshirani R., 8.2 Bagging, Random Forests, Boosting](http://faculty.marshall.usc.edu/gareth-james/ISL/) +//! +//! Example: +//! +//! ``` +//! use smartcore::ensemble::max_features::MaxFeatures; +//! use smartcore::ensemble::random_forest_classifier::RandomForestClassifierParameters; +//! +//! // Use the default: the square root of the number of features. +//! let params = RandomForestClassifierParameters::default(); +//! assert_eq!(params.m, MaxFeatures::Sqrt); +//! +//! // Use half of the features at each node. +//! let params = RandomForestClassifierParameters::default() +//! .with_m(MaxFeatures::Fraction(0.5)); +//! +//! // Use exactly 3 features at each node. A plain `usize` gives `Count`. +//! let params = RandomForestClassifierParameters::default().with_m(3); +//! assert_eq!(params.m, MaxFeatures::Count(3)); +//! ``` +use crate::error::{Failed, FailedError}; + +#[cfg(feature = "serde")] +use serde::{Deserialize, Serialize}; + +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Debug, Clone, Copy, PartialEq, Default)] +/// Specify the number of features to use in each node while fitting the tree. +pub enum MaxFeatures { + #[default] + /// Consider sqrt(n_features) in each node + Sqrt, // max(1, floor(sqrt(p))) — default + /// Consider the given fraction of the features in each node + Fraction(f64), // max(1, floor(f * p)), 0 < f <= 1 + /// Consider the given number of features in each node + Count(usize), // n, 1 <= n <= p + /// Consider all the features in each node + All, // p +} + +impl MaxFeatures { + pub(crate) fn resolve(&self, n_features: usize) -> Result { + match *self { + MaxFeatures::Sqrt => Ok(1.max(n_features.isqrt())), + MaxFeatures::Fraction(f) => { + if 0.0 < f && f <= 1.0 { + Ok(1.max((f * n_features as f64) as usize)) + } else { + Err(Failed::because( + FailedError::ParametersError, + &format!( + "fraction must be between 0 (exclusive) and 1 (inclusive), got {f}." + ), + )) + } + } + MaxFeatures::Count(count) => { + if (1..=n_features).contains(&count) { + Ok(count) + } else { + Err(Failed::because( + FailedError::ParametersError, + &format!( + "number of features to use must be between 1 and {n_features}, got {count}." + ), + )) + } + } + MaxFeatures::All => Ok(n_features), + } + } +} + +impl From for MaxFeatures { + fn from(n: usize) -> Self { + MaxFeatures::Count(n) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn default_is_sqrt() { + assert_eq!(MaxFeatures::default(), MaxFeatures::Sqrt); + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn from_usize_gives_count() { + assert_eq!(MaxFeatures::from(3), MaxFeatures::Count(3)); + let m: MaxFeatures = 7.into(); + assert_eq!(m, MaxFeatures::Count(7)); + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn sqrt_rounds_down() { + // (n_features, expected) + let cases = [ + (1, 1), + (2, 1), + (3, 1), + (4, 2), + (8, 2), + (9, 3), + (10, 3), + (100, 10), + ]; + for (p, expected) in cases { + assert_eq!(MaxFeatures::Sqrt.resolve(p).unwrap(), expected, "p = {p}"); + } + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn fraction_rounds_down() { + assert_eq!(MaxFeatures::Fraction(0.5).resolve(7).unwrap(), 3); + assert_eq!(MaxFeatures::Fraction(0.25).resolve(8).unwrap(), 2); + assert_eq!(MaxFeatures::Fraction(1.0).resolve(8).unwrap(), 8); + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn small_fraction_gives_at_least_one() { + assert_eq!(MaxFeatures::Fraction(0.01).resolve(10).unwrap(), 1); + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn fraction_out_of_range_is_error() { + for f in [0.0, -0.5, 1.5, f64::NAN, f64::INFINITY] { + assert!(MaxFeatures::Fraction(f).resolve(10).is_err(), "f = {f}"); + } + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn fraction_error_message() { + let expected = Failed::because( + FailedError::ParametersError, + "fraction must be between 0 (exclusive) and 1 (inclusive), got 1.5.", + ); + assert_eq!(MaxFeatures::Fraction(1.5).resolve(10).err(), Some(expected)); + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn count_in_range_is_unchanged() { + assert_eq!(MaxFeatures::Count(1).resolve(5).unwrap(), 1); + assert_eq!(MaxFeatures::Count(3).resolve(5).unwrap(), 3); + assert_eq!(MaxFeatures::Count(5).resolve(5).unwrap(), 5); + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn count_out_of_range_is_error() { + assert!(MaxFeatures::Count(0).resolve(5).is_err()); + assert!(MaxFeatures::Count(6).resolve(5).is_err()); + + let expected = Failed::because( + FailedError::ParametersError, + "number of features to use must be between 1 and 5, got 6.", + ); + assert_eq!(MaxFeatures::Count(6).resolve(5).err(), Some(expected)); + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn all_gives_every_feature() { + assert_eq!(MaxFeatures::All.resolve(1).unwrap(), 1); + assert_eq!(MaxFeatures::All.resolve(12).unwrap(), 12); + } + + #[cfg(feature = "serde")] + #[test] + fn serde_round_trip() { + for m in [ + MaxFeatures::Sqrt, + MaxFeatures::Fraction(0.5), + MaxFeatures::Count(3), + MaxFeatures::All, + ] { + let json = serde_json::to_string(&m).unwrap(); + let back: MaxFeatures = serde_json::from_str(&json).unwrap(); + assert_eq!(back, m, "json = {json}"); + } + } +} diff --git a/src/ensemble/mod.rs b/src/ensemble/mod.rs index 282fda7a..ee0c2d92 100644 --- a/src/ensemble/mod.rs +++ b/src/ensemble/mod.rs @@ -19,6 +19,7 @@ mod base_forest; pub mod extra_trees_classifier; pub mod extra_trees_regressor; +pub mod max_features; /// Random forest classifier pub mod random_forest_classifier; /// Random forest regressor diff --git a/src/ensemble/random_forest_classifier.rs b/src/ensemble/random_forest_classifier.rs index c675d118..a0531cfa 100644 --- a/src/ensemble/random_forest_classifier.rs +++ b/src/ensemble/random_forest_classifier.rs @@ -57,6 +57,7 @@ use crate::api::{Predictor, SupervisedEstimator}; use crate::ensemble::base_forest::{ BaseForest, BaseForestParameters, predict_mode, predict_mode_oob, }; +use crate::ensemble::max_features::MaxFeatures; use crate::error::Failed; use crate::linalg::basic::arrays::{Array1, Array2}; use crate::numbers::basenum::Number; @@ -88,8 +89,8 @@ pub struct RandomForestClassifierParameters { /// The number of trees in the forest. pub n_trees: u16, #[cfg_attr(feature = "serde", serde(default))] - /// Number of random sample of predictors to use as split candidates. - pub m: Option, + /// Number of features to test at each split. See [`MaxFeatures`]. + pub m: MaxFeatures, #[cfg_attr(feature = "serde", serde(default))] /// Whether to keep samples used for tree generation. This is required for OOB prediction. pub keep_samples: bool, @@ -138,9 +139,9 @@ impl RandomForestClassifierParameters { self.n_trees = n_trees; self } - /// Number of random sample of predictors to use as split candidates. - pub fn with_m(mut self, m: usize) -> Self { - self.m = Some(m); + /// Number of features to test at each split. See [`MaxFeatures`]. + pub fn with_m(mut self, m: impl Into) -> Self { + self.m = m.into(); self } @@ -173,7 +174,7 @@ impl Default for RandomForestClassifierParameters { min_samples_leaf: 1, min_samples_split: 2, n_trees: 100, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Option::None, } @@ -226,7 +227,7 @@ pub struct RandomForestClassifierSearchParameters { pub n_trees: Vec, #[cfg_attr(feature = "serde", serde(default))] /// Number of random sample of predictors to use as split candidates. - pub m: Vec>, + pub m: Vec, #[cfg_attr(feature = "serde", serde(default))] /// Whether to keep samples used for tree generation. This is required for OOB prediction. pub keep_samples: Vec, @@ -633,22 +634,22 @@ mod tests { fn search_parameters() { let parameters = RandomForestClassifierSearchParameters { n_trees: vec![10, 100], - m: vec![None, Some(1)], + m: vec![MaxFeatures::Sqrt, MaxFeatures::Count(1)], ..Default::default() }; let mut iter = parameters.into_iter(); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 10); - assert_eq!(next.m, None); + assert_eq!(next.m, MaxFeatures::Sqrt); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 100); - assert_eq!(next.m, None); + assert_eq!(next.m, MaxFeatures::Sqrt); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 10); - assert_eq!(next.m, Some(1)); + assert_eq!(next.m, MaxFeatures::Count(1)); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 100); - assert_eq!(next.m, Some(1)); + assert_eq!(next.m, MaxFeatures::Count(1)); assert!(iter.next().is_none()); } @@ -715,7 +716,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 100, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(87), }, @@ -772,7 +773,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(87), }, @@ -800,7 +801,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 100, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(87), }, @@ -826,7 +827,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(0), }, @@ -875,7 +876,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 100, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: true, seed: Some(87), }, @@ -1041,7 +1042,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Some(1), + m: MaxFeatures::Count(1), keep_samples: false, seed: Some(42), }; diff --git a/src/ensemble/random_forest_regressor.rs b/src/ensemble/random_forest_regressor.rs index 3cba88c7..7509732b 100644 --- a/src/ensemble/random_forest_regressor.rs +++ b/src/ensemble/random_forest_regressor.rs @@ -53,6 +53,7 @@ use crate::api::{Predictor, SupervisedEstimator}; use crate::ensemble::base_forest::{ BaseForest, BaseForestParameters, predict_mean, predict_mean_oob, }; +use crate::ensemble::max_features::MaxFeatures; use crate::error::Failed; use crate::linalg::basic::arrays::{Array1, Array2}; use crate::numbers::basenum::Number; @@ -79,8 +80,8 @@ pub struct RandomForestRegressorParameters { /// The number of trees in the forest. pub n_trees: usize, #[cfg_attr(feature = "serde", serde(default))] - /// Number of random sample of predictors to use as split candidates. - pub m: Option, + /// Number of features to test at each split. See [`MaxFeatures`]. + pub m: MaxFeatures, #[cfg_attr(feature = "serde", serde(default))] /// Whether to keep samples used for tree generation. This is required for OOB prediction. pub keep_samples: bool, @@ -122,9 +123,9 @@ impl RandomForestRegressorParameters { self.n_trees = n_trees; self } - /// Number of random sample of predictors to use as split candidates. - pub fn with_m(mut self, m: usize) -> Self { - self.m = Some(m); + /// Number of features to test at each split. See [`MaxFeatures`]. + pub fn with_m(mut self, m: impl Into) -> Self { + self.m = m.into(); self } @@ -147,7 +148,7 @@ impl Default for RandomForestRegressorParameters { min_samples_leaf: 1, min_samples_split: 2, n_trees: 10, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Option::None, } @@ -204,7 +205,7 @@ pub struct RandomForestRegressorSearchParameters { pub n_trees: Vec, #[cfg_attr(feature = "serde", serde(default))] /// Number of random sample of predictors to use as split candidates. - pub m: Vec>, + pub m: Vec, #[cfg_attr(feature = "serde", serde(default))] /// Whether to keep samples used for tree generation. This is required for OOB prediction. pub keep_samples: Vec, @@ -500,22 +501,22 @@ mod tests { fn search_parameters() { let parameters = RandomForestRegressorSearchParameters { n_trees: vec![10, 100], - m: vec![None, Some(1)], + m: vec![MaxFeatures::Sqrt, MaxFeatures::Count(1)], ..Default::default() }; let mut iter = parameters.into_iter(); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 10); - assert_eq!(next.m, None); + assert_eq!(next.m, MaxFeatures::Sqrt); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 100); - assert_eq!(next.m, None); + assert_eq!(next.m, MaxFeatures::Sqrt); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 10); - assert_eq!(next.m, Some(1)); + assert_eq!(next.m, MaxFeatures::Count(1)); let next = iter.next().unwrap(); assert_eq!(next.n_trees, 100); - assert_eq!(next.m, Some(1)); + assert_eq!(next.m, MaxFeatures::Count(1)); assert!(iter.next().is_none()); } @@ -542,6 +543,30 @@ mod tests { assert!(iter.next().is_none()); } + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn search_parameters_max_features() { + let seq = [ + MaxFeatures::Sqrt, + MaxFeatures::Count(1), + MaxFeatures::Count(2), + MaxFeatures::Fraction(0.1), + MaxFeatures::All, + ]; + let parameters = RandomForestRegressorSearchParameters { + m: seq.to_vec(), + ..Default::default() + }; + let iter = parameters.into_iter(); + + for (idx, rf_params) in iter.enumerate() { + assert_eq!(seq[idx], rf_params.m); + } + } + #[cfg_attr( all(target_arch = "wasm32", not(target_os = "wasi")), wasm_bindgen_test::wasm_bindgen_test @@ -580,7 +605,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 1000, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(87), }, @@ -608,7 +633,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 1000, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: false, seed: Some(87), }, @@ -655,7 +680,7 @@ mod tests { min_samples_leaf: 1, min_samples_split: 2, n_trees: 1000, - m: Option::None, + m: MaxFeatures::Sqrt, keep_samples: true, seed: Some(87), }, @@ -1349,4 +1374,102 @@ mod tests { .unwrap(); assert_eq!(rf.compute_feature_importances().unwrap(), vec![0.0; 3]); } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn fit_returns_error_for_invalid_max_features() { + let n_rows = 10; + let n_features = 4; + let x: DenseMatrix = DenseMatrix::from_iterator( + (0..n_features * n_rows).map(|k| ((k * 7919) % 101) as f64), + n_rows, + n_features, + 0, + ); + let y: Vec = (0..n_rows).map(|i| i as f64).collect(); + + let invalid = [ + MaxFeatures::Count(0), + MaxFeatures::Count(n_features + 1), + MaxFeatures::Fraction(0.0), + MaxFeatures::Fraction(-0.5), + MaxFeatures::Fraction(1.5), + MaxFeatures::Fraction(f64::NAN), + MaxFeatures::Fraction(f64::INFINITY), + ]; + + for m in invalid { + let params = RandomForestRegressorParameters::default().with_m(m); + let result = RandomForestRegressor::fit(&x, &y, params); + let err = result + .err() + .unwrap_or_else(|| panic!("fit must fail for m = {m:?}")); + assert_eq!(err.error(), FailedError::ParametersError, "m = {m:?}"); + } + } + + #[cfg_attr( + all(target_arch = "wasm32", not(target_os = "wasi")), + wasm_bindgen_test::wasm_bindgen_test + )] + #[test] + fn equivalent_max_features_give_same_predictions() { + let n_rows = 10; + let n_features = 4; + let x: DenseMatrix = DenseMatrix::from_iterator( + (0..n_features * n_rows).map(|k| ((k * 7919) % 101) as f64), + n_rows, + n_features, + 0, + ); + let y: Vec = (0..n_rows).map(|i| i as f64).collect(); + let mut params = RandomForestRegressorParameters::default() + .with_m(MaxFeatures::All) + .with_seed(42); + + let forest_all = + RandomForestRegressor::fit(&x, &y, params.clone()).expect("Fit should work"); + let yhat_all = forest_all.predict(&x).expect("predict should work"); + + params.m = MaxFeatures::Fraction(1.0); + + let forest_fraction = + RandomForestRegressor::fit(&x, &y, params.clone()).expect("Fit should work"); + let yhat_fraction = forest_fraction.predict(&x).expect("predict should work"); + + params.m = MaxFeatures::Count(4); + + let forest_count = + RandomForestRegressor::fit(&x, &y, params.clone()).expect("Fit should work"); + let yhat_count = forest_count.predict(&x).expect("predict should work"); + + assert_eq!( + yhat_all, yhat_fraction, + "Predictions made by 'all' and 'fraction' are different", + ); + assert_eq!( + yhat_all, yhat_count, + "Predictions made by 'all' and 'count' are different", + ); + + params.m = MaxFeatures::Sqrt; + + let forest_sqrt = + RandomForestRegressor::fit(&x, &y, params.clone()).expect("Fit should work"); + let yhat_sqrt = forest_sqrt.predict(&x).expect("predict should work"); + + params.m = MaxFeatures::Count(2); + + let forest_count_sqrt = + RandomForestRegressor::fit(&x, &y, params.clone()).expect("Fit should work"); + let yhat_count_sqrt = forest_count_sqrt.predict(&x).expect("predict should work"); + + assert_eq!( + yhat_sqrt, yhat_count_sqrt, + "Predictions made 'sqrt' and 'count' are different" + ); + } }