Merge potential next release v0.4 (#187) Breaking Changes

* First draft of the new n-dimensional arrays + NB use case
* Improves default implementation of multiple Array methods
* Refactors tree methods
* Adds matrix decomposition routines
* Adds matrix decomposition methods to ndarray and nalgebra bindings
* Refactoring + linear regression now uses array2
* Ridge & Linear regression
* LBFGS optimizer & logistic regression
* LBFGS optimizer & logistic regression
* Changes linear methods, metrics and model selection methods to new n-dimensional arrays
* Switches KNN and clustering algorithms to new n-d array layer
* Refactors distance metrics
* Optimizes knn and clustering methods
* Refactors metrics module
* Switches decomposition methods to n-dimensional arrays
* Linalg refactoring - cleanup rng merge (#172)
* Remove legacy DenseMatrix and BaseMatrix implementation. Port the new Number, FloatNumber and Array implementation into module structure.
* Exclude AUC metrics. Needs reimplementation
* Improve developers walkthrough

New traits system in place at `src/numbers` and `src/linalg`
Co-authored-by: Lorenzo <tunedconsulting@gmail.com>

* Provide SupervisedEstimator with a constructor to avoid explicit dynamical box allocation in 'cross_validate' and 'cross_validate_predict' as required by the use of 'dyn' as per Rust 2021
* Implement getters to use as_ref() in src/neighbors
* Implement getters to use as_ref() in src/naive_bayes
* Implement getters to use as_ref() in src/linear
* Add Clone to src/naive_bayes
* Change signature for cross_validate and other model_selection functions to abide to use of dyn in Rust 2021
* Implement ndarray-bindings. Remove FloatNumber from implementations
* Drop nalgebra-bindings support (as decided in conf-call to go for ndarray)
* Remove benches. Benches will have their own repo at smartcore-benches
* Implement SVC
* Implement SVC serialization. Move search parameters in dedicated module
* Implement SVR. Definitely too slow
* Fix compilation issues for wasm (#202)

Co-authored-by: Luis Moreno <morenol@users.noreply.github.com>
* Fix tests (#203)

* Port linalg/traits/stats.rs
* Improve methods naming
* Improve Display for DenseMatrix

Co-authored-by: Montana Low <montanalow@users.noreply.github.com>
Co-authored-by: VolodymyrOrlov <volodymyr.orlov@gmail.com>
This commit is contained in:
Lorenzo
2022-10-31 10:44:57 +00:00
committed by morenol
parent a32eb66a6a
commit a7fa0585eb
110 changed files with 10327 additions and 9107 deletions
+135 -112
View File
@@ -6,7 +6,7 @@
//! Example:
//!
//! ```
//! use smartcore::linalg::naive::dense_matrix::*;
//! use smartcore::linalg::basic::matrix::DenseMatrix;
//! use smartcore::naive_bayes::gaussian::GaussianNB;
//!
//! let x = DenseMatrix::from_2d_array(&[
@@ -17,51 +17,53 @@
//! &[ 2., 1.],
//! &[ 3., 2.],
//! ]);
//! let y = vec![1., 1., 1., 2., 2., 2.];
//! let y: Vec<u32> = vec![1, 1, 1, 2, 2, 2];
//!
//! let nb = GaussianNB::fit(&x, &y, Default::default()).unwrap();
//! let y_hat = nb.predict(&x).unwrap();
//! ```
use num_traits::Unsigned;
use crate::api::{Predictor, SupervisedEstimator};
use crate::error::Failed;
use crate::linalg::row_iter;
use crate::linalg::BaseVector;
use crate::linalg::Matrix;
use crate::math::num::RealNumber;
use crate::math::vector::RealNumberVector;
use crate::linalg::basic::arrays::{Array1, Array2, ArrayView1};
use crate::naive_bayes::{BaseNaiveBayes, NBDistribution};
use crate::numbers::basenum::Number;
use crate::numbers::realnum::RealNumber;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
/// Naive Bayes classifier using Gaussian distribution
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, PartialEq)]
struct GaussianNBDistribution<T: RealNumber> {
#[derive(Debug, PartialEq, Clone)]
struct GaussianNBDistribution<T: Number> {
/// class labels known to the classifier
class_labels: Vec<T>,
/// number of training samples observed in each class
class_count: Vec<usize>,
/// probability of each class.
class_priors: Vec<T>,
class_priors: Vec<f64>,
/// variance of each feature per class
var: Vec<Vec<T>>,
var: Vec<Vec<f64>>,
/// mean of each feature per class
theta: Vec<Vec<T>>,
theta: Vec<Vec<f64>>,
}
impl<T: RealNumber, M: Matrix<T>> NBDistribution<T, M> for GaussianNBDistribution<T> {
fn prior(&self, class_index: usize) -> T {
impl<X: Number + RealNumber, Y: Number + Ord + Unsigned> NBDistribution<X, Y>
for GaussianNBDistribution<Y>
{
fn prior(&self, class_index: usize) -> f64 {
if class_index >= self.class_labels.len() {
T::zero()
0f64
} else {
self.class_priors[class_index]
}
}
fn log_likelihood(&self, class_index: usize, j: &M::RowVector) -> T {
let mut likelihood = T::zero();
for feature in 0..j.len() {
let value = j.get(feature);
fn log_likelihood<'a>(&self, class_index: usize, j: &'a Box<dyn ArrayView1<X> + 'a>) -> f64 {
let mut likelihood = 0f64;
for feature in 0..j.shape() {
let value = X::to_f64(j.get(feature)).unwrap();
let mean = self.theta[class_index][feature];
let variance = self.var[class_index][feature];
likelihood += self.calculate_log_probability(value, mean, variance);
@@ -69,52 +71,54 @@ impl<T: RealNumber, M: Matrix<T>> NBDistribution<T, M> for GaussianNBDistributio
likelihood
}
fn classes(&self) -> &Vec<T> {
fn classes(&self) -> &Vec<Y> {
&self.class_labels
}
}
/// `GaussianNB` parameters. Use `Default::default()` for default values.
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone)]
pub struct GaussianNBParameters<T: RealNumber> {
#[derive(Debug, Default, Clone)]
pub struct GaussianNBParameters {
#[cfg_attr(feature = "serde", serde(default))]
/// Prior probabilities of the classes. If specified the priors are not adjusted according to the data
pub priors: Option<Vec<T>>,
pub priors: Option<Vec<f64>>,
}
impl<T: RealNumber> GaussianNBParameters<T> {
impl GaussianNBParameters {
/// Prior probabilities of the classes. If specified the priors are not adjusted according to the data
pub fn with_priors(mut self, priors: Vec<T>) -> Self {
pub fn with_priors(mut self, priors: Vec<f64>) -> Self {
self.priors = Some(priors);
self
}
}
impl<T: RealNumber> Default for GaussianNBParameters<T> {
impl GaussianNBParameters {
fn default() -> Self {
Self { priors: None }
Self {
priors: Option::None,
}
}
}
/// GaussianNB grid search parameters
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, Clone)]
pub struct GaussianNBSearchParameters<T: RealNumber> {
pub struct GaussianNBSearchParameters {
#[cfg_attr(feature = "serde", serde(default))]
/// Prior probabilities of the classes. If specified the priors are not adjusted according to the data
pub priors: Vec<Option<Vec<T>>>,
pub priors: Vec<Option<Vec<f64>>>,
}
/// GaussianNB grid search iterator
pub struct GaussianNBSearchParametersIterator<T: RealNumber> {
gaussian_nb_search_parameters: GaussianNBSearchParameters<T>,
pub struct GaussianNBSearchParametersIterator {
gaussian_nb_search_parameters: GaussianNBSearchParameters,
current_priors: usize,
}
impl<T: RealNumber> IntoIterator for GaussianNBSearchParameters<T> {
type Item = GaussianNBParameters<T>;
type IntoIter = GaussianNBSearchParametersIterator<T>;
impl IntoIterator for GaussianNBSearchParameters {
type Item = GaussianNBParameters;
type IntoIter = GaussianNBSearchParametersIterator;
fn into_iter(self) -> Self::IntoIter {
GaussianNBSearchParametersIterator {
@@ -124,8 +128,8 @@ impl<T: RealNumber> IntoIterator for GaussianNBSearchParameters<T> {
}
}
impl<T: RealNumber> Iterator for GaussianNBSearchParametersIterator<T> {
type Item = GaussianNBParameters<T>;
impl Iterator for GaussianNBSearchParametersIterator {
type Item = GaussianNBParameters;
fn next(&mut self) -> Option<Self::Item> {
if self.current_priors == self.gaussian_nb_search_parameters.priors.len() {
@@ -142,7 +146,7 @@ impl<T: RealNumber> Iterator for GaussianNBSearchParametersIterator<T> {
}
}
impl<T: RealNumber> Default for GaussianNBSearchParameters<T> {
impl Default for GaussianNBSearchParameters {
fn default() -> Self {
let default_params = GaussianNBParameters::default();
@@ -152,19 +156,19 @@ impl<T: RealNumber> Default for GaussianNBSearchParameters<T> {
}
}
impl<T: RealNumber> GaussianNBDistribution<T> {
impl<TY: Number + Ord + Unsigned> GaussianNBDistribution<TY> {
/// Fits the distribution to a NxM matrix where N is number of samples and M is number of features.
/// * `x` - training data.
/// * `y` - vector with target values (classes) of length N.
/// * `priors` - Optional vector with prior probabilities of the classes. If not defined,
/// priors are adjusted according to the data.
pub fn fit<M: Matrix<T>>(
x: &M,
y: &M::RowVector,
priors: Option<Vec<T>>,
pub fn fit<TX: Number + RealNumber, X: Array2<TX>, Y: Array1<TY>>(
x: &X,
y: &Y,
priors: Option<Vec<f64>>,
) -> Result<Self, Failed> {
let (n_samples, n_features) = x.shape();
let y_samples = y.len();
let (n_samples, _) = x.shape();
let y_samples = y.shape();
if y_samples != n_samples {
return Err(Failed::fit(&format!(
"Size of x should equal size of y; |x|=[{}], |y|=[{}]",
@@ -178,14 +182,14 @@ impl<T: RealNumber> GaussianNBDistribution<T> {
n_samples
)));
}
let y = y.to_vec();
let (class_labels, indices) = <Vec<T> as RealNumberVector<T>>::unique_with_indices(&y);
let (class_labels, indices) = y.unique_with_indices();
let mut class_count = vec![0_usize; class_labels.len()];
let mut subdataset: Vec<Vec<Vec<T>>> = vec![vec![]; class_labels.len()];
let mut subdataset: Vec<Vec<Box<dyn ArrayView1<TX>>>> =
(0..class_labels.len()).map(|_| vec![]).collect();
for (row, class_index) in row_iter(x).zip(indices.iter()) {
for (row, class_index) in x.row_iter().zip(indices.iter()) {
class_count[*class_index] += 1;
subdataset[*class_index].push(row);
}
@@ -200,26 +204,25 @@ impl<T: RealNumber> GaussianNBDistribution<T> {
} else {
class_count
.iter()
.map(|&c| T::from(c).unwrap() / T::from(n_samples).unwrap())
.map(|&c| c as f64 / n_samples as f64)
.collect()
};
let subdataset: Vec<M> = subdataset
.into_iter()
let subdataset: Vec<X> = subdataset
.iter()
.map(|v| {
let mut m = M::zeros(v.len(), n_features);
for (row_i, v_i) in v.iter().enumerate() {
for (col_j, v_i_j) in v_i.iter().enumerate().take(n_features) {
m.set(row_i, col_j, *v_i_j);
}
}
m
X::concatenate_1d(
&v.iter()
.map(|v| v.as_ref())
.collect::<Vec<&dyn ArrayView1<TX>>>(),
0,
)
})
.collect();
let (var, theta): (Vec<Vec<T>>, Vec<Vec<T>>) = subdataset
let (var, theta): (Vec<Vec<f64>>, Vec<Vec<f64>>) = subdataset
.iter()
.map(|data| (data.var(0), data.mean(0)))
.map(|data| (data.variance(0), data.mean_by(0)))
.unzip();
Ok(Self {
@@ -233,11 +236,11 @@ impl<T: RealNumber> GaussianNBDistribution<T> {
/// Calculate probability of x equals to a value of a Gaussian distribution given its mean and its
/// variance.
fn calculate_log_probability(&self, value: T, mean: T, variance: T) -> T {
let pi = T::from(std::f64::consts::PI).unwrap();
-((value - mean).powf(T::two()) / (T::two() * variance))
- (T::two() * pi).ln() / T::two()
- (variance).ln() / T::two()
fn calculate_log_probability(&self, value: f64, mean: f64, variance: f64) -> f64 {
let pi = std::f64::consts::PI;
-((value - mean).powf(2.0) / (2.0 * variance))
- (2.0 * pi).ln() / 2.0
- (variance).ln() / 2.0
}
}
@@ -245,82 +248,101 @@ impl<T: RealNumber> GaussianNBDistribution<T> {
/// distribution.
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[derive(Debug, PartialEq)]
pub struct GaussianNB<T: RealNumber, M: Matrix<T>> {
inner: BaseNaiveBayes<T, M, GaussianNBDistribution<T>>,
pub struct GaussianNB<
TX: Number + RealNumber + RealNumber,
TY: Number + Ord + Unsigned,
X: Array2<TX>,
Y: Array1<TY>,
> {
inner: Option<BaseNaiveBayes<TX, TY, X, Y, GaussianNBDistribution<TY>>>,
}
impl<T: RealNumber, M: Matrix<T>> SupervisedEstimator<M, M::RowVector, GaussianNBParameters<T>>
for GaussianNB<T, M>
impl<
TX: Number + RealNumber + RealNumber,
TY: Number + Ord + Unsigned,
X: Array2<TX>,
Y: Array1<TY>,
> SupervisedEstimator<X, Y, GaussianNBParameters> for GaussianNB<TX, TY, X, Y>
{
fn fit(x: &M, y: &M::RowVector, parameters: GaussianNBParameters<T>) -> Result<Self, Failed> {
fn new() -> Self {
Self {
inner: Option::None,
}
}
fn fit(x: &X, y: &Y, parameters: GaussianNBParameters) -> Result<Self, Failed> {
GaussianNB::fit(x, y, parameters)
}
}
impl<T: RealNumber, M: Matrix<T>> Predictor<M, M::RowVector> for GaussianNB<T, M> {
fn predict(&self, x: &M) -> Result<M::RowVector, Failed> {
impl<
TX: Number + RealNumber + RealNumber,
TY: Number + Ord + Unsigned,
X: Array2<TX>,
Y: Array1<TY>,
> Predictor<X, Y> for GaussianNB<TX, TY, X, Y>
{
fn predict(&self, x: &X) -> Result<Y, Failed> {
self.predict(x)
}
}
impl<T: RealNumber, M: Matrix<T>> GaussianNB<T, M> {
impl<TX: Number + RealNumber, TY: Number + Ord + Unsigned, X: Array2<TX>, Y: Array1<TY>>
GaussianNB<TX, TY, X, Y>
{
/// Fits GaussianNB with given data
/// * `x` - training data of size NxM where N is the number of samples and M is the number of
/// features.
/// * `y` - vector with target values (classes) of length N.
/// * `parameters` - additional parameters like class priors.
pub fn fit(
x: &M,
y: &M::RowVector,
parameters: GaussianNBParameters<T>,
) -> Result<Self, Failed> {
pub fn fit(x: &X, y: &Y, parameters: GaussianNBParameters) -> Result<Self, Failed> {
let distribution = GaussianNBDistribution::fit(x, y, parameters.priors)?;
let inner = BaseNaiveBayes::fit(distribution)?;
Ok(Self { inner })
Ok(Self { inner: Some(inner) })
}
/// Estimates the class labels for the provided data.
/// * `x` - data of shape NxM where N is number of data points to estimate and M is number of features.
/// Returns a vector of size N with class estimates.
pub fn predict(&self, x: &M) -> Result<M::RowVector, Failed> {
self.inner.predict(x)
pub fn predict(&self, x: &X) -> Result<Y, Failed> {
self.inner.as_ref().unwrap().predict(x)
}
/// Class labels known to the classifier.
/// Returns a vector of size n_classes.
pub fn classes(&self) -> &Vec<T> {
&self.inner.distribution.class_labels
pub fn classes(&self) -> &Vec<TY> {
&self.inner.as_ref().unwrap().distribution.class_labels
}
/// Number of training samples observed in each class.
/// Returns a vector of size n_classes.
pub fn class_count(&self) -> &Vec<usize> {
&self.inner.distribution.class_count
&self.inner.as_ref().unwrap().distribution.class_count
}
/// Probability of each class
/// Returns a vector of size n_classes.
pub fn class_priors(&self) -> &Vec<T> {
&self.inner.distribution.class_priors
pub fn class_priors(&self) -> &Vec<f64> {
&self.inner.as_ref().unwrap().distribution.class_priors
}
/// Mean of each feature per class
/// Returns a 2d vector of shape (n_classes, n_features).
pub fn theta(&self) -> &Vec<Vec<T>> {
&self.inner.distribution.theta
pub fn theta(&self) -> &Vec<Vec<f64>> {
&self.inner.as_ref().unwrap().distribution.theta
}
/// Variance of each feature per class
/// Returns a 2d vector of shape (n_classes, n_features).
pub fn var(&self) -> &Vec<Vec<T>> {
&self.inner.distribution.var
pub fn var(&self) -> &Vec<Vec<f64>> {
&self.inner.as_ref().unwrap().distribution.var
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::linalg::naive::dense_matrix::DenseMatrix;
use crate::linalg::basic::matrix::DenseMatrix;
#[test]
fn search_parameters() {
@@ -347,13 +369,13 @@ mod tests {
&[2., 1.],
&[3., 2.],
]);
let y = vec![1., 1., 1., 2., 2., 2.];
let y: Vec<u32> = vec![1, 1, 1, 2, 2, 2];
let gnb = GaussianNB::fit(&x, &y, Default::default()).unwrap();
let y_hat = gnb.predict(&x).unwrap();
assert_eq!(y_hat, y);
assert_eq!(gnb.classes(), &[1., 2.]);
assert_eq!(gnb.classes(), &[1, 2]);
assert_eq!(gnb.class_count(), &[3, 3]);
@@ -384,7 +406,7 @@ mod tests {
&[2., 1.],
&[3., 2.],
]);
let y = vec![1., 1., 1., 2., 2., 2.];
let y: Vec<u32> = vec![1, 1, 1, 2, 2, 2];
let priors = vec![0.3, 0.7];
let parameters = GaussianNBParameters::default().with_priors(priors.clone());
@@ -393,24 +415,25 @@ mod tests {
assert_eq!(gnb.class_priors(), &priors);
}
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
#[test]
#[cfg(feature = "serde")]
fn serde() {
let x = DenseMatrix::<f64>::from_2d_array(&[
&[-1., -1.],
&[-2., -1.],
&[-3., -2.],
&[1., 1.],
&[2., 1.],
&[3., 2.],
]);
let y = vec![1., 1., 1., 2., 2., 2.];
// TODO: implement serialization
// #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
// #[test]
// #[cfg(feature = "serde")]
// fn serde() {
// let x = DenseMatrix::<f64>::from_2d_array(&[
// &[-1., -1.],
// &[-2., -1.],
// &[-3., -2.],
// &[1., 1.],
// &[2., 1.],
// &[3., 2.],
// ]);
// let y: Vec<u32> = vec![1, 1, 1, 2, 2, 2];
let gnb = GaussianNB::fit(&x, &y, Default::default()).unwrap();
let deserialized_gnb: GaussianNB<f64, DenseMatrix<f64>> =
serde_json::from_str(&serde_json::to_string(&gnb).unwrap()).unwrap();
// let gnb = GaussianNB::fit(&x, &y, Default::default()).unwrap();
// let deserialized_gnb: GaussianNB<f64, u32, DenseMatrix<f64>, Vec<u32>> =
// serde_json::from_str(&serde_json::to_string(&gnb).unwrap()).unwrap();
assert_eq!(gnb, deserialized_gnb);
}
// assert_eq!(gnb, deserialized_gnb);
// }
}