work/smartcore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge potential next release v0.4 (#187) Breaking Changes

Browse Source 
* First draft of the new n-dimensional arrays + NB use case
* Improves default implementation of multiple Array methods
* Refactors tree methods
* Adds matrix decomposition routines
* Adds matrix decomposition methods to ndarray and nalgebra bindings
* Refactoring + linear regression now uses array2
* Ridge & Linear regression
* LBFGS optimizer & logistic regression
* LBFGS optimizer & logistic regression
* Changes linear methods, metrics and model selection methods to new n-dimensional arrays
* Switches KNN and clustering algorithms to new n-d array layer
* Refactors distance metrics
* Optimizes knn and clustering methods
* Refactors metrics module
* Switches decomposition methods to n-dimensional arrays
* Linalg refactoring - cleanup rng merge (#172)
* Remove legacy DenseMatrix and BaseMatrix implementation. Port the new Number, FloatNumber and Array implementation into module structure.
* Exclude AUC metrics. Needs reimplementation
* Improve developers walkthrough

New traits system in place at `src/numbers` and `src/linalg`
Co-authored-by: Lorenzo <tunedconsulting@gmail.com>

* Provide SupervisedEstimator with a constructor to avoid explicit dynamical box allocation in 'cross_validate' and 'cross_validate_predict' as required by the use of 'dyn' as per Rust 2021
* Implement getters to use as_ref() in src/neighbors
* Implement getters to use as_ref() in src/naive_bayes
* Implement getters to use as_ref() in src/linear
* Add Clone to src/naive_bayes
* Change signature for cross_validate and other model_selection functions to abide to use of dyn in Rust 2021
* Implement ndarray-bindings. Remove FloatNumber from implementations
* Drop nalgebra-bindings support (as decided in conf-call to go for ndarray)
* Remove benches. Benches will have their own repo at smartcore-benches
* Implement SVC
* Implement SVC serialization. Move search parameters in dedicated module
* Implement SVR. Definitely too slow
* Fix compilation issues for wasm (#202)

Co-authored-by: Luis Moreno <morenol@users.noreply.github.com>
* Fix tests (#203)

* Port linalg/traits/stats.rs
* Improve methods naming
* Improve Display for DenseMatrix

Co-authored-by: Montana Low <montanalow@users.noreply.github.com>
Co-authored-by: VolodymyrOrlov <volodymyr.orlov@gmail.com>
This commit is contained in:
Lorenzo
2022-10-31 10:44:57 +00:00
committed by GitHub
parent bb71656137
commit 52eb6ce023
110 changed files with 10327 additions and 9107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/optimization/first_order/gradient_descent.rs
+35 -30
View File
@@ -1,29 +1,38 @@
// TODO: missing documentation
use std::default::Default;
use crate::linalg::Matrix;
use crate::math::num::RealNumber;
use crate::linalg::basic::arrays::Array1;
use crate::numbers::floatnum::FloatNumber;
use crate::optimization::first_order::{FirstOrderOptimizer, OptimizerResult};
use crate::optimization::line_search::LineSearchMethod;
use crate::optimization::{DF, F};
pub struct GradientDescent<T: RealNumber> {
///
pub struct GradientDescent {
///
pub max_iter: usize,
pub g_rtol: T,
pub g_atol: T,
///
pub g_rtol: f64,
///
pub g_atol: f64,
}
impl<T: RealNumber> Default for GradientDescent<T> {
///
impl Default for GradientDescent {
fn default() -> Self {
GradientDescent {
max_iter: 10000,
g_rtol: T::epsilon().sqrt(),
g_atol: T::epsilon(),
g_rtol: std::f64::EPSILON.sqrt(),
g_atol: std::f64::EPSILON,
}
}
}
impl<T: RealNumber> FirstOrderOptimizer<T> for GradientDescent<T> {
fn optimize<'a, X: Matrix<T>, LS: LineSearchMethod<T>>(
///
impl<T: FloatNumber> FirstOrderOptimizer<T> for GradientDescent {
///
fn optimize<'a, X: Array1<T>, LS: LineSearchMethod<T>>(
&self,
f: &'a F<'_, T, X>,
df: &'a DF<'_, X>,
@@ -45,19 +54,21 @@ impl<T: RealNumber> FirstOrderOptimizer<T> for GradientDescent<T> {
while iter < self.max_iter && (iter == 0 || gnorm > gtol) {
iter += 1;
let mut step = gvec.negative();
let mut step = gvec.neg();
let f_alpha = |alpha: T| -> T {
let mut dx = step.clone();
dx.mul_scalar_mut(alpha);
f(dx.add_mut(&x)) // f(x) = f(x .+ gvec .* alpha)
dx.add_mut(&x);
f(&dx) // f(x) = f(x .+ gvec .* alpha)
};
let df_alpha = |alpha: T| -> T {
let mut dx = step.clone();
let mut dg = gvec.clone();
dx.mul_scalar_mut(alpha);
df(&mut dg, dx.add_mut(&x)); //df(x) = df(x .+ gvec .* alpha)
dx.add_mut(&x);
df(&mut dg, &dx); //df(x) = df(x .+ gvec .* alpha)
gvec.dot(&dg)
};
@@ -66,7 +77,8 @@ impl<T: RealNumber> FirstOrderOptimizer<T> for GradientDescent<T> {
let ls_r = ls.search(&f_alpha, &df_alpha, alpha, fx, df0);
alpha = ls_r.alpha;
fx = ls_r.f_x;
x.add_mut(step.mul_scalar_mut(alpha));
step.mul_scalar_mut(alpha);
x.add_mut(&step);
df(&mut gvec, &x);
gnorm = gvec.norm2();
}
@@ -84,36 +96,29 @@ impl<T: RealNumber> FirstOrderOptimizer<T> for GradientDescent<T> {
#[cfg(test)]
mod tests {
use super::*;
use crate::linalg::naive::dense_matrix::*;
use crate::optimization::line_search::Backtracking;
use crate::optimization::FunctionOrder;
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
#[test]
fn gradient_descent() {
let x0 = DenseMatrix::row_vector_from_array(&[-1., 1.]);
let f = |x: &DenseMatrix<f64>| {
(1.0 - x.get(0, 0)).powf(2.) + 100.0 * (x.get(0, 1) - x.get(0, 0).powf(2.)).powf(2.)
};
let x0 = vec![-1., 1.];
let f = |x: &Vec<f64>| (1.0 - x[0]).powf(2.) + 100.0 * (x[1] - x[0].powf(2.)).powf(2.);
let df = |g: &mut DenseMatrix<f64>, x: &DenseMatrix<f64>| {
g.set(
0,
0,
-2. * (1. - x.get(0, 0))
- 400. * (x.get(0, 1) - x.get(0, 0).powf(2.)) * x.get(0, 0),
);
g.set(0, 1, 200. * (x.get(0, 1) - x.get(0, 0).powf(2.)));
let df = |g: &mut Vec<f64>, x: &Vec<f64>| {
g[0] = -2. * (1. - x[0]) - 400. * (x[1] - x[0].powf(2.)) * x[0];
g[1] = 200. * (x[1] - x[0].powf(2.));
};
let mut ls: Backtracking<f64> = Default::default();
ls.order = FunctionOrder::THIRD;
let optimizer: GradientDescent<f64> = Default::default();
let optimizer: GradientDescent = Default::default();
let result = optimizer.optimize(&f, &df, &x0, &ls);
println!("{:?}", result);
assert!((result.f_x - 0.0).abs() < 1e-5);
assert!((result.x.get(0, 0) - 1.0).abs() < 1e-2);
assert!((result.x.get(0, 1) - 1.0).abs() < 1e-2);
assert!((result.x[0] - 1.0).abs() < 1e-2);
assert!((result.x[1] - 1.0).abs() < 1e-2);
}
}
src/optimization/first_order/lbfgs.rs
+83 -52
View File
@@ -1,44 +1,60 @@
#![allow(clippy::suspicious_operation_groupings)]
// TODO: Add documentation
use std::default::Default;
use std::fmt::Debug;
use crate::linalg::Matrix;
use crate::math::num::RealNumber;
use crate::linalg::basic::arrays::Array1;
use crate::numbers::floatnum::FloatNumber;
use crate::numbers::realnum::RealNumber;
use crate::optimization::first_order::{FirstOrderOptimizer, OptimizerResult};
use crate::optimization::line_search::LineSearchMethod;
use crate::optimization::{DF, F};
#[allow(clippy::upper_case_acronyms)]
pub struct LBFGS<T: RealNumber> {
///
pub struct LBFGS {
///
pub max_iter: usize,
pub g_rtol: T,
pub g_atol: T,
pub x_atol: T,
pub x_rtol: T,
pub f_abstol: T,
pub f_reltol: T,
///
pub g_rtol: f64,
///
pub g_atol: f64,
///
pub x_atol: f64,
///
pub x_rtol: f64,
///
pub f_abstol: f64,
///
pub f_reltol: f64,
///
pub successive_f_tol: usize,
///
pub m: usize,
}
impl<T: RealNumber> Default for LBFGS<T> {
///
impl Default for LBFGS {
///
fn default() -> Self {
LBFGS {
max_iter: 1000,
g_rtol: T::from(1e-8).unwrap(),
g_atol: T::from(1e-8).unwrap(),
x_atol: T::zero(),
x_rtol: T::zero(),
f_abstol: T::zero(),
f_reltol: T::zero(),
g_rtol: 1e-8,
g_atol: 1e-8,
x_atol: 0f64,
x_rtol: 0f64,
f_abstol: 0f64,
f_reltol: 0f64,
successive_f_tol: 1,
m: 10,
}
}
}
impl<T: RealNumber> LBFGS<T> {
fn two_loops<X: Matrix<T>>(&self, state: &mut LBFGSState<T, X>) {
///
impl LBFGS {
///
fn two_loops<T: FloatNumber + RealNumber, X: Array1<T>>(&self, state: &mut LBFGSState<T, X>) {
let lower = state.iteration.max(self.m) - self.m;
let upper = state.iteration;
@@ -58,7 +74,9 @@ impl<T: RealNumber> LBFGS<T> {
let i = (upper - 1).rem_euclid(self.m);
let dxi = &state.dx_history[i];
let dgi = &state.dg_history[i];
let scaling = dxi.dot(dgi) / dgi.abs().pow_mut(T::two()).sum();
let mut div = dgi.abs();
div.pow_mut(RealNumber::two());
let scaling = dxi.dot(dgi) / div.sum();
state.s.copy_from(&state.twoloop_q.mul_scalar(scaling));
} else {
state.s.copy_from(&state.twoloop_q);
@@ -77,7 +95,8 @@ impl<T: RealNumber> LBFGS<T> {
state.s.mul_scalar_mut(-T::one());
}
fn init_state<X: Matrix<T>>(&self, x: &X) -> LBFGSState<T, X> {
///
fn init_state<T: FloatNumber + RealNumber, X: Array1<T>>(&self, x: &X) -> LBFGSState<T, X> {
LBFGSState {
x: x.clone(),
x_prev: x.clone(),
@@ -100,7 +119,8 @@ impl<T: RealNumber> LBFGS<T> {
}
}
fn update_state<'a, X: Matrix<T>, LS: LineSearchMethod<T>>(
///
fn update_state<'a, T: FloatNumber + RealNumber, X: Array1<T>, LS: LineSearchMethod<T>>(
&self,
f: &'a F<'_, T, X>,
df: &'a DF<'_, X>,
@@ -118,53 +138,69 @@ impl<T: RealNumber> LBFGS<T> {
let f_alpha = |alpha: T| -> T {
let mut dx = state.s.clone();
dx.mul_scalar_mut(alpha);
f(dx.add_mut(&state.x)) // f(x) = f(x .+ gvec .* alpha)
dx.add_mut(&state.x);
f(&dx) // f(x) = f(x .+ gvec .* alpha)
};
let df_alpha = |alpha: T| -> T {
let mut dx = state.s.clone();
let mut dg = state.x_df.clone();
dx.mul_scalar_mut(alpha);
df(&mut dg, dx.add_mut(&state.x)); //df(x) = df(x .+ gvec .* alpha)
dx.add_mut(&state.x);
df(&mut dg, &dx); //df(x) = df(x .+ gvec .* alpha)
state.x_df.dot(&dg)
};
let ls_r = ls.search(&f_alpha, &df_alpha, T::one(), state.x_f_prev, df0);
state.alpha = ls_r.alpha;
state.dx.copy_from(state.s.mul_scalar_mut(state.alpha));
state.s.mul_scalar_mut(state.alpha);
state.dx.copy_from(&state.s);
state.x.add_mut(&state.dx);
state.x_f = f(&state.x);
df(&mut state.x_df, &state.x);
}
fn assess_convergence<X: Matrix<T>>(&self, state: &mut LBFGSState<T, X>) -> bool {
///
fn assess_convergence<T: FloatNumber, X: Array1<T>>(
&self,
state: &mut LBFGSState<T, X>,
) -> bool {
let (mut x_converged, mut g_converged) = (false, false);
if state.x.max_diff(&state.x_prev) <= self.x_atol {
if state.x.max_diff(&state.x_prev) <= T::from_f64(self.x_atol).unwrap() {
x_converged = true;
}
if state.x.max_diff(&state.x_prev) <= self.x_rtol * state.x.norm(T::infinity()) {
if state.x.max_diff(&state.x_prev)
<= T::from_f64(self.x_rtol * state.x.norm(std::f64::INFINITY)).unwrap()
{
x_converged = true;
}
if (state.x_f - state.x_f_prev).abs() <= self.f_abstol {
if (state.x_f - state.x_f_prev).abs() <= T::from_f64(self.f_abstol).unwrap() {
state.counter_f_tol += 1;
}
if (state.x_f - state.x_f_prev).abs() <= self.f_reltol * state.x_f.abs() {
if (state.x_f - state.x_f_prev).abs()
<= T::from_f64(self.f_reltol).unwrap() * state.x_f.abs()
{
state.counter_f_tol += 1;
}
if state.x_df.norm(T::infinity()) <= self.g_atol {
if state.x_df.norm(std::f64::INFINITY) <= self.g_atol {
g_converged = true;
}
g_converged || x_converged || state.counter_f_tol > self.successive_f_tol
}
fn update_hessian<'a, X: Matrix<T>>(&self, _: &'a DF<'_, X>, state: &mut LBFGSState<T, X>) {
///
fn update_hessian<'a, T: FloatNumber, X: Array1<T>>(
&self,
_: &'a DF<'_, X>,
state: &mut LBFGSState<T, X>,
) {
state.dg = state.x_df.sub(&state.x_df_prev);
let rho_iteration = T::one() / state.dx.dot(&state.dg);
if !rho_iteration.is_infinite() {
@@ -176,8 +212,9 @@ impl<T: RealNumber> LBFGS<T> {
}
}
///
#[derive(Debug)]
struct LBFGSState<T: RealNumber, X: Matrix<T>> {
struct LBFGSState<T: FloatNumber, X: Array1<T>> {
x: X,
x_prev: X,
x_f: T,
@@ -197,8 +234,10 @@ struct LBFGSState<T: RealNumber, X: Matrix<T>> {
alpha: T,
}
impl<T: RealNumber> FirstOrderOptimizer<T> for LBFGS<T> {
fn optimize<'a, X: Matrix<T>, LS: LineSearchMethod<T>>(
///
impl<T: FloatNumber + RealNumber> FirstOrderOptimizer<T> for LBFGS {
///
fn optimize<'a, X: Array1<T>, LS: LineSearchMethod<T>>(
&self,
f: &F<'_, T, X>,
df: &'a DF<'_, X>,
@@ -209,7 +248,7 @@ impl<T: RealNumber> FirstOrderOptimizer<T> for LBFGS<T> {
df(&mut state.x_df, x0);
let g_converged = state.x_df.norm(T::infinity()) < self.g_atol;
let g_converged = state.x_df.norm(std::f64::INFINITY) < self.g_atol;
let mut converged = g_converged;
let stopped = false;
@@ -236,36 +275,28 @@ impl<T: RealNumber> FirstOrderOptimizer<T> for LBFGS<T> {
#[cfg(test)]
mod tests {
use super::*;
use crate::linalg::naive::dense_matrix::*;
use crate::optimization::line_search::Backtracking;
use crate::optimization::FunctionOrder;
#[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
#[test]
fn lbfgs() {
let x0 = DenseMatrix::row_vector_from_array(&[0., 0.]);
let f = |x: &DenseMatrix<f64>| {
(1.0 - x.get(0, 0)).powf(2.) + 100.0 * (x.get(0, 1) - x.get(0, 0).powf(2.)).powf(2.)
};
let x0 = vec![0., 0.];
let f = |x: &Vec<f64>| (1.0 - x[0]).powf(2.) + 100.0 * (x[1] - x[0].powf(2.)).powf(2.);
let df = |g: &mut DenseMatrix<f64>, x: &DenseMatrix<f64>| {
g.set(
0,
0,
-2. * (1. - x.get(0, 0))
- 400. * (x.get(0, 1) - x.get(0, 0).powf(2.)) * x.get(0, 0),
);
g.set(0, 1, 200. * (x.get(0, 1) - x.get(0, 0).powf(2.)));
let df = |g: &mut Vec<f64>, x: &Vec<f64>| {
g[0] = -2. * (1. - x[0]) - 400. * (x[1] - x[0].powf(2.)) * x[0];
g[1] = 200. * (x[1] - x[0].powf(2.));
};
let mut ls: Backtracking<f64> = Default::default();
ls.order = FunctionOrder::THIRD;
let optimizer: LBFGS<f64> = Default::default();
let optimizer: LBFGS = Default::default();
let result = optimizer.optimize(&f, &df, &x0, &ls);
assert!((result.f_x - 0.0).abs() < std::f64::EPSILON);
assert!((result.x.get(0, 0) - 1.0).abs() < 1e-8);
assert!((result.x.get(0, 1) - 1.0).abs() < 1e-8);
assert!((result.x[0] - 1.0).abs() < 1e-8);
assert!((result.x[1] - 1.0).abs() < 1e-8);
assert!(result.iterations <= 24);
}
}
src/optimization/first_order/mod.rs
+13 -5
View File
@@ -1,16 +1,20 @@
///
pub mod gradient_descent;
///
pub mod lbfgs;
use std::clone::Clone;
use std::fmt::Debug;
use crate::linalg::Matrix;
use crate::math::num::RealNumber;
use crate::linalg::basic::arrays::Array1;
use crate::numbers::floatnum::FloatNumber;
use crate::optimization::line_search::LineSearchMethod;
use crate::optimization::{DF, F};
pub trait FirstOrderOptimizer<T: RealNumber> {
fn optimize<'a, X: Matrix<T>, LS: LineSearchMethod<T>>(
///
pub trait FirstOrderOptimizer<T: FloatNumber> {
///
fn optimize<'a, X: Array1<T>, LS: LineSearchMethod<T>>(
&self,
f: &F<'_, T, X>,
df: &'a DF<'_, X>,
@@ -19,9 +23,13 @@ pub trait FirstOrderOptimizer<T: RealNumber> {
) -> OptimizerResult<T, X>;
}
///
#[derive(Debug, Clone)]
pub struct OptimizerResult<T: RealNumber, X: Matrix<T>> {
pub struct OptimizerResult<T: FloatNumber, X: Array1<T>> {
///
pub x: X,
///
pub f_x: T,
///
pub iterations: usize,
}
src/optimization/line_search.rs
+17
View File
@@ -1,7 +1,11 @@
// TODO: missing documentation
use crate::optimization::FunctionOrder;
use num_traits::Float;
///
pub trait LineSearchMethod<T: Float> {
///
fn search(
&self,
f: &(dyn Fn(T) -> T),
@@ -12,21 +16,32 @@ pub trait LineSearchMethod<T: Float> {
) -> LineSearchResult<T>;
}
///
#[derive(Debug, Clone)]
pub struct LineSearchResult<T: Float> {
///
pub alpha: T,
///
pub f_x: T,
}
///
pub struct Backtracking<T: Float> {
///
pub c1: T,
///
pub max_iterations: usize,
///
pub max_infinity_iterations: usize,
///
pub phi: T,
///
pub plo: T,
///
pub order: FunctionOrder,
}
///
impl<T: Float> Default for Backtracking<T> {
fn default() -> Self {
Backtracking {
@@ -40,7 +55,9 @@ impl<T: Float> Default for Backtracking<T> {
}
}
///
impl<T: Float> LineSearchMethod<T> for Backtracking<T> {
///
fn search(
&self,
f: &(dyn Fn(T) -> T),
src/optimization/mod.rs
+9
View File
@@ -1,12 +1,21 @@
// TODO: missing documentation
///
pub mod first_order;
///
pub mod line_search;
///
pub type F<'a, T, X> = dyn for<'b> Fn(&'b X) -> T + 'a;
///
pub type DF<'a, X> = dyn for<'b> Fn(&'b mut X, &'b X) + 'a;
///
#[allow(clippy::upper_case_acronyms)]
#[derive(Debug, PartialEq, Eq)]
pub enum FunctionOrder {
///
SECOND,
///
THIRD,
}