work/smartcore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: adds 3 more SVM kernels, linalg refactoring

Browse Source
This commit is contained in:
Volodymyr Orlov
2020-10-28 17:10:17 -07:00
parent 1773ed0e6e
commit cf4f658f01
6 changed files with 568 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/svm/mod.rs
+146 -1
View File
@@ -1,5 +1,7 @@
//! # Support Vector Machines
//!
//! <script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
//! <script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
pub mod svc;
pub mod svr;
@@ -9,18 +11,161 @@ use serde::{Deserialize, Serialize};
use crate::linalg::BaseVector;
use crate::math::num::RealNumber;
/// Kernel
/// Defines a kernel function
pub trait Kernel<T: RealNumber, V: BaseVector<T>> {
/// Apply kernel function to x_i and x_j
fn apply(&self, x_i: &V, x_j: &V) -> T;
}
/// Pre-defined kernel functions
pub struct Kernels {}
impl Kernels {
/// Linear kernel
pub fn linear() -> LinearKernel {
LinearKernel {}
}
/// Radial basis function kernel (Gaussian)
pub fn rbf<T: RealNumber>(gamma: T) -> RBFKernel<T> {
RBFKernel { gamma: gamma }
}
/// Polynomial kernel
/// * `degree` - degree of the polynomial
/// * `gamma` - kernel coefficient
/// * `coef0` - independent term in kernel function
pub fn polynomial<T: RealNumber>(degree: T, gamma: T, coef0: T) -> PolynomialKernel<T> {
PolynomialKernel {
degree: degree,
gamma: gamma,
coef0: coef0,
}
}
/// Polynomial kernel
/// * `degree` - degree of the polynomial
/// * `n_features` - number of features in vector
pub fn polynomial_with_degree<T: RealNumber>(
degree: T,
n_features: usize,
) -> PolynomialKernel<T> {
let coef0 = T::one();
let gamma = T::one() / T::from_usize(n_features).unwrap();
Kernels::polynomial(degree, gamma, coef0)
}
/// Sigmoid kernel
/// * `gamma` - kernel coefficient
/// * `coef0` - independent term in kernel function
pub fn sigmoid<T: RealNumber>(gamma: T, coef0: T) -> SigmoidKernel<T> {
SigmoidKernel {
gamma: gamma,
coef0: coef0,
}
}
/// Sigmoid kernel
/// * `gamma` - kernel coefficient
pub fn sigmoid_with_gamma<T: RealNumber>(gamma: T) -> SigmoidKernel<T> {
SigmoidKernel {
gamma: gamma,
coef0: T::one(),
}
}
}
/// Linear Kernel
#[derive(Serialize, Deserialize, Debug)]
pub struct LinearKernel {}
/// Radial basis function (Gaussian) kernel
pub struct RBFKernel<T: RealNumber> {
/// kernel coefficient
pub gamma: T,
}
/// Polynomial kernel
pub struct PolynomialKernel<T: RealNumber> {
/// degree of the polynomial
pub degree: T,
/// kernel coefficient
pub gamma: T,
/// independent term in kernel function
pub coef0: T,
}
/// Sigmoid (hyperbolic tangent) kernel
pub struct SigmoidKernel<T: RealNumber> {
/// kernel coefficient
pub gamma: T,
/// independent term in kernel function
pub coef0: T,
}
impl<T: RealNumber, V: BaseVector<T>> Kernel<T, V> for LinearKernel {
fn apply(&self, x_i: &V, x_j: &V) -> T {
x_i.dot(x_j)
}
}
impl<T: RealNumber, V: BaseVector<T>> Kernel<T, V> for RBFKernel<T> {
fn apply(&self, x_i: &V, x_j: &V) -> T {
let v_diff = x_i.sub(x_j);
(-self.gamma * v_diff.mul(&v_diff).sum()).exp()
}
}
impl<T: RealNumber, V: BaseVector<T>> Kernel<T, V> for PolynomialKernel<T> {
fn apply(&self, x_i: &V, x_j: &V) -> T {
let dot = x_i.dot(x_j);
(self.gamma * dot + self.coef0).powf(self.degree)
}
}
impl<T: RealNumber, V: BaseVector<T>> Kernel<T, V> for SigmoidKernel<T> {
fn apply(&self, x_i: &V, x_j: &V) -> T {
let dot = x_i.dot(x_j);
(self.gamma * dot + self.coef0).tanh()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn linear_kernel() {
let v1 = vec![1., 2., 3.];
let v2 = vec![4., 5., 6.];
assert_eq!(32f64, Kernels::linear().apply(&v1, &v2));
}
#[test]
fn rbf_kernel() {
let v1 = vec![1., 2., 3.];
let v2 = vec![4., 5., 6.];
assert!((0.2265f64 - Kernels::rbf(0.055).apply(&v1, &v2)).abs() < 1e-4);
}
#[test]
fn polynomial_kernel() {
let v1 = vec![1., 2., 3.];
let v2 = vec![4., 5., 6.];
assert!(
(4913f64 - Kernels::polynomial(3.0, 0.5, 1.0).apply(&v1, &v2)).abs()
< std::f64::EPSILON
);
}
#[test]
fn sigmoid_kernel() {
let v1 = vec![1., 2., 3.];
let v2 = vec![4., 5., 6.];
assert!((0.3969f64 - Kernels::sigmoid(0.01, 0.1).apply(&v1, &v2)).abs() < 1e-4);
}
}
src/svm/svc.rs
+81 -8
View File
@@ -5,7 +5,7 @@
//! ```
//! use smartcore::linalg::naive::dense_matrix::*;
//! use smartcore::linear::linear_regression::*;
//! use smartcore::svm::LinearKernel;
//! use smartcore::svm::Kernels;
//! use smartcore::svm::svc::{SVC, SVCParameters};
//!
//! // Iris dataset
@@ -31,11 +31,11 @@
//! &[6.6, 2.9, 4.6, 1.3],
//! &[5.2, 2.7, 3.9, 1.4],
//! ]);
//! let y = vec![ -1., -1., -1., -1., -1., -1., -1., -1.,
//! let y = vec![ 0., 0., 0., 0., 0., 0., 0., 0.,
//! 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.];
//!
//! let svr = SVC::fit(&x, &y,
//! LinearKernel {},
//! Kernels::linear(),
//! SVCParameters {
//! epoch: 2,
//! c: 200.0,
@@ -83,6 +83,7 @@ pub struct SVCParameters<T: RealNumber> {
))]
/// Support Vector Classifier
pub struct SVC<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> {
classes: Vec<T>,
kernel: K,
instances: Vec<M::RowVector>,
w: Vec<T>,
@@ -150,11 +151,32 @@ impl<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> SVC<T, M, K> {
)));
}
let optimizer = Optimizer::new(x, y, &kernel, &parameters);
let classes = y.unique();
if classes.len() != 2 {
return Err(Failed::fit(&format!(
"Incorrect number of classes {}", classes.len()
)));
}
// Make sure class labels are either 1 or -1
let mut y = y.clone();
for i in 0..y.len() {
let y_v = y.get(i);
if y_v != -T::one() || y_v != T::one() {
match y_v == classes[0] {
true => y.set(i, -T::one()),
false => y.set(i, T::one())
}
}
}
let optimizer = Optimizer::new(x, &y, &kernel, &parameters);
let (support_vectors, weight, b) = optimizer.optimize();
Ok(SVC {
classes: classes,
kernel: kernel,
instances: support_vectors,
w: weight,
@@ -170,7 +192,11 @@ impl<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> SVC<T, M, K> {
let mut y_hat = M::RowVector::zeros(n);
for i in 0..n {
y_hat.set(i, self.predict_for_row(x.get_row(i)));
let cls_idx = match self.predict_for_row(x.get_row(i)) == T::one() {
false => self.classes[0],
true => self.classes[1]
};
y_hat.set(i, cls_idx);
}
Ok(y_hat)
@@ -647,13 +673,13 @@ mod tests {
]);
let y: Vec<f64> = vec![
-1., -1., -1., -1., -1., -1., -1., -1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
];
let y_hat = SVC::fit(
&x,
&y,
LinearKernel {},
Kernels::linear(),
SVCParameters {
epoch: 2,
c: 200.0,
@@ -663,6 +689,53 @@ mod tests {
.and_then(|lr| lr.predict(&x))
.unwrap();
println!("{:?}", y_hat);
assert!(accuracy(&y_hat, &y) >= 0.9);
}
#[test]
fn svc_fit_predict_rbf() {
let x = DenseMatrix::from_2d_array(&[
&[5.1, 3.5, 1.4, 0.2],
&[4.9, 3.0, 1.4, 0.2],
&[4.7, 3.2, 1.3, 0.2],
&[4.6, 3.1, 1.5, 0.2],
&[5.0, 3.6, 1.4, 0.2],
&[5.4, 3.9, 1.7, 0.4],
&[4.6, 3.4, 1.4, 0.3],
&[5.0, 3.4, 1.5, 0.2],
&[4.4, 2.9, 1.4, 0.2],
&[4.9, 3.1, 1.5, 0.1],
&[7.0, 3.2, 4.7, 1.4],
&[6.4, 3.2, 4.5, 1.5],
&[6.9, 3.1, 4.9, 1.5],
&[5.5, 2.3, 4.0, 1.3],
&[6.5, 2.8, 4.6, 1.5],
&[5.7, 2.8, 4.5, 1.3],
&[6.3, 3.3, 4.7, 1.6],
&[4.9, 2.4, 3.3, 1.0],
&[6.6, 2.9, 4.6, 1.3],
&[5.2, 2.7, 3.9, 1.4],
]);
let y: Vec<f64> = vec![
-1., -1., -1., -1., -1., -1., -1., -1., -1., -1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
];
let y_hat = SVC::fit(
&x,
&y,
Kernels::rbf(0.7),
SVCParameters {
epoch: 2,
c: 1.0,
tol: 1e-3,
},
)
.and_then(|lr| lr.predict(&x))
.unwrap();
assert!(accuracy(&y_hat, &y) >= 0.9);
}
@@ -695,7 +768,7 @@ mod tests {
-1., -1., -1., -1., -1., -1., -1., -1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,
];
let svr = SVC::fit(&x, &y, LinearKernel {}, Default::default()).unwrap();
let svr = SVC::fit(&x, &y, Kernels::linear(), Default::default()).unwrap();
let deserialized_svr: SVC<f64, DenseMatrix<f64>, LinearKernel> =
serde_json::from_str(&serde_json::to_string(&svr).unwrap()).unwrap();