Complete grid search params (#166)

* grid search draft

* hyperparam search for linear estimators

* grid search for ensembles

* support grid search for more algos

* grid search for unsupervised algos

* minor cleanup

src/svm/mod.rs

@@ -33,7 +33,7 @@ use crate::linalg::BaseVector;
 use crate::math::num::RealNumber;
 
 /// Defines a kernel function
-pub trait Kernel<T: RealNumber, V: BaseVector<T>> {
+pub trait Kernel<T: RealNumber, V: BaseVector<T>>: Clone {
     /// Apply kernel function to x_i and x_j
     fn apply(&self, x_i: &V, x_j: &V) -> T;
 }
@@ -95,12 +95,12 @@ impl Kernels {
 
 /// Linear Kernel
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct LinearKernel {}
 
 /// Radial basis function (Gaussian) kernel
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct RBFKernel<T: RealNumber> {
     /// kernel coefficient
     pub gamma: T,
@@ -108,7 +108,7 @@ pub struct RBFKernel<T: RealNumber> {
 
 /// Polynomial kernel
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct PolynomialKernel<T: RealNumber> {
     /// degree of the polynomial
     pub degree: T,
@@ -120,7 +120,7 @@ pub struct PolynomialKernel<T: RealNumber> {
 
 /// Sigmoid (hyperbolic tangent) kernel
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct SigmoidKernel<T: RealNumber> {
     /// kernel coefficient
     pub gamma: T,

src/svm/svc.rs

@@ -102,6 +102,109 @@ pub struct SVCParameters<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>
     m: PhantomData<M>,
 }
 
+/// SVC grid search parameters
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone)]
+pub struct SVCSearchParameters<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> {
+    /// Number of epochs.
+    pub epoch: Vec<usize>,
+    /// Regularization parameter.
+    pub c: Vec<T>,
+    /// Tolerance for stopping epoch.
+    pub tol: Vec<T>,
+    /// The kernel function.
+    pub kernel: Vec<K>,
+    /// Unused parameter.
+    m: PhantomData<M>,
+}
+
+/// SVC grid search iterator
+pub struct SVCSearchParametersIterator<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> {
+    svc_search_parameters: SVCSearchParameters<T, M, K>,
+    current_epoch: usize,
+    current_c: usize,
+    current_tol: usize,
+    current_kernel: usize,
+}
+
+impl<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> IntoIterator
+    for SVCSearchParameters<T, M, K>
+{
+    type Item = SVCParameters<T, M, K>;
+    type IntoIter = SVCSearchParametersIterator<T, M, K>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        SVCSearchParametersIterator {
+            svc_search_parameters: self,
+            current_epoch: 0,
+            current_c: 0,
+            current_tol: 0,
+            current_kernel: 0,
+        }
+    }
+}
+
+impl<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> Iterator
+    for SVCSearchParametersIterator<T, M, K>
+{
+    type Item = SVCParameters<T, M, K>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_epoch == self.svc_search_parameters.epoch.len()
+            && self.current_c == self.svc_search_parameters.c.len()
+            && self.current_tol == self.svc_search_parameters.tol.len()
+            && self.current_kernel == self.svc_search_parameters.kernel.len()
+        {
+            return None;
+        }
+
+        let next = SVCParameters::<T, M, K> {
+            epoch: self.svc_search_parameters.epoch[self.current_epoch],
+            c: self.svc_search_parameters.c[self.current_c],
+            tol: self.svc_search_parameters.tol[self.current_tol],
+            kernel: self.svc_search_parameters.kernel[self.current_kernel].clone(),
+            m: PhantomData,
+        };
+
+        if self.current_epoch + 1 < self.svc_search_parameters.epoch.len() {
+            self.current_epoch += 1;
+        } else if self.current_c + 1 < self.svc_search_parameters.c.len() {
+            self.current_epoch = 0;
+            self.current_c += 1;
+        } else if self.current_tol + 1 < self.svc_search_parameters.tol.len() {
+            self.current_epoch = 0;
+            self.current_c = 0;
+            self.current_tol += 1;
+        } else if self.current_kernel + 1 < self.svc_search_parameters.kernel.len() {
+            self.current_epoch = 0;
+            self.current_c = 0;
+            self.current_tol = 0;
+            self.current_kernel += 1;
+        } else {
+            self.current_epoch += 1;
+            self.current_c += 1;
+            self.current_tol += 1;
+            self.current_kernel += 1;
+        }
+
+        Some(next)
+    }
+}
+
+impl<T: RealNumber, M: Matrix<T>> Default for SVCSearchParameters<T, M, LinearKernel> {
+    fn default() -> Self {
+        let default_params: SVCParameters<T, M, LinearKernel> = SVCParameters::default();
+
+        SVCSearchParameters {
+            epoch: vec![default_params.epoch],
+            c: vec![default_params.c],
+            tol: vec![default_params.tol],
+            kernel: vec![default_params.kernel],
+            m: PhantomData,
+        }
+    }
+}
+
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[derive(Debug)]
 #[cfg_attr(
@@ -737,6 +840,24 @@ mod tests {
     #[cfg(feature = "serde")]
     use crate::svm::*;
 
+    #[test]
+    fn search_parameters() {
+        let parameters: SVCSearchParameters<f64, DenseMatrix<f64>, LinearKernel> =
+            SVCSearchParameters {
+                epoch: vec![10, 100],
+                kernel: vec![LinearKernel {}],
+                ..Default::default()
+            };
+        let mut iter = parameters.into_iter();
+        let next = iter.next().unwrap();
+        assert_eq!(next.epoch, 10);
+        assert_eq!(next.kernel, LinearKernel {});
+        let next = iter.next().unwrap();
+        assert_eq!(next.epoch, 100);
+        assert_eq!(next.kernel, LinearKernel {});
+        assert!(iter.next().is_none());
+    }
+
     #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
     #[test]
     fn svc_fit_predict() {

src/svm/svr.rs

@@ -94,6 +94,109 @@ pub struct SVRParameters<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>
     m: PhantomData<M>,
 }
 
+/// SVR grid search parameters
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone)]
+pub struct SVRSearchParameters<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> {
+    /// Epsilon in the epsilon-SVR model.
+    pub eps: Vec<T>,
+    /// Regularization parameter.
+    pub c: Vec<T>,
+    /// Tolerance for stopping eps.
+    pub tol: Vec<T>,
+    /// The kernel function.
+    pub kernel: Vec<K>,
+    /// Unused parameter.
+    m: PhantomData<M>,
+}
+
+/// SVR grid search iterator
+pub struct SVRSearchParametersIterator<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> {
+    svr_search_parameters: SVRSearchParameters<T, M, K>,
+    current_eps: usize,
+    current_c: usize,
+    current_tol: usize,
+    current_kernel: usize,
+}
+
+impl<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> IntoIterator
+    for SVRSearchParameters<T, M, K>
+{
+    type Item = SVRParameters<T, M, K>;
+    type IntoIter = SVRSearchParametersIterator<T, M, K>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        SVRSearchParametersIterator {
+            svr_search_parameters: self,
+            current_eps: 0,
+            current_c: 0,
+            current_tol: 0,
+            current_kernel: 0,
+        }
+    }
+}
+
+impl<T: RealNumber, M: Matrix<T>, K: Kernel<T, M::RowVector>> Iterator
+    for SVRSearchParametersIterator<T, M, K>
+{
+    type Item = SVRParameters<T, M, K>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.current_eps == self.svr_search_parameters.eps.len()
+            && self.current_c == self.svr_search_parameters.c.len()
+            && self.current_tol == self.svr_search_parameters.tol.len()
+            && self.current_kernel == self.svr_search_parameters.kernel.len()
+        {
+            return None;
+        }
+
+        let next = SVRParameters::<T, M, K> {
+            eps: self.svr_search_parameters.eps[self.current_eps],
+            c: self.svr_search_parameters.c[self.current_c],
+            tol: self.svr_search_parameters.tol[self.current_tol],
+            kernel: self.svr_search_parameters.kernel[self.current_kernel].clone(),
+            m: PhantomData,
+        };
+
+        if self.current_eps + 1 < self.svr_search_parameters.eps.len() {
+            self.current_eps += 1;
+        } else if self.current_c + 1 < self.svr_search_parameters.c.len() {
+            self.current_eps = 0;
+            self.current_c += 1;
+        } else if self.current_tol + 1 < self.svr_search_parameters.tol.len() {
+            self.current_eps = 0;
+            self.current_c = 0;
+            self.current_tol += 1;
+        } else if self.current_kernel + 1 < self.svr_search_parameters.kernel.len() {
+            self.current_eps = 0;
+            self.current_c = 0;
+            self.current_tol = 0;
+            self.current_kernel += 1;
+        } else {
+            self.current_eps += 1;
+            self.current_c += 1;
+            self.current_tol += 1;
+            self.current_kernel += 1;
+        }
+
+        Some(next)
+    }
+}
+
+impl<T: RealNumber, M: Matrix<T>> Default for SVRSearchParameters<T, M, LinearKernel> {
+    fn default() -> Self {
+        let default_params: SVRParameters<T, M, LinearKernel> = SVRParameters::default();
+
+        SVRSearchParameters {
+            eps: vec![default_params.eps],
+            c: vec![default_params.c],
+            tol: vec![default_params.tol],
+            kernel: vec![default_params.kernel],
+            m: PhantomData,
+        }
+    }
+}
+
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
 #[derive(Debug)]
 #[cfg_attr(
@@ -536,6 +639,24 @@ mod tests {
     #[cfg(feature = "serde")]
     use crate::svm::*;
 
+    #[test]
+    fn search_parameters() {
+        let parameters: SVRSearchParameters<f64, DenseMatrix<f64>, LinearKernel> =
+            SVRSearchParameters {
+                eps: vec![0., 1.],
+                kernel: vec![LinearKernel {}],
+                ..Default::default()
+            };
+        let mut iter = parameters.into_iter();
+        let next = iter.next().unwrap();
+        assert_eq!(next.eps, 0.);
+        assert_eq!(next.kernel, LinearKernel {});
+        let next = iter.next().unwrap();
+        assert_eq!(next.eps, 1.);
+        assert_eq!(next.kernel, LinearKernel {});
+        assert!(iter.next().is_none());
+    }
+
     #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)]
     #[test]
     fn svr_fit_predict() {