diff --git a/src/lib.rs b/src/lib.rs index 7d2b089..6e6205f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -91,6 +91,8 @@ pub mod naive_bayes; /// Supervised neighbors-based learning methods pub mod neighbors; pub(crate) mod optimization; +/// Preprocessing utilities +pub mod preprocessing; /// Support Vector Machines pub mod svm; /// Supervised tree-based learning methods diff --git a/src/preprocessing/categorical.rs b/src/preprocessing/categorical.rs new file mode 100644 index 0000000..8571e74 --- /dev/null +++ b/src/preprocessing/categorical.rs @@ -0,0 +1,329 @@ +//! # One-hot Encoding For [RealNumber](../../math/num/trait.RealNumber.html) Matricies +//! Transform a data [Matrix](../../linalg/trait.BaseMatrix.html) by replacing all categorical variables with their one-hot equivalents +//! +//! Internally OneHotEncoder treats every categorical column as a series and transforms it using [CategoryMapper](../series_encoder/struct.CategoryMapper.html) +//! +//! ### Usage Example +//! ``` +//! use smartcore::linalg::naive::dense_matrix::DenseMatrix; +//! use smartcore::preprocessing::categorical::{OneHotEncoder, OneHotEncoderParams}; +//! let data = DenseMatrix::from_2d_array(&[ +//! &[1.5, 1.0, 1.5, 3.0], +//! &[1.5, 2.0, 1.5, 4.0], +//! &[1.5, 1.0, 1.5, 5.0], +//! &[1.5, 2.0, 1.5, 6.0], +//! ]); +//! let encoder_params = OneHotEncoderParams::from_cat_idx(&[1, 3]); +//! // Infer number of categories from data and return a reusable encoder +//! let encoder = OneHotEncoder::fit(&data, encoder_params).unwrap(); +//! // Transform categorical to one-hot encoded (can transform similar) +//! let oh_data = encoder.transform(&data).unwrap(); +//! // Produces the following: +//! // &[1.5, 1.0, 0.0, 1.5, 1.0, 0.0, 0.0, 0.0] +//! // &[1.5, 0.0, 1.0, 1.5, 0.0, 1.0, 0.0, 0.0] +//! // &[1.5, 1.0, 0.0, 1.5, 0.0, 0.0, 1.0, 0.0] +//! // &[1.5, 0.0, 1.0, 1.5, 0.0, 0.0, 0.0, 1.0] +//! ``` +use std::iter; + +use crate::error::Failed; +use crate::linalg::Matrix; + +use crate::preprocessing::data_traits::{CategoricalFloat, Categorizable}; +use crate::preprocessing::series_encoder::CategoryMapper; + +/// OneHotEncoder Parameters +#[derive(Debug, Clone)] +pub struct OneHotEncoderParams { + /// Column number that contain categorical variable + pub col_idx_categorical: Option>, + /// (Currently not implemented) Try and infer which of the matrix columns are categorical variables + infer_categorical: bool, +} + +impl OneHotEncoderParams { + /// Generate parameters from categorical variable column numbers + pub fn from_cat_idx(categorical_params: &[usize]) -> Self { + Self { + col_idx_categorical: Some(categorical_params.to_vec()), + infer_categorical: false, + } + } +} + +/// Calculate the offset to parameters to due introduction of one-hot encoding +fn find_new_idxs(num_params: usize, cat_sizes: &[usize], cat_idxs: &[usize]) -> Vec { + // This functions uses iterators and returns a vector. + // In case we get a huge amount of paramenters this might be a problem + // todo: Change this such that it will return an iterator + + let cat_idx = cat_idxs.iter().copied().chain((num_params..).take(1)); + + // Offset is constant between two categorical values, here we calculate the number of steps + // that remain constant + let repeats = cat_idx.scan(0, |a, v| { + let im = v + 1 - *a; + *a = v; + Some(im) + }); + + // Calculate the offset to parameter idx due to newly intorduced one-hot vectors + let offset_ = cat_sizes.iter().scan(0, |a, &v| { + *a = *a + v - 1; + Some(*a) + }); + let offset = (0..1).chain(offset_); + + let new_param_idxs: Vec = (0..num_params) + .zip( + repeats + .zip(offset) + .map(|(r, o)| iter::repeat(o).take(r)) + .flatten(), + ) + .map(|(idx, ofst)| idx + ofst) + .collect(); + new_param_idxs +} + +fn validate_col_is_categorical(data: &[T]) -> bool { + for v in data { + if !v.is_valid() { + return false; + } + } + true +} + +/// Encode Categorical variavbles of data matrix to one-hot +#[derive(Debug, Clone)] +pub struct OneHotEncoder { + category_mappers: Vec>, + col_idx_categorical: Vec, +} + +impl OneHotEncoder { + /// Create an encoder instance with categories infered from data matrix + pub fn fit(data: &M, params: OneHotEncoderParams) -> Result + where + T: Categorizable, + M: Matrix, + { + match (params.col_idx_categorical, params.infer_categorical) { + (None, false) => Err(Failed::fit( + "Must pass categorical series ids or infer flag", + )), + + (Some(_idxs), true) => Err(Failed::fit( + "Ambigous parameters, got both infer and categroy ids", + )), + + (Some(mut idxs), false) => { + // make sure categories have same order as data columns + idxs.sort_unstable(); + + let (nrows, _) = data.shape(); + + // col buffer to avoid allocations + let mut col_buf: Vec = iter::repeat(T::zero()).take(nrows).collect(); + + let mut res: Vec> = Vec::with_capacity(idxs.len()); + + for &idx in &idxs { + data.copy_col_as_vec(idx, &mut col_buf); + if !validate_col_is_categorical(&col_buf) { + let msg = format!( + "Column {} of data matrix containts non categorizable (integer) values", + idx + ); + return Err(Failed::fit(&msg[..])); + } + let hashable_col = col_buf.iter().map(|v| v.to_category()); + res.push(CategoryMapper::fit_to_iter(hashable_col)); + } + + Ok(Self { + category_mappers: res, + col_idx_categorical: idxs, + }) + } + + (None, true) => { + todo!("Auto-Inference for Categorical Variables not yet implemented") + } + } + } + + /// Transform categorical variables to one-hot encoded and return a new matrix + pub fn transform(&self, x: &M) -> Result + where + T: Categorizable, + M: Matrix, + { + let (nrows, p) = x.shape(); + let additional_params: Vec = self + .category_mappers + .iter() + .map(|enc| enc.num_categories()) + .collect(); + + // Eac category of size v adds v-1 params + let expandws_p: usize = p + additional_params.iter().fold(0, |cs, &v| cs + v - 1); + + let new_col_idx = find_new_idxs(p, &additional_params[..], &self.col_idx_categorical[..]); + let mut res = M::zeros(nrows, expandws_p); + + for (pidx, &old_cidx) in self.col_idx_categorical.iter().enumerate() { + let cidx = new_col_idx[old_cidx]; + let col_iter = (0..nrows).map(|r| x.get(r, old_cidx).to_category()); + let sencoder = &self.category_mappers[pidx]; + let oh_series = col_iter.map(|c| sencoder.get_one_hot::>(&c)); + + for (row, oh_vec) in oh_series.enumerate() { + match oh_vec { + None => { + // Since we support T types, bad value in a series causes in to be invalid + let msg = format!("At least one value in column {} doesn't conform to category definition", old_cidx); + return Err(Failed::transform(&msg[..])); + } + Some(v) => { + // copy one hot vectors to their place in the data matrix; + for (col_ofst, &val) in v.iter().enumerate() { + res.set(row, cidx + col_ofst, val); + } + } + } + } + } + + // copy old data in x to their new location while skipping catergorical vars (already treated) + let mut skip_idx_iter = self.col_idx_categorical.iter(); + let mut cur_skip = skip_idx_iter.next(); + + for (old_p, &new_p) in new_col_idx.iter().enumerate() { + // if found treated varible, skip it + if let Some(&v) = cur_skip { + if v == old_p { + cur_skip = skip_idx_iter.next(); + continue; + } + } + + for r in 0..nrows { + let val = x.get(r, old_p); + res.set(r, new_p, val); + } + } + + Ok(res) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::linalg::naive::dense_matrix::DenseMatrix; + use crate::preprocessing::series_encoder::CategoryMapper; + + #[test] + fn adjust_idxs() { + assert_eq!(find_new_idxs(0, &[], &[]), Vec::::new()); + // [0,1,2] -> [0, 1, 1, 1, 2] + assert_eq!(find_new_idxs(3, &[3], &[1]), vec![0, 1, 4]); + } + + fn build_cat_first_and_last() -> (DenseMatrix, DenseMatrix) { + let orig = DenseMatrix::from_2d_array(&[ + &[1.0, 1.5, 3.0], + &[2.0, 1.5, 4.0], + &[1.0, 1.5, 5.0], + &[2.0, 1.5, 6.0], + ]); + + let oh_enc = DenseMatrix::from_2d_array(&[ + &[1.0, 0.0, 1.5, 1.0, 0.0, 0.0, 0.0], + &[0.0, 1.0, 1.5, 0.0, 1.0, 0.0, 0.0], + &[1.0, 0.0, 1.5, 0.0, 0.0, 1.0, 0.0], + &[0.0, 1.0, 1.5, 0.0, 0.0, 0.0, 1.0], + ]); + + (orig, oh_enc) + } + + fn build_fake_matrix() -> (DenseMatrix, DenseMatrix) { + // Categorical first and last + let orig = DenseMatrix::from_2d_array(&[ + &[1.5, 1.0, 1.5, 3.0], + &[1.5, 2.0, 1.5, 4.0], + &[1.5, 1.0, 1.5, 5.0], + &[1.5, 2.0, 1.5, 6.0], + ]); + + let oh_enc = DenseMatrix::from_2d_array(&[ + &[1.5, 1.0, 0.0, 1.5, 1.0, 0.0, 0.0, 0.0], + &[1.5, 0.0, 1.0, 1.5, 0.0, 1.0, 0.0, 0.0], + &[1.5, 1.0, 0.0, 1.5, 0.0, 0.0, 1.0, 0.0], + &[1.5, 0.0, 1.0, 1.5, 0.0, 0.0, 0.0, 1.0], + ]); + + (orig, oh_enc) + } + + #[test] + fn hash_encode_f64_series() { + let series = vec![3.0, 1.0, 2.0, 1.0]; + let hashable_series: Vec = + series.iter().map(|v| v.to_category()).collect(); + let enc = CategoryMapper::from_positional_category_vec(hashable_series); + let inv = enc.invert_one_hot(vec![0.0, 0.0, 1.0]); + let orig_val: f64 = inv.unwrap().into(); + assert_eq!(orig_val, 2.0); + } + #[test] + fn test_fit() { + let (x, _) = build_fake_matrix(); + let params = OneHotEncoderParams::from_cat_idx(&[1, 3]); + let oh_enc = OneHotEncoder::fit(&x, params).unwrap(); + assert_eq!(oh_enc.category_mappers.len(), 2); + + let num_cat: Vec = oh_enc + .category_mappers + .iter() + .map(|a| a.num_categories()) + .collect(); + assert_eq!(num_cat, vec![2, 4]); + } + + #[test] + fn matrix_transform_test() { + let (x, expected_x) = build_fake_matrix(); + let params = OneHotEncoderParams::from_cat_idx(&[1, 3]); + let oh_enc = OneHotEncoder::fit(&x, params).unwrap(); + let nm = oh_enc.transform(&x).unwrap(); + assert_eq!(nm, expected_x); + + let (x, expected_x) = build_cat_first_and_last(); + let params = OneHotEncoderParams::from_cat_idx(&[0, 2]); + let oh_enc = OneHotEncoder::fit(&x, params).unwrap(); + let nm = oh_enc.transform(&x).unwrap(); + assert_eq!(nm, expected_x); + } + + #[test] + fn fail_on_bad_category() { + let m = DenseMatrix::from_2d_array(&[ + &[1.0, 1.5, 3.0], + &[2.0, 1.5, 4.0], + &[1.0, 1.5, 5.0], + &[2.0, 1.5, 6.0], + ]); + + let params = OneHotEncoderParams::from_cat_idx(&[1]); + match OneHotEncoder::fit(&m, params) { + Err(_) => { + assert!(true); + } + _ => assert!(false), + } + } +} diff --git a/src/preprocessing/data_traits.rs b/src/preprocessing/data_traits.rs new file mode 100644 index 0000000..38d9e3e --- /dev/null +++ b/src/preprocessing/data_traits.rs @@ -0,0 +1,43 @@ +//! Traits to indicate that float variables can be viewed as categorical +//! This module assumes + +use crate::math::num::RealNumber; + +pub type CategoricalFloat = u16; + +// pub struct CategoricalFloat(u16); +const ERROR_MARGIN: f64 = 0.001; + +pub trait Categorizable: RealNumber { + type A; + + fn to_category(self) -> CategoricalFloat; + + fn is_valid(self) -> bool; +} + +impl Categorizable for f32 { + type A = CategoricalFloat; + + fn to_category(self) -> CategoricalFloat { + self as CategoricalFloat + } + + fn is_valid(self) -> bool { + let a = self.to_category(); + (a as f32 - self).abs() < (ERROR_MARGIN as f32) + } +} + +impl Categorizable for f64 { + type A = CategoricalFloat; + + fn to_category(self) -> CategoricalFloat { + self as CategoricalFloat + } + + fn is_valid(self) -> bool { + let a = self.to_category(); + (a as f64 - self).abs() < ERROR_MARGIN + } +} diff --git a/src/preprocessing/mod.rs b/src/preprocessing/mod.rs new file mode 100644 index 0000000..32a0cfa --- /dev/null +++ b/src/preprocessing/mod.rs @@ -0,0 +1,5 @@ +/// Transform a data matrix by replaceing all categorical variables with their one-hot vector equivalents +pub mod categorical; +mod data_traits; +/// Encode a series (column, array) of categorical variables as one-hot vectors +pub mod series_encoder; diff --git a/src/preprocessing/series_encoder.rs b/src/preprocessing/series_encoder.rs new file mode 100644 index 0000000..e24eca1 --- /dev/null +++ b/src/preprocessing/series_encoder.rs @@ -0,0 +1,278 @@ +#![allow(clippy::ptr_arg)] +//! # Series Encoder +//! Encode a series of categorical features as a one-hot numeric array. + +use crate::error::Failed; +use crate::linalg::BaseVector; +use crate::math::num::RealNumber; +use std::collections::HashMap; +use std::hash::Hash; + +/// ## Bi-directional map category <-> label num. +/// Turn Hashable objects into a one-hot vectors or ordinal values. +/// This struct encodes single class per exmample +/// +/// You can fit_to_iter a category enumeration by passing an iterator of categories. +/// category numbers will be assigned in the order they are encountered +/// +/// Example: +/// ``` +/// use std::collections::HashMap; +/// use smartcore::preprocessing::series_encoder::CategoryMapper; +/// +/// let fake_categories: Vec = vec![1, 2, 3, 4, 5, 3, 5, 3, 1, 2, 4]; +/// let it = fake_categories.iter().map(|&a| a); +/// let enc = CategoryMapper::::fit_to_iter(it); +/// let oh_vec: Vec = enc.get_one_hot(&1).unwrap(); +/// // notice that 1 is actually a zero-th positional category +/// assert_eq!(oh_vec, vec![1.0, 0.0, 0.0, 0.0, 0.0]); +/// ``` +/// +/// You can also pass a predefined category enumeration such as a hashmap `HashMap` or a vector `Vec` +/// +/// +/// ``` +/// use std::collections::HashMap; +/// use smartcore::preprocessing::series_encoder::CategoryMapper; +/// +/// let category_map: HashMap<&str, usize> = +/// vec![("cat", 2), ("background",0), ("dog", 1)] +/// .into_iter() +/// .collect(); +/// let category_vec = vec!["background", "dog", "cat"]; +/// +/// let enc_lv = CategoryMapper::<&str>::from_positional_category_vec(category_vec); +/// let enc_lm = CategoryMapper::<&str>::from_category_map(category_map); +/// +/// // ["background", "dog", "cat"] +/// println!("{:?}", enc_lv.get_categories()); +/// let lv: Vec = enc_lv.get_one_hot(&"dog").unwrap(); +/// let lm: Vec = enc_lm.get_one_hot(&"dog").unwrap(); +/// assert_eq!(lv, lm); +/// ``` +#[derive(Debug, Clone)] +pub struct CategoryMapper { + category_map: HashMap, + categories: Vec, + num_categories: usize, +} + +impl CategoryMapper +where + C: Hash + Eq + Clone, +{ + /// Get the number of categories in the mapper + pub fn num_categories(&self) -> usize { + self.num_categories + } + + /// Fit an encoder to a lable iterator + pub fn fit_to_iter(categories: impl Iterator) -> Self { + let mut category_map: HashMap = HashMap::new(); + let mut category_num = 0usize; + let mut unique_lables: Vec = Vec::new(); + + for l in categories { + if !category_map.contains_key(&l) { + category_map.insert(l.clone(), category_num); + unique_lables.push(l.clone()); + category_num += 1; + } + } + Self { + category_map, + num_categories: category_num, + categories: unique_lables, + } + } + + /// Build an encoder from a predefined (category -> class number) map + pub fn from_category_map(category_map: HashMap) -> Self { + let mut _unique_cat: Vec<(C, usize)> = + category_map.iter().map(|(k, v)| (k.clone(), *v)).collect(); + _unique_cat.sort_by(|a, b| a.1.cmp(&b.1)); + let categories: Vec = _unique_cat.into_iter().map(|a| a.0).collect(); + Self { + num_categories: categories.len(), + categories, + category_map, + } + } + + /// Build an encoder from a predefined positional category-class num vector + pub fn from_positional_category_vec(categories: Vec) -> Self { + let category_map: HashMap = categories + .iter() + .enumerate() + .map(|(v, k)| (k.clone(), v)) + .collect(); + Self { + num_categories: categories.len(), + category_map, + categories, + } + } + + /// Get label num of a category + pub fn get_num(&self, category: &C) -> Option<&usize> { + self.category_map.get(category) + } + + /// Return category corresponding to label num + pub fn get_cat(&self, num: usize) -> &C { + &self.categories[num] + } + + /// List all categories (position = category number) + pub fn get_categories(&self) -> &[C] { + &self.categories[..] + } + + /// Get one-hot encoding of the category + pub fn get_one_hot(&self, category: &C) -> Option + where + U: RealNumber, + V: BaseVector, + { + match self.get_num(category) { + None => None, + Some(&idx) => Some(make_one_hot::(idx, self.num_categories)), + } + } + + /// Invert one-hot vector, back to the category + pub fn invert_one_hot(&self, one_hot: V) -> Result + where + U: RealNumber, + V: BaseVector, + { + let pos = U::one(); + + let oh_it = (0..one_hot.len()).map(|idx| one_hot.get(idx)); + + let s: Vec = oh_it + .enumerate() + .filter_map(|(idx, v)| if v == pos { Some(idx) } else { None }) + .collect(); + + if s.len() == 1 { + let idx = s[0]; + return Ok(self.get_cat(idx).clone()); + } + let pos_entries = format!( + "Expected a single positive entry, {} entires found", + s.len() + ); + Err(Failed::transform(&pos_entries[..])) + } + + /// Get ordinal encoding of the catergory + pub fn get_ordinal(&self, category: &C) -> Option + where + U: RealNumber, + { + match self.get_num(category) { + None => None, + Some(&idx) => U::from_usize(idx), + } + } +} + +/// Make a one-hot encoded vector from a categorical variable +/// +/// Example: +/// ``` +/// use smartcore::preprocessing::series_encoder::make_one_hot; +/// let one_hot: Vec = make_one_hot(2, 3); +/// assert_eq!(one_hot, vec![0.0, 0.0, 1.0]); +/// ``` +pub fn make_one_hot(category_idx: usize, num_categories: usize) -> V +where + T: RealNumber, + V: BaseVector, +{ + let pos = T::one(); + let mut z = V::zeros(num_categories); + z.set(category_idx, pos); + z +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn from_categories() { + let fake_categories: Vec = vec![1, 2, 3, 4, 5, 3, 5, 3, 1, 2, 4]; + let it = fake_categories.iter().map(|&a| a); + let enc = CategoryMapper::::fit_to_iter(it); + let oh_vec: Vec = match enc.get_one_hot(&1) { + None => panic!("Wrong categories"), + Some(v) => v, + }; + let res: Vec = vec![1f64, 0f64, 0f64, 0f64, 0f64]; + assert_eq!(oh_vec, res); + } + + fn build_fake_str_enc<'a>() -> CategoryMapper<&'a str> { + let fake_category_pos = vec!["background", "dog", "cat"]; + let enc = CategoryMapper::<&str>::from_positional_category_vec(fake_category_pos); + enc + } + #[test] + fn ordinal_encoding() { + let enc = build_fake_str_enc(); + assert_eq!(1f64, enc.get_ordinal::(&"dog").unwrap()) + } + + #[test] + fn category_map_and_vec() { + let category_map: HashMap<&str, usize> = vec![("background", 0), ("dog", 1), ("cat", 2)] + .into_iter() + .collect(); + let enc = CategoryMapper::<&str>::from_category_map(category_map); + let oh_vec: Vec = match enc.get_one_hot(&"dog") { + None => panic!("Wrong categories"), + Some(v) => v, + }; + let res: Vec = vec![0f64, 1f64, 0f64]; + assert_eq!(oh_vec, res); + } + + #[test] + fn positional_categories_vec() { + let enc = build_fake_str_enc(); + let oh_vec: Vec = match enc.get_one_hot(&"dog") { + None => panic!("Wrong categories"), + Some(v) => v, + }; + let res: Vec = vec![0.0, 1.0, 0.0]; + assert_eq!(oh_vec, res); + } + + #[test] + fn invert_label_test() { + let enc = build_fake_str_enc(); + let res: Vec = vec![0.0, 1.0, 0.0]; + let lab = enc.invert_one_hot(res).unwrap(); + assert_eq!(lab, "dog"); + if let Err(e) = enc.invert_one_hot(vec![0.0, 0.0, 0.0]) { + let pos_entries = format!("Expected a single positive entry, 0 entires found"); + assert_eq!(e, Failed::transform(&pos_entries[..])); + }; + } + + #[test] + fn test_many_categorys() { + let enc = build_fake_str_enc(); + let cat_it = ["dog", "cat", "fish", "background"].iter().cloned(); + let res: Vec>> = cat_it.map(|v| enc.get_one_hot(&v)).collect(); + let v = vec![ + Some(vec![0.0, 1.0, 0.0]), + Some(vec![0.0, 0.0, 1.0]), + None, + Some(vec![1.0, 0.0, 0.0]), + ]; + assert_eq!(res, v) + } +}