simplify SeriesEncoder trait
This commit is contained in:
@@ -10,19 +10,22 @@ use std::hash::Hash;
|
|||||||
|
|
||||||
/// Bi-directional map category <-> label num.
|
/// Bi-directional map category <-> label num.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct CategoryMapper<CategoryType> {
|
pub struct CategoryMapper<C> {
|
||||||
category_map: HashMap<CategoryType, usize>,
|
category_map: HashMap<C, usize>,
|
||||||
categories: Vec<CategoryType>,
|
categories: Vec<C>,
|
||||||
num_categories: usize,
|
num_categories: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a, CategoryType: 'a + Hash + Eq + Clone> CategoryMapper<CategoryType> {
|
impl<'a, C> CategoryMapper<C>
|
||||||
|
where
|
||||||
|
C: 'a + Hash + Eq + Clone
|
||||||
|
{
|
||||||
|
|
||||||
/// Fit an encoder to a lable iterator
|
/// Fit an encoder to a lable iterator
|
||||||
pub fn fit_to_iter(categories: impl Iterator<Item = CategoryType>) -> Self {
|
pub fn fit_to_iter(categories: impl Iterator<Item = C>) -> Self {
|
||||||
let mut category_map: HashMap<CategoryType, usize> = HashMap::new();
|
let mut category_map: HashMap<C, usize> = HashMap::new();
|
||||||
let mut category_num = 0usize;
|
let mut category_num = 0usize;
|
||||||
let mut unique_lables: Vec<CategoryType> = Vec::new();
|
let mut unique_lables: Vec<C> = Vec::new();
|
||||||
|
|
||||||
for l in categories {
|
for l in categories {
|
||||||
if !category_map.contains_key(&l) {
|
if !category_map.contains_key(&l) {
|
||||||
@@ -39,11 +42,11 @@ impl<'a, CategoryType: 'a + Hash + Eq + Clone> CategoryMapper<CategoryType> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Build an encoder from a predefined (category -> class number) map
|
/// Build an encoder from a predefined (category -> class number) map
|
||||||
pub fn from_category_map(category_map: HashMap<CategoryType, usize>) -> Self {
|
pub fn from_category_map(category_map: HashMap<C, usize>) -> Self {
|
||||||
let mut _unique_cat: Vec<(CategoryType, usize)> =
|
let mut _unique_cat: Vec<(C, usize)> =
|
||||||
category_map.iter().map(|(k, v)| (k.clone(), *v)).collect();
|
category_map.iter().map(|(k, v)| (k.clone(), *v)).collect();
|
||||||
_unique_cat.sort_by(|a, b| a.1.cmp(&b.1));
|
_unique_cat.sort_by(|a, b| a.1.cmp(&b.1));
|
||||||
let categories: Vec<CategoryType> = _unique_cat.into_iter().map(|a| a.0).collect();
|
let categories: Vec<C> = _unique_cat.into_iter().map(|a| a.0).collect();
|
||||||
Self {
|
Self {
|
||||||
num_categories: categories.len(),
|
num_categories: categories.len(),
|
||||||
categories,
|
categories,
|
||||||
@@ -52,8 +55,8 @@ impl<'a, CategoryType: 'a + Hash + Eq + Clone> CategoryMapper<CategoryType> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Build an encoder from a predefined positional category-class num vector
|
/// Build an encoder from a predefined positional category-class num vector
|
||||||
pub fn from_positional_category_vec(categories: Vec<CategoryType>) -> Self {
|
pub fn from_positional_category_vec(categories: Vec<C>) -> Self {
|
||||||
let category_map: HashMap<CategoryType, usize> = categories
|
let category_map: HashMap<C, usize> = categories
|
||||||
.iter()
|
.iter()
|
||||||
.enumerate()
|
.enumerate()
|
||||||
.map(|(v, k)| (k.clone(), v))
|
.map(|(v, k)| (k.clone(), v))
|
||||||
@@ -66,64 +69,49 @@ impl<'a, CategoryType: 'a + Hash + Eq + Clone> CategoryMapper<CategoryType> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Get label num of a category
|
/// Get label num of a category
|
||||||
pub fn get_num(&self, category: &CategoryType) -> Option<&usize> {
|
pub fn get_num(&self, category: &C) -> Option<&usize> {
|
||||||
self.category_map.get(category)
|
self.category_map.get(category)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return category corresponding to label num
|
/// Return category corresponding to label num
|
||||||
pub fn get_cat(&self, num: usize) -> &CategoryType {
|
pub fn get_cat(&self, num: usize) -> &C {
|
||||||
&self.categories[num]
|
&self.categories[num]
|
||||||
}
|
}
|
||||||
|
|
||||||
/// List all categories (position = category number)
|
/// List all categories (position = category number)
|
||||||
pub fn get_categories(&self) -> &[CategoryType] {
|
pub fn get_categories(&self) -> &[C] {
|
||||||
&self.categories[..]
|
&self.categories[..]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Defines common behavior for series encoders(e.g. OneHot, Ordinal)
|
/// Defines common behavior for series encoders(e.g. OneHot, Ordinal)
|
||||||
pub trait SeriesEncoder<CategoryType>:
|
pub trait SeriesEncoder<C>:
|
||||||
where
|
where
|
||||||
CategoryType:Hash + Eq + Clone
|
C: Hash + Eq + Clone
|
||||||
{
|
{
|
||||||
/// Fit an encoder to a lable iterator
|
/// Fit an encoder to a lable iterator
|
||||||
fn fit_to_iter(categories: impl Iterator<Item = CategoryType>) -> Self;
|
fn fit_to_iter(categories: impl Iterator<Item = C>) -> Self;
|
||||||
|
|
||||||
/// Number of categories for categorical variable
|
/// Number of categories for categorical variable
|
||||||
fn num_categories(&self) -> usize;
|
fn num_categories(&self) -> usize;
|
||||||
|
|
||||||
/// Build an encoder from a predefined (category -> class number) map
|
|
||||||
fn from_category_map(category_map: HashMap<CategoryType, usize>) -> Self;
|
|
||||||
|
|
||||||
/// Build an encoder from a predefined positional category-class num vector
|
|
||||||
fn from_positional_category_vec(categories: Vec<CategoryType>) -> Self;
|
|
||||||
|
|
||||||
/// Transform a single category type into a one-hot vector
|
/// Transform a single category type into a one-hot vector
|
||||||
fn transform_one<U: RealNumber, V: BaseVector<U>>(&self, category: &CategoryType) -> Option<V>;
|
fn transform_one<U: RealNumber, V: BaseVector<U>>(&self, category: &C) -> Option<V>;
|
||||||
|
|
||||||
/// Invert one-hot vector, back to the category
|
/// Invert one-hot vector, back to the category
|
||||||
fn invert_one<U: RealNumber, V: BaseVector<U>>(&self, one_hot: V) -> Result<CategoryType, Failed>;
|
fn invert_one<U: RealNumber, V: BaseVector<U>>(&self, one_hot: V) -> Result<C, Failed>;
|
||||||
|
|
||||||
/// Get categories ordered by encoder's category enumeration
|
/// Get categories ordered by encoder's category enumeration
|
||||||
fn get_categories(&self) -> &[CategoryType];
|
fn get_categories(&self) -> &[C];
|
||||||
|
|
||||||
/// Take an iterator as a series to transform
|
/// Take an iterator as a series to transform
|
||||||
|
/// None is returned if unknown category is encountered
|
||||||
fn transform_iter<U: RealNumber, V: BaseVector<U>>(
|
fn transform_iter<U: RealNumber, V: BaseVector<U>>(
|
||||||
&self,
|
&self,
|
||||||
cat_it: impl Iterator<Item = CategoryType>,
|
cat_it: impl Iterator<Item = C>,
|
||||||
) -> Vec<Option<V>> {
|
) -> Vec<Option<V>> {
|
||||||
cat_it.map(|l| self.transform_one(&l)).collect()
|
cat_it.map(|l| self.transform_one(&l)).collect()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Transform a slice of category types into one-hot vectors
|
|
||||||
/// None is returned if unknown category is encountered
|
|
||||||
fn transfrom_series<U: RealNumber,V: BaseVector<U>>(
|
|
||||||
&self,
|
|
||||||
categories: &[CategoryType],
|
|
||||||
) -> Vec<Option<V>> {
|
|
||||||
let v = categories.iter().cloned();
|
|
||||||
self.transform_iter(v)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Make a one-hot encoded vector from a categorical variable
|
/// Make a one-hot encoded vector from a categorical variable
|
||||||
@@ -153,22 +141,22 @@ pub fn make_one_hot<T: RealNumber, V: BaseVector<T>>(
|
|||||||
/// Example:
|
/// Example:
|
||||||
/// ```
|
/// ```
|
||||||
/// use std::collections::HashMap;
|
/// use std::collections::HashMap;
|
||||||
/// use smartcore::preprocessing::series_encoder::SeriesOneHotEncoder;
|
/// use smartcore::preprocessing::series_encoder::{SeriesOneHotEncoder, SeriesEncoder};
|
||||||
///
|
///
|
||||||
/// let fake_categories: Vec<usize> = vec![1, 2, 3, 4, 5, 3, 5, 3, 1, 2, 4];
|
/// let fake_categories: Vec<usize> = vec![1, 2, 3, 4, 5, 3, 5, 3, 1, 2, 4];
|
||||||
/// let it = fake_categories.iter().map(|&a| a);
|
/// let it = fake_categories.iter().map(|&a| a);
|
||||||
/// let enc = SeriesOneHotEncoder::<usize>::fit_to_iter(it);
|
/// let enc: SeriesOneHotEncoder::<usize> = SeriesEncoder::fit_to_iter(it);
|
||||||
/// let oh_vec: Vec<f64> = enc.transform_one(&1).unwrap();
|
/// let oh_vec: Vec<f64> = enc.transform_one(&1).unwrap();
|
||||||
/// // notice that 1 is actually a zero-th positional category
|
/// // notice that 1 is actually a zero-th positional category
|
||||||
/// assert_eq!(oh_vec, vec![1.0, 0.0, 0.0, 0.0, 0.0]);
|
/// assert_eq!(oh_vec, vec![1.0, 0.0, 0.0, 0.0, 0.0]);
|
||||||
/// ```
|
/// ```
|
||||||
///
|
///
|
||||||
/// You can also pass a predefined category enumeration such as a hashmap `HashMap<CategoryType, usize>` or a vector `Vec<CategoryType>`
|
/// You can also pass a predefined category enumeration such as a hashmap `HashMap<C, usize>` or a vector `Vec<C>`
|
||||||
///
|
///
|
||||||
///
|
///
|
||||||
/// ```
|
/// ```
|
||||||
/// use std::collections::HashMap;
|
/// use std::collections::HashMap;
|
||||||
/// use smartcore::preprocessing::series_encoder::SeriesOneHotEncoder;
|
/// use smartcore::preprocessing::series_encoder::{SeriesOneHotEncoder, SeriesEncoder, CategoryMapper};
|
||||||
///
|
///
|
||||||
/// let category_map: HashMap<&str, usize> =
|
/// let category_map: HashMap<&str, usize> =
|
||||||
/// vec![("cat", 2), ("background",0), ("dog", 1)]
|
/// vec![("cat", 2), ("background",0), ("dog", 1)]
|
||||||
@@ -176,43 +164,53 @@ pub fn make_one_hot<T: RealNumber, V: BaseVector<T>>(
|
|||||||
/// .collect();
|
/// .collect();
|
||||||
/// let category_vec = vec!["background", "dog", "cat"];
|
/// let category_vec = vec!["background", "dog", "cat"];
|
||||||
///
|
///
|
||||||
/// let enc_lv = SeriesOneHotEncoder::<&str>::from_positional_category_vec(category_vec);
|
/// let enc_lv = SeriesOneHotEncoder::<&str>::new(CategoryMapper::from_positional_category_vec(category_vec));
|
||||||
/// let enc_lm = SeriesOneHotEncoder::<&str>::from_category_map(category_map);
|
/// let enc_lm = SeriesOneHotEncoder::<&str>::new(CategoryMapper::from_category_map(category_map));
|
||||||
///
|
///
|
||||||
/// // ["background", "dog", "cat"]
|
/// // ["background", "dog", "cat"]
|
||||||
/// println!("{:?}", enc_lv.get_categories());
|
/// println!("{:?}", enc_lv.get_categories());
|
||||||
/// assert_eq!(enc_lv.transform_one::<f64>(&"dog"), enc_lm.transform_one::<f64>(&"dog"))
|
/// let lv: Vec<f64> = enc_lv.transform_one(&"dog").unwrap();
|
||||||
|
/// let lm: Vec<f64> = enc_lm.transform_one(&"dog").unwrap();
|
||||||
|
/// assert_eq!(lv, lm);
|
||||||
/// ```
|
/// ```
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct SeriesOneHotEncoder<CategoryType> {
|
pub struct SeriesOneHotEncoder<C> {
|
||||||
mapper: CategoryMapper<CategoryType>,
|
mapper: CategoryMapper<C>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<CategoryType: Hash + Eq + Clone> SeriesEncoder<CategoryType> for SeriesOneHotEncoder<CategoryType> {
|
impl<C> SeriesOneHotEncoder<C>
|
||||||
|
where
|
||||||
|
C: Hash + Eq + Clone
|
||||||
|
{
|
||||||
|
/// Create SeriesEncoder form existing mapper
|
||||||
|
pub fn new(mapper: CategoryMapper<C>) -> Self {
|
||||||
|
Self {mapper}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C> SeriesEncoder<C> for SeriesOneHotEncoder<C>
|
||||||
|
where
|
||||||
|
C: Hash + Eq + Clone
|
||||||
|
{
|
||||||
|
|
||||||
fn fit_to_iter(categories: impl Iterator<Item = CategoryType>) -> Self {
|
|
||||||
|
fn fit_to_iter(categories: impl Iterator<Item = C>) -> Self {
|
||||||
Self {mapper:CategoryMapper::fit_to_iter(categories)}
|
Self {mapper:CategoryMapper::fit_to_iter(categories)}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Build an encoder from a predefined (category -> class number) map
|
|
||||||
fn from_category_map(category_map: HashMap<CategoryType, usize>) -> Self {
|
|
||||||
Self {mapper: CategoryMapper::from_category_map(category_map)}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Build an encoder from a predefined positional category-class num vector
|
|
||||||
fn from_positional_category_vec(categories: Vec<CategoryType>) -> Self {
|
|
||||||
Self {mapper:CategoryMapper::from_positional_category_vec(categories)}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn num_categories(&self) -> usize {
|
fn num_categories(&self) -> usize {
|
||||||
self.mapper.num_categories
|
self.mapper.num_categories
|
||||||
}
|
}
|
||||||
|
|
||||||
fn get_categories(&self) -> &[CategoryType] {
|
fn get_categories(&self) -> &[C] {
|
||||||
self.mapper.get_categories()
|
self.mapper.get_categories()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn invert_one<U: RealNumber, V: BaseVector<U>>(&self, one_hot: V) -> Result<CategoryType, Failed>
|
fn invert_one<U, V>(&self, one_hot: V) -> Result<C, Failed>
|
||||||
|
where
|
||||||
|
U: RealNumber,
|
||||||
|
V: BaseVector<U>
|
||||||
|
|
||||||
{
|
{
|
||||||
let pos = U::from_f64(1f64).unwrap();
|
let pos = U::from_f64(1f64).unwrap();
|
||||||
|
|
||||||
@@ -234,7 +232,11 @@ impl<CategoryType: Hash + Eq + Clone> SeriesEncoder<CategoryType> for SeriesOneH
|
|||||||
Err(Failed::transform(&pos_entries[..]))
|
Err(Failed::transform(&pos_entries[..]))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn transform_one<U: RealNumber, V: BaseVector<U>>(&self, category: &CategoryType) -> Option<V> {
|
fn transform_one<U, V>(&self, category: &C) -> Option<V>
|
||||||
|
where
|
||||||
|
U: RealNumber,
|
||||||
|
V: BaseVector<U>
|
||||||
|
{
|
||||||
match self.mapper.get_num(category) {
|
match self.mapper.get_num(category) {
|
||||||
None => None,
|
None => None,
|
||||||
Some(&idx) => Some(make_one_hot(idx, self.num_categories())),
|
Some(&idx) => Some(make_one_hot(idx, self.num_categories())),
|
||||||
@@ -262,7 +264,7 @@ mod tests {
|
|||||||
|
|
||||||
fn build_fake_str_enc<'a>() -> SeriesOneHotEncoder<&'a str> {
|
fn build_fake_str_enc<'a>() -> SeriesOneHotEncoder<&'a str> {
|
||||||
let fake_category_pos = vec!["background", "dog", "cat"];
|
let fake_category_pos = vec!["background", "dog", "cat"];
|
||||||
let enc = SeriesOneHotEncoder::<&str>::from_positional_category_vec(fake_category_pos);
|
let enc = SeriesOneHotEncoder::<&str>::new( CategoryMapper::from_positional_category_vec(fake_category_pos));
|
||||||
enc
|
enc
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -271,7 +273,7 @@ mod tests {
|
|||||||
let category_map: HashMap<&str, usize> = vec![("background", 0), ("dog", 1), ("cat", 2)]
|
let category_map: HashMap<&str, usize> = vec![("background", 0), ("dog", 1), ("cat", 2)]
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.collect();
|
.collect();
|
||||||
let enc = SeriesOneHotEncoder::<&str>::from_category_map(category_map);
|
let enc = SeriesOneHotEncoder::<&str>::new( CategoryMapper::from_category_map(category_map));
|
||||||
let oh_vec: Vec<f64> = match enc.transform_one(&"dog") {
|
let oh_vec: Vec<f64> = match enc.transform_one(&"dog") {
|
||||||
None => panic!("Wrong categories"),
|
None => panic!("Wrong categories"),
|
||||||
Some(v) => v,
|
Some(v) => v,
|
||||||
@@ -306,8 +308,8 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn test_many_categorys() {
|
fn test_many_categorys() {
|
||||||
let enc = build_fake_str_enc();
|
let enc = build_fake_str_enc();
|
||||||
let res: Vec<Option<Vec<f64>>> =
|
let cat_it = ["dog", "cat", "fish", "background"].iter().cloned();
|
||||||
enc.transfrom_series(&["dog", "cat", "fish", "background"]);
|
let res: Vec<Option<Vec<f64>>> = enc.transform_iter(cat_it);
|
||||||
let v = vec![
|
let v = vec![
|
||||||
Some(vec![0.0, 1.0, 0.0]),
|
Some(vec![0.0, 1.0, 0.0]),
|
||||||
Some(vec![0.0, 0.0, 1.0]),
|
Some(vec![0.0, 0.0, 1.0]),
|
||||||
|
|||||||
Reference in New Issue
Block a user