diff --git a/src/decomposition/mod.rs b/src/decomposition/mod.rs index bdbd3e9..a01c114 100644 --- a/src/decomposition/mod.rs +++ b/src/decomposition/mod.rs @@ -9,7 +9,7 @@ //! Dimension reduction is also used for the purposes of data visualization. //! //! ## References -//! * ["An Introduction to Statistical Learning", James G., Witten D., Hastie T., Tibshirani R., 10.3.1 K-Means Clustering, 6.3 Dimension Reduction Methods](http://faculty.marshall.usc.edu/gareth-james/ISL/) +//! * ["An Introduction to Statistical Learning", James G., Witten D., Hastie T., Tibshirani R., 6.3 Dimension Reduction Methods](http://faculty.marshall.usc.edu/gareth-james/ISL/) /// PCA is a popular approach for deriving a low-dimensional set of features from a large set of variables. pub mod pca; diff --git a/src/linear/linear_regression.rs b/src/linear/linear_regression.rs index 78cab25..f2aad51 100644 --- a/src/linear/linear_regression.rs +++ b/src/linear/linear_regression.rs @@ -1,3 +1,63 @@ +//! # Linear Regression +//! +//! Linear regression is a very straightforward approach for predicting a quantitative response \\(y\\) on the basis of a linear combination of explanatory variables \\(X\\). +//! Linear regression assumes that there is approximately a linear relationship between \\(X\\) and \\(y\\). Formally, we can write this linear relationship as +//! +//! \\[y \approx \beta_0 + \sum_{i=1}^n \beta_iX_i + \epsilon\\] +//! +//! where \\(\epsilon\\) is a mean-zero random error term and the regression coefficients \\(\beta_0, \beta_0, ... \beta_n\\) are unknown, and must be estimated. +//! +//! While regression coefficients can be estimated directly by solving +//! +//! \\[\hat{\beta} = (X^TX)^{-1}X^Ty \\] +//! +//! the \\((X^TX)^{-1}\\) term is both computationally expensive and numerically unstable. An alternative approach is to use a matrix decomposition to avoid this operation. +//! SmartCore uses [SVD](../../linalg/svd/index.html) and [QR](../../linalg/qr/index.html) matrix decomposition to find estimates of \\(\hat{\beta}\\). +//! The QR decomposition is more computationally efficient and more numerically stable than calculating the normal equation directly, +//! but does not work for all data matrices. Unlike the QR decomposition, all matrices have an SVD decomposition. +//! +//! Example: +//! +//! ``` +//! use smartcore::linalg::naive::dense_matrix::*; +//! use smartcore::linear::linear_regression::*; +//! +//! // Longley dataset (https://www.statsmodels.org/stable/datasets/generated/longley.html) +//! let x = DenseMatrix::from_array(&[ +//! &[234.289, 235.6, 159.0, 107.608, 1947., 60.323], +//! &[259.426, 232.5, 145.6, 108.632, 1948., 61.122], +//! &[258.054, 368.2, 161.6, 109.773, 1949., 60.171], +//! &[284.599, 335.1, 165.0, 110.929, 1950., 61.187], +//! &[328.975, 209.9, 309.9, 112.075, 1951., 63.221], +//! &[346.999, 193.2, 359.4, 113.270, 1952., 63.639], +//! &[365.385, 187.0, 354.7, 115.094, 1953., 64.989], +//! &[363.112, 357.8, 335.0, 116.219, 1954., 63.761], +//! &[397.469, 290.4, 304.8, 117.388, 1955., 66.019], +//! &[419.180, 282.2, 285.7, 118.734, 1956., 67.857], +//! &[442.769, 293.6, 279.8, 120.445, 1957., 68.169], +//! &[444.546, 468.1, 263.7, 121.950, 1958., 66.513], +//! &[482.704, 381.3, 255.2, 123.366, 1959., 68.655], +//! &[502.601, 393.1, 251.4, 125.368, 1960., 69.564], +//! &[518.173, 480.6, 257.2, 127.852, 1961., 69.331], +//! &[554.894, 400.7, 282.7, 130.081, 1962., 70.551], +//! ]); +//! +//! let y: Vec = vec![83.0, 88.5, 88.2, 89.5, 96.2, 98.1, 99.0, +//! 100.0, 101.2, 104.6, 108.4, 110.8, 112.6, 114.2, 115.7, 116.9]; +//! +//! let lr = LinearRegression::fit(&x, &y, LinearRegressionParameters { +//! solver: LinearRegressionSolverName::QR, // or SVD +//! }); +//! +//! let y_hat = lr.predict(&x); +//! ``` +//! +//! ## References: +//! +//! * ["An Introduction to Statistical Learning", James G., Witten D., Hastie T., Tibshirani R., 3. Linear Regression](http://faculty.marshall.usc.edu/gareth-james/ISL/) +//! * ["Numerical Recipes: The Art of Scientific Computing", Press W.H., Teukolsky S.A., Vetterling W.T, Flannery B.P, 3rd ed., Section 15.4 General Linear Least Squares](http://numerical.recipes/) +//! +//! use std::fmt::Debug; use serde::{Deserialize, Serialize}; @@ -6,16 +66,22 @@ use crate::linalg::Matrix; use crate::math::num::RealNumber; #[derive(Serialize, Deserialize, Debug)] +/// Approach to use for estimation of regression coefficients. QR is more efficient but SVD is more stable. pub enum LinearRegressionSolverName { + /// QR decomposition, see [QR](../../linalg/qr/index.html) QR, + /// SVD decomposition, see [SVD](../../linalg/svd/index.html) SVD, } +/// Linear Regression parameters #[derive(Serialize, Deserialize, Debug)] pub struct LinearRegressionParameters { + /// Solver to use for estimation of regression coefficients. pub solver: LinearRegressionSolverName, } +/// Linear Regression #[derive(Serialize, Deserialize, Debug)] pub struct LinearRegression> { coefficients: M, @@ -39,6 +105,10 @@ impl> PartialEq for LinearRegression { } impl> LinearRegression { + /// Fits Linear Regression to your data. + /// * `x` - _NxM_ matrix with _N_ observations and _M_ features in each observation. + /// * `y` - target values + /// * `parameters` - other parameters, use `Default::default()` to set parameters to default values. pub fn fit( x: &M, y: &M::RowVector, @@ -69,12 +139,24 @@ impl> LinearRegression { } } + /// Predict target values from `x` + /// * `x` - _KxM_ data where _K_ is number of observations and _M_ is number of features. pub fn predict(&self, x: &M) -> M::RowVector { let (nrows, _) = x.shape(); let mut y_hat = x.dot(&self.coefficients); y_hat.add_mut(&M::fill(nrows, 1, self.intercept)); y_hat.transpose().to_row_vector() } + + /// Get estimates regression coefficients + pub fn coefficients(&self) -> M { + self.coefficients.clone() + } + + /// Get estimate of intercept + pub fn intercept(&self) -> T { + self.intercept + } } #[cfg(test)] diff --git a/src/linear/logistic_regression.rs b/src/linear/logistic_regression.rs index a643758..259760c 100644 --- a/src/linear/logistic_regression.rs +++ b/src/linear/logistic_regression.rs @@ -1,3 +1,54 @@ +//! # Logistic Regression +//! +//! As [Linear Regression](../linear_regression/index.html), logistic regression explains your outcome as a linear combination of predictor variables \\(X\\) but rather than modeling this response directly, +//! logistic regression models the probability that \\(y\\) belongs to a particular category, \\(Pr(y = 1|X) \\), as: +//! +//! \\[ Pr(y=1) \approx \frac{e^{\beta_0 + \sum_{i=1}^n \beta_iX_i}}{1 + e^{\beta_0 + \sum_{i=1}^n \beta_iX_i}} \\] +//! +//! SmartCore uses [limited memory BFGS](https://en.wikipedia.org/wiki/Limited-memory_BFGS) method to find estimates of regression coefficients, \\(\beta\\) +//! +//! Example: +//! +//! ``` +//! use smartcore::linalg::naive::dense_matrix::*; +//! use smartcore::linear::logistic_regression::*; +//! +//! //Iris data +//! let x = DenseMatrix::from_array(&[ +//! &[5.1, 3.5, 1.4, 0.2], +//! &[4.9, 3.0, 1.4, 0.2], +//! &[4.7, 3.2, 1.3, 0.2], +//! &[4.6, 3.1, 1.5, 0.2], +//! &[5.0, 3.6, 1.4, 0.2], +//! &[5.4, 3.9, 1.7, 0.4], +//! &[4.6, 3.4, 1.4, 0.3], +//! &[5.0, 3.4, 1.5, 0.2], +//! &[4.4, 2.9, 1.4, 0.2], +//! &[4.9, 3.1, 1.5, 0.1], +//! &[7.0, 3.2, 4.7, 1.4], +//! &[6.4, 3.2, 4.5, 1.5], +//! &[6.9, 3.1, 4.9, 1.5], +//! &[5.5, 2.3, 4.0, 1.3], +//! &[6.5, 2.8, 4.6, 1.5], +//! &[5.7, 2.8, 4.5, 1.3], +//! &[6.3, 3.3, 4.7, 1.6], +//! &[4.9, 2.4, 3.3, 1.0], +//! &[6.6, 2.9, 4.6, 1.3], +//! &[5.2, 2.7, 3.9, 1.4], +//! ]); +//! let y: Vec = vec![ +//! 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., +//! ]; +//! +//! let lr = LogisticRegression::fit(&x, &y); +//! +//! let y_hat = lr.predict(&x); +//! ``` +//! +//! ## References: +//! * ["An Introduction to Statistical Learning", James G., Witten D., Hastie T., Tibshirani R., 4.3 Logistic Regression](http://faculty.marshall.usc.edu/gareth-james/ISL/) +//! * ["On the Limited Memory Method for Large Scale Optimization", Nocedal et al., Mathematical Programming, 1989](http://users.iems.northwestern.edu/~nocedal/PDFfiles/limited.pdf) +//! use std::fmt::Debug; use std::marker::PhantomData; @@ -10,6 +61,7 @@ use crate::optimization::first_order::{FirstOrderOptimizer, OptimizerResult}; use crate::optimization::line_search::Backtracking; use crate::optimization::FunctionOrder; +/// Logistic Regression #[derive(Serialize, Deserialize, Debug)] pub struct LogisticRegression> { weights: M, @@ -150,6 +202,9 @@ impl<'a, T: RealNumber, M: Matrix> ObjectiveFunction } impl> LogisticRegression { + /// Fits Logistic Regression to your data. + /// * `x` - _NxM_ matrix with _N_ observations and _M_ features in each observation. + /// * `y` - target class values pub fn fit(x: &M, y: &M::RowVector) -> LogisticRegression { let y_m = M::from_row_vector(y.clone()); let (x_nrows, num_attributes) = x.shape(); @@ -212,6 +267,8 @@ impl> LogisticRegression { } } + /// Predict class labels for samples in `x`. + /// * `x` - _KxM_ data where _K_ is number of observations and _M_ is number of features. pub fn predict(&self, x: &M) -> M::RowVector { let n = x.shape().0; let mut result = M::zeros(1, n); @@ -238,11 +295,13 @@ impl> LogisticRegression { result.to_row_vector() } + /// Get estimates regression coefficients pub fn coefficients(&self) -> M { self.weights .slice(0..self.num_classes, 0..self.num_attributes) } + /// Get estimate of intercept pub fn intercept(&self) -> M { self.weights.slice( 0..self.num_classes, diff --git a/src/linear/mod.rs b/src/linear/mod.rs index 244671e..1965d42 100644 --- a/src/linear/mod.rs +++ b/src/linear/mod.rs @@ -1,2 +1,23 @@ +//! # Linear Models +//! Linear models describe a continuous response variable as a function of one or more predictor variables. +//! The model describes the relationship between a dependent variable y (also called the response) as a function of one or more independent, or explanatory variables \\(X_i\\). The general equation for a linear model is: +//! \\[y = \beta_0 + \sum_{i=1}^n \beta_iX_i + \epsilon\\] +//! +//! where \\(\beta_0 \\) is the intercept term (the expected value of Y when X = 0), \\(\epsilon \\) is an error term that is is independent of X and \\(\beta_i \\) +//! is the average increase in y associated with a one-unit increase in \\(X_i\\) +//! +//! Model assumptions: +//! * _Linearity_. The relationship between X and the mean of y is linear. +//! * _Constant variance_. The variance of residual is the same for any value of X. +//! * _Normality_. For any fixed value of X, Y is normally distributed. +//! * _Independence_. Observations are independent of each other. +//! +//! ## References: +//! +//! * ["An Introduction to Statistical Learning", James G., Witten D., Hastie T., Tibshirani R., 3. Linear Regression](http://faculty.marshall.usc.edu/gareth-james/ISL/) +//! * ["The Statistical Sleuth, A Course in Methods of Data Analysis", Ramsey F.L., Schafer D.W., Ch 7, 8, 3rd edition, 2013](http://www.statisticalsleuth.com/) +//! +//! + pub mod linear_regression; pub mod logistic_regression;