work/smartcore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: adds elastic net

Browse Source
This commit is contained in:
Volodymyr Orlov
2020-12-11 18:55:07 -08:00
parent 2650416235
commit 78673b597f
8 changed files with 647 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/linear/lasso.rs
+10 -237
View File
@@ -29,7 +29,7 @@ use serde::{Deserialize, Serialize};
use crate::error::Failed;
use crate::linalg::BaseVector;
use crate::linalg::Matrix;
use crate::linear::bg_solver::BiconjugateGradientSolver;
use crate::linear::lasso_optimizer::InteriorPointOptimizer;
use crate::math::num::RealNumber;
/// Lasso regression parameters
@@ -53,14 +53,6 @@ pub struct Lasso<T: RealNumber, M: Matrix<T>> {
intercept: T,
}
struct InteriorPointOptimizer<T: RealNumber, M: Matrix<T>> {
ata: M,
d1: Vec<T>,
d2: Vec<T>,
prb: Vec<T>,
prs: Vec<T>,
}
impl<T: RealNumber> Default for LassoParameters<T> {
fn default() -> Self {
LassoParameters {
@@ -118,7 +110,13 @@ impl<T: RealNumber, M: Matrix<T>> Lasso<T, M> {
let mut optimizer = InteriorPointOptimizer::new(&scaled_x, p);
let mut w = optimizer.optimize(&scaled_x, y, &parameters)?;
let mut w = optimizer.optimize(
&scaled_x,
y,
parameters.alpha,
parameters.max_iter,
parameters.tol,
)?;
for j in 0..p {
w.set(j, 0, w.get(j, 0) / col_std[j]);
@@ -135,7 +133,8 @@ impl<T: RealNumber, M: Matrix<T>> Lasso<T, M> {
} else {
let mut optimizer = InteriorPointOptimizer::new(x, p);
let w = optimizer.optimize(x, y, &parameters)?;
let w =
optimizer.optimize(x, y, parameters.alpha, parameters.max_iter, parameters.tol)?;
(w, y.mean())
};
@@ -184,232 +183,6 @@ impl<T: RealNumber, M: Matrix<T>> Lasso<T, M> {
}
}
impl<T: RealNumber, M: Matrix<T>> InteriorPointOptimizer<T, M> {
fn new(a: &M, n: usize) -> InteriorPointOptimizer<T, M> {
InteriorPointOptimizer {
ata: a.ab(true, a, false),
d1: vec![T::zero(); n],
d2: vec![T::zero(); n],
prb: vec![T::zero(); n],
prs: vec![T::zero(); n],
}
}
fn optimize(
&mut self,
x: &M,
y: &M::RowVector,
parameters: &LassoParameters<T>,
) -> Result<M, Failed> {
let (n, p) = x.shape();
let p_f64 = T::from_usize(p).unwrap();
//parameters
let pcgmaxi = 5000;
let min_pcgtol = T::from_f64(0.1).unwrap();
let eta = T::from_f64(1E-3).unwrap();
let alpha = T::from_f64(0.01).unwrap();
let beta = T::from_f64(0.5).unwrap();
let gamma = T::from_f64(-0.25).unwrap();
let mu = T::two();
let y = M::from_row_vector(y.sub_scalar(y.mean())).transpose();
let mut max_ls_iter = 100;
let mut pitr = 0;
let mut w = M::zeros(p, 1);
let mut neww = w.clone();
let mut u = M::ones(p, 1);
let mut newu = u.clone();
let mut f = M::fill(p, 2, -T::one());
let mut newf = f.clone();
let mut q1 = vec![T::zero(); p];
let mut q2 = vec![T::zero(); p];
let mut dx = M::zeros(p, 1);
let mut du = M::zeros(p, 1);
let mut dxu = M::zeros(2 * p, 1);
let mut grad = M::zeros(2 * p, 1);
let mut nu = M::zeros(n, 1);
let mut dobj = T::zero();
let mut s = T::infinity();
let mut t = T::one()
.max(T::one() / parameters.alpha)
.min(T::two() * p_f64 / T::from(1e-3).unwrap());
for ntiter in 0..parameters.max_iter {
let mut z = x.matmul(&w);
for i in 0..n {
z.set(i, 0, z.get(i, 0) - y.get(i, 0));
nu.set(i, 0, T::two() * z.get(i, 0));
}
// CALCULATE DUALITY GAP
let xnu = x.ab(true, &nu, false);
let max_xnu = xnu.norm(T::infinity());
if max_xnu > parameters.alpha {
let lnu = parameters.alpha / max_xnu;
nu.mul_scalar_mut(lnu);
}
let pobj = z.dot(&z) + parameters.alpha * w.norm(T::one());
dobj = dobj.max(gamma * nu.dot(&nu) - nu.dot(&y));
let gap = pobj - dobj;
// STOPPING CRITERION
if gap / dobj < parameters.tol {
break;
}
// UPDATE t
if s >= T::half() {
t = t.max((T::two() * p_f64 * mu / gap).min(mu * t));
}
// CALCULATE NEWTON STEP
for i in 0..p {
let q1i = T::one() / (u.get(i, 0) + w.get(i, 0));
let q2i = T::one() / (u.get(i, 0) - w.get(i, 0));
q1[i] = q1i;
q2[i] = q2i;
self.d1[i] = (q1i * q1i + q2i * q2i) / t;
self.d2[i] = (q1i * q1i - q2i * q2i) / t;
}
let mut gradphi = x.ab(true, &z, false);
for i in 0..p {
let g1 = T::two() * gradphi.get(i, 0) - (q1[i] - q2[i]) / t;
let g2 = parameters.alpha - (q1[i] + q2[i]) / t;
gradphi.set(i, 0, g1);
grad.set(i, 0, -g1);
grad.set(i + p, 0, -g2);
}
for i in 0..p {
self.prb[i] = T::two() + self.d1[i];
self.prs[i] = self.prb[i] * self.d1[i] - self.d2[i] * self.d2[i];
}
let normg = grad.norm2();
let mut pcgtol = min_pcgtol.min(eta * gap / T::one().min(normg));
if ntiter != 0 && pitr == 0 {
pcgtol *= min_pcgtol;
}
let error = self.solve_mut(x, &grad, &mut dxu, pcgtol, pcgmaxi)?;
if error > pcgtol {
pitr = pcgmaxi;
}
for i in 0..p {
dx.set(i, 0, dxu.get(i, 0));
du.set(i, 0, dxu.get(i + p, 0));
}
// BACKTRACKING LINE SEARCH
let phi = z.dot(&z) + parameters.alpha * u.sum() - Self::sumlogneg(&f) / t;
s = T::one();
let gdx = grad.dot(&dxu);
let lsiter = 0;
while lsiter < max_ls_iter {
for i in 0..p {
neww.set(i, 0, w.get(i, 0) + s * dx.get(i, 0));
newu.set(i, 0, u.get(i, 0) + s * du.get(i, 0));
newf.set(i, 0, neww.get(i, 0) - newu.get(i, 0));
newf.set(i, 1, -neww.get(i, 0) - newu.get(i, 0));
}
if newf.max() < T::zero() {
let mut newz = x.matmul(&neww);
for i in 0..n {
newz.set(i, 0, newz.get(i, 0) - y.get(i, 0));
}
let newphi = newz.dot(&newz) + parameters.alpha * newu.sum()
- Self::sumlogneg(&newf) / t;
if newphi - phi <= alpha * s * gdx {
break;
}
}
s = beta * s;
max_ls_iter += 1;
}
if lsiter == max_ls_iter {
return Err(Failed::fit(
"Exceeded maximum number of iteration for interior point optimizer",
));
}
w.copy_from(&neww);
u.copy_from(&newu);
f.copy_from(&newf);
}
Ok(w)
}
fn sumlogneg(f: &M) -> T {
let (n, _) = f.shape();
let mut sum = T::zero();
for i in 0..n {
sum += (-f.get(i, 0)).ln();
sum += (-f.get(i, 1)).ln();
}
sum
}
}
impl<'a, T: RealNumber, M: Matrix<T>> BiconjugateGradientSolver<T, M>
for InteriorPointOptimizer<T, M>
{
fn solve_preconditioner(&self, a: &M, b: &M, x: &mut M) {
let (_, p) = a.shape();
for i in 0..p {
x.set(
i,
0,
(self.d1[i] * b.get(i, 0) - self.d2[i] * b.get(i + p, 0)) / self.prs[i],
);
x.set(
i + p,
0,
(-self.d2[i] * b.get(i, 0) + self.prb[i] * b.get(i + p, 0)) / self.prs[i],
);
}
}
fn mat_vec_mul(&self, _: &M, x: &M, y: &mut M) {
let (_, p) = self.ata.shape();
let atax = self.ata.matmul(&x.slice(0..p, 0..1));
for i in 0..p {
y.set(
i,
0,
T::two() * atax.get(i, 0) + self.d1[i] * x.get(i, 0) + self.d2[i] * x.get(i + p, 0),
);
y.set(
i + p,
0,
self.d2[i] * x.get(i, 0) + self.d1[i] * x.get(i + p, 0),
);
}
}
fn mat_t_vec_mul(&self, a: &M, x: &M, y: &mut M) {
self.mat_vec_mul(a, x, y);
}
}
#[cfg(test)]
mod tests {
use super::*;