Ridge Regression - L2 Regularization

ridge <- function(X, Y, rm_na=T, standardise=F, lambda, method="matrix") {
  
  if (!is.matrix(X)) {
    X <- as.matrix(X)
  }
  
  if (rm_na) {
    index <- complete.cases(X) & complete.cases(Y)
    X <- X[index,]
    Y <- Y[index]
  }
  
  if (standardise) {
    X <- apply(X, 2, function(x) (x - mean(x))/sd(x))
  }
  
  # if (intercept) {
  #   X <- cbind(rep(1,nrow(X)), X)
  #   colnames(X)[1] <- "(Intercept)"
  # }
  
  k <- ncol(X)
  
  if (method=="matrix") {
    b <- solve(crossprod(X) + lambda*diag(k)) %*% crossprod(X, Y)
  } else if (method=="MLE") {
    start <- numeric(k)
    ridgeMin <- function(b) {
      crossprod(Y - X%*%b) + lambda * crossprod(b)
    }
    out <- optim(start, ridgeMin, method="BFGS")
    b <- out$par
    names(b) <- colnames(X)
  }
  
  return(b)
  
}

Testing the function:

library(glmnet)

X <- as.matrix(mtcars[, -1])
X_standard <- apply(X, 2, function(x) (x-mean(x)) / sd(x))
Y <- mtcars[[1]]

data.frame(
  glmnet = glmnet(X, Y, alpha=0, lambda=0.5, intercept = F, standardize=F)$beta[,1],
  Ridge_Matrix = ridge(X, Y, lambda=0.5),
  Ridge_MLE = ridge(X, Y, lambda=0.5, method="MLE")
)

##           glmnet Ridge_Matrix    Ridge_MLE
## cyl   0.44401246  0.369554729  0.369555066
## disp  0.00457116  0.008205706  0.008205705
## hp   -0.01691490 -0.017783212 -0.017783227
## drat  1.35027698  1.392774438  1.392772924
## wt   -2.82334377 -3.210741706 -3.210737396
## qsec  1.01961140  1.083489203  1.083488388
## vs    0.08513161  0.078258643  0.078262480
## am    2.29971390  2.425849951  2.425855958
## gear  1.48674773  1.299391499  1.299392402
## carb -0.59355808 -0.469904311 -0.469904608

data.frame(
  glmnet = glmnet(X_standard, Y, alpha=0, lambda=0.5, intercept = F, standardize=F)$beta[,1],
  Ridge_Matrix = ridge(X_standard, Y, lambda=0.5),
  Ridge_MLE = ridge(X_standard, Y, lambda=0.5, method="MLE")
)

##          glmnet Ridge_Matrix  Ridge_MLE
## cyl  -0.2664494   -0.2279229 -0.2278197
## disp  0.4681466    0.6833358  0.6833124
## hp   -1.0616275   -1.1445907 -1.1446305
## drat  0.4776239    0.4749628  0.4749703
## wt   -2.5629699   -2.7815105 -2.7814765
## qsec  0.9656145    1.0764747  1.0764682
## vs    0.1892789    0.1724660  0.1725038
## am    1.1745870    1.1941804  1.1941926
## gear  0.5221163    0.5117097  0.5117464
## carb -0.8771296   -0.7712839 -0.7713067