Project 1.Rmd

---
title: "Project 1"
output: html_document
date: "2022-09-27"
---
```{r}
list.of.packages <- c("caret", "xgboost", "randomForest", "glmnet")
new.packages <- list.of.packages[!(list.of.packages %in% installed.packages()[,"Package"])]
if(length(new.packages)) install.packages(new.packages)

library(caret)
library(xgboost)
library(randomForest)
library(glmnet)
```

```{r}
# Get data
data <- read.csv("Ames_data.csv")
testIDs <- read.table("project1_testIDs.dat")

# Drop unwanted columns
#drop <- c('Street', 'Utilities', 'Condition_2', 'Roof_Matl', 'Heating', 'Pool_QC', 'Misc_Feature', 'Low_Qual_Fin_SF', 'Pool_Area', 'Longitude','Latitude')
#housingData = data[,!(names(data) %in% drop)]

# Set NA values to 0
#housingData[is.na(housingData)] = 0

# Create train and test datasets
# for (split_number in 1:10) {

#drop <- c('Street', 'Utilities', 'Condition_2', 'Roof_Matl', 'Heating', 'Pool_QC', 'Misc_Feature', 'Low_Qual_Fin_SF', 'Pool_Area', 'Longitude','Latitude')
#train = train[,!(names(train) %in% drop)]
#test = test[,!(names(test) %in% drop)]

#print("TEST")

for (j in 1:10) {
  train <- data[-testIDs[,j], ]
  test <- data[testIDs[,j], ]
  test.y <- test[, c(1, 83)]
  test <- test[, -83]
  
  directory = paste0("test_runs/test_run", j)
  
  
  write.csv(train, paste0(directory, "/train.csv"),row.names=FALSE)
  write.csv(test, paste0(directory, "/test.csv"),row.names=FALSE)
  write.csv(test.y, paste0(directory, "/test_y.csv"),row.names=FALSE)
}

```

```{r}
train <- read.csv("train.csv")
test <- read.csv("test.csv")
PIDs <- test[,1]

drop <- c('Street', 'Utilities', 'Condition_2', 'Roof_Matl', 'Heating', 'Pool_QC', 'Misc_Feature', 'Low_Qual_Fin_SF', 'Pool_Area', 'Longitude','Latitude')
train = train[,!(names(train) %in% drop)]
test = test[,!(names(test) %in% drop)]

train[is.na(train)] = 0
test[is.na(test)] = 0

# Preprocessing
x_train_drop <- c('PID','Sale_Price')
train.x  = train[,!(names(train) %in% x_train_drop)] # train data without "PID" and "Sale_Price"
train.y = log(train['Sale_Price'])# log transformed "Sale_Price"

# Process train data
categorical.vars <- colnames(train.x)[
  which(sapply(train.x,
                 function(x) mode(x)=="character"))]
train.matrix <- train.x[, !colnames(train.x) %in% categorical.vars, 
                          drop=FALSE]
n.train <- nrow(train.matrix)

train_levels <- list()

#Save train.x levels to compare with test.x levels later
for(var in categorical.vars){
    mylevels <- sort(unique(train.x[, var]))
    train_levels <- c(train_levels, mylevels)
    m <- length(mylevels)
    m <- ifelse(m>2, m, 1)
    tmp.train <- matrix(0, n.train, m)
    col.names <- NULL
    for(j in 1:m){
      tmp.train[train.x[, var]==mylevels[j], j] <- 1
      col.names <- c(col.names, paste(var, '_', mylevels[j], sep=''))
      }
    colnames(tmp.train) <- col.names
    train.matrix <- cbind(train.matrix, tmp.train)
}

# Process test data
x_test_drop <- 'PID'
test.x = test[,!(names(test) %in% x_test_drop)] # test data without "PID" and "Sale_Price"

test_categorical.vars <- colnames(test.x)[
which(sapply(test.x, function(x) mode(x)=="character"))]

test.matrix <- test.x[, !colnames(test.x) %in% test_categorical.vars, 
                          drop=FALSE]
n.test <- nrow(test.matrix)

for(var in categorical.vars){
    testlevels <- sort(unique(test.x[, var]))
    m <- length(testlevels)
    m <- ifelse(m>2, m, 1)
    tmp.test <- matrix(0, n.test, m)
    col.names <- NULL
    for(j in 1:m){
      tmp.test[test.x[, var]==testlevels[j], j] <- 1
      col.names <- c(col.names, paste(var, '_', testlevels[j], sep=''))
      }
    colnames(tmp.test) <- col.names
    test.matrix <- cbind(test.matrix, tmp.test)
}

# We need to match the columns for train.matrix to test.matrix. 
# If there is a column in train.matrix that isnt in test.matrix create the column with all 0 values in the test.matrix. 
# If there is a column in test.matrix that is not in train.matrix remove that column from test.matrix. 
# Columns have to be the same order for both

test_col_names = colnames(test.matrix)
train_col_names = colnames(train.matrix)

columns_to_drop_from_test_matrix = c()

for (test_name in test_col_names) {
  found = 0
  
  for (train_name in train_col_names) {
    if(train_name == test_name) {
      found = 1
    }
  }
  
  if(found == 0) {
    columns_to_drop_from_test_matrix = c(columns_to_drop_from_test_matrix, test_name)
  }
}

test.matrix = test.matrix[,!(names(test.matrix) %in% columns_to_drop_from_test_matrix)]

test_matrix_df = as.data.frame(test.matrix)
train_matrix_df = as.data.frame(train.matrix)


for (train_name in train_col_names) {
  found = 0
  
  for (test_name in test_col_names) {
    if(test_name == train_name) {
      found = 1
    }
  }
  
  if(found == 0) {
    test_matrix_df[train_name] = rep(0, dim(test.matrix)[1])
  }
}

#print(ncol(test_matrix_df))
#print(ncol(train_matrix_df))

#old_test_matrix_df = test_matrix_df[,sort(names(test_matrix_df))]
#old_train_matrix_df = train_matrix_df[,sort(names(train_matrix_df))]



#drop_high_threshold_cols = c("MS_Zoning", "Alley", "Lot_Shape", "Land_Contour", "Lot_Config", "Land_Slope", "Condition_1", "Bldg_Type", "House_Style", "Overall_Cond", "Roof_Style", "Mas_Vnr_Type", "Exter_Qual", "Exter_Cond", "Bsmt_Cond", "Bsmt_Exposure", "BsmtFin_Type_2", "Heating_QC", "Central_Air", "Electrical", "Kitchen_Qual", "Functional", "Garage_Type", "Garage_Qual", "Garage_Cond", "Paved_Drive", "Fence", "Sale_Type", "Sale_Condition")
#drop_high_threshold_cols = c('MS_Zoning')
#test_matrix_df = test_matrix_df[,!(names(test_matrix_df) %in% drop_high_threshold_cols)]
#train_matrix_df = train_matrix_df[,!(names(train_matrix_df) %in% drop_high_threshold_cols)]

#new_test_matrix_df = subset(test_matrix_df, select = !(names(test_matrix_df) %in% drop_high_threshold_cols))
#new_train_matrix_df = subset(train_matrix_df, select = !(names(train_matrix_df) %in% drop_high_threshold_cols))


test_matrix_df = test_matrix_df[,sort(names(test_matrix_df))]
train_matrix_df = train_matrix_df[,sort(names(train_matrix_df))]

#print(ncol(new_test_matrix_df))
#print(ncol(new_train_matrix_df))

# for(i in ncol(test_matrix_df)) {
#   if (colnames(test_matrix_df)[i] != colnames(train_matrix_df)[i]) {
#    print("HERE")
#     print(colnames(test_matrix_df)[i])
#     print(colnames(train_matrix_df)[i])
#   }
# }

# Remember to set a seed so we can reproduce your results; 
# the seed does not need to be related to your UIN. 
set.seed(1852)

```

```{r}

# Decision Tree

xgb.model <- xgboost(data = as.matrix(train_matrix_df), 
                       label = as.matrix(train.y), max_depth = 6,
                       eta = 0.05, nrounds = 5000,
                       subsample = 0.5,
                       verbose = FALSE)

#predict(xgb.model, as.matrix(test_matrix_df))

# From Campuswire post 490 we are training the model on the log of train.y (line 161) and then writing to the csv the exp of the prediction (line 170)

df = data.frame(PID = PIDs, Sale_Price = exp(predict(xgb.model, as.matrix(test_matrix_df))))

write.csv(df,"mysubmission1.txt", row.names = FALSE, quote=FALSE)

pred <- read.csv("mysubmission1.txt")
names(test.y)[2] <- "True_Sale_Price"
pred <- merge(pred, test.y, by="PID")
sqrt(mean((log(pred$Sale_Price) - log(pred$True_Sale_Price))^2))

```

```{r}
# Random Forest

# rf <- randomForest(x=as.matrix(train_matrix_df), y=as.matrix(train.y), 
#                        mtry=14, importance=TRUE, ntree = 100)
# 
# #p1 <- predict(rf, as.matrix(test_matrix_df))
# 
# randomForest_df = data.frame(PID = test.y[1], Sale_Price = exp(predict(rf, as.matrix(test_matrix_df))))
# 
# write.csv(randomForest_df,"mysubmission2.txt", row.names = FALSE, quote=FALSE)
# 
# pred <- read.csv("mysubmission2.txt")
# names(test.y)[2] <- "True_Sale_Price"
# pred <- merge(pred, test.y, by="PID")
# sqrt(mean((log(pred$Sale_Price) - log(pred$True_Sale_Price))^2))
```

```{r}
# Ridge Regression

# mylasso.lambda.seq = exp(seq(-10, 1, length.out = 100))
# cv.out = cv.glmnet(as.matrix(train_matrix_df), as.matrix(train.y), alpha = 0, 
#                     lambda = mylasso.lambda.seq)
#   
# best.lam = cv.out$lambda.min
# Ytest.pred = exp(predict(cv.out, s = best.lam, newx = as.matrix(test_matrix_df)))
# 
# colnames(Ytest.pred)[1]<-"Sale_Price"
# 
# ridge_df = data.frame(PID = test.y[1], Sale_Price = Ytest.pred)
# 
# write.csv(ridge_df,"mysubmission3.txt", row.names = FALSE, quote=FALSE)
# 
# pred <- read.csv("mysubmission3.txt")
# names(test.y)[2] <- "True_Sale_Price"
# pred <- merge(pred, test.y, by="PID")
# sqrt(mean((log(pred$Sale_Price) - log(pred$True_Sale_Price))^2))
```
```{r}
#Lasso Regression

cv.out = cv.glmnet(as.matrix(train_matrix_df), as.matrix(train.y), alpha = 1)
best.lam = cv.out$lambda.min
Ytest.pred = exp(predict(cv.out, s = best.lam, newx = as.matrix(test_matrix_df)))

colnames(Ytest.pred)[1]<-"Sale_Price"

lasso_df = data.frame(PID = test.y[1], Sale_Price = Ytest.pred)

write.csv(lasso_df,"mysubmission4.txt", row.names = FALSE, quote=FALSE)

pred <- read.csv("mysubmission4.txt")
names(test.y)[2] <- "True_Sale_Price"
pred <- merge(pred, test.y, by="PID")
sqrt(mean((log(pred$Sale_Price) - log(pred$True_Sale_Price))^2))
```
```{r}
mylasso.lambda.seq = exp(seq(-10, 1, length.out = 100))
cv.out = cv.glmnet(as.matrix(train_matrix_df), as.matrix(train.y), alpha = 0.77, 
                    lambda = mylasso.lambda.seq)
  
best.lam = cv.out$lambda.min
Ytest.pred = exp(predict(cv.out, s = best.lam, newx = as.matrix(test_matrix_df)))
colnames(Ytest.pred)[1]<-"Sale_Price"
ridge_df = data.frame(PID = test.y[1], Sale_Price = Ytest.pred)
write.csv(ridge_df,"mysubmission5.txt", row.names = FALSE, quote=FALSE)

pred <- read.csv("mysubmission5.txt")
names(test.y)[2] <- "True_Sale_Price"
pred <- merge(pred, test.y, by="PID")
sqrt(mean((log(pred$Sale_Price) - log(pred$True_Sale_Price))^2))
```
```{r}
# cv.out <- cv.glmnet(as.matrix(train_matrix_df), as.matrix(train.y), alpha = 1)
# 
# sel.vars <- predict(cv.out, type="nonzero", s = cv.out$lambda.min)$s1
# 
# cv.out <- cv.glmnet(as.matrix(as.matrix(train_matrix_df)[, sel.vars]), as.matrix(train.y), alpha = 0)
# 
# Ytest.pred <- exp(predict(cv.out, s = cv.out$lambda.min, newx = as.matrix(test_matrix_df[, sel.vars])))
# 
# colnames(Ytest.pred)[1]<-"Sale_Price"
# 
# ridge_df = data.frame(PID = test.y[1], Sale_Price = Ytest.pred)
# 
# write.csv(ridge_df,"mysubmission6.txt", row.names = FALSE, quote=FALSE)
# 
# pred <- read.csv("mysubmission6.txt")
# names(test.y)[2] <- "True_Sale_Price"
# pred <- merge(pred, test.y, by="PID")
# sqrt(mean((log(pred$Sale_Price) - log(pred$True_Sale_Price))^2))
```