log_regression_basics.py

# Author: Hitesh Verma

import numpy as np
import pandas as pd
import time, math
from sklearn import metrics
from scipy import special
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler

def importdata(file):
    data = pd.read_csv(file, sep=',', header=None)

    #for missing values remove row or column with (atleast 1) any Nan Null present
    # spam_data = spam_data.dropna(axis=0, how='any')
    print(data.shape)
    return data.values


# Function to split the dataset
def splitdataset(data):
    # Seperating the target variable
    x = data[:, 0:data.shape[1]-1]
    y = data[:, data.shape[1]-1:data.shape[1]]
    print(np.shape(x),np.shape(y))
    return x, y

# Calculate accuracy
def accuracy_val(actual, predicted):
    correct = 0
    for i in range(len(actual)):
        if actual[i] == predicted[i]:
            correct += 1
    return correct / float(len(actual)) * 100.0


def sigmoid(z):
    z = np.array(z, dtype=np.float32)
    # zexp = np.exp(-z)
    # return (1.0 / (1 + np.exp(-z)))
    return special.expit(z)


# test prediction
def predict(X_test,params):
    X_test_b = np.c_[np.ones((len(X_test), 1)), X_test]
    score = sigmoid(X_test_b.dot(params))
    prediction = np.round(score)
    return prediction


def feature_normalization(x):
    mu = np.mean(x,axis=0)
    sigma = np.std(x,axis=0)
    return mu, sigma

def normalization(x,mu,sigma):
    x = np.subtract(x, mu)
    x = np.divide(x, sigma)
    return x

# def compute_likelihood(X, y, w):
#     J = 0
#     h_w = sigmoid(X.dot(w))
#     J = - np.sum(np.dot(y.T,np.log(h_w)) + np.dot((1 - y).T,np.log(1 - h_w)))
#     return J


def gradient_descent(X, y, w, learning_rate, num_iters):
    m = len(X)
    prediction = sigmoid(X.dot(w))
    reg = 0.2
    for iter in range(num_iters):
        w = w - (1 / m) * learning_rate * (X.T.dot((prediction - y)))
        print("iter ",iter)
        # if np.linalg.norm(j_old - j) < 0.0001: #0.001
        #     print("convergence")
        #     break
        # j_old = j
    return w


def minibatch_gradient_descent(X, y, w, learning_rate=0.01, iterations=200, batch_size=20):
   # X -> Matrix of X without added bias units
    m = len(y)
    cost_history = np.zeros(iterations)
    n_batches = int(m / batch_size)
    for it in range(iterations):
        cost = 0.0
        indices = np.random.permutation(m)
        X = X[indices]
        y = y[indices]
        for i in range(0, m, batch_size):
            X_i = X[i:i + batch_size]
            y_i = y[i:i + batch_size]
            X_i = np.c_[np.ones(len(X_i)), X_i]  #adding bias
            prediction = sigmoid(np.dot(X_i, w))
            w = w - (1 / m) * learning_rate * (X_i.T.dot((prediction - y_i)))
            # cost += cal_cost(w, X_i, y_i)
        # cost_history[it] = cost
    return w


def stocashtic_gradient_descent(X, y, w, learning_rate=0.5, iterations=50):
    #X -> Matrix of X with added bias units
    m = len(y)
    for it in range(iterations):
        for i in range(m):
            rand_ind = np.random.randint(0, m)
            X_i = X[rand_ind, :].reshape(1, X.shape[1])
            y_i = y[rand_ind].reshape(1, 1)
            prediction = sigmoid(np.dot(X_i, w))
            w = w - (1 / m) * learning_rate * (X_i.T.dot((prediction - y_i)))
    return w



def log_reg(X, y, alpha, num_iters):
    X_b = np.c_[np.ones((len(X), 1)), X]  # set bias term to 1 for each sample
    size = (X_b.shape[1],1)
    w = np.random.uniform(size=size)
    # w = np.random.normal(size=size)
    # params = gradient_descent(X_b, y, w, alpha, num_iters) #batch
    # params = minibatch_gradient_descent(X, y, w, alpha, num_iters) #minibatch
    params = stocashtic_gradient_descent(X_b, y, w, alpha, num_iters) #stochastic
    return params





def main(i):
    np.random.seed(1)
    s = time.time()
    alpha =  0.01
    num_iters = 1000
    l1 = [0.0001, 0.001, 0.01, 0.1, 0.5, 1, 10, 1000]
    #======================================================================================================

    data = importdata('http://archive.ics.uci.edu/ml/machine-learning-databases/spambase/spambase.data')
    X, y = splitdataset(data)
    X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2)
    y_train = y_train.astype(int)
    y_test = y_test.astype(int)
    print(np.shape(X_train),np.shape(y_train))

    scaler = StandardScaler()
    scaler.fit(X_train)
    X_train = scaler.transform(X_train)
    X_test = scaler.transform(X_test)
    #======================================================================================================

    params = log_reg(X_train, y_train, alpha, num_iters)

    # after training model, make predictions
    train_predicted_label = predict(X_train, params)
    # print(train_predicted_label)
    train_acc = metrics.accuracy_score(y_train, train_predicted_label)
    print('Train Accuracy',train_acc)

    test_predicted_label = predict(X_test, params)
    # print(test_predicted_label)
    test_acc = metrics.accuracy_score(y_test, test_predicted_label)
    print('Test Accuracy',test_acc)
    e = time.time()
    # print("time",e-s)
    return int(test_acc * 100)




if __name__ == '__main__':
    max = 85
    max_i = 0
    #seeding trick
    # for i in range(100):
    #     accuracy = main(i)
    #     print(i, accuracy)
    #     if accuracy > max_i:
    #         max_i = accuracy
    #     if accuracy > max:
    #         print(i)
    #         break
    # print("max_i",max_i) #19 seed -> 80 score
    main(19)