linear.cpp

/***************************************************
 * linear_model.cpp - Self-Contained Example
 * 
 * Build & Run:
 *   g++ -o linear.exe linear_model.cpp
 *   ./linear.exe
 ***************************************************/

#include <iostream>
#include <fstream>
#include <vector>
#include <cmath>
#include <algorithm>
#include <random>
#include <string>

using namespace std;

/******************************************
 * 1) Data Structures & Utility
 *****************************************/
// A simple Image struct to store:
//  - 3D pixel data [channels][height][width]
//  - label as a one-hot vector of size 10
struct Image {
    vector<vector<vector<float>>> rgb;
    vector<float> label; // e.g. [1,0,0,0,0,0,0,0,0,0] for class 0
};

int readData(ifstream 
    &readFile,vector<Image> 
    &images){

    for(int i=0;i<10000;i++){
        uint8_t byte_value;
        struct Image img = Image();

        for(int i=0;i<10;i++){
            img.label.push_back(0.0f);
        }
        // label is now an array that has all 0s and only the label index as 1. 
        readFile.read(reinterpret_cast<char*>(&byte_value), 1);
        // cout<< byte_value << endl;
        img.label[byte_value] = 1;

        // cout << "Label: ";
        // for (int i = 0; i < 10; ++i) cout << img.label[i] << " ";
        // cout << endl;

        img.rgb.resize(3, vector<vector<float>>(32, vector<float>(32)));
        for (int i = 0; i < 32; i++) {
            for (int j = 0; j < 32; j++) {
                readFile.read(reinterpret_cast<char*>(&byte_value), 1);
                float float_value = static_cast<float>(byte_value) / 255.0f;
                img.rgb[0][i][j] = float_value;
            }
        }
        for (int i = 0; i < 32; i++) {
            for (int j = 0; j < 32; j++) {
                readFile.read(reinterpret_cast<char*>(&byte_value), 1);
                float float_value = static_cast<float>(byte_value) / 255.0f;
                img.rgb[1][i][j] = float_value;
            }
        }
        for (int i = 0; i < 32; i++) {
            for (int j = 0; j < 32; j++) {
                readFile.read(reinterpret_cast<char*>(&byte_value), 1);
                float float_value = static_cast<float>(byte_value) / 255.0f;
                img.rgb[2][i][j] = float_value;
            }
        }
        images.push_back(img);
    }
    return 0;
}

// Utility to flatten a 3D image into 1D vector
int flatten_image(const Image &image, vector<float> &flat) {
    flat.clear();
    for (auto &channel : image.rgb) {
        for (auto &row : channel) {
            for (auto val : row) {
                flat.push_back(val);
            }
        }
    }
    return 0;
}

// Argmax for class prediction
int argmax(const vector<float> &vec) {
    return max_element(vec.begin(), vec.end()) - vec.begin();
}

/******************************************
 * 2) Core Linear Model Functions
 *****************************************/
// Forward pass for a linear classifier: logit = W*x + b
int forward_pass_linear(
    const vector<float> &flat, 
    const vector<vector<float>> &weights, 
    const vector<float> &bias, 
    vector<float> &logits
) {
    int num_classes = weights.size();
    int num_inputs = flat.size();

    logits.assign(num_classes, 0.0f);
    for (int i = 0; i < num_classes; ++i) {
        for (int j = 0; j < num_inputs; ++j) {
            logits[i] += weights[i][j] * flat[j];
        }
        logits[i] += bias[i];
    }
    return 0;
}

// Softmax
int softmax(const vector<float> &logits, vector<float> &probs) {
    float max_logit = *max_element(logits.begin(), logits.end());
    float sum_exp = 0.0f;
    probs.resize(logits.size());

    for (int i = 0; i < (int)logits.size(); ++i) {
        probs[i] = exp(logits[i] - max_logit);
        sum_exp += probs[i];
    }
    for (int i = 0; i < (int)logits.size(); ++i) {
        probs[i] /= sum_exp;
    }
    return 0;
}

// Cross-entropy loss
float cross_entropy_loss(const vector<float> &probs, const vector<float> &label) {
    float loss = 0.0f;
    for (int i = 0; i < (int)probs.size(); ++i) {
        // -label[i] * log(probs[i])
        loss -= label[i] * log(probs[i] + 1e-9f);
    }
    return loss;
}

// Backward pass for linear model: 
// grad_w[i][j] = (probs[i] - label[i]) * flat[j]
int backward_pass_linear(
    const vector<float> &flat, 
    const vector<float> &probs, 
    const vector<float> &label, 
    vector<vector<float>> &grad_weights, 
    vector<float> &grad_bias
) {
    int num_classes = probs.size();
    int input_size = flat.size();

    grad_weights.assign(num_classes, vector<float>(input_size, 0.0f));
    grad_bias.assign(num_classes, 0.0f);

    for (int i = 0; i < num_classes; ++i) {
        float delta = probs[i] - label[i];
        // Weight gradients
        for (int j = 0; j < input_size; ++j) {
            grad_weights[i][j] += delta * flat[j];
        }
        // Bias gradient
        grad_bias[i] += delta;
    }
    return 0;
}

// Update weights: w -= lr * dw
int update_weights_linear(
    vector<vector<float>> &weights, 
    vector<float> &bias, 
    const vector<vector<float>> &grad_weights, 
    const vector<float> &grad_bias, 
    float learning_rate
) {
    for (int i = 0; i < (int)weights.size(); ++i) {
        for (int j = 0; j < (int)weights[i].size(); ++j) {
            weights[i][j] -= learning_rate * grad_weights[i][j];
        }
        bias[i] -= learning_rate * grad_bias[i];
    }
    return 0;
}

/******************************************
 * 3) Mini-batch Training & Testing
 *****************************************/
// Train on one batch
int process_batch_linear(
    vector<Image> &batch, 
    vector<vector<float>> &weights, 
    vector<float> &bias, 
    float &batch_loss,
    float learning_rate = 0.01f
) {
    int batch_size = batch.size();
    int num_classes = weights.size();
    int input_size  = weights[0].size();

    vector<vector<float>> total_grad_weights(
        num_classes, vector<float>(input_size, 0.0f));
    vector<float> total_grad_bias(num_classes, 0.0f);

    batch_loss = 0.0f;

    // Accumulate gradients
    for (auto &image : batch) {
        vector<float> flat, logits, probs;
        flatten_image(image, flat);
        forward_pass_linear(flat, weights, bias, logits);
        softmax(logits, probs);
        batch_loss += cross_entropy_loss(probs, image.label);

        // local gradients
        vector<vector<float>> grad_weights;
        vector<float> grad_bias;
        backward_pass_linear(flat, probs, image.label, grad_weights, grad_bias);

        // Accumulate
        for (int i = 0; i < num_classes; ++i) {
            for (int j = 0; j < input_size; ++j) {
                total_grad_weights[i][j] += grad_weights[i][j];
            }
            total_grad_bias[i] += grad_bias[i];
        }
    }

    // Average over batch
    for (int i = 0; i < num_classes; ++i) {
        for (int j = 0; j < input_size; ++j) {
            total_grad_weights[i][j] /= batch_size;
        }
        total_grad_bias[i] /= batch_size;
    }

    // Update
    update_weights_linear(weights, bias, total_grad_weights, total_grad_bias, learning_rate);
    batch_loss /= batch_size;

    return 0;
}

// Evaluate on test set
float process_test_linear(
    vector<Image> &images, 
    vector<vector<float>> &weights, 
    vector<float> &bias
) {
    int correct = 0;
    for (auto &image : images) {
        vector<float> flat, logits, probs;
        flatten_image(image, flat);
        forward_pass_linear(flat, weights, bias, logits);
        softmax(logits, probs);
        int predicted = argmax(probs);
        int actual = argmax(image.label);
        if (predicted == actual) correct++;
    }
    return static_cast<float>(correct) / images.size();
}

/******************************************
 * 4) MAIN - Demonstration
 *****************************************/
int main() {
    // Hyperparameters
    const int num_classes = 10;
    const int input_size = 32 * 32 * 3; // for a 32x32 RGB image
    const int epochs = 30;
    const int batch_size = 64;
    const float learning_rate = 0.01f;

    // 1) Initialize weights & bias
    vector<vector<float>> weights(num_classes, vector<float>(input_size));
    vector<float> bias(num_classes, 0.0f);

    // Random initialization in [-0.01, 0.01]
    random_device rd;
    mt19937 gen(rd());
    uniform_real_distribution<> dis(-0.01, 0.01);
    for (auto &row : weights) {
        for (auto &w : row) {
            w = dis(gen);
        }
    }

    // 2) Load or create training & test data
    // In a real use-case, you'd load CIFAR-10 or another dataset.
    // We'll make dummy data here to illustrate usage.
    ifstream MyTrainFile1("data\\cifar-10-binary\\data_batch_1.bin", ios::binary);
    if (!MyTrainFile1.is_open()) {
        cerr << "Error opening data_batch_1.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    ifstream MyTrainFile2("data\\cifar-10-binary\\data_batch_2.bin", ios::binary);
    if (!MyTrainFile2.is_open()) {
        cerr << "Error opening data_batch_2.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    ifstream MyTrainFile3("data\\cifar-10-binary\\data_batch_3.bin", ios::binary);
    if (!MyTrainFile3.is_open()) {
        cerr << "Error opening data_batch_3.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    ifstream MyTrainFile4("data\\cifar-10-binary\\data_batch_4.bin", ios::binary);
    if (!MyTrainFile4.is_open()) {
        cerr << "Error opening data_batch_4.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    ifstream MyTrainFile5("data\\cifar-10-binary\\data_batch_5.bin", ios::binary);
    if (!MyTrainFile5.is_open()) {
        cerr << "Error opening data_batch_5.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    ifstream MyTestFile("data\\cifar-10-binary\\test_batch.bin", ios::binary);
    if (!MyTestFile.is_open()) {
        cerr << "Error opening test_batch.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    vector<Image> train_images;
    vector<Image> test_images;
    readData(MyTrainFile1,train_images);
    cout<<train_images.size()<<endl;
    readData(MyTrainFile2,train_images);
    cout<<train_images.size()<<endl;
    readData(MyTrainFile3,train_images);
    cout<<train_images.size()<<endl;
    readData(MyTrainFile4,train_images);
    cout<<train_images.size()<<endl;
    readData(MyTrainFile5,train_images);
    cout<<train_images.size()<<endl;
    readData(MyTestFile,test_images);
    cout<<test_images.size()<<endl;

    MyTrainFile1.close();
    MyTrainFile2.close();
    MyTrainFile3.close();
    MyTestFile.close();


    // 3) Training loop
    for (int e = 1; e <= epochs; ++e) {
        float total_loss = 0.0f;
        int total_batches = train_images.size() / batch_size;

        for (int i = 0; i < total_batches; ++i) {
            vector<Image> batch(
                train_images.begin() + i * batch_size, 
                train_images.begin() + (i + 1) * batch_size
            );
            float batch_loss = 0.0f;
            // Train on this batch
            process_batch_linear(batch, weights, bias, batch_loss, learning_rate);
            total_loss += batch_loss;
        }

        // Evaluate on test set
        float acc = process_test_linear(test_images, weights, bias);
        float avg_loss = (total_batches > 0) ? total_loss / total_batches : 0.0f;
        cout << "Epoch " << e 
             << " | Loss: " << avg_loss
             << " | Test Accuracy: " << acc * 100.0f << "%" << endl;
    }

    return 0;
}