main.cpp

#include <iostream>
#include <fstream>
#include <cstdint>
#include <vector>
#include <string>
#include <stdio.h>
#include <stdlib.h>  
#include <algorithm>   
#include <random> 
#include <cassert>
#include<chrono>
#include <omp.h>
#include <openacc.h>
#include "consts.h"
#include "hidden_layer.h"
#include "execute.h"
#include "fcl_layer.h"
#include "backprop_fcl.h"
#include "backprop_conv.h"

// g++ main.cpp execute.cpp hidden_layer.cpp consts.cpp fcl_layer.cpp backprop_conv.cpp backprop_fcl.cpp -o run.exe

// dont use recursive include, only include whats needed

using namespace std;

float get_scheduled_learning_rate(int epoch, float learning_r) {
    if ((epoch + 1) % 30 == 0) {
        return learning_r * 0.5f;
        std::cout << "[Scheduler] Updated learning rate: " << learning_r * 0.5f << std::endl;
    }
    else return learning_r;
}

int readData(ifstream 
    &readFile,vector<Image> 
    &images){

    for(int i=0;i<10000;i++){
        uint8_t byte_value;
        struct Image img = Image();

        for(int i=0;i<10;i++){
            img.label[i] = 0;
        }
        // label is now an array that has all 0s and only the label index as 1. 
        readFile.read(reinterpret_cast<char*>(&byte_value), 1);
        // cout<< byte_value << endl;
        img.label[byte_value] = 1;

        // cout << "Label: ";
        // for (int i = 0; i < 10; ++i) cout << img.label[i] << " ";
        // cout << endl;

        img.rgb.resize(3, vector<vector<float>>(32, vector<float>(32)));
        for (int i = 0; i < 32; i++) {
            for (int j = 0; j < 32; j++) {
                readFile.read(reinterpret_cast<char*>(&byte_value), 1);
                float float_value = static_cast<float>(byte_value) / 255.0f;
                img.rgb[0][i][j] = float_value;
            }
        }
        for (int i = 0; i < 32; i++) {
            for (int j = 0; j < 32; j++) {
                readFile.read(reinterpret_cast<char*>(&byte_value), 1);
                float float_value = static_cast<float>(byte_value) / 255.0f;
                img.rgb[1][i][j] = float_value;
            }
        }
        for (int i = 0; i < 32; i++) {
            for (int j = 0; j < 32; j++) {
                readFile.read(reinterpret_cast<char*>(&byte_value), 1);
                float float_value = static_cast<float>(byte_value) / 255.0f;
                img.rgb[2][i][j] = float_value;
            }
        }
        images.push_back(img);
    }
    return 0;
}

int readBatch(std::ifstream &file, vector<Image> &images, int num_images_to_read = 10000) {
    images.clear();
    uint8_t byte_value;

    for (int i = 0; i < num_images_to_read && file.peek() != EOF; ++i) {
        Image img;
        std::fill(img.label, img.label + 10, 0);

        file.read(reinterpret_cast<char*>(&byte_value), 1);
        img.label[byte_value] = 1;

        img.rgb.resize(3, vector<vector<float>>(32, vector<float>(32)));
        for (int c = 0; c < 3; c++) {
            for (int row = 0; row < 32; row++) {
                for (int col = 0; col < 32; col++) {
                    file.read(reinterpret_cast<char*>(&byte_value), 1);
                    img.rgb[c][row][col] = static_cast<float>(byte_value) / 255.0f;
                }
            }
        }

        images.push_back(img);
    }
    return images.size();
}


int main() {
    // Read from the text file
    ifstream MyTrainFile1("data\\cifar-10-binary\\data_batch_1.bin", ios::binary);
    if (!MyTrainFile1.is_open()) {
        cerr << "Error opening data_batch_1.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    ifstream MyTrainFile2("data\\cifar-10-binary\\data_batch_2.bin", ios::binary);
    if (!MyTrainFile1.is_open()) {
        cerr << "Error opening data_batch_2.bin" << endl;
        return 1; // Or handle the error appropriately
    }
    // ifstream MyTrainFile3("data\\cifar-10-binary\\data_batch_3.bin", ios::binary);
    // if (!MyTrainFile1.is_open()) {
    //     cerr << "Error opening data_batch_3.bin" << endl;
    //     return 1; // Or handle the error appropriately
    // }
    // ifstream MyTrainFile4("data\\cifar-10-binary\\data_batch_4.bin", ios::binary);
    // if (!MyTrainFile1.is_open()) {
    //     cerr << "Error opening data_batch_4.bin" << endl;
    //     return 1; // Or handle the error appropriately
    // }
    ifstream MyTrainFile5("data\\cifar-10-binary\\data_batch_5.bin", ios::binary);
    if (!MyTrainFile5.is_open()) {
        cerr << "Error opening data_batch_5.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    ifstream MyTestFile("data\\cifar-10-binary\\test_batch.bin", ios::binary);
    if (!MyTestFile.is_open()) {
        cerr << "Error opening test_batch.bin" << endl;
        return 1; // Or handle the error appropriately
    }

    vector<Image> train_images;
    vector<Image> test_images;
    
    cout << "Starting to read training data..." << endl;
    
    readData(MyTrainFile1,train_images);
    // cout << "Read " << train_images.size() << " images from data_batch_1.bin" << endl;
    // cout << "First pixel value (R) of image " << train_images.size()-1 << ": "
    //  << train_images.back().rgb[0][0][0] << endl;
    // for (int l = 0; l < 10; l++) {
    //     if (train_images.back().label[l] == 1) {
    //         cout << "Last loaded image label: " << l << endl;
    //         break;
    //     }
    // }
    // assert(train_images.back().rgb[0][0].size() == 3);  // RGB check
    // assert(train_images.back().rgb.size() == 32);       // Height
    // assert(train_images.back().rgb[0].size() == 32);    // Width

    
    readData(MyTrainFile2,train_images);
    // cout << "Read " << train_images.size() << " images from data_batch_2.bin" << endl;
    // // cout << "First pixel value (R) of image " << train_images.size()-1 << ": "
    // //  << train_images.back().rgb[0][0][0] << endl;
    // // for (int l = 0; l < 10; l++) {
    // //     if (train_images.back().label[l] == 1) {
    // //         cout << "Last loaded image label: " << l << endl;
    // //         break;
    // //     }
    // // }
    // // assert(train_images.back().rgb[0][0].size() == 3);  // RGB check
    // // assert(train_images.back().rgb.size() == 32);       // Height
    // // assert(train_images.back().rgb[0].size() == 32);    // Width

    
    // readData(MyTrainFile3,train_images);
    // cout << "Read " << train_images.size() << " images from data_batch_3.bin" << endl;
    // // cout << "First pixel value (R) of image " << train_images.size()-1 << ": "
    // //  << train_images.back().rgb[0][0][0] << endl;
    // // for (int l = 0; l < 10; l++) {
    // //     if (train_images.back().label[l] == 1) {
    // //         cout << "Last loaded image label: " << l << endl;
    // //         break;
    // //     }
    // // }
    // // assert(train_images.back().rgb[0][0].size() == 3);  // RGB check
    // // assert(train_images.back().rgb.size() == 32);       // Height
    // // assert(train_images.back().rgb[0].size() == 32);    // Width

    
    // readData(MyTrainFile4,train_images);
    // cout << "Read " << train_images.size() << " images from data_batch_4.bin" << endl;
    // // cout << "First pixel value (R) of image " << train_images.size()-1 << ": "
    // //  << train_images.back().rgb[0][0][0] << endl;
    // // for (int l = 0; l < 10; l++) {
    // //     if (train_images.back().label[l] == 1) {
    // //         cout << "Last loaded image label: " << l << endl;
    // //         break;
    // //     }
    // // }
    // // assert(train_images.back().rgb[0][0].size() == 3);  // RGB check
    // // assert(train_images.back().rgb.size() == 32);       // Height
    // // assert(train_images.back().rgb[0].size() == 32);    // Width

    
    readData(MyTrainFile5,train_images);
    cout << "Read " << train_images.size() << " images from data_batch_5.bin" << endl;
    // cout << "First pixel value (R) of image " << train_images.size()-1 << ": "
    //  << train_images.back().rgb[0][0][0] << endl;
    // cout<<train_images.back().rgb[0][1][1]<<endl;
    // cout<<train_images.back().rgb[1][2][1]<<endl;

    // cout<<train_images.back().rgb[0][1][4]<<endl;
    // cout<<train_images.back().rgb[2][1][7]<<endl;

    // cout<<train_images.back().rgb[0][1][3]<<endl;

    // for (int l = 0; l < 10; l++) {
    //     if (train_images.back().label[l] == 1) {
    //         cout << "Last loaded image label: " << l << endl;
    //         break;
    //     }
    // }
    // // assert(train_images.back().rgb[0][0].size() == 3);  // RGB check
    // // assert(train_images.back().rgb.size() == 32);       // Height
    // // assert(train_images.back().rgb[0].size() == 32);    // Width
    
    readData(MyTestFile,test_images);
    cout << "Read " << test_images.size() << " images from test_batch.bin" << endl;
    // // cout << "First pixel value (R) of image " << test_images.size()-1 << ": "
    // //  << test_images.back().rgb[0][0][0] << endl;
    // // for (int l = 0; l < 10; l++) {
    // //     if (test_images.back().label[l] == 1) {
    // //         cout << "Last loaded image label: " << l << endl;
    // //         break;
    // //     }
    // // }
    // // assert(test_images.back().rgb[0][0].size() == 3);  // RGB check
    // // assert(test_images.back().rgb.size() == 32);       // Height
    // // assert(test_images.back().rgb[0].size() == 32);    // Width

    // Close the file
    MyTrainFile1.close();
    MyTrainFile2.close();
    // MyTrainFile3.close();
    // MyTrainFile4.close();
    MyTrainFile5.close();
    MyTestFile.close();

    // for(int i=0;i<100;i++){
    //     cout<< train_images[i].label << endl;
    // }
    vector<vector<vector<vector<vector<float>>>>> kernel_list;
    vector<vector<float>> bias_list;
    vector<vector<vector<float>>> weights;
    vector<vector<float>> weights_bias;
    vector<vector<vector<vector<vector<float>>>>> rotated_kernel_list;
    vector<vector<vector<float>>> velocity_fcl;  // store momentum for FCL
    vector<vector<float>> velocity_fcl_bias;     // shape [layers][neurons]
    vector<vector<vector<vector<vector<float>>>>> velocity_kernels;
    vector<vector<float>> velocity_kernels_bias;
    // vector<Flat> output;

    random_device rd;
    mt19937 g(rd());
    
    initialise(kernel_list, rotated_kernel_list, bias_list, weights, weights_bias, velocity_kernels, velocity_kernels_bias, velocity_fcl, velocity_fcl_bias);
    cout << "Initialized kernels: " << kernel_list.size() << " conv layers" << endl;

    float initial_weight = kernel_list[1][1][1][1][1];
    cout << "[Before Training] Initial weight: " << initial_weight << endl;

    ofstream logFile("training_log_gpu.txt");  // creates or overwrites
    for (int epoch = 0; epoch < epochs; ++epoch) {
        learning_rate = get_scheduled_learning_rate(epoch, learning_rate);
        using namespace std::chrono;
        auto epoch_start = high_resolution_clock::now();
        std::vector<std::string> file_paths = {
            "data\\cifar-10-binary\\data_batch_1.bin",
            "data\\cifar-10-binary\\data_batch_2.bin",
            "data\\cifar-10-binary\\data_batch_3.bin",
            "data\\cifar-10-binary\\data_batch_4.bin",
            "data\\cifar-10-binary\\data_batch_5.bin"
        };
        std::shuffle(file_paths.begin(), file_paths.end(), std::mt19937{std::random_device{}()});
        float epoch_loss = 0.0f;
        bool first_itr = (epoch == 0); 
    
        for (const auto& path : file_paths) {
            ifstream data_file(path, ios::binary);
            if (!data_file.is_open()) {
                cerr << "Could not open " << path << endl;
                continue;
            }
    
            // while (!data_file.eof()) {
                vector<Image> image_buffer;
                int loaded = readBatch(data_file, image_buffer, 10000);
                if (loaded == 0) break;
    
                shuffle(image_buffer.begin(), image_buffer.end(), std::mt19937{std::random_device{}()});
    
                for (int i = 0; i < image_buffer.size(); i += batch_size) {
                    vector<Image> batch(image_buffer.begin() + i, image_buffer.begin() + min(i + batch_size, (int)image_buffer.size()));
                    float batch_loss = 0.0f;
                    auto t1 = chrono::high_resolution_clock::now();
                    process_batch(batch, kernel_list, bias_list, weights, weights_bias, batch_loss, first_itr, rotated_kernel_list, velocity_kernels, velocity_kernels_bias, velocity_fcl, velocity_fcl_bias);
                    auto t2 = chrono::high_resolution_clock::now();
                    cout << "Batch time: " << chrono::duration_cast<chrono::milliseconds>(t2 - t1).count() << " ms\n";
                    first_itr = false;  // Only true for the first batch of the first epoch
                    cout << "Batch " << i/batch_size + 1<< ": Avg Loss = " << batch_loss/batch_size << endl;
                    epoch_loss += batch_loss;                    
                }
            // }
            data_file.close();
        }
        auto epoch_end = high_resolution_clock::now();
        auto epoch_duration = duration_cast<minutes>(epoch_end - epoch_start).count();
        cout << "[Epoch " << epoch + 1 << "] Duration: " << epoch_duration << " minutes\n";
        logFile << "[Epoch " << epoch + 1 << "] Duration: " << epoch_duration << " minutes\n";
        cout << "Epoch " << epoch + 1 << ": Average Loss = " << epoch_loss / (50000 / batch_size) << endl;
        logFile << "Epoch " << epoch + 1 << ": Average Loss = " << epoch_loss / (50000 / batch_size) << endl;

        float updated_weight = kernel_list[1][1][1][1][1];
        cout << "[Epoch " << epoch + 1 << "] Tracked weight: " << updated_weight << endl;
        logFile << "[Epoch " << epoch + 1 << "] Tracked weight: " << updated_weight << endl;
        float updated_fcl_weight = weights[1][1][1];
        cout << "[Epoch " << epoch + 1 << "] FCL weight: " << updated_fcl_weight << endl;
        logFile << "[Epoch " << epoch + 1 << "] FCL weight: " << updated_fcl_weight << endl;

        float batch_loss = 0.0f;
        float train_accuracy = process_test(train_images, kernel_list, bias_list, weights, weights_bias, batch_loss);
        cout << "[Epoch " << epoch + 1 << "] Training accuracy: " << train_accuracy << endl;
        logFile << "[Epoch " << epoch + 1 << "] Training accuracy: " << train_accuracy << endl;

        batch_loss = 0.0f;
        float test_accuracy = process_test(test_images, kernel_list, bias_list, weights, weights_bias, batch_loss);
        cout << "[Epoch " << epoch + 1 << "] Test accuracy: " << test_accuracy << endl;
        logFile << "[Epoch " << epoch + 1 << "] Test accuracy: " << test_accuracy << endl;
    }
        
    return 0;
}