fcl_layer.cpp

#include <cstdint>
#include <iostream>
#include <vector>
#include <string>
#include <stdio.h>
#include <stdlib.h>
#include <cmath>
#include <algorithm>
#include <cassert>
#include <chrono>
#include "consts.h"
#include "fcl_layer.h"

using namespace std;

int flatten(Image &pool, Flat &flat){
    flat.flattened.resize(pool.rgb.size() * pool.rgb[0].size() * pool.rgb[0][0].size());
    int H = pool.rgb[0].size();
    int W = pool.rgb[0][0].size();

    for(int i=0;i<pool.rgb.size();i++){
        for(int j=0;j<pool.rgb[i].size();j++){
            for(int k=0;k<pool.rgb[i][j].size();k++){
                flat.flattened[i*H*W + j*W + k] = pool.rgb[i][j][k];
            }
        }
    }
    for (int i = 0; i < 10; i++) {
        flat.label[i] = pool.label[i];
    }
    return 0;
}

int initialise_weights(vector<vector<float>> &velocity_fcl, vector<vector<float>> &weights, int num_output, Flat &flat){
    // number of output layers should be equal to the number of classes in the final layer = 10 
    // weights.resize(num_output, vector<float>(flat.flattened.size()));
    // for(int i=0;i<num_output;i++){
    //     for(int j=0;j<flat.flattened.size();j++){
    //         weights[i][j] =  0.2f * ((float)rand() / RAND_MAX) - 0.1f;
    //     }
    // }
    // return 0;

    // xavier initialisation
    // int fan_in = flat.flattened.size();   // inputs
    // int fan_out = num_output;             // outputs
    // float limit = sqrt(6.0f / (fan_in + fan_out));

    // he initialisation
    int fan_in = flat.flattened.size();   // inputs
    float limit = sqrt(2.0f / fan_in);

    weights.resize(num_output, vector<float>(fan_in));
    velocity_fcl.resize(num_output, vector<float>(fan_in, 0.0f));
    for(int i = 0; i < num_output; i++){
        for(int j = 0; j < fan_in; j++){
            weights[i][j] = ((float)rand() / RAND_MAX) * 2 * limit - limit; // [-limit, limit]
        }
    }
    return 0;
}

int initialise_weights_bias(vector<float> &velocity_fcl_bias, vector<float> &weights_bias, int num_output){
    weights_bias.resize(num_output);
    velocity_fcl_bias.resize(num_output, 0.0f);
    for(int j=0;j<num_output;j++){
        weights_bias[j] = 0.2f * ((float)rand() / RAND_MAX) - 0.1f; // limiting float values between -0.1 to 0.1 can change if needed
    }
    return 0;
}

// output vector should be resized to num_output
int fully_connected_layer(Flat &flat, vector<vector<float>> &weights, Flat &output, vector<float> &weights_bias, bool first_itr){
    if (output.flattened.size() != weights.size()) {
        output.flattened.resize(weights.size());
    }

    // Always zero out the output
    output.flattened.resize(weights.size());
    output.pre_activation.resize(weights.size());
    
    for (int i = 0; i < weights.size(); ++i) {  // loop over output neurons
        float sum = 0.0f;
        for (int j = 0; j < weights[i].size(); ++j) {
            sum += weights[i][j] * flat.flattened[j];
        }
        float z = sum + weights_bias[i];
        output.pre_activation[i] = z;     // Save before ReLU
        output.flattened[i] = z;   // Apply ReLU
    }

    for (int i = 0; i < 10; i++) {
        output.label[i] = flat.label[i];
    }
    return 0;
}


int softmax(vector<float> &output, vector<float> &softmax_output){
    float max_logit = *max_element(output.begin(), output.end());

    float sum = 0.0f;
    for(int i = 0; i < output.size(); i++){
        softmax_output[i] = exp(output[i] - max_logit);  // shift for numerical stability
        sum += softmax_output[i];
    }

    for(int i = 0; i < output.size(); i++){
        softmax_output[i] /= sum;
    }

    return 0;
}

//this should measure difference between what the model predicted and what the actual label is
// the losses are added up and then the average is taken
int cross_entropy(vector<float> &softmax_output, float &loss, int (&label)[10]){
    float total_loss = 0.0f;
    for(int i=0;i<10;i++){
        if(label[i] == 1) {
            total_loss -= log(softmax_output[i] + 1e-8f);  // Prevent log(0)
        }
    }
    // cout << "[Debug] Image loss = " << total_loss << endl;
    loss += total_loss;
    return 0;
}

int relu(vector<float> &output){
    for(int i=0;i<output.size();i++){
        if(output[i] > 0.0f){
            output[i];
        }else{
            output[i] = 0.0f;
        }
    }
    return 0;
}

int output_layer(Flat &flat, 
        vector<vector<vector<float>>> &weights, 
        vector<vector<float>> &weights_bias,
        vector<vector<vector<float>>> &velocity_fcl, 
        vector<vector<float>> &velocity_fcl_bias, 
        vector<Flat> &output, 
        float &loss, bool first_itr){
    Flat passed = flat;
    for(int i=0;i<num_fcl;i++){
        if(first_itr){
            initialise_weights(velocity_fcl[i], weights[i], num_neurons[i], passed);
            initialise_weights_bias(velocity_fcl_bias[i], weights_bias[i], num_neurons[i]);
        }
        // cout<<"reacching here before fcl is called"<<endl;
        auto t1_fully_connected_layer = std::chrono::high_resolution_clock::now();
        fully_connected_layer(passed, weights[i], output[i], weights_bias[i], first_itr);
        auto t2_fully_connected_layer = std::chrono::high_resolution_clock::now();
        // std::cout << "fcl forward time: " 
        //         << std::chrono::duration_cast<std::chrono::milliseconds>(t2_fully_connected_layer - t1_fully_connected_layer).count() 
        //         << " ms" << std::endl;

        // cout<<"fcl done?"<<endl;
        if(i<num_fcl-1){
            relu(output[i].flattened);
        }
        passed = output[i];
        // After ReLU on a conv or FCL layer:
        // int zero_count = count_if(output[i].flattened.begin(), output[i].flattened.end(), [](float x) { return x == 0.0f; });
        // float zero_ratio = 100.0f * zero_count / output[i].flattened.size();
        // cout << "FCL ReLU Zero Count: " << zero_count << " / " << output[i].flattened.size() << "(" << zero_ratio<<" % zeroes)" <<endl;
    }
    // for(int i=0;i<num_fcl;i++){
    //     cout << "Output " << i << ": " << endl;
    //     for (int j = 0; j < 10; j++) {
    //         cout << output[i].flattened[j] << " ";
    //     }
    //     cout << endl;
    // }

    softmax(passed.flattened, output[num_fcl-1].flattened);
    for (int i = 0; i < 10; ++i) {
        assert(flat.label[i] == 0 || flat.label[i] == 1);
    }
    cross_entropy(output[num_fcl-1].flattened, loss, flat.label);
    return 0;
}