execute.cpp

#include <iostream>
#include <cstdint>
#include <vector>
#include <string>
#include <stdio.h>
#include <stdlib.h> 
#include <algorithm>
#include <numeric>
#include <limits>
#include <cmath>
#include <chrono>
#include <omp.h>
#include <openacc.h>
#include <random>
#include "consts.h"
#include "hidden_layer.h"
#include "execute.h"
#include "fcl_layer.h"
#include "backprop_fcl.h"
#include "backprop_conv.h"

using namespace std;

int argmax(vector<float> &vec) {
    return max_element(vec.begin(), vec.end()) - vec.begin();
}

void normalize_image(Image &img) {
    int C = img.rgb.size();    // 3
    int H = img.rgb[0].size(); // 32
    int W = img.rgb[0][0].size(); // 32

    // If you're storing the above constants somewhere global or static:
    static float mean[3] = {0.4914f, 0.4822f, 0.4465f};
    static float stdv[3] = {0.2470f, 0.2435f, 0.2616f};

    for (int c = 0; c < C; ++c) {
        for (int i = 0; i < H; ++i) {
            for (int j = 0; j < W; ++j) {
                img.rgb[c][i][j] = (img.rgb[c][i][j] - mean[c]) / stdv[c];
            }
        }
    }
}

void augment_image(Image &img) {
    static std::random_device rd;
    static std::mt19937 gen(rd());
    std::uniform_real_distribution<float> dist01(0.0f, 1.0f);

    float flip_prob = dist01(gen);
    if (flip_prob < 0.5f) {
        int C = img.rgb.size();    // should be 3
        int H = img.rgb[0].size(); // 32
        int W = img.rgb[0][0].size(); // 32
        for (int c = 0; c < C; ++c) {
            for (int row = 0; row < H; ++row) {
                // reverse the row in-place
                for (int col = 0; col < W / 2; ++col) {
                    std::swap(img.rgb[c][row][col], img.rgb[c][row][W - 1 - col]);
                }
            }
        }
    }

    const int pad = 4;
    int origH = 32, origW = 32;
    int paddedH = origH + 2 * pad; // 40
    int paddedW = origW + 2 * pad; // 40

    vector<vector<vector<float>>> padded(3,
        vector<vector<float>>(paddedH, vector<float>(paddedW, 0.0f)) );

    // Copy original image into the center
    for (int c = 0; c < 3; ++c) {
        for (int i = 0; i < origH; ++i) {
            for (int j = 0; j < origW; ++j) {
                padded[c][i + pad][j + pad] = img.rgb[c][i][j];
            }
        }
    }

    // Now pick a random top-left corner for a 32x32 crop
    int max_offset = 2 * pad; // 8
    int rand_i = (int)(dist01(gen) * (max_offset + 1)); 
    int rand_j = (int)(dist01(gen) * (max_offset + 1));

    for (int c = 0; c < 3; ++c) {
        for (int i = 0; i < origH; ++i) {
            for (int j = 0; j < origW; ++j) {
                img.rgb[c][i][j] = padded[c][i + rand_i][j + rand_j];
            }
        }
    }
}


int initialise(vector<vector<vector<vector<vector<float>>>>> &kernels_list, 
    vector<vector<vector<vector<vector<float>>>>> &rotated_kernel_list,
    vector<vector<float>> &bias_list, 
    vector<vector<vector<float>>> &weights, 
    vector<vector<float>> &weights_bias,
    vector<vector<vector<vector<vector<float>>>>> &velocity_kernels,
    vector<vector<float>> &velocity_kernels_bias,
    vector<vector<vector<float>>> &velocity_fcl,
    vector<vector<float>> &velocity_fcl_bias
    // vector<vector<Flat>> &output
){
    // initialise sets up all kernels and biases for all hidden layers
    for(int i=0;i<hidden_layers;i++){
        kernels_list.push_back(vector<vector<vector<vector<float>>>>());
        rotated_kernel_list.push_back(vector<vector<vector<vector<float>>>>());
        velocity_kernels.push_back(vector<vector<vector<vector<float>>>>());
        initialise_kernel(velocity_kernels[i],kernels_list[i], i==0?3:kernels_list[i-1].size(), 3, num_kernels_hidden_layer[i]);
        bias_list.push_back(vector<float>());
        velocity_kernels_bias.push_back(vector<float>());
        initialise_bias(velocity_kernels_bias[i], bias_list[i], num_kernels_hidden_layer[i]);
       
        int num_kernels = kernels_list[i].size();
        rotated_kernel_list[i].resize(num_kernels);
        
        // Because each kernel_list[i][k] = [ in_channels ][ kH ][ kW ]
        for (int k = 0; k < num_kernels; k++) {
            int in_channels = kernels_list[i][k].size();
            rotated_kernel_list[i][k].resize(in_channels);
            
            // Now each kernels_list[i][k][c] = a 2D matrix
            for (int c = 0; c < in_channels; c++) {
                int kH = kernels_list[i][k][c].size();
                int kW = kernels_list[i][k][c][0].size();

                // Prepare an empty 2D matrix for the rotated kernel
                rotated_kernel_list[i][k][c].resize(kH, vector<float>(kW, 0.0f));

                // Immediately rotate it once (initial rotation)
                rotate_180(kernels_list[i][k][c], rotated_kernel_list[i][k][c]);
            }
        }
    }

    for(int i=0;i<num_fcl;i++){
        weights.push_back(vector<vector<float>>());
        velocity_fcl.push_back(vector<vector<float>>());
        weights_bias.push_back(vector<float>());
        velocity_fcl_bias.push_back(vector<float>());
        // output.push_back(vector<Flat>());
    }
    return 0;
}


int initialise_accumulators(vector<vector<float>> &db_accum, 
    vector<vector<float>> &db_fcl_accum, 
    vector<vector<vector<vector<vector<float>>>>> &dK_accum, 
    vector<vector<vector<float>>> &dW_accum,
    const vector<vector<vector<float>>> &weights, 
    const vector<vector<vector<vector<vector<float>>>>> &kernel_list) {

    int num_fcl = weights.size();
    int hidden_layers = kernel_list.size();

    // === Fully Connected Layers ===
    db_fcl_accum.resize(num_fcl);
    dW_accum.resize(num_fcl);

    for (int i = 0; i < num_fcl; ++i) {
        // cout<<"fcl initialised?"<<endl;
        int out_dim = weights[i].size();       // neurons
        int in_dim = weights[i][0].size();     // inputs per neuron
        // cout<<"weights an issue?"<<endl;
        db_fcl_accum[i].resize(out_dim, 0.0f);
        dW_accum[i].resize(out_dim, vector<float>(in_dim, 0.0f));
        // cout << "dW_accum[" << i << "]: " << dW_accum[i].size() << "x" << dW_accum[i][0].size() << endl;

    }

    // === Convolutional Layers ===
    db_accum.resize(hidden_layers);
    dK_accum.resize(hidden_layers);

    for (int l = 0; l < hidden_layers; ++l) {
        int num_kernels = kernel_list[l].size();
        int in_channels = kernel_list[l][0].size();
        int kH = kernel_list[l][0][0].size();
        int kW = kernel_list[l][0][0][0].size();

        db_accum[l].resize(num_kernels, 0.0f);
        dK_accum[l].resize(num_kernels);

        for (int k = 0; k < num_kernels; ++k) {
            dK_accum[l][k].resize(in_channels);
            for (int c = 0; c < in_channels; ++c) {
                dK_accum[l][k][c].resize(kH, vector<float>(kW, 0.0f));
            }
        }
    }

    return 0;
}

int update_rotated_kernel(vector<vector<vector<vector<vector<float>>>>> &kernels_list,vector<vector<vector<vector<vector<float>>>>> &rotated_kernel_list){

    for(int i=0;i<hidden_layers;i++){
        int num_filters = kernels_list[i].size();
        int in_channels = kernels_list[i][0].size();
        int kH = kernels_list[i][0][0].size();
        int kW = kernels_list[i][0][0][0].size();
    
        for (int k = 0; k < num_filters; ++k) {
            for (int c = 0; c < in_channels; ++c) {
                rotate_180(kernels_list[i][k][c], rotated_kernel_list[i][k][c]);
                if (rotated_kernel_list[i][k][c].size() != kH || rotated_kernel_list[i][k][c][0].size() != kW) {
                    cerr << "Kernel " << k << ", channel " << c << " has wrong size." << endl;
                    exit(1);
                }
            }
        }
    }

    return 0;
}
int process_batch(vector<Image> &batch, 
                vector<vector<vector<vector<vector<float>>>>> &kernels_list, 
                vector<vector<float>> &bias_list, 
                vector<vector<vector<float>>> &weights, 
                vector<vector<float>> &weights_bias, 
                float &batch_loss, bool first_itr,
                vector<vector<vector<vector<vector<float>>>>> &rotated_kernel_list,
                vector<vector<vector<vector<vector<float>>>>> &velocity_kernels,
                vector<vector<float>> &velocity_kernels_bias,
                vector<vector<vector<float>>> &velocity_fcl,
                vector<vector<float>> &velocity_fcl_bias 
                // vector<vector<Flat>> &output
        ){
    
    vector<vector<float>> db_accum;
    vector<vector<vector<vector<vector<float>>>>> dK_accum;
    vector<vector<float>> db_fcl_accum;
    vector<vector<vector<float>>> dW_accum;     
      
    // cout<< "batch size "<<batch.size()<<endl;
    for(int i=0;i<batch.size();i++){
        vector<Flat> output(num_fcl);
        // output[i].resize(num_fcl);
        // cout<<"forward pass start "<<i<<endl;
        vector<Image> image_map(hidden_layers);
        vector<Image> pool(hidden_layers);
        Flat unpool;
        vector<Image> conv_output(hidden_layers);
        augment_image(batch[i]);
        normalize_image(batch[i]);
        forward_pass(batch[i], kernels_list, image_map, pool, bias_list, weights, weights_bias, velocity_fcl, velocity_fcl_bias, batch_loss, i==0?first_itr:false, output, unpool, conv_output);
        if (i == 0) {
            initialise_accumulators(db_accum, db_fcl_accum, dK_accum, dW_accum, weights, kernels_list);
        }
        // now run the backprop step, but dont update the weights 
        backward_pass(batch[i], unpool, kernels_list, weights, weights_bias, image_map, pool, db_fcl_accum, dW_accum, db_accum, dK_accum, output, conv_output, rotated_kernel_list, i==0?first_itr:false);
        // Example: right after backprop_fcl() or backprop_conv() in process_batch()
        // cout << "[Check] dW_accum[0][0][0] = " << dW_accum[0][0][0] << endl;
        // cout << "[Check] dK_accum[0][0][0][0][0] = " << dK_accum[0][0][0][0][0] << endl;
        // cout<<"back pass done "<<i<<endl;
    }
    // cout << "[Check] dW_accum[1][1][1] = " << dW_accum[1][1][1] << endl;
    // cout << "[Check] dK_accum[2][1][4][1][3] = " << dK_accum[2][1][4][1][3] << endl;
    // cout << ">> Calling update_weights" << endl;
    update_weights(weights, weights_bias,kernels_list, bias_list, batch_loss, dW_accum, db_fcl_accum, dK_accum, db_accum,velocity_kernels,velocity_kernels_bias,velocity_fcl,velocity_fcl_bias); 
    update_rotated_kernel(kernels_list, rotated_kernel_list);
    // write the function to update all the weights
    return 0;
}

int forward_pass(Image &image, 
                vector<vector<vector<vector<vector<float>>>>> &kernels_list, 
                vector<Image> &image_map, 
                vector<Image> &pool, 
                vector<vector<float>> &bias_list, 
                vector<vector<vector<float>>> &weights, 
                vector<vector<float>> &weights_bias, 
                vector<vector<vector<float>>> &velocity_fcl, 
                vector<vector<float>> &velocity_fcl_bias, 
                float &batch_loss, bool first_itr, vector<Flat> &output, Flat &unpool,
            vector<Image> &conv_output){
    
    Image temp = image;
    for(int i=0;i<hidden_layers;i++){
        auto t1_apply_kernel = std::chrono::high_resolution_clock::now();
        apply_kernel(temp, kernels_list[i], image_map[i], bias_list[i]);
        auto t2_apply_kernel = std::chrono::high_resolution_clock::now();
        // std::cout << "apply_kernel time: " 
        //         << std::chrono::duration_cast<std::chrono::milliseconds>(t2_apply_kernel - t1_apply_kernel).count() 
        //         << " ms" << std::endl;

        // cout<<"apply kernel over"<<endl;
        auto t1_max_pool = std::chrono::high_resolution_clock::now();
        max_pool(image_map[i], pool[i]);
        auto t2_max_pool = std::chrono::high_resolution_clock::now();
        // std::cout << "max_pool time: " 
        //         << std::chrono::duration_cast<std::chrono::milliseconds>(t2_max_pool - t1_max_pool).count() 
        //         << " ms" << std::endl;

        conv_output[i] = image_map[i];
        // cout<<"max pool over"<<endl;
        temp = pool[i];
    }
    flatten(pool[hidden_layers-1], unpool);
    // cout<<"flattening?"<<endl;
    // cout<<first_itr<<endl;
    output_layer(unpool, weights, weights_bias, velocity_fcl, velocity_fcl_bias, output, batch_loss, first_itr);
    // cout<<"output layer done?"<<endl;
    return 0;
}

int backward_pass(
                Image &original_input,
                Flat &flat_input, 
                vector<vector<vector<vector<vector<float>>>>> &kernel_list,
                vector<vector<vector<float>>> &weights, 
                vector<vector<float>> &weights_bias, 
                vector<Image> &image_map, 
                vector<Image> &pool, 
                vector<vector<float>> &db_fcl_accum,
                vector<vector<vector<float>>> &dW_accum,
                vector<vector<float>> &db_accum,
                vector<vector<vector<vector<vector<float>>>>> &dK_accum,
                vector<Flat> &flat_output,
                vector<Image> &conv_output,
                vector<vector<vector<vector<vector<float>>>>> &rotated_kernel_list,
                bool first_itr){
    vector<float> delta_inner;
    
    // cout << "Running fcl_backward_pass..." << endl;
    fcl_backward_pass(flat_input, flat_output, weights, weights_bias, delta_inner, db_fcl_accum, dW_accum);
    // cout << ">> fcl_backward_pass completed" << endl;
    // cout << ">> Calling unflatten..." << endl;
    vector<vector<vector<float>>> loss;
    vector<vector<vector<float>>> unpool;

    int C = pool[hidden_layers - 1].rgb.size();
    int H = pool[hidden_layers - 1].rgb[0].size();
    int W = pool[hidden_layers - 1].rgb[0][0].size();

    unflatten(delta_inner, loss, H, W, C);
    conv_backward_pass(original_input, loss, image_map, pool, kernel_list, db_accum, dK_accum, conv_output, rotated_kernel_list, first_itr);
    // cout<<"backprop over"<<endl;
    return 0;
}

int fcl_backward_pass(
    Flat &flat, 
    vector<Flat> &output, 
    vector<vector<vector<float>>> &weights, 
    vector<vector<float>> &weights_bias, 
    vector<float> &delta_inner,
    vector<vector<float>> &db_fcl_accum,
    vector<vector<vector<float>>> &dW_accum){
                        
    // cout << "fcl_backward_pass -> output size: " << output.size() << endl;
    // if (output.size() < num_fcl) {
    //     cerr << "ERROR: output size (" << output.size() << ") < num_fcl (" << num_fcl << ")" << endl;
    //     exit(1);
    // }    
    // cout << "  output[num_fcl-1].flattened.size(): " << output[num_fcl-1].flattened.size() << endl;
    // cout << "  output[num_fcl-2].flattened.size(): " << output[num_fcl - 2].flattened.size() << endl;
    vector<float> delta_outer = output[num_fcl-1].flattened;
    // cout << "[Debug] Output logits: ";
    // for (float val : output[num_fcl - 1].flattened) {
    //     cout << val << " ";
    // }
    // cout << endl;

    // cout << "[Debug] Layer before final FCL (input): ";
    // for (int i = 0; i < 10; i++) cout << output[num_fcl-2].flattened[i] << " ";
    // cout << endl;

    backpropogation_outer_layer(delta_outer, weights[num_fcl-1], weights_bias[num_fcl-1], output[num_fcl-2].pre_activation, output[num_fcl-1].label, delta_inner, db_fcl_accum[num_fcl-1], dW_accum[num_fcl-1]);
    // cout << "backprop_outer done" << endl;
    for(int j=num_fcl-2;j>=0;j--){
        // theres an index issue here
        // cout<< "running backpropogation_fcl" << j << endl;
        backpropogation_fcl(delta_inner, weights[j], weights_bias[j], (j == 0) ? flat.flattened : output[j-1].pre_activation, delta_inner, db_fcl_accum[j], dW_accum[j]); // call as many times as number of fcl layers apart from outer layer
    }
    return 0;
}


int conv_backward_pass(
                    Image &original_input,
                    vector<vector<vector<float>>> &loss, 
                    vector<Image> &image_map, 
                    vector<Image> &pool, 
                    vector<vector<vector<vector<vector<float>>>>> &kernel_list, 
                    vector<vector<float>> &db_accum, 
                    vector<vector<vector<vector<vector<float>>>>> &dK_accum,
                    vector<Image> &conv_output,
                    vector<vector<vector<vector<vector<float>>>>> &rotated_kernel_list,
                    bool first_itr){
    vector<vector<vector<float>>> passed = loss;
    for(int i=hidden_layers-1;i>=0;i--){
        vector<vector<vector<float>>> unpool;
        reverse_max_pool(passed, image_map[i].rgb, unpool, stride, window);
        Image &activation_layer = image_map[i];
        for (int ch = 0; ch < unpool.size(); ++ch) {
            for (int row = 0; row < unpool[0].size(); ++row) {
                for (int col = 0; col < unpool[0][0].size(); ++col) {
                    if (activation_layer.pre_activation[ch][row][col] <= 0.0f) {
                        unpool[ch][row][col] = 0.0f;  // ReLU gradient mask
                    }
                }
            }
        }
        vector<vector<vector<float>>> &input_map = (i == 0) ? original_input.rgb : conv_output[i-1].rgb;
        backpropagation_conv(unpool, input_map, kernel_list[i], db_accum[i], dK_accum[i], passed, rotated_kernel_list[i], first_itr);
        // cout<<"backprop conv done"<<endl;
        passed = unpool;
    }
    return 0;
}

int update_weights(
    vector<vector<vector<float>>> &weights, vector<vector<float>> &weights_bias,
    vector<vector<vector<vector<vector<float>>>>> &kernel_list, 
    vector<vector<float>> &bias_list, 
    float batch_loss,  // we dont even use ts
    vector<vector<vector<float>>> &dW_accum, 
    vector<vector<float>> &db_fcl_accum, 
    vector<vector<vector<vector<vector<float>>>>> &dK_accum, 
    vector<vector<float>> &db_accum,
    vector<vector<vector<vector<vector<float>>>>> &velocity_kernels,
    vector<vector<float>> &velocity_kernels_bias,
    vector<vector<vector<float>>> &velocity_fcl,
    vector<vector<float>> &velocity_fcl_bias){
    update_weights_fcl(weights, weights_bias, dW_accum, db_fcl_accum,velocity_fcl,velocity_fcl_bias);
    update_weights_conv(kernel_list, bias_list, dK_accum, db_accum,velocity_kernels,velocity_kernels_bias);
    return 0;
}

int update_weights_fcl(vector<vector<vector<float>>> &weights, 
    vector<vector<float>> &weights_bias, 
    vector<vector<vector<float>>> &dW_accum, 
    vector<vector<float>> &db_fcl_accum,
    vector<vector<vector<float>>> &velocity_fcl,
    vector<vector<float>> &velocity_fcl_bias){
    
    for(int i=0;i<weights.size();i++){
        for(int j=0;j<weights[i].size();j++){
            for(int k=0;k<weights[i][j].size();k++){
                // if (i == 1 && j == 1 && k == 1) {
                //     cout << "BEFORE: w = " << weights[i][j][k]
                //          << ", grad = " << dW_accum[i][j][k]
                //          << ", batch_size = " << batch_size << endl;
                // }
                // weights[i][j][k] -= learning_rate * dW_accum[i][j][k]/batch_size;
                // if (i == 1 && j == 1 && k == 1) {
                float grad = (dW_accum[i][j][k] / batch_size) + (lambda * weights[i][j][k]);
                velocity_fcl[i][j][k] = momentum * velocity_fcl[i][j][k] + (learning_rate * grad);
                weights[i][j][k] -= velocity_fcl[i][j][k];
            }
        }
    }
    // cout<<"fcl vecloty done"<<endl;
    for(int i=0;i<weights_bias.size();i++){
        for(int j=0;j<weights_bias[i].size();j++){
            // weights_bias[i][j] -= learning_rate * db_fcl_accum[i][j]/batch_size;
            velocity_fcl_bias[i][j] =
                momentum * velocity_fcl_bias[i][j] +
                (learning_rate * db_fcl_accum[i][j] / batch_size);

            weights_bias[i][j] -= velocity_fcl_bias[i][j];
        }
    }
    // cout<<"velocity fcl bias also done"<<endl;
    return 0;
}

int update_weights_conv(vector<vector<vector<vector<vector<float>>>>> &kernel_list, 
    vector<vector<float>> &bias_list, 
    vector<vector<vector<vector<vector<float>>>>> &dK_accum, 
    vector<vector<float>> &db_accum,
    vector<vector<vector<vector<vector<float>>>>> &velocity_kernels,
    vector<vector<float>> &velocity_kernels_bias){
    
    for(int i=0;i<hidden_layers;i++){
        for(int j=0;j<kernel_list[i].size();j++){
            for(int k=0;k<kernel_list[i][j].size();k++){
                for(int l=0;l<kernel_list[i][j][k].size();l++){
                    for(int m=0;m<kernel_list[i][j][k][l].size();m++){
                        // if (i==1 && j==1 && k==1 && l==1 && m==1) {
                        //     cout << "BEFORE kernel: " << kernel_list[i][j][k][l][m] << endl;
                        //     cout << "grad = " << dK_accum[i][j][k][l][m] << ", batch_size = " << batch_size << endl;
                        // }
                        // kernel_list[i][j][k][l][m] -= learning_rate * dK_accum[i][j][k][l][m]/batch_size;
                        // if (i==1 && j==1 && k==1 && l==1 && m==1) {
                        //     cout << "AFTER kernel: " << kernel_list[i][j][k][l][m] << endl;
                        // }

                        float grad = (dK_accum[i][j][k][l][m] / batch_size) + (lambda * kernel_list[i][j][k][l][m]);
                        velocity_kernels[i][j][k][l][m] = momentum * velocity_kernels[i][j][k][l][m] + (learning_rate * grad);

                        kernel_list[i][j][k][l][m] -= velocity_kernels[i][j][k][l][m];
                    }
                }
            }
        }
    }    
    for(int i=0;i<bias_list.size();i++){
        for(int j=0;j<bias_list[i].size();j++){
            // bias_list[i][j] -= learning_rate * db_accum[i][j]/batch_size;
            velocity_kernels_bias[i][j] =
                momentum * velocity_kernels_bias[i][j] +
                (learning_rate * db_accum[i][j] / batch_size);

            bias_list[i][j] -= velocity_kernels_bias[i][j];
        }
    }
    return 0;
}

float process_test(vector<Image> &images, 
                vector<vector<vector<vector<vector<float>>>>> &kernel_list, 
                vector<vector<float>> &bias_list, 
                vector<vector<vector<float>>> &weights, 
                vector<vector<float>> &weights_bias, 
                float &batch_loss){
    int correct = 0;
    vector<Image> image_map(hidden_layers);
    vector<Image> pool(hidden_layers);
    for(int i=0;i<images.size();i++){
        vector<Flat> output(num_fcl);
        int true_label = -1;
        for (int j = 0; j < 10; ++j) {
            if (images[i].label[j] == 1) {
                true_label = j;
                break;
            }
        }
        Flat flat;
        vector<Image> conv_output(hidden_layers);
        vector<vector<vector<float>>> velocity_fcl(num_fcl);
        vector<vector<float>> velocity_fcl_bias(num_fcl);
        Image temp = images[i];
        forward_pass(temp, kernel_list, image_map, pool, bias_list, weights, weights_bias, velocity_fcl, velocity_fcl_bias, batch_loss, false, output, flat, conv_output);
        // cout << "forward pass "<< i << endl;
        int predicted_class = argmax(output[num_fcl-1].flattened);
        // cout<<"predicted class "<< predicted_class<<endl;
        // cout <<" true label "<< true_label<<endl;
        if (predicted_class == true_label) {
            // cout << "Correct prediction: " << predicted_class << endl;
            correct++;
        }
    }
    float accuracy = (float)correct / images.size();
    return accuracy;
}

// // // kernels_list is the list of all different layers' kernels
// // // kernel_list is the list of all kernels for a specific layer
// // // fcl_dim is the number of fully connected layers