NumericalStabilityFuzzing/softmax.cpp at main · anicolellis/NumericalStabilityFuzzing · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
#include <iostream>
#include <math.h>
#include <vector>
#include <algorithm>
#include <cstdlib>
#include <ctime>
using namespace std;

#define LB 100.0
#define UB 100000.0

#define DEBUG 1

vector<float> softmax_unstable(const vector<float> &x)
{
    float sum = 0;
    vector<float> result;
    result.resize(x.size());

    for (size_t i = 0; i < x.size(); i++)
    {
        sum += exp(x[i]);
        //cout << exp(x[i]) <<endl;
    }
    //cout << sum << endl;
    for (size_t j = 0; j < x.size(); j++)
    {
        result[j] = exp(x[j]) / sum;
    }
    return result;
}


vector<float> softmax_stable(const vector<float> &x)
{
    float sum = 0;
    vector<float> result;
    result.resize(x.size());
    float max = *max_element(x.begin(), x.end());

    for (size_t i = 0; i < x.size(); i++)
    {
        sum += exp(x[i] - max);
    }
    for (size_t j = 0; j < x.size(); j++)
    {
        result[j] = exp(x[j] - max) / sum;
    }
    return result;
}

vector<float> softmax_herbie(const vector<float> &x)
{
    float sum = 0;
    vector<float> result;
    result.resize(x.size());
    float max = *max_element(x.begin(), x.end());

    for (size_t i = 0; i < x.size(); i++)
    {
        sum += exp(x[i] - max);
    }
    for (size_t j = 0; j < x.size(); j++)
    {
        result[j] = exp(x[j] - log(sum));
    }
    return result;
}

float getFloatInRange()
{
    float val = LB + static_cast <float> (rand()) /( static_cast <float> (RAND_MAX/(UB-LB)));
    return val;
}

int main() {
    srand (static_cast <unsigned> (time(0)));
    //vector<float> x{10.0, 100.0, 1000.0};
    //vector<float> y{-1000.0, -10000.0, -1000000.0};
    vector<float> x(3);
    //vector<float> y(3);

    generate(begin(x), end(x), getFloatInRange);
    //generate(begin(y), end(y), getFloatInRange);

    if(DEBUG)
    {
        cout << "Values of x: " << endl;
        for (const auto &i : x) {
            cout << i << endl;
        }
        /*
        cout << "Values of y: " << endl;
        for (const auto &i : y) {
            cout << i << endl;
        }
        */
    }

    vector<float> result_unstable_x = softmax_unstable(x);
    cout << "Unstable softmax given x: " << endl;
    for (const auto &i : result_unstable_x) {
        cout << i << endl;
    }
    vector<float> result_stable_x = softmax_stable(x);
    cout << "Stable softmax given x: " << endl;
    for (const auto &i : result_stable_x) {
        cout << i << endl;
    }
    vector<float> result_herbie_x = softmax_herbie(x);
    cout << "Herbie softmax given x: " << endl;
    for (const auto &i : result_herbie_x) {
        cout << i << endl;
    }
    /*
    vector<float> result_unstable_y = softmax_unstable(y);
    cout << "Unstable softmax given y: " << endl;
    for (const auto &i : result_unstable_y) {
        cout << i << endl;
    }
    vector<float> result_stable_y = softmax_stable(y);
    cout << "Stable softmax given y: " << endl;
    for (const auto &i : result_stable_y) {
        cout << i << endl;
    }
    */
}