optim/rprop.lua at master · ywpre/optim · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
----------------------------------------------------------------------
-- A plain implementation of RPROP
--
-- ARGS:
-- opfunc : a function that takes a single input (X), the point of
--          evaluation, and returns f(X) and df/dX
-- x      : the initial point
-- state  : a table describing the state of the optimizer; after each
--          call the state is modified
--   state.learningRate      : learning rate
--   state.learningRateDecay : learning rate decay
--   state.weightDecay       : weight decay
--   state.momentum          : momentum
--   state.learningRates     : vector of individual learning rates
--
-- RETURN:
-- x     : the new x vector
-- f(x)  : the function, evaluated before the update
--
-- (Martin Riedmiller, Koray Kavukcuoglu 2013)
--
function optim.rprop(opfunc, x, config, state)
    if config == nil and state == nil then
        print('no state table RPROP initializing')
    end
    -- (0) get/update state
    local config = config or {}
    local state = state or config
    local stepsize = config.stepsize or 0.1
    local etaplus = config.etaplus or 1.2
    local etaminus = config.etaminus or 0.5
    local stepsizemax = config.stepsizemax or 50.0
    local stepsizemin = config.stepsizemin or 1E-06
    local niter = config.niter or 1

    local hfx = {}

    for i=1,niter do

        -- (1) evaluate f(x) and df/dx
        local fx,dfdx = opfunc(x)

        -- init temp storage
        if not state.delta then
            state.delta    = dfdx.new(dfdx:size()):zero()
            state.stepsize = dfdx.new(dfdx:size()):fill(stepsize)
            state.sign     = dfdx.new(dfdx:size())
            state.psign    = torch.ByteTensor(dfdx:size())
            state.nsign    = torch.ByteTensor(dfdx:size())
            state.zsign    = torch.ByteTensor(dfdx:size())
            state.dminmax  = torch.ByteTensor(dfdx:size())
        end

        -- sign of derivative from last step to this one
        torch.cmul(state.sign, dfdx, state.delta)
        torch.sign(state.sign, state.sign)

        -- get indices of >0, <0 and ==0 entries
        state.sign.gt(state.psign, state.sign, 0)
        state.sign.lt(state.nsign, state.sign, 0)
        state.sign.eq(state.zsign, state.sign, 0)

        -- get step size updates
        state.sign[state.psign] = etaplus
        state.sign[state.nsign] = etaminus
        state.sign[state.zsign] = 1

        -- update stepsizes with step size updates
        state.stepsize:cmul(state.sign)

        -- threshold step sizes
        -- >50 => 50
        state.stepsize.gt(state.dminmax, state.stepsize, stepsizemax)
        state.stepsize[state.dminmax] = stepsizemax
        -- <1e-6 ==> 1e-6
        state.stepsize.lt(state.dminmax, state.stepsize, stepsizemin)
        state.stepsize[state.dminmax] = stepsizemin

        -- for dir<0, dfdx=0
        -- for dir>=0 dfdx=dfdx
        dfdx[state.nsign] = 0
        -- state.sign = sign(dfdx)
        torch.sign(state.sign,dfdx)

        -- update weights
        x:addcmul(-1,state.sign,state.stepsize)

        -- update state.dfdx with current dfdx
        state.delta:copy(dfdx)

        table.insert(hfx,fx)
    end

   -- return x*, f(x) before optimization
   return x,hfx
end