TastyPancakes/Flux.jl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

moved epsilon into sqrt

Browse Source
This commit is contained in:
tejank10 2018-04-04 15:25:20 +05:30
parent 3ead662987
commit 65847bb745
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/optimise/optimisers.jl
View File

@ -27,7 +27,7 @@ function rmsprop(p::Param; η::Real = 0.001, ρ::Real = 0.9, ϵ::Real = 1e-8)
acc = zeros(p.x)
function ()
@. acc = ρ * acc + (1 - ρ) * p.Δ^2
@. p.Δ *= η / (√acc + ϵ)
@. p.Δ *= η / √(acc + ϵ)
end
end
@ -35,7 +35,7 @@ function adagrad(p::Param; η::Real = 0.01, ϵ::Real = 1e-8)
acc = zeros(p.x) .+ ϵ
function ()
@. acc += p.Δ^2
@. p.Δ *= η / √acc
@. p.Δ *= η / √(acc + ϵ)
end
end
@ -56,7 +56,7 @@ function adam(p::Param; η::Real = 0.001, β1::Real = 0.9, β2::Real = 0.999, ϵ
function ()
@. mt = β1 * mt + (1 - β1) * p.Δ
@. vt = β2 * vt + (1 - β2) * p.Δ^2
@. p.Δ = mt / (1 - β1p) / (√(vt / (1 - β2p)) + ϵ) * η
@. p.Δ = mt / (1 - β1p) / √(vt / (1 - β2p) + ϵ) * η
β1p *= β1
β2p *= β2
end