Fix bug in rmsprop and adadelta

`@. p.Δ = η * p.Δ / √acc` parses correctly while `@. p.Δ /= √acc*η` seems to parse like `@. p.Δ /= (√acc*η)`, hence the step size was de facto interpreted as `1/η`
This commit is contained in:
Fredrik Bagge Carlson 2017-11-14 17:32:16 +01:00 committed by GitHub
parent e5d99d784e
commit 8991ce028c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -38,7 +38,7 @@ function rmsprop(p::Param; η::Real = 0.001, ρ::Real = 0.9, ϵ::Real = 1e-8)
acc = zeros(p.x) .+ ϵ
function ()
@. acc = ρ * acc + (1 - ρ) * p.Δ ^ 2
@. p.Δ /= acc * η
@. p.Δ = η * p.Δ / acc
end
end
@ -46,7 +46,7 @@ function adagrad(p::Param; η::Real = 0.01, ϵ::Real = 1e-8)
acc = zeros(p.x) .+ ϵ
function ()
@. acc += p.Δ ^ 2
@. p.Δ /= acc * η
@. p.Δ = η * p.Δ / acc
end
end