From 8991ce028ca02ed9d4c3286eba3468d2fe6e9ec1 Mon Sep 17 00:00:00 2001 From: Fredrik Bagge Carlson Date: Tue, 14 Nov 2017 17:32:16 +0100 Subject: [PATCH] Fix bug in rmsprop and adadelta MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `@. p.Δ = η * p.Δ / √acc` parses correctly while `@. p.Δ /= √acc*η` seems to parse like `@. p.Δ /= (√acc*η)`, hence the step size was de facto interpreted as `1/η` --- src/optimise/optimisers.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/optimise/optimisers.jl b/src/optimise/optimisers.jl index 95b31b98..1ffd8982 100644 --- a/src/optimise/optimisers.jl +++ b/src/optimise/optimisers.jl @@ -38,7 +38,7 @@ function rmsprop(p::Param; η::Real = 0.001, ρ::Real = 0.9, ϵ::Real = 1e-8) acc = zeros(p.x) .+ ϵ function () @. acc = ρ * acc + (1 - ρ) * p.Δ ^ 2 - @. p.Δ /= √acc * η + @. p.Δ = η * p.Δ / √acc end end @@ -46,7 +46,7 @@ function adagrad(p::Param; η::Real = 0.01, ϵ::Real = 1e-8) acc = zeros(p.x) .+ ϵ function () @. acc += p.Δ ^ 2 - @. p.Δ /= √acc * η + @. p.Δ = η * p.Δ / √acc end end