From 67f18663d96668ec9a905149fd5c4c6a9dabc9ad Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Sun, 16 Jun 2019 19:06:59 +0530 Subject: [PATCH 1/2] pick beta from state in NADAM --- src/optimise/optimisers.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/optimise/optimisers.jl b/src/optimise/optimisers.jl index aa2db1c5..ea33c1bd 100644 --- a/src/optimise/optimisers.jl +++ b/src/optimise/optimisers.jl @@ -214,7 +214,7 @@ NADAM(η = 0.001, β = (0.9, 0.999)) = NADAM(η, β, IdDict()) function apply!(o::NADAM, x, Δ) η, β = o.eta, o.beta β1p, β2p = o.beta - mt, vt = get!(o.state, x, (zero(x), zero(x))) + mt, vt, (β1p, β2p) = get!(o.state, x, (zero(x), zero(x), o.beta)) @. mt = β[1] * mt + (1 - β[1]) * Δ @. vt = β[2] * vt + (1 - β[2]) * Δ^2 @. Δ = (β[1] * mt / (1 - β[1] * β1p) + (1 - β[1]) * Δ / (1 - β1p)) / (√(vt * β[2] / (1 - β2p)) + ϵ) * η From dd9cdbef14a2779e166a757b4ba3e9e7b7a4b093 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Sun, 16 Jun 2019 19:09:50 +0530 Subject: [PATCH 2/2] remove uncessary call to beta --- src/optimise/optimisers.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/optimise/optimisers.jl b/src/optimise/optimisers.jl index ea33c1bd..2319cfdb 100644 --- a/src/optimise/optimisers.jl +++ b/src/optimise/optimisers.jl @@ -213,7 +213,6 @@ NADAM(η = 0.001, β = (0.9, 0.999)) = NADAM(η, β, IdDict()) function apply!(o::NADAM, x, Δ) η, β = o.eta, o.beta - β1p, β2p = o.beta mt, vt, (β1p, β2p) = get!(o.state, x, (zero(x), zero(x), o.beta)) @. mt = β[1] * mt + (1 - β[1]) * Δ @. vt = β[2] * vt + (1 - β[2]) * Δ^2