TastyPancakes/Flux.jl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Made sure Gradients are not lost.

Browse Source
This commit is contained in:
Manjunath Bhat 2019-03-04 22:17:19 +05:30 committed by GitHub
parent 922e9c9bc2
commit 29b853e0bb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/layers/normalise.jl
View File

@ -63,13 +63,11 @@ function (a::AlphaDropout)(x)
a.active || return x
α = -1.75813631
noise = randn(Float64, size(x.data))
y = collect(x)
y .= y .* (noise .> (1 - a.p)) + α .* (noise .<= (1 - a.p))
x.data .= x.data .* (noise .> (1 - a.p)) + α .* (noise .<= (1 - a.p))
A = (a.p + a.p * (1 - a.p) * α ^ 2)^0.5
B = -A * α * (1 - a.p)
y .= A .* y .+ B
x1 = param(y)
return x1
x.data .= A .* x.data .+ B
return x
end
_testmode!(a::AlphaDropout, test) = (a.active = !test)