TastyPancakes/Flux.jl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

batchnorm: batch σ correct coefficient

Browse Source
This commit is contained in:
Iblis Lin 2017-10-30 12:57:30 +08:00
parent e0201be770
commit b3356cc6bb
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/layers/normalisation.jl
View File

@ -92,12 +92,12 @@ function (BN::BatchNorm)(x)
ϵ = T(BN.ϵ) ϵ = T(BN.ϵ)
m = size(x, 2) # batch size m = size(x, 2) # batch size
μ = sum(x, 2) ./ m μ = sum(x, 2) ./ m
σ = sqrt.(sum((x .- μ).^2, 2) ./ (m - 1) .+ ϵ) σ = sqrt.(sum((x .- μ).^2, 2) ./ m .+ ϵ)
# update moving mean/std # update moving mean/std
mtm = T(BN.momentum) mtm = T(BN.momentum)
BN.μ = mtm .* μ.data .+ (1 - mtm) .* BN.μ BN.μ = (1 - mtm) .* BN.μ .+ mtm .* μ.data
BN.σ = mtm .* σ.data .+ (1 - mtm) .* BN.σ BN.σ = (1 - mtm) .* BN.σ .+ mtm .* σ.data .* m ./ (m - 1)
end end
BN.λ.(BN.γ .* ((x .- μ) ./ σ) .+ BN.β) BN.λ.(BN.γ .* ((x .- μ) ./ σ) .+ BN.β)