add `active` helpers for other normalise layers

This commit is contained in:
Ian 2020-03-03 11:55:39 -05:00
parent 0def352383
commit d9ea5fba76
1 changed files with 8 additions and 0 deletions

View File

@ -40,6 +40,8 @@ mutable struct Dropout{F,D}
active::Union{Bool, Nothing} active::Union{Bool, Nothing}
end end
Dropout(p, dims) = Dropout(p, dims, nothing)
function Dropout(p; dims = :) function Dropout(p; dims = :)
@assert 0 p 1 @assert 0 p 1
Dropout{typeof(p),typeof(dims)}(p, dims, nothing) Dropout{typeof(p),typeof(dims)}(p, dims, nothing)
@ -77,6 +79,8 @@ mutable struct AlphaDropout{F}
end end
end end
AlphaDropout(p) = AlphaDropout(p, nothing)
function (a::AlphaDropout)(x) function (a::AlphaDropout)(x)
_isactive(a) || return x _isactive(a) || return x
λ = eltype(x)(1.0507009873554804934193349852946) λ = eltype(x)(1.0507009873554804934193349852946)
@ -253,6 +257,8 @@ mutable struct InstanceNorm{F,V,W,N}
active::Union{Bool, Nothing} active::Union{Bool, Nothing}
end end
InstanceNorm(λ, β, γ, μ, σ², ϵ, momentum) = InstanceNorm(λ, β, γ, μ, σ², ϵ, momentum, nothing)
InstanceNorm(chs::Integer, λ = identity; InstanceNorm(chs::Integer, λ = identity;
initβ = (i) -> zeros(Float32, i), initγ = (i) -> ones(Float32, i), ϵ = 1f-5, momentum = 0.1f0) = initβ = (i) -> zeros(Float32, i), initγ = (i) -> ones(Float32, i), ϵ = 1f-5, momentum = 0.1f0) =
InstanceNorm(λ, initβ(chs), initγ(chs), InstanceNorm(λ, initβ(chs), initγ(chs),
@ -344,6 +350,8 @@ mutable struct GroupNorm{F,V,W,N,T}
active::Union{Bool, Nothing} active::Union{Bool, Nothing}
end end
GroupNorm(G, λ, β, γ, μ, σ², ϵ, momentum) = GroupNorm(G, λ, β, γ, μ, σ², ϵ, momentum, nothing)
GroupNorm(chs::Integer, G::Integer, λ = identity; GroupNorm(chs::Integer, G::Integer, λ = identity;
initβ = (i) -> zeros(Float32, i), initγ = (i) -> ones(Float32, i), ϵ = 1f-5, momentum = 0.1f0) = initβ = (i) -> zeros(Float32, i), initγ = (i) -> ones(Float32, i), ϵ = 1f-5, momentum = 0.1f0) =
GroupNorm(G, λ, initβ(chs), initγ(chs), GroupNorm(G, λ, initβ(chs), initγ(chs),