Flux.jl/src/layers/stateless.jl

# Cost functions
"""
    mae(ŷ, y)

Return the mean of absolute error; calculated as
`sum(abs.(ŷ .- y)) / length(y)`.
"""
mae(ŷ, y) = sum(abs.(ŷ .- y)) * 1 // length(y)


"""
    mse(ŷ, y)

Return the mean squared error between ŷ and y; calculated as
`sum((ŷ .- y).^2) / length(y)`.

# Examples
```jldoctest
julia> Flux.mse([0, 2], [1, 1])
1//1
```
"""
mse(ŷ, y) = sum((ŷ .- y).^2) * 1 // length(y)


"""
    msle(ŷ, y; ϵ=eps(eltype(ŷ)))

Return the mean of the squared logarithmic errors; calculated as
`sum((log.(ŷ .+ ϵ) .- log.(y .+ ϵ)).^2) / length(y)`.
The `ϵ` term provides numerical stability.

Penalizes an under-predicted estimate greater than an over-predicted estimate.
"""
msle(ŷ, y; ϵ=eps(eltype(ŷ))) = sum((log.(ŷ .+ ϵ) .- log.(y .+ ϵ)).^2) * 1 // length(y)


"""
    huber_loss(ŷ, y; δ=1.0)

Return the mean of the [Huber loss](https://en.wikipedia.org/wiki/Huber_loss)
given the prediction `ŷ` and true values `y`.

                 | 0.5 * |ŷ - y|,            for |ŷ - y| <= δ
    Huber loss = |
                 |  δ * (|ŷ - y| - 0.5 * δ), otherwise
"""
function huber_loss(ŷ, y;  δ=eltype(ŷ)(1))
   abs_error = abs.(ŷ .- y)
   temp = abs_error .<  δ
   x = eltype(ŷ)(0.5)
   hub_loss = sum(((abs_error.^2) .* temp) .* x .+ δ*(abs_error .- x*δ) .* (1 .- temp)) * 1 // length(y)
end

function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::Nothing)
  return -sum(xlogy.(y, ŷ)) * 1 // size(y, 2)
end

function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::Number)
  return -sum(xlogy.(y, ŷ)) .* weight * 1 // size(y, 2)
end

function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::AbstractVector)
  return -sum(xlogy.(y, ŷ) .* weight) * 1 // size(y, 2)
end

"""
    crossentropy(ŷ, y; weight = nothing)

Return the cross entropy between the given probability distributions;
calculated as `-sum(y .* log.(ŷ) .* weight) / size(y, 2)`.

`weight` can be `Nothing`, a `Number` or an `AbstractVector`.
`weight=nothing` acts like `weight=1` but is faster.

See also: [`Flux.logitcrossentropy`](@ref), [`Flux.binarycrossentropy`](@ref), [`Flux.logitbinarycrossentropy`](@ref)

# Examples
```jldoctest
julia> Flux.crossentropy(softmax([-1.1491, 0.8619, 0.3127]), [1, 1, 0])
3.085467254747739
```
"""
crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight=nothing) = _crossentropy(ŷ, y, weight)

"""
    logitcrossentropy(ŷ, y; weight = 1)

Return the crossentropy computed after a [`Flux.logsoftmax`](@ref) operation;
calculated as `-sum(y .* logsoftmax(ŷ) .* weight) / size(y, 2)`.

`logitcrossentropy(ŷ, y)` is mathematically equivalent to
[`Flux.crossentropy(softmax(ŷ), y)`](@ref) but it is more numerically stable.

See also: [`Flux.crossentropy`](@ref), [`Flux.binarycrossentropy`](@ref), [`Flux.logitbinarycrossentropy`](@ref)

# Examples
```jldoctest
julia> Flux.logitcrossentropy([-1.1491, 0.8619, 0.3127], [1, 1, 0])
3.085467254747738
```
"""
function logitcrossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight = 1)
  return -sum(y .* logsoftmax(ŷ) .* weight) * 1 // size(y, 2)
end

"""
    binarycrossentropy(ŷ, y; ϵ=eps(ŷ))

Return ``-y*\\log(ŷ + ϵ) - (1-y)*\\log(1-ŷ + ϵ)``. The `ϵ` term provides numerical stability.

Typically, the prediction `ŷ` is given by the output of a [`sigmoid`](@ref) activation.

See also: [`Flux.crossentropy`](@ref), [`Flux.logitcrossentropy`](@ref), [`Flux.logitbinarycrossentropy`](@ref)

# Examples
```jldoctest
julia> Flux.binarycrossentropy.(σ.([-1.1491, 0.8619, 0.3127]), [1, 1, 0])
3-element Array{Float64,1}:
 1.424397097347566
 0.35231664672364077
 0.8616703662235441
```
"""
binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -xlogy(y, ŷ + ϵ) - xlogy(1 - y, 1 - ŷ + ϵ)

# Re-definition to fix interaction with CuArrays.
CuArrays.@cufunc binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -y*log(ŷ + ϵ) - (1 - y)*log(1 - ŷ + ϵ)

"""
    logitbinarycrossentropy(ŷ, y)

`logitbinarycrossentropy(ŷ, y)` is mathematically equivalent to
[`Flux.binarycrossentropy(σ(ŷ), y)`](@ref) but it is more numerically stable.

See also: [`Flux.crossentropy`](@ref), [`Flux.logitcrossentropy`](@ref), [`Flux.binarycrossentropy`](@ref)

# Examples
```jldoctest
julia> Flux.logitbinarycrossentropy.([-1.1491, 0.8619, 0.3127], [1, 1, 0])
3-element Array{Float64,1}:
 1.4243970973475661
 0.35231664672364094
 0.8616703662235443
```
"""
logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)

# Re-definition to fix interaction with CuArrays.
CuArrays.@cufunc logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)

"""
    normalise(x; dims=1)

Normalise `x` to mean 0 and standard deviation 1 across the dimensions given by `dims`.
Defaults to normalising over columns.

```jldoctest
julia> a = reshape(collect(1:9), 3, 3)
3×3 Array{Int64,2}:
 1  4  7
 2  5  8
 3  6  9

julia> Flux.normalise(a)
3×3 Array{Float64,2}:
 -1.22474  -1.22474  -1.22474
  0.0       0.0       0.0
  1.22474   1.22474   1.22474

julia> Flux.normalise(a, dims=2)
3×3 Array{Float64,2}:
 -1.22474  0.0  1.22474
 -1.22474  0.0  1.22474
 -1.22474  0.0  1.22474
```
"""
function normalise(x::AbstractArray; dims=1)
  μ′ = mean(x, dims = dims)
  σ′ = std(x, dims = dims, mean = μ′, corrected=false)
  return (x .- μ′) ./ σ′
end

"""
    kldivergence(ŷ, y)

Return the
[Kullback-Leibler divergence](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence)
between the given probability distributions.

KL divergence is a measure of how much one probability distribution is different
from the other.
It is always non-negative and zero only when both the distributions are equal
everywhere.
"""
function kldivergence(ŷ, y)
  entropy = sum(xlogx.(y)) * 1 //size(y,2)
  cross_entropy = crossentropy(ŷ, y)
  return entropy + cross_entropy
end

"""
    poisson(ŷ, y)

Return how much the predicted distribution `ŷ` diverges from the expected Poisson
distribution `y`; calculated as `sum(ŷ .- y .* log.(ŷ)) / size(y, 2)`.

[More information.](https://peltarion.com/knowledge-center/documentation/modeling-view/build-an-ai-model/loss-functions/poisson).
"""
poisson(ŷ, y) = sum(ŷ .- xlogy.(y, ŷ)) * 1 // size(y,2)

"""
    hinge(ŷ, y)

Return the [hinge loss](https://en.wikipedia.org/wiki/Hinge_loss) given the
prediction `ŷ` and true labels `y` (containing 1 or -1); calculated as
`sum(max.(0, 1 .- ŷ .* y)) / size(y, 2)`.

See also: [`squared_hinge`](@ref)
"""
hinge(ŷ, y) = sum(max.(0, 1 .-  ŷ .* y)) * 1 // size(y, 2)

"""
    squared_hinge(ŷ, y)

Return the squared hinge loss given the prediction `ŷ` and true labels `y`
(containing 1 or -1); calculated as `sum((max.(0, 1 .- ŷ .* y)).^2) / size(y, 2)`.

See also: [`hinge`](@ref)
"""
squared_hinge(ŷ, y) = sum((max.(0, 1 .- ŷ .* y)).^2) * 1 // size(y, 2)

"""
    dice_coeff_loss(ŷ, y; smooth=1)

Return a loss based on the dice coefficient.
Used in the [V-Net](https://arxiv.org/pdf/1606.04797v1.pdf) image segmentation
architecture.
Similar to the F1_score. Calculated as:
    1 - 2*sum(|ŷ .* y| + smooth) / (sum(ŷ.^2) + sum(y.^2) + smooth)`
"""
dice_coeff_loss(ŷ, y; smooth=eltype(ŷ)(1.0)) = 1 - (2*sum(y .* ŷ) + smooth) / (sum(y.^2) + sum(ŷ.^2) + smooth)

"""
    tversky_loss(ŷ, y; β=0.7)

Return the [Tversky loss](https://arxiv.org/pdf/1706.05721.pdf).
Used with imbalanced data to give more weight to false negatives.
Larger β weigh recall higher than precision (by placing more emphasis on false negatives)
Calculated as:
    1 - sum(|y .* ŷ| + 1) / (sum(y .* ŷ + β*(1 .- y) .* ŷ + (1 - β)*y .* (1 .- ŷ)) + 1)
"""
tversky_loss(ŷ, y; β=eltype(ŷ)(0.7)) = 1 - (sum(y .* ŷ) + 1) / (sum(y .* ŷ + β*(1 .- y) .* ŷ + (1 - β)*y .* (1 .- ŷ)) + 1)

"""
    flatten(x::AbstractArray)

Transform (w, h, c, b)-shaped input into (w × h × c, b)-shaped output
by linearizing all values for each element in the batch.
"""
function flatten(x::AbstractArray)
  return reshape(x, :, size(x)[end])
end

"""
    xlogx(x)
Return `x * log(x)` for `x ≥ 0`, handling `x = 0` by taking the downward limit.
"""
function xlogx(x)
  result = x * log(x)
  ifelse(iszero(x), zero(result), result)
end
CuArrays.@cufunc function xlogx(x)
  result = x * log(x)
  ifelse(iszero(x), zero(result), result)
end

"""
    xlogy(x, y)
Return `x * log(y)` for `y > 0` with correct limit at `x = 0`.
"""
function xlogy(x, y)
  result = x * log(y)
  ifelse(iszero(x), zero(result), result)
end
CuArrays.@cufunc function xlogy(x, y)
  result = x * log(y)
  ifelse(iszero(x), zero(result), result)
end

@adjoint function broadcasted(::typeof(xlogy), x::Zygote.Numeric, y::Zygote.Numeric)
  res = xlogy.(x, y)
  res, Δ -> (nothing, Zygote.unbroadcast(x, xlogy.(Δ, y)), Zygote.unbroadcast(y, Δ .* x ./ y))
end
-												simplify organisation

											
										
										
											2017-08-19 19:52:29 +00:00
+								# Cost functions
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
 								    mae(ŷ, y)
-												Merge branch 'master' into patch-1
											
										
										
											2020-03-01 06:19:33 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the mean of absolute error; calculated as
 								`sum(abs.(ŷ .- y)) / length(y)`.
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
-												Removed spurious promotions
											
										
										
											2020-02-05 19:36:41 +00:00
+								mae(ŷ, y) = sum(abs.(ŷ .- y)) * 1 // length(y)
-												simplify organisation

											
										
										
											2017-08-19 19:52:29 +00:00
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
 								"""
 								    mse(ŷ, y)
-												simplify organisation

											
										
										
											2017-08-19 19:52:29 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the mean squared error between ŷ and y; calculated as
 								`sum((ŷ .- y).^2) / length(y)`.
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
 								# Examples
 								```jldoctest
 								julia> Flux.mse([0, 2], [1, 1])
 //1
 								```
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
-												fix promotion by avoiding integer division in mse and crossentropy

oops

add tests

											
										
										
											2019-01-06 19:29:30 +00:00
+								mse(ŷ, y) = sum((ŷ .- y).^2) * 1 // length(y)
-												simplify organisation

											
										
										
											2017-08-19 19:52:29 +00:00
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
 								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								    msle(ŷ, y; ϵ=eps(eltype(ŷ)))
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the mean of the squared logarithmic errors; calculated as
 								`sum((log.(ŷ .+ ϵ) .- log.(y .+ ϵ)).^2) / length(y)`.
-												split up Flatten layer to use the flatten function

											
										
										
											2019-12-05 13:16:12 +00:00
+								The `ϵ` term provides numerical stability.
-												Added consistency with ŷ and unicode chars
											
										
										
											2020-03-02 14:30:47 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Penalizes an under-predicted estimate greater than an over-predicted estimate.
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								msle(ŷ, y; ϵ=eps(eltype(ŷ))) = sum((log.(ŷ .+ ϵ) .- log.(y .+ ϵ)).^2) * 1 // length(y)
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
 								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								    huber_loss(ŷ, y; δ=1.0)
-												Updated huber_loss with other minute changes
											
										
										
											2020-03-02 07:55:23 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the mean of the [Huber loss](https://en.wikipedia.org/wiki/Huber_loss)
 								given the prediction `ŷ` and true values `y`.
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								                 | 0.5 * |ŷ - y|,            for |ŷ - y| <= δ
 								    Huber loss = |
 								                 |  δ * (|ŷ - y| - 0.5 * δ), otherwise
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								function huber_loss(ŷ, y;  δ=eltype(ŷ)(1))
 								   abs_error = abs.(ŷ .- y)
 								   temp = abs_error .<  δ
-												Updated huber_loss with other minute changes
											
										
										
											2020-03-02 07:55:23 +00:00
+								   x = eltype(ŷ)(0.5)
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								   hub_loss = sum(((abs_error.^2) .* temp) .* x .+ δ*(abs_error .- x*δ) .* (1 .- temp)) * 1 // length(y)
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								end
-												Fix problem in crossentropy breaking GPU compilation

											
										
										
											2019-10-17 15:01:28 +00:00
+								function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::Nothing)
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								  return -sum(xlogy.(y, ŷ)) * 1 // size(y, 2)
-												Add `weighted_crossentropy` for imbalanced classification problems

											
										
										
											2017-12-05 23:38:15 +00:00
+								end
-												Fix problem in crossentropy breaking GPU compilation

											
										
										
											2019-10-17 15:01:28 +00:00
+								function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::Number)
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								  return -sum(xlogy.(y, ŷ)) .* weight * 1 // size(y, 2)
-												Fix problem in crossentropy breaking GPU compilation

											
										
										
											2019-10-17 15:01:28 +00:00
+								end
 								function _crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat, weight::AbstractVector)
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								  return -sum(xlogy.(y, ŷ) .* weight) * 1 // size(y, 2)
-												Fix problem in crossentropy breaking GPU compilation

											
										
										
											2019-10-17 15:01:28 +00:00
+								end
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
+								    crossentropy(ŷ, y; weight = nothing)
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
+								Return the cross entropy between the given probability distributions;
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								calculated as `-sum(y .* log.(ŷ) .* weight) / size(y, 2)`.
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
 								`weight` can be `Nothing`, a `Number` or an `AbstractVector`.
 								`weight=nothing` acts like `weight=1` but is faster.
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								See also: [`Flux.logitcrossentropy`](@ref), [`Flux.binarycrossentropy`](@ref), [`Flux.logitbinarycrossentropy`](@ref)
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
 								# Examples
 								```jldoctest
 								julia> Flux.crossentropy(softmax([-1.1491, 0.8619, 0.3127]), [1, 1, 0])
 .085467254747739
 								```
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
-												Fix problem in crossentropy breaking GPU compilation

											
										
										
											2019-10-17 15:01:28 +00:00
+								crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight=nothing) = _crossentropy(ŷ, y, weight)
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								"""
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
+								    logitcrossentropy(ŷ, y; weight = 1)
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								Return the crossentropy computed after a [`Flux.logsoftmax`](@ref) operation;
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								calculated as `-sum(y .* logsoftmax(ŷ) .* weight) / size(y, 2)`.
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
+								`logitcrossentropy(ŷ, y)` is mathematically equivalent to
-												fix logitcrossentropy doc string
											
										
										
											2020-05-12 15:18:29 +00:00
+								[`Flux.crossentropy(softmax(ŷ), y)`](@ref) but it is more numerically stable.
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								See also: [`Flux.crossentropy`](@ref), [`Flux.binarycrossentropy`](@ref), [`Flux.logitbinarycrossentropy`](@ref)
-												`src/layers/stateless.jl`: Add missing docstrings

											
										
										
											2019-08-31 09:08:25 +00:00
 								# Examples
 								```jldoctest
 								julia> Flux.logitcrossentropy([-1.1491, 0.8619, 0.3127], [1, 1, 0])
 .085467254747738
 								```
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								"""
 								function logitcrossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight = 1)
 								  return -sum(y .* logsoftmax(ŷ) .* weight) * 1 // size(y, 2)
-												logit cross entropy

											
										
										
											2017-10-17 16:57:10 +00:00
+								end
-												adding layer normalization

											
										
										
											2017-10-10 20:33:37 +00:00
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
+								"""
-												Change epsilon value to eps(ŷ)

											
										
										
											2018-06-26 18:29:06 +00:00
+								    binarycrossentropy(ŷ, y; ϵ=eps(ŷ))
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								Return ``-y*\\log(ŷ + ϵ) - (1-y)*\\log(1-ŷ + ϵ)``. The `ϵ` term provides numerical stability.
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								Typically, the prediction `ŷ` is given by the output of a [`sigmoid`](@ref) activation.
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
 								See also: [`Flux.crossentropy`](@ref), [`Flux.logitcrossentropy`](@ref), [`Flux.logitbinarycrossentropy`](@ref)
 								# Examples
 								```jldoctest
 								julia> Flux.binarycrossentropy.(σ.([-1.1491, 0.8619, 0.3127]), [1, 1, 0])
 -element Array{Float64,1}:
 .424397097347566
 .35231664672364077
 .8616703662235441
 								```
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
+								"""
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -xlogy(y, ŷ + ϵ) - xlogy(1 - y, 1 - ŷ + ϵ)
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
-												Fix binarycrossentropy on CuArrays

											
										
										
											2019-11-08 15:48:11 +00:00
+								# Re-definition to fix interaction with CuArrays.
 								CuArrays.@cufunc binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -y*log(ŷ + ϵ) - (1 - y)*log(1 - ŷ + ϵ)
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
+								"""
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								    logitbinarycrossentropy(ŷ, y)
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								`logitbinarycrossentropy(ŷ, y)` is mathematically equivalent to
-												Fix docstring of logitbinarycrossentropy and logitcrossentropy
											
										
										
											2020-05-05 14:29:29 +00:00
+								[`Flux.binarycrossentropy(σ(ŷ), y)`](@ref) but it is more numerically stable.
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								See also: [`Flux.crossentropy`](@ref), [`Flux.logitcrossentropy`](@ref), [`Flux.binarycrossentropy`](@ref)
 								# Examples
 								```jldoctest
 								julia> Flux.logitbinarycrossentropy.([-1.1491, 0.8619, 0.3127], [1, 1, 0])
 -element Array{Float64,1}:
 .4243970973475661
 .35231664672364094
 .8616703662235443
 								```
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
+								"""
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)
-												Register back! for logsigmoid and implement (logit)binarycrossentropy

											
										
										
											2018-02-06 11:32:46 +00:00
-												Fix logitbinarycrossentropy on CuArrays

											
										
										
											2019-11-22 05:23:24 +00:00
+								# Re-definition to fix interaction with CuArrays.
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								CuArrays.@cufunc logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)
-												Fix logitbinarycrossentropy on CuArrays

											
										
										
											2019-11-22 05:23:24 +00:00
-												adding layer normalization

											
										
										
											2017-10-10 20:33:37 +00:00
+								"""
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								    normalise(x; dims=1)
-												adding layer normalization

											
										
										
											2017-10-10 20:33:37 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								Normalise `x` to mean 0 and standard deviation 1 across the dimensions given by `dims`.
 								Defaults to normalising over columns.
-												Backticks and examples for normalise

											
										
										
											2019-10-21 14:31:44 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								```jldoctest
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								julia> a = reshape(collect(1:9), 3, 3)
 ×3 Array{Int64,2}:
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+4  7
 5  8
 6  9
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								julia> Flux.normalise(a)
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+×3 Array{Float64,2}:
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								 -1.22474  -1.22474  -1.22474
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+.0       0.0       0.0
 .22474   1.22474   1.22474
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								julia> Flux.normalise(a, dims=2)
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+×3 Array{Float64,2}:
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								 -1.22474  0.0  1.22474
 								 -1.22474  0.0  1.22474
 								 -1.22474  0.0  1.22474
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
+								```
-												adding layer normalization

											
										
										
											2017-10-10 20:33:37 +00:00
+								"""
-												drop dims type restriction
											
										
										
											2019-02-08 13:15:37 +00:00
+								function normalise(x::AbstractArray; dims=1)
-												add support for n-dimensional input to normalise layer

											
										
										
											2019-02-05 11:39:22 +00:00
+								  μ′ = mean(x, dims = dims)
-												fix std arguments in normalise

											
										
										
											2019-02-05 13:06:04 +00:00
+								  σ′ = std(x, dims = dims, mean = μ′, corrected=false)
-												LayerNorm tweaks

											
										
										
											2017-10-23 11:53:07 +00:00
+								  return (x .- μ′) ./ σ′
-												adding layer normalization

											
										
										
											2017-10-10 20:33:37 +00:00
+								end
-												`dims` kwarg
											
										
										
											2019-02-08 13:00:32 +00:00
-												Added new loss functions.
											
										
										
											2019-03-11 21:01:42 +00:00
+								"""
-												Changes to docs

											
										
										
											2019-10-09 09:23:03 +00:00
+								    kldivergence(ŷ, y)
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the
 								[Kullback-Leibler divergence](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence)
 								between the given probability distributions.
-												Merge branch 'master' into patch-1
											
										
										
											2020-03-01 06:19:33 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								KL divergence is a measure of how much one probability distribution is different
 								from the other.
 								It is always non-negative and zero only when both the distributions are equal
 								everywhere.
-												Added new loss functions.
											
										
										
											2019-03-11 21:01:42 +00:00
+								"""
-												Loss function names in lowercase

											
										
										
											2019-03-25 21:39:48 +00:00
+								function kldivergence(ŷ, y)
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								  entropy = sum(xlogx.(y)) * 1 //size(y,2)
-												Added new loss functions.
											
										
										
											2019-03-11 21:01:42 +00:00
+								  cross_entropy = crossentropy(ŷ, y)
 								  return entropy + cross_entropy
 								end
 								"""
-												Changes to docs

											
										
										
											2019-10-09 09:23:03 +00:00
+								    poisson(ŷ, y)
-												NNlib docs + misc docs improvements

											
										
										
											2020-02-29 10:14:48 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return how much the predicted distribution `ŷ` diverges from the expected Poisson
 								distribution `y`; calculated as `sum(ŷ .- y .* log.(ŷ)) / size(y, 2)`.
-												Merge branch 'master' into patch-1
											
										
										
											2020-03-01 06:19:33 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								[More information.](https://peltarion.com/knowledge-center/documentation/modeling-view/build-an-ai-model/loss-functions/poisson).
-												Added new loss functions.
											
										
										
											2019-03-11 21:01:42 +00:00
+								"""
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								poisson(ŷ, y) = sum(ŷ .- xlogy.(y, ŷ)) * 1 // size(y,2)
-												Added new loss functions.
											
										
										
											2019-03-11 21:01:42 +00:00
-												Added docstring for hinge loss

											
										
										
											2019-10-03 15:43:09 +00:00
+								"""
-												Changes to docs

											
										
										
											2019-10-09 09:23:03 +00:00
+								    hinge(ŷ, y)
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the [hinge loss](https://en.wikipedia.org/wiki/Hinge_loss) given the
 								prediction `ŷ` and true labels `y` (containing 1 or -1); calculated as
 								`sum(max.(0, 1 .- ŷ .* y)) / size(y, 2)`.
-												Merge branch 'master' into patch-1
											
										
										
											2020-03-01 06:19:33 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								See also: [`squared_hinge`](@ref)
-												Added docstring for hinge loss

											
										
										
											2019-10-03 15:43:09 +00:00
+								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								hinge(ŷ, y) = sum(max.(0, 1 .-  ŷ .* y)) * 1 // size(y, 2)
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
 								"""
 								    squared_hinge(ŷ, y)
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the squared hinge loss given the prediction `ŷ` and true labels `y`
 								(containing 1 or -1); calculated as `sum((max.(0, 1 .- ŷ .* y)).^2) / size(y, 2)`.
-												Merge branch 'master' into patch-1
											
										
										
											2020-03-01 06:19:33 +00:00
-												Improve docstrings

Improvements like...
   - fixing typos,
   - removing trailing and double whitespaces,
   - using `jldoctest` blocks where applicable,
   - fixing, updating or correctly setting up existing doctests,
   - improving consistency (for example, always use "# Examples" instead
     of other variants),
   - removing empty lines between docstrings and functions,
   - instead of mentioning keywords, put them into the docstring,
   - adding some missing but useful keywords,
   - adding references (`@ref`),
   - using LaTeX math where applicable, and
   - linking papers.

Debatable stuff that is untouched:
   - BE/AE s/z irregularities ("normalise" versus "normalize") since
     most papers use the AE version while the Flux source code was
     written with BE spelling.
   - Names of normalization functions are capitalized
     ("Batch Normalization" instead of "batch normalization").

											
										
										
											2019-08-31 09:39:28 +00:00
+								See also: [`hinge`](@ref)
-												Added loss functions
											
										
										
											2020-02-05 16:59:15 +00:00
+								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								squared_hinge(ŷ, y) = sum((max.(0, 1 .- ŷ .* y)).^2) * 1 // size(y, 2)
-												Added tversky and dice loss
											
										
										
											2020-02-26 20:30:28 +00:00
 								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								    dice_coeff_loss(ŷ, y; smooth=1)
-												Added tversky and dice loss
											
										
										
											2020-02-26 20:30:28 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return a loss based on the dice coefficient.
 								Used in the [V-Net](https://arxiv.org/pdf/1606.04797v1.pdf) image segmentation
 								architecture.
 								Similar to the F1_score. Calculated as:
 - 2*sum(|ŷ .* y| + smooth) / (sum(ŷ.^2) + sum(y.^2) + smooth)`
-												Added tversky and dice loss
											
										
										
											2020-02-26 20:30:28 +00:00
+								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								dice_coeff_loss(ŷ, y; smooth=eltype(ŷ)(1.0)) = 1 - (2*sum(y .* ŷ) + smooth) / (sum(y.^2) + sum(ŷ.^2) + smooth)
-												Added tversky and dice loss
											
										
										
											2020-02-26 20:30:28 +00:00
 								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								    tversky_loss(ŷ, y; β=0.7)
-												Added tversky and dice loss
											
										
										
											2020-02-26 20:30:28 +00:00
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Return the [Tversky loss](https://arxiv.org/pdf/1706.05721.pdf).
 								Used with imbalanced data to give more weight to false negatives.
-												Added consistency with ŷ and unicode chars
											
										
										
											2020-03-02 14:30:47 +00:00
+								Larger β weigh recall higher than precision (by placing more emphasis on false negatives)
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Calculated as:
 - sum(|y .* ŷ| + 1) / (sum(y .* ŷ + β*(1 .- y) .* ŷ + (1 - β)*y .* (1 .- ŷ)) + 1)
-												Added tversky and dice loss
											
										
										
											2020-02-26 20:30:28 +00:00
+								"""
-												Updated loss functions
											
										
										
											2020-03-03 10:32:57 +00:00
+								tversky_loss(ŷ, y; β=eltype(ŷ)(0.7)) = 1 - (sum(y .* ŷ) + 1) / (sum(y .* ŷ + β*(1 .- y) .* ŷ + (1 - β)*y .* (1 .- ŷ)) + 1)
-												split up Flatten layer to use the flatten function

											
										
										
											2019-12-05 13:16:12 +00:00
 								"""
 								    flatten(x::AbstractArray)
-												Further docstring improvements in src/

Some had to be re-done after the rebase

											
										
										
											2020-04-04 20:59:45 +00:00
+								Transform (w, h, c, b)-shaped input into (w × h × c, b)-shaped output
-												split up Flatten layer to use the flatten function

											
										
										
											2019-12-05 13:16:12 +00:00
+								by linearizing all values for each element in the batch.
 								"""
 								function flatten(x::AbstractArray)
 								  return reshape(x, :, size(x)[end])
 								end
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
 								"""
-												generic

											
										
										
											2020-05-05 17:23:05 +00:00
+								    xlogx(x)
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								Return `x * log(x)` for `x ≥ 0`, handling `x = 0` by taking the downward limit.
 								"""
-												generic

											
										
										
											2020-05-05 17:23:05 +00:00
+								function xlogx(x)
 								  result = x * log(x)
-												NaN

											
										
										
											2020-05-07 10:44:32 +00:00
+								  ifelse(iszero(x), zero(result), result)
-												generic

											
										
										
											2020-05-05 17:23:05 +00:00
+								end
-												cufunc

											
										
										
											2020-05-07 07:54:05 +00:00
+								CuArrays.@cufunc function xlogx(x)
 								  result = x * log(x)
-												NaN

											
										
										
											2020-05-07 10:44:32 +00:00
+								  ifelse(iszero(x), zero(result), result)
-												cufunc

											
										
										
											2020-05-07 07:54:05 +00:00
+								end
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
 								"""
-												generic

											
										
										
											2020-05-05 17:23:05 +00:00
+								    xlogy(x, y)
-												xlogy

											
										
										
											2020-05-05 16:29:51 +00:00
+								Return `x * log(y)` for `y > 0` with correct limit at `x = 0`.
 								"""
-												generic

											
										
										
											2020-05-05 17:23:05 +00:00
+								function xlogy(x, y)
 								  result = x * log(y)
-												NaN

											
										
										
											2020-05-07 10:44:32 +00:00
+								  ifelse(iszero(x), zero(result), result)
-												generic

											
										
										
											2020-05-05 17:23:05 +00:00
+								end
-												cufunc

											
										
										
											2020-05-07 07:54:05 +00:00
+								CuArrays.@cufunc function xlogy(x, y)
 								  result = x * log(y)
-												NaN

											
										
										
											2020-05-07 10:44:32 +00:00
+								  ifelse(iszero(x), zero(result), result)
-												cufunc

											
										
										
											2020-05-07 07:54:05 +00:00
+								end
-												xlogy broadcast adjoint

											
										
										
											2020-05-12 16:29:35 +00:00
 								@adjoint function broadcasted(::typeof(xlogy), x::Zygote.Numeric, y::Zygote.Numeric)
 								  res = xlogy.(x, y)
 								  res, Δ -> (nothing, Zygote.unbroadcast(x, xlogy.(Δ, y)), Zygote.unbroadcast(y, Δ .* x ./ y))
 								end