2017-05-01 15:57:51 +00:00
|
|
|
|
# Arrays
|
2018-09-07 00:25:32 +00:00
|
|
|
|
glorot_uniform(dims...) = (rand(Float32, dims...) .- 0.5f0) .* sqrt(24.0f0/sum(dims))
|
|
|
|
|
glorot_normal(dims...) = randn(Float32, dims...) .* sqrt(2.0f0/sum(dims))
|
|
|
|
|
|
|
|
|
|
ones(T::Type, dims...) = Base.ones(T, dims...)
|
|
|
|
|
zeros(T::Type, dims...) = Base.zeros(T, dims...)
|
|
|
|
|
|
|
|
|
|
ones(dims...) = Base.ones(Float32, dims...)
|
|
|
|
|
zeros(dims...) = Base.zeros(Float32, dims...)
|
2016-08-25 16:25:33 +00:00
|
|
|
|
|
2017-09-06 22:58:55 +00:00
|
|
|
|
unsqueeze(xs, dim) = reshape(xs, (size(xs)[1:dim-1]..., 1, size(xs)[dim:end]...))
|
2017-08-19 19:52:29 +00:00
|
|
|
|
|
2019-01-04 01:32:11 +00:00
|
|
|
|
stack(xs, dim) = cat(unsqueeze.(xs, dim)..., dims=dim)
|
|
|
|
|
unstack(xs, dim) = [copy(selectdim(xs, dim, i)) for i in 1:size(xs, dim)]
|
2017-05-01 15:57:51 +00:00
|
|
|
|
|
2017-10-18 16:07:58 +00:00
|
|
|
|
"""
|
|
|
|
|
chunk(xs, n)
|
|
|
|
|
|
|
|
|
|
Split `xs` into `n` parts.
|
|
|
|
|
|
|
|
|
|
```julia
|
|
|
|
|
julia> chunk(1:10, 3)
|
|
|
|
|
3-element Array{Array{Int64,1},1}:
|
|
|
|
|
[1, 2, 3, 4]
|
|
|
|
|
[5, 6, 7, 8]
|
|
|
|
|
[9, 10]
|
|
|
|
|
```
|
|
|
|
|
"""
|
|
|
|
|
chunk(xs, n) = collect(Iterators.partition(xs, ceil(Int, length(xs)/n)))
|
|
|
|
|
|
2018-10-02 19:39:00 +00:00
|
|
|
|
batchindex(xs, i) = (reverse(Base.tail(reverse(axes(xs))))..., i)
|
2017-10-15 22:44:40 +00:00
|
|
|
|
|
2018-02-17 11:19:14 +00:00
|
|
|
|
"""
|
|
|
|
|
frequencies(xs)
|
|
|
|
|
|
|
|
|
|
Count the number of times that each element of `xs` appears.
|
|
|
|
|
|
|
|
|
|
```julia
|
|
|
|
|
julia> frequencies(['a','b','b'])
|
|
|
|
|
Dict{Char,Int64} with 2 entries:
|
|
|
|
|
'b' => 2
|
|
|
|
|
'a' => 1
|
|
|
|
|
```
|
|
|
|
|
"""
|
|
|
|
|
function frequencies(xs)
|
|
|
|
|
fs = Dict{eltype(xs),Int}()
|
|
|
|
|
for x in xs
|
|
|
|
|
fs[x] = get(fs, x, 0) + 1
|
|
|
|
|
end
|
|
|
|
|
return fs
|
|
|
|
|
end
|
|
|
|
|
|
2018-02-28 14:25:32 +00:00
|
|
|
|
head(x::Tuple) = reverse(Base.tail(reverse(x)))
|
|
|
|
|
|
|
|
|
|
squeezebatch(x) = reshape(x, head(size(x)))
|
|
|
|
|
|
2017-10-18 15:21:15 +00:00
|
|
|
|
"""
|
|
|
|
|
batch(xs)
|
|
|
|
|
|
|
|
|
|
Batch the arrays in `xs` into a single array.
|
|
|
|
|
|
|
|
|
|
```julia
|
|
|
|
|
julia> batch([[1,2,3],[4,5,6]])
|
|
|
|
|
3×2 Array{Int64,2}:
|
|
|
|
|
1 4
|
|
|
|
|
2 5
|
|
|
|
|
3 6
|
|
|
|
|
```
|
|
|
|
|
"""
|
2017-10-15 22:44:40 +00:00
|
|
|
|
function batch(xs)
|
2017-10-18 15:21:15 +00:00
|
|
|
|
data = first(xs) isa AbstractArray ?
|
|
|
|
|
similar(first(xs), size(first(xs))..., length(xs)) :
|
2018-10-02 19:39:00 +00:00
|
|
|
|
Vector{eltype(xs)}(undef, length(xs))
|
2017-10-15 22:44:40 +00:00
|
|
|
|
for (i, x) in enumerate(xs)
|
|
|
|
|
data[batchindex(data, i)...] = x
|
|
|
|
|
end
|
|
|
|
|
return data
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
Base.rpad(v::AbstractVector, n::Integer, p) = [v; fill(p, max(n - length(v), 0))]
|
|
|
|
|
|
2017-10-18 15:21:15 +00:00
|
|
|
|
"""
|
|
|
|
|
batchseq(seqs, pad)
|
|
|
|
|
|
|
|
|
|
Take a list of `N` sequences, and turn them into a single sequence where each
|
|
|
|
|
item is a batch of `N`. Short sequences will be padded by `pad`.
|
|
|
|
|
|
|
|
|
|
```julia
|
|
|
|
|
julia> batchseq([[1, 2, 3], [4, 5]], 0)
|
|
|
|
|
3-element Array{Array{Int64,1},1}:
|
|
|
|
|
[1, 4]
|
|
|
|
|
[2, 5]
|
|
|
|
|
[3, 0]
|
|
|
|
|
```
|
|
|
|
|
"""
|
|
|
|
|
function batchseq(xs, pad = nothing, n = maximum(length(x) for x in xs))
|
2017-10-15 22:44:40 +00:00
|
|
|
|
xs_ = [rpad(x, n, pad) for x in xs]
|
|
|
|
|
[batch([xs_[j][i] for j = 1:length(xs_)]) for i = 1:n]
|
|
|
|
|
end
|
|
|
|
|
|
2017-05-01 15:57:51 +00:00
|
|
|
|
# Other
|
2017-05-01 12:46:23 +00:00
|
|
|
|
|
2017-08-18 00:04:50 +00:00
|
|
|
|
"""
|
|
|
|
|
Returns a function that when invoked, will only be triggered at most once
|
|
|
|
|
during `timeout` seconds. Normally, the throttled function will run
|
|
|
|
|
as much as it can, without ever going more than once per `wait` duration;
|
|
|
|
|
but if you'd like to disable the execution on the leading edge, pass
|
|
|
|
|
`leading=false`. To enable execution on the trailing edge, ditto.
|
|
|
|
|
"""
|
|
|
|
|
function throttle(f, timeout; leading=true, trailing=false)
|
|
|
|
|
cooldown = true
|
|
|
|
|
later = nothing
|
2017-12-13 18:24:56 +00:00
|
|
|
|
result = nothing
|
2017-08-18 00:04:50 +00:00
|
|
|
|
|
|
|
|
|
function throttled(args...; kwargs...)
|
|
|
|
|
yield()
|
|
|
|
|
|
|
|
|
|
if cooldown
|
|
|
|
|
if leading
|
2017-12-13 18:24:56 +00:00
|
|
|
|
result = f(args...; kwargs...)
|
2017-08-18 00:04:50 +00:00
|
|
|
|
else
|
|
|
|
|
later = () -> f(args...; kwargs...)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
cooldown = false
|
2018-06-20 14:18:07 +00:00
|
|
|
|
@async try
|
2017-08-18 00:04:50 +00:00
|
|
|
|
while (sleep(timeout); later != nothing)
|
|
|
|
|
later()
|
|
|
|
|
later = nothing
|
|
|
|
|
end
|
|
|
|
|
finally
|
|
|
|
|
cooldown = true
|
|
|
|
|
end
|
|
|
|
|
elseif trailing
|
2017-12-13 18:24:56 +00:00
|
|
|
|
later = () -> (result = f(args...; kwargs...))
|
2017-08-18 00:04:50 +00:00
|
|
|
|
end
|
|
|
|
|
|
2017-12-13 18:24:56 +00:00
|
|
|
|
return result
|
2017-08-18 00:04:50 +00:00
|
|
|
|
end
|
|
|
|
|
end
|
2017-12-08 13:46:12 +00:00
|
|
|
|
|
2019-11-07 11:23:41 +00:00
|
|
|
|
import Base: +, -, *, reshape, size
|
|
|
|
|
import Base.Broadcast: broadcasted, Broadcasted, BroadcastStyle
|
2019-10-08 11:47:36 +00:00
|
|
|
|
|
2019-10-08 14:32:04 +00:00
|
|
|
|
"""
|
|
|
|
|
Zeros()
|
2019-11-07 11:23:41 +00:00
|
|
|
|
Zeros(size...)
|
|
|
|
|
Zeros(Type, size...)
|
2019-10-08 14:32:04 +00:00
|
|
|
|
|
2019-10-22 10:41:27 +00:00
|
|
|
|
Acts as a stand-in for an array of zeros that can be
|
|
|
|
|
used during training which is ignored by the optimisers.
|
2019-10-08 11:47:36 +00:00
|
|
|
|
|
2019-11-07 11:23:41 +00:00
|
|
|
|
Useful to turn bias off for a forward pass of a layer.
|
|
|
|
|
|
|
|
|
|
!!! warning
|
|
|
|
|
Zeros acts a scalar while broadcasting, so does not
|
|
|
|
|
expand dims. Checks for shape compatibility by default.
|
2019-10-08 14:32:04 +00:00
|
|
|
|
|
2019-10-22 10:41:27 +00:00
|
|
|
|
## Examples
|
2019-10-08 14:32:04 +00:00
|
|
|
|
|
2019-10-22 10:41:27 +00:00
|
|
|
|
```julia
|
2019-11-07 11:23:41 +00:00
|
|
|
|
julia> Flux.Zeros(3,3)
|
|
|
|
|
3×3 Flux.Zeros{Bool,2}:
|
|
|
|
|
false false false
|
|
|
|
|
false false false
|
|
|
|
|
false false false
|
|
|
|
|
|
|
|
|
|
julia> Flux.Zeros(Float32, 3,3)
|
|
|
|
|
3×3 Flux.Zeros{Float32,2}:
|
|
|
|
|
0.0 0.0 0.0
|
|
|
|
|
0.0 0.0 0.0
|
|
|
|
|
0.0 0.0 0.0
|
|
|
|
|
|
2019-10-22 10:41:27 +00:00
|
|
|
|
julia> rand(3,3) .+ Flux.Zeros()
|
|
|
|
|
3×3 Array{Float64,2}:
|
|
|
|
|
0.198739 0.490459 0.785386
|
|
|
|
|
0.779074 0.39986 0.66383
|
|
|
|
|
0.854981 0.447292 0.314497
|
|
|
|
|
|
2019-11-07 11:23:41 +00:00
|
|
|
|
julia> bias_less_conv = Conv((2,2), 1=>3, bias = Flux.Zeros())
|
2019-10-22 10:41:27 +00:00
|
|
|
|
Conv((2, 2), 1=>3)
|
|
|
|
|
```
|
|
|
|
|
"""
|
2019-10-23 14:32:15 +00:00
|
|
|
|
struct Zeros{T,N} <: AbstractArray{T,N}
|
|
|
|
|
size::Tuple
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
Zeros(::Type{T}, sz...) where T = Zeros{T,length(sz)}(sz)
|
|
|
|
|
Zeros(sz::Integer...) = Zeros(Bool, sz...)
|
|
|
|
|
|
|
|
|
|
Base.size(xs::Zeros) = xs.size
|
|
|
|
|
Base.axes(xs::Zeros) = Base.OneTo.(size(xs))
|
|
|
|
|
|
2019-11-07 11:23:41 +00:00
|
|
|
|
Base.IndexStyle(::Type{<:Zeros}) = IndexCartesian()
|
|
|
|
|
|
|
|
|
|
Base.getindex(xs::Zeros{T,N}, I::Vararg{Int, N}) where {T,N} = zero(T)
|
|
|
|
|
Base.getindex(xs::Zeros{T,N}, inds::Union{Base.OneTo, Base.UnitRange}) where {T,N} =
|
|
|
|
|
Zeros(T, inds.stop)
|
|
|
|
|
|
2019-10-23 14:32:15 +00:00
|
|
|
|
Base.setindex(xs::Zeros, args...) =
|
|
|
|
|
error("setindex disallowed on Zeros Array")
|
|
|
|
|
Base.setindex!(xs::Zeros, args...) =
|
|
|
|
|
error("setindex! disallowed on Zeros Array")
|
|
|
|
|
|
|
|
|
|
Base.collect(xs::Zeros{T,N}) where {T,N} = fill(zero(T), size(xs))
|
|
|
|
|
|
2019-11-07 11:23:41 +00:00
|
|
|
|
# Ignore during backwards pass
|
2019-10-23 14:32:15 +00:00
|
|
|
|
@adjoint reshape(xs::Zeros{T}, dims...) where T =
|
|
|
|
|
reshape(xs, dims...), _ -> nothing
|
|
|
|
|
|
2019-11-07 11:23:41 +00:00
|
|
|
|
# Define basic ops
|
2019-10-22 10:41:27 +00:00
|
|
|
|
for f in (:+, :-)
|
2019-10-23 14:32:15 +00:00
|
|
|
|
@eval $f(a::Union{AbstractArray{<:Number}, Zeros}, b::Zeros) = a
|
2019-10-22 10:41:27 +00:00
|
|
|
|
end
|
2019-11-07 11:23:41 +00:00
|
|
|
|
Base.:+(a::Zeros, b::AbstractArray) = b
|
|
|
|
|
Base.:-(a::Zeros, b::AbstractArray) = -b
|
2019-10-23 14:32:15 +00:00
|
|
|
|
Base.:*(a::Union{AbstractArray{<:Number}, Zeros}, b::Zeros) = zero(a)
|
2019-11-07 11:23:41 +00:00
|
|
|
|
Base.:*(a::Zeros, b::AbstractArray) = zero(a)
|
|
|
|
|
|
|
|
|
|
# Hook into broadcasting API - to allow using as a regular array
|
|
|
|
|
Base.BroadcastStyle(::Type{<:Zeros}) = Broadcast.ArrayStyle{Zeros}()
|
|
|
|
|
Broadcast.broadcastable(xs::Zeros) = xs
|
|
|
|
|
Base.BroadcastStyle(::Broadcast.ArrayStyle{Zeros}, ::Broadcast.DefaultArrayStyle{N}) where N =
|
|
|
|
|
Broadcast.ArrayStyle{Zeros}()
|
2019-10-08 14:32:04 +00:00
|
|
|
|
|
2019-11-07 11:23:41 +00:00
|
|
|
|
function Base.similar(bc::Broadcasted{Broadcast.ArrayStyle{Flux.Zeros}}, ::Type{T}) where T
|
|
|
|
|
similar(Array{T}, axes(bc))
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
Base.copy(xs::Zeros{T,N}) where {T,N} = Zeros(T, size(xs)...)
|
|
|
|
|
|
|
|
|
|
isZeros(x::Zeros) = true
|
|
|
|
|
isZeros(x) = false
|
|
|
|
|
|
|
|
|
|
function Base.copyto!(dest::AbstractArray, bc::Broadcasted{Broadcast.ArrayStyle{Flux.Zeros}})
|
|
|
|
|
bc = Broadcast.flatten(bc)
|
|
|
|
|
|
|
|
|
|
i = isZeros(first(bc.args)) ? 2 : 1 # findfirst(!isZeros, bc.args)
|
|
|
|
|
dest .= bc.args[i]
|
|
|
|
|
end
|
2019-10-08 11:47:36 +00:00
|
|
|
|
|
2018-10-05 13:02:00 +00:00
|
|
|
|
"""
|
|
|
|
|
@jit ...
|
|
|
|
|
|
|
|
|
|
The `@jit` annotation can be applied to any code, and the code will be compiled
|
|
|
|
|
for performance.
|
|
|
|
|
|
|
|
|
|
@jit f(x) = @jit(x) + @jit(x)
|
|
|
|
|
|
|
|
|
|
Note that compilation happens regardless of the `@jit` macro, so it should only
|
|
|
|
|
be used for aesthetic purposes, or by recovering Python users.
|
|
|
|
|
"""
|
|
|
|
|
macro jit(ex)
|
|
|
|
|
esc(ex)
|
|
|
|
|
end
|