Merge branch 'master' into dg/gradtests
This commit is contained in:
Dhairya Gandhi
GitHub
commit
9b6155c77d
|
|
@ -1,5 +1,3 @@
|
|||
# This file is machine-generated - editing it directly is not advised
|
||||
|
||||
[[AbstractFFTs]]
|
||||
deps = ["LinearAlgebra"]
|
||||
git-tree-sha1 = "380e36c66edfa099cd90116b24c1ce8cafccac40"
|
||||
|
|
@ -94,7 +92,6 @@ version = "1.3.0"
|
|||
|
||||
[[CuArrays]]
|
||||
deps = ["AbstractFFTs", "Adapt", "CEnum", "CUDAapi", "CUDAdrv", "CUDAnative", "DataStructures", "GPUArrays", "Libdl", "LinearAlgebra", "MacroTools", "NNlib", "Printf", "Random", "Requires", "SparseArrays", "TimerOutputs"]
|
||||
git-tree-sha1 = "dc6e4f2bf3d84861ca00face3f0209ab71a0bc00"
|
||||
repo-rev = "master"
|
||||
repo-url = "https://github.com/JuliaGPU/CuArrays.jl.git"
|
||||
uuid = "3a865a2d-5b23-5a0f-bc46-62713ec82fae"
|
||||
|
|
@ -107,9 +104,9 @@ version = "1.1.0"
|
|||
|
||||
[[DataStructures]]
|
||||
deps = ["InteractiveUtils", "OrderedCollections"]
|
||||
git-tree-sha1 = "1fe8fad5fc84686dcbc674aa255bc867a64f8132"
|
||||
git-tree-sha1 = "a1b652fb77ae8ca7ea328fa7ba5aa151036e5c10"
|
||||
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
|
||||
version = "0.17.5"
|
||||
version = "0.17.6"
|
||||
|
||||
[[Dates]]
|
||||
deps = ["Printf"]
|
||||
|
|
@ -126,13 +123,13 @@ uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5"
|
|||
version = "0.0.4"
|
||||
|
||||
[[DiffRules]]
|
||||
deps = ["Random", "Test"]
|
||||
git-tree-sha1 = "dc0869fb2f5b23466b32ea799bd82c76480167f7"
|
||||
deps = ["NaNMath", "Random", "SpecialFunctions"]
|
||||
git-tree-sha1 = "f734b5f6bc9c909027ef99f6d91d5d9e4b111eed"
|
||||
uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
|
||||
version = "0.0.10"
|
||||
version = "0.1.0"
|
||||
|
||||
[[Distributed]]
|
||||
deps = ["Random", "Serialization", "Sockets"]
|
||||
deps = ["LinearAlgebra", "Random", "Serialization", "Sockets"]
|
||||
uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
|
||||
|
||||
[[FFTW]]
|
||||
|
|
@ -143,9 +140,9 @@ version = "1.0.1"
|
|||
|
||||
[[FillArrays]]
|
||||
deps = ["LinearAlgebra", "Random", "SparseArrays"]
|
||||
git-tree-sha1 = "b2cf74f09216cfe3c241e8484178ec0ea941870f"
|
||||
git-tree-sha1 = "1a9fe4e1323f38de0ba4da49eafd15b25ec62298"
|
||||
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
|
||||
version = "0.8.1"
|
||||
version = "0.8.2"
|
||||
|
||||
[[FixedPointNumbers]]
|
||||
git-tree-sha1 = "d14a6fa5890ea3a7e5dcab6811114f132fec2b4b"
|
||||
|
|
@ -154,9 +151,9 @@ version = "0.6.1"
|
|||
|
||||
[[ForwardDiff]]
|
||||
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"]
|
||||
git-tree-sha1 = "4407e7b76999eca2646abdb68203bd4302476168"
|
||||
git-tree-sha1 = "da46ac97b17793eba44ff366dc6cb70f1238a738"
|
||||
uuid = "f6369f11-7733-5829-9624-2563aa707210"
|
||||
version = "0.10.6"
|
||||
version = "0.10.7"
|
||||
|
||||
[[GPUArrays]]
|
||||
deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"]
|
||||
|
|
@ -171,7 +168,7 @@ uuid = "7869d1d1-7146-5819-86e3-90919afe41df"
|
|||
version = "0.3.0"
|
||||
|
||||
[[InteractiveUtils]]
|
||||
deps = ["Markdown"]
|
||||
deps = ["LinearAlgebra", "Markdown"]
|
||||
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
|
||||
|
||||
[[JSON]]
|
||||
|
|
@ -239,10 +236,9 @@ uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
|
|||
version = "0.6.0"
|
||||
|
||||
[[NaNMath]]
|
||||
deps = ["Compat"]
|
||||
git-tree-sha1 = "ce3b85e484a5d4c71dd5316215069311135fa9f2"
|
||||
git-tree-sha1 = "928b8ca9b2791081dc71a51c55347c27c618760f"
|
||||
uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
|
||||
version = "0.3.2"
|
||||
version = "0.3.3"
|
||||
|
||||
[[OrderedCollections]]
|
||||
deps = ["Random", "Serialization", "Test"]
|
||||
|
|
@ -252,12 +248,12 @@ version = "1.1.0"
|
|||
|
||||
[[Parsers]]
|
||||
deps = ["Dates", "Test"]
|
||||
git-tree-sha1 = "c56ecb484f286639f161e712b8311f5ab77e8d32"
|
||||
git-tree-sha1 = "0139ba59ce9bc680e2925aec5b7db79065d60556"
|
||||
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
|
||||
version = "0.3.8"
|
||||
version = "0.3.10"
|
||||
|
||||
[[Pkg]]
|
||||
deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
|
||||
deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
|
||||
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
|
||||
|
||||
[[Printf]]
|
||||
|
|
@ -356,7 +352,7 @@ uuid = "30578b45-9adc-5946-b283-645ec420af67"
|
|||
version = "0.4.0"
|
||||
|
||||
[[UUIDs]]
|
||||
deps = ["Random", "SHA"]
|
||||
deps = ["Random"]
|
||||
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
|
||||
|
||||
[[Unicode]]
|
||||
|
|
|
|||
13
NEWS.md
13
NEWS.md
|
|
@ -1,3 +1,16 @@
|
|||
# v0.10.0
|
||||
* The default AD engine has switched from [Tracker to Zygote.jl](https://github.com/FluxML/Flux.jl/pull/669)
|
||||
- The dependency on Tracker.jl has been removed.
|
||||
- This means Flux now does not depend on using a specialised `TrackedArray` type, and can be used with normal Array implementations directly.
|
||||
- Tracker compatibility is maintained in most common cases, but Zygote will be the preferred AD backend for Flux from now on.
|
||||
* The CUDNN wrappers have been [moved from Flux into CuArrays](https://github.com/FluxML/Flux.jl/pull/874), to allow for better supporting the CUDA backend, and improve user experience, not to mention making Flux lean.
|
||||
* `*crossentropy` functions now [work as expected with CuArrays](https://github.com/FluxML/Flux.jl/pull/926). [PR for binarycrossentropy](https://github.com/FluxML/Flux.jl/pull/940).
|
||||
* Added [clearer docs](https://github.com/FluxML/Flux.jl/pull/904) around training and the Optimiser interface.
|
||||
* [Layer initialisations](https://github.com/FluxML/Flux.jl/pull/937) have been improved with a clearer API on how to extend it for other purposes.
|
||||
* [Better messaging around CUDA availability](https://github.com/FluxML/Flux.jl/pull/924), with hooks to initialize the GPU as default where possible.
|
||||
* `@treelike` has been formalised as a [functor](https://github.com/FluxML/Flux.jl/pull/865), with an effective deprecation.
|
||||
* `testmode!` is deprecated in favour of [istraining](https://github.com/FluxML/Flux.jl/pull/669)
|
||||
|
||||
# v0.9.0
|
||||
* [Depthwise convolutional layer API changes](https://github.com/FluxML/Flux.jl/pull/756) from `in => mult` channel specification to `in => out` channel specification, and deprecates implicit `out` constructor.
|
||||
* New [SkipConnection](https://github.com/FluxML/Flux.jl/pull/446), which can be used to train residual neural network architectures.
|
||||
|
|
|
|||
11
Project.toml
11
Project.toml
|
|
@ -1,6 +1,6 @@
|
|||
name = "Flux"
|
||||
uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c"
|
||||
version = "0.9.0"
|
||||
version = "0.10.0"
|
||||
|
||||
[deps]
|
||||
AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
|
||||
|
|
@ -24,8 +24,17 @@ ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
|
|||
Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
|
||||
|
||||
[compat]
|
||||
AbstractTrees = "0.2"
|
||||
Adapt = "1"
|
||||
CodecZlib = "0.5, 0.6"
|
||||
Colors = "0.8, 0.9"
|
||||
CuArrays = "1.4.3"
|
||||
Juno = "0.5, 0.6, 0.7"
|
||||
MacroTools = "0.3, 0.4, 0.5"
|
||||
NNlib = "0.6"
|
||||
Reexport = "0.2"
|
||||
StatsBase = "0"
|
||||
ZipFile = "0.7, 0.8"
|
||||
Zygote = "0.4"
|
||||
julia = "1"
|
||||
|
||||
|
|
|
|||
88
README.md
88
README.md
|
|
@ -7,93 +7,9 @@
|
|||
Flux is an elegant approach to machine learning. It's a 100% pure-Julia stack, and provides lightweight abstractions on top of Julia's native GPU and AD support. Flux makes the easy things easy while remaining fully hackable.
|
||||
|
||||
```julia
|
||||
julia> Pkg.add("Flux")
|
||||
] add Flux
|
||||
```
|
||||
|
||||
See the [documentation](https://fluxml.github.io/Flux.jl/) or the [model zoo](https://github.com/FluxML/model-zoo/) for examples.
|
||||
|
||||
If you use Flux in research, please cite the following paper:
|
||||
|
||||
```
|
||||
@article{innes:2018,
|
||||
author = {Mike Innes},
|
||||
title = {Flux: Elegant Machine Learning with Julia},
|
||||
journal = {Journal of Open Source Software},
|
||||
year = {2018},
|
||||
doi = {10.21105/joss.00602},
|
||||
}
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
Flux has powerful high-level features, and common architectures can be defined in a few lines.
|
||||
|
||||
```julia
|
||||
model = Chain(
|
||||
Dense(768, 128, σ),
|
||||
LSTM(128, 256),
|
||||
LSTM(256, 128),
|
||||
Dense(128, 10),
|
||||
softmax)
|
||||
|
||||
loss(x, y) = crossentropy(model(x), y)
|
||||
|
||||
Flux.train!(loss, params(model), data, ADAM(...))
|
||||
```
|
||||
|
||||
Yet you can easily strip away the layers, and directly write the mathematics for your problem. Flux will seamlessly take gradients of any Julia code, so your model looks just like the paper.
|
||||
|
||||
```julia
|
||||
W = param(randn(2, 10))
|
||||
b = param(randn(2))
|
||||
|
||||
y(x) = σ.(W * x .+ b)
|
||||
```
|
||||
|
||||
If that's *still* not enough, you can go as deep as you want, even writing your own CUDA kernels with [CUDAnative](https://github.com/JuliaGPU/CUDAnative.jl)! All this can be freely mixed-and-matched in a single model or script, and it all runs interactively via Jupyter or Juno.
|
||||
|
||||
```julia
|
||||
function gpu_add(a, b, c)
|
||||
i = (blockIdx().x-1) * blockDim().x + threadIdx().x
|
||||
c[i] = a[i] + b[i]
|
||||
return nothing
|
||||
end
|
||||
```
|
||||
|
||||
Unusual architectures are no problem in Flux, as you can use all the loops, control flow and even macros that you're used to. Here's a Tree RNN in 4 lines.
|
||||
|
||||
```julia
|
||||
tree() = rand() < 0.5 ? rand(10) : (tree(), tree()) # dummy data
|
||||
|
||||
shrink = Dense(20, 10)
|
||||
combine(a, b) = shrink([a; b])
|
||||
|
||||
model(x) = x
|
||||
model(x::Tuple) = combine(model(x[1]), model(x[2]))
|
||||
|
||||
model(tree()) # Sample output
|
||||
```
|
||||
|
||||
Despite this flexibility, Julia's advanced compiler lets us do some powerful optimisations. For example, this definition of `sigmoid` automatically gets fused into a *single* GPU kernel – so it's really fast.
|
||||
|
||||
```julia
|
||||
sigmoid(xs) = 1 ./ (1 .+ exp.(.-xs))
|
||||
```
|
||||
|
||||
Similarly, Flux is the first dynamic framework to support [compiling to the browser](https://fluxml.github.io/experiments/) and model import via [formats like ONNX](https://github.com/FluxML/ONNX.jl/), both of which are thinly-veiled compiler problems.
|
||||
|
||||
For more on our philosophy on machine learning, check out our article [On Machine Learning & Programming Languages](https://julialang.org/blog/2017/12/ml&pl).
|
||||
|
||||
## Contributing & Help
|
||||
|
||||
For general questions and help, check out Julia's [community forum](https://discourse.julialang.org/c/domain/ML).
|
||||
|
||||
Flux development is carried out via our [GitHub issues](https://github.com/FluxML/Flux.jl/issues), so feel free to open feature requests or PRs here.
|
||||
|
||||
For more informal discussions we'd love to have you on the [Julia slack](https://slackinvite.julialang.org/), where we hang out on the #machine-learning channel.
|
||||
|
||||
## Related Packages
|
||||
|
||||
Check out [Metalhead.jl](https://github.com/FluxML/Metalhead.jl) for common computer vision datasets and trained models.
|
||||
|
||||
[MLDatasets.jl](https://github.com/JuliaML/MLDatasets.jl) provides further common datasets.
|
||||
If you use Flux in research, please see [our papers](CITATION.bib) for appropriate citations.
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ include("data/Data.jl")
|
|||
include("deprecations.jl")
|
||||
|
||||
function __init__()
|
||||
precompiling = ccall(:jl_generating_output, Cint, ()) != 0
|
||||
|
||||
# we don't want to include the CUDA module when precompiling,
|
||||
# or we could end up replacing it at run time (triggering a warning)
|
||||
precompiling && return
|
||||
|
||||
if !CuArrays.functional()
|
||||
# nothing to do here, and either CuArrays or one of its dependencies will have warned
|
||||
else
|
||||
|
|
|
|||
|
|
@ -53,6 +53,9 @@ but it is more numerically stable.
|
|||
"""
|
||||
logitbinarycrossentropy(logŷ, y) = (1 - y)*logŷ - logσ(logŷ)
|
||||
|
||||
# Re-definition to fix interaction with CuArrays.
|
||||
CuArrays.@cufunc logitbinarycrossentropy(logŷ, y) = (1 - y)*logŷ - logσ(logŷ)
|
||||
|
||||
"""
|
||||
normalise(x::AbstractArray; dims=1)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
# Arrays
|
||||
glorot_uniform(dims...) = (rand(Float32, dims...) .- 0.5f0) .* sqrt(24.0f0/sum(dims))
|
||||
glorot_normal(dims...) = randn(Float32, dims...) .* sqrt(2.0f0/sum(dims))
|
||||
nfan() = 1, 1 #fan_in, fan_out
|
||||
nfan(n) = 1, n #A vector is treated as a n×1 matrix
|
||||
nfan(n_out, n_in) = n_in, n_out #In case of Dense kernels: arranged as matrices
|
||||
nfan(dims...) = prod(dims[1:end-2]) .* (dims[end-1], dims[end]) #In case of convolution kernels
|
||||
|
||||
glorot_uniform(dims...) = (rand(Float32, dims...) .- 0.5f0) .* sqrt(24.0f0 / sum(nfan(dims...)))
|
||||
glorot_normal(dims...) = randn(Float32, dims...) .* sqrt(2.0f0 / sum(nfan(dims...)))
|
||||
|
||||
ones(T::Type, dims...) = Base.ones(T, dims...)
|
||||
zeros(T::Type, dims...) = Base.zeros(T, dims...)
|
||||
|
|
|
|||
|
|
@ -31,9 +31,10 @@ cx = gpu(x)
|
|||
@test Flux.crossentropy(x,x, weight=1.0) ≈ Flux.crossentropy(cx,cx, weight=1.0)
|
||||
@test Flux.crossentropy(x,x, weight=[1.0;2.0;3.0]) ≈ Flux.crossentropy(cx,cx, weight=cu([1.0;2.0;3.0]))
|
||||
|
||||
x = σ.([-1.1491, 0.8619, 0.3127])
|
||||
x = [-1.1491, 0.8619, 0.3127]
|
||||
y = [1, 1, 0.]
|
||||
@test Flux.binarycrossentropy.(x,y) ≈ Flux.binarycrossentropy.(cu(x),cu(y))
|
||||
@test Flux.binarycrossentropy.(σ.(x),y) ≈ Flux.binarycrossentropy.(cu(σ.(x)),cu(y))
|
||||
@test Flux.logitbinarycrossentropy.(x,y) ≈ Flux.logitbinarycrossentropy.(cu(x),cu(y))
|
||||
|
||||
xs = rand(5, 5)
|
||||
ys = Flux.onehotbatch(1:5,1:5)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
using Flux
|
||||
using Flux: throttle, glorot_uniform, glorot_normal, stack, unstack
|
||||
using StatsBase: std
|
||||
using Flux: throttle, nfan, glorot_uniform, glorot_normal, stack, unstack
|
||||
using StatsBase: var
|
||||
using Random
|
||||
using Test
|
||||
|
||||
|
|
@ -56,18 +56,26 @@ end
|
|||
# Set random seed so that these tests don't fail randomly
|
||||
Random.seed!(0)
|
||||
|
||||
# glorot_uniform should yield a kernel with stddev ~= sqrt(6/(n_in + n_out)),
|
||||
# and glorot_normal should yield a kernel with stddev != 2/(n_in _ n_out)
|
||||
for (n_in, n_out) in [(100, 100), (100, 400)]
|
||||
v = glorot_uniform(n_in, n_out)
|
||||
@test minimum(v) > -1.1*sqrt(6/(n_in + n_out))
|
||||
@test minimum(v) < -0.9*sqrt(6/(n_in + n_out))
|
||||
@test maximum(v) > 0.9*sqrt(6/(n_in + n_out))
|
||||
@test maximum(v) < 1.1*sqrt(6/(n_in + n_out))
|
||||
@testset "Fan in/out" begin
|
||||
@test nfan() == (1, 1) #For a constant
|
||||
@test nfan(100) == (1, 100) #For vector
|
||||
@test nfan(100, 200) == (200, 100) #For Dense layer
|
||||
@test nfan(2, 30, 40) == (2 * 30, 2 * 40) #For 1D Conv layer
|
||||
@test nfan(2, 3, 40, 50) == (2 * 3 * 40, 2 * 3 * 50) #For 2D Conv layer
|
||||
@test nfan(2, 3, 4, 50, 60) == (2 * 3 * 4 * 50, 2 * 3 * 4 * 60) #For 3D Conv layer
|
||||
end
|
||||
|
||||
v = glorot_normal(n_in, n_out)
|
||||
@test std(v) > 0.9*sqrt(2/(n_in + n_out))
|
||||
@test std(v) < 1.1*sqrt(2/(n_in + n_out))
|
||||
@testset "glorot" begin
|
||||
# glorot_uniform and glorot_normal should both yield a kernel with
|
||||
# variance ≈ 2/(fan_in + fan_out)
|
||||
for dims ∈ [(1000,), (100, 100), (100, 400), (2, 3, 32, 64), (2, 3, 4, 32, 64)]
|
||||
for init ∈ [glorot_uniform, glorot_normal]
|
||||
v = init(dims...)
|
||||
fan_in, fan_out = nfan(dims...)
|
||||
σ2 = 2 / (fan_in + fan_out)
|
||||
@test 0.9σ2 < var(v) < 1.1σ2
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue