diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
new file mode 100644
index 00000000..9777033d
--- /dev/null
+++ b/.github/workflows/CompatHelper.yml
@@ -0,0 +1,24 @@
+name: CompatHelper
+
+on:
+ schedule:
+ - cron: '00 00 * * *'
+
+jobs:
+ CompatHelper:
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ julia-version: [1.3]
+ julia-arch: [x64]
+ os: [ubuntu-latest]
+ steps:
+ - uses: julia-actions/setup-julia@latest
+ with:
+ version: ${{ matrix.julia-version }}
+ - name: Pkg.add("CompatHelper")
+ run: julia -e 'using Pkg; Pkg.add("CompatHelper")'
+ - name: CompatHelper.main()
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ run: julia -e 'using CompatHelper; CompatHelper.main()'
diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml
new file mode 100644
index 00000000..d77d3a0c
--- /dev/null
+++ b/.github/workflows/TagBot.yml
@@ -0,0 +1,11 @@
+name: TagBot
+on:
+ schedule:
+ - cron: 0 * * * *
+jobs:
+ TagBot:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: JuliaRegistries/TagBot@v1
+ with:
+ token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/Manifest.toml b/Manifest.toml
index 12986ccd..788e5354 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -8,15 +8,15 @@ version = "0.5.0"
[[AbstractTrees]]
deps = ["Markdown"]
-git-tree-sha1 = "8201f932428d25a2e2903300764515754847d87d"
+git-tree-sha1 = "86d092c2599f1f7bb01668bf8eb3412f98d61e47"
uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
-version = "0.3.0"
+version = "0.3.2"
[[Adapt]]
deps = ["LinearAlgebra"]
-git-tree-sha1 = "82dab828020b872fa9efd3abec1152b075bc7cbf"
+git-tree-sha1 = "c88cfc7f9c1f9f8633cddf0b56e86302b70f64c5"
uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
-version = "1.0.0"
+version = "1.0.1"
[[Base64]]
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -34,21 +34,21 @@ version = "0.2.0"
[[CUDAapi]]
deps = ["Libdl", "Logging"]
-git-tree-sha1 = "56a813440ac98a1aa64672ab460a1512552211a7"
+git-tree-sha1 = "d7ceadd8f821177d05b897c0517e94633db535fe"
uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3"
-version = "2.1.0"
+version = "3.1.0"
[[CUDAdrv]]
deps = ["CEnum", "CUDAapi", "Printf"]
-git-tree-sha1 = "1fce616fa0806c67c133eb1d2f68f0f1a7504665"
+git-tree-sha1 = "01e90fa34e25776bc7c8661183d4519149ebfe59"
uuid = "c5f51814-7f29-56b8-a69c-e4d8f6be1fde"
-version = "5.0.1"
+version = "6.0.0"
[[CUDAnative]]
deps = ["Adapt", "CEnum", "CUDAapi", "CUDAdrv", "DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Printf", "TimerOutputs"]
-git-tree-sha1 = "6e11d5c2c91fc623952e94c4fb73f9c4db74795a"
+git-tree-sha1 = "f86269ff60ebe082a2806ecbce51f3cadc68afe9"
uuid = "be33ccc6-a3ff-5ff2-a52e-74243cff1e17"
-version = "2.7.0"
+version = "2.10.2"
[[CodecZlib]]
deps = ["BinaryProvider", "Libdl", "TranscodingStreams"]
@@ -58,15 +58,15 @@ version = "0.6.0"
[[ColorTypes]]
deps = ["FixedPointNumbers", "Random"]
-git-tree-sha1 = "7b62b728a5f3dd6ee3b23910303ccf27e82fad5e"
+git-tree-sha1 = "b9de8dc6106e09c79f3f776c27c62360d30e5eb8"
uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
-version = "0.8.1"
+version = "0.9.1"
[[Colors]]
deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Printf", "Reexport"]
-git-tree-sha1 = "c9c1845d6bf22e34738bee65c357a69f416ed5d1"
+git-tree-sha1 = "177d8b959d3c103a6d57574c38ee79c81059c31b"
uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
-version = "0.9.6"
+version = "0.11.2"
[[CommonSubexpressions]]
deps = ["Test"]
@@ -74,11 +74,17 @@ git-tree-sha1 = "efdaf19ab11c7889334ca247ff4c9f7c322817b0"
uuid = "bbf7d656-a473-5ed7-a52c-81e309532950"
version = "0.2.0"
+[[CompilerSupportLibraries_jll]]
+deps = ["Libdl", "Pkg"]
+git-tree-sha1 = "b57c5d019367c90f234a7bc7e24ff0a84971da5d"
+uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
+version = "0.2.0+1"
+
[[CuArrays]]
deps = ["AbstractFFTs", "Adapt", "CEnum", "CUDAapi", "CUDAdrv", "CUDAnative", "DataStructures", "GPUArrays", "Libdl", "LinearAlgebra", "MacroTools", "NNlib", "Printf", "Random", "Requires", "SparseArrays", "TimerOutputs"]
-git-tree-sha1 = "51fbe053dea29ed2513e02d38380007310cf4c4b"
+git-tree-sha1 = "7c20c5a45bb245cf248f454d26966ea70255b271"
uuid = "3a865a2d-5b23-5a0f-bc46-62713ec82fae"
-version = "1.6.0"
+version = "1.7.2"
[[DataAPI]]
git-tree-sha1 = "674b67f344687a88310213ddfa8a2b3c76cc4252"
@@ -87,9 +93,9 @@ version = "1.1.0"
[[DataStructures]]
deps = ["InteractiveUtils", "OrderedCollections"]
-git-tree-sha1 = "f784254f428fb8fd7ac15982e5862a38a44523d3"
+git-tree-sha1 = "5a431d46abf2ef2a4d5d00bd0ae61f651cf854c8"
uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
-version = "0.17.7"
+version = "0.17.10"
[[Dates]]
deps = ["Printf"]
@@ -107,9 +113,9 @@ version = "1.0.2"
[[DiffRules]]
deps = ["NaNMath", "Random", "SpecialFunctions"]
-git-tree-sha1 = "10dca52cf6d4a62d82528262921daf63b99704a2"
+git-tree-sha1 = "eb0c34204c8410888844ada5359ac8b96292cfd1"
uuid = "b552c78f-8df3-52c6-915a-8e097449b14b"
-version = "1.0.0"
+version = "1.0.1"
[[Distributed]]
deps = ["Random", "Serialization", "Sockets"]
@@ -123,26 +129,26 @@ version = "1.2.0"
[[FFTW_jll]]
deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "05674f209a6e3387dd103a945b0113eeb64b1a58"
+git-tree-sha1 = "ddb57f4cf125243b4aa4908c94d73a805f3cbf2c"
uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a"
-version = "3.3.9+3"
+version = "3.3.9+4"
[[FillArrays]]
deps = ["LinearAlgebra", "Random", "SparseArrays"]
-git-tree-sha1 = "fec413d4fc547992eb62a5c544cedb6d7853c1f5"
+git-tree-sha1 = "85c6b57e2680fa28d5c8adc798967377646fbf66"
uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
-version = "0.8.4"
+version = "0.8.5"
[[FixedPointNumbers]]
-git-tree-sha1 = "d14a6fa5890ea3a7e5dcab6811114f132fec2b4b"
+git-tree-sha1 = "4aaea64dd0c30ad79037084f8ca2b94348e65eaa"
uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
-version = "0.6.1"
+version = "0.7.1"
[[ForwardDiff]]
deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "NaNMath", "Random", "SpecialFunctions", "StaticArrays"]
-git-tree-sha1 = "840700059391d36e2498d89c2e82c08f261f2a2a"
+git-tree-sha1 = "88b082d492be6b63f967b6c96b352e25ced1a34c"
uuid = "f6369f11-7733-5829-9624-2563aa707210"
-version = "0.10.8"
+version = "0.10.9"
[[GPUArrays]]
deps = ["AbstractFFTs", "Adapt", "LinearAlgebra", "Printf", "Random", "Serialization"]
@@ -152,9 +158,9 @@ version = "2.0.1"
[[IRTools]]
deps = ["InteractiveUtils", "MacroTools", "Test"]
-git-tree-sha1 = "72421971e60917b8cd7737f9577c4f0f87eab306"
+git-tree-sha1 = "1a4355e4b5b50be2311ebb644f34f3306dbd0410"
uuid = "7869d1d1-7146-5819-86e3-90919afe41df"
-version = "0.3.0"
+version = "0.3.1"
[[IntelOpenMP_jll]]
deps = ["Libdl", "Pkg"]
@@ -167,10 +173,10 @@ deps = ["Markdown"]
uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
[[Juno]]
-deps = ["Base64", "Logging", "Media", "Profile", "Test"]
-git-tree-sha1 = "30d94657a422d09cb97b6f86f04f750fa9c50df8"
+deps = ["Base64", "Logging", "Media", "Profile"]
+git-tree-sha1 = "4f2249fb58cfb140eeb89428e31791e2f8959d8c"
uuid = "e5e0dc1b-0480-54bc-9374-aad01c23163d"
-version = "0.7.2"
+version = "0.8.0"
[[LLVM]]
deps = ["CEnum", "Libdl", "Printf", "Unicode"]
@@ -192,16 +198,16 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
[[MKL_jll]]
-deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "61069ae718b8ab1e325bbfb4e5268902e7ea08e3"
+deps = ["IntelOpenMP_jll", "Libdl", "Pkg"]
+git-tree-sha1 = "720629cc8cbd12c146ca01b661fd1a6cf66e2ff4"
uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
-version = "2019.0.117+0"
+version = "2019.0.117+2"
[[MacroTools]]
deps = ["DataStructures", "Markdown", "Random"]
-git-tree-sha1 = "e2fc7a55bb2224e203bbd8b59f72b91323233458"
+git-tree-sha1 = "07ee65e03e28ca88bc9a338a3726ae0c3efaa94b"
uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.3"
+version = "0.5.4"
[[Markdown]]
deps = ["Base64"]
@@ -224,9 +230,9 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
[[NNlib]]
deps = ["BinaryProvider", "Libdl", "LinearAlgebra", "Requires", "Statistics"]
-git-tree-sha1 = "135c0de4794d5e214b06f1fb4787af4a72896e61"
+git-tree-sha1 = "755c0bab3912ff782167e1b4b774b833f8a0e550"
uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
-version = "0.6.2"
+version = "0.6.4"
[[NaNMath]]
git-tree-sha1 = "928b8ca9b2791081dc71a51c55347c27c618760f"
@@ -234,10 +240,10 @@ uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3"
version = "0.3.3"
[[OpenSpecFun_jll]]
-deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "65f672edebf3f4e613ddf37db9dcbd7a407e5e90"
+deps = ["CompilerSupportLibraries_jll", "Libdl", "Pkg"]
+git-tree-sha1 = "d110040968b9afe95c6bd9c6233570b0fe8abd22"
uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
-version = "0.5.3+1"
+version = "0.5.3+2"
[[OrderedCollections]]
deps = ["Random", "Serialization", "Test"]
@@ -246,7 +252,7 @@ uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
version = "1.1.0"
[[Pkg]]
-deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
+deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Test", "UUIDs"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
[[Printf]]
@@ -273,9 +279,9 @@ version = "0.2.0"
[[Requires]]
deps = ["UUIDs"]
-git-tree-sha1 = "999513b7dea8ac17359ed50ae8ea089e4464e35e"
+git-tree-sha1 = "d37400976e98018ee840e0ca4f9d20baa231dc6b"
uuid = "ae029012-a4dd-5104-9daa-d747884805df"
-version = "1.0.0"
+version = "1.0.1"
[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
@@ -298,9 +304,9 @@ uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
[[SpecialFunctions]]
deps = ["OpenSpecFun_jll"]
-git-tree-sha1 = "268052ee908b2c086cc0011f528694f02f3e2408"
+git-tree-sha1 = "e19b98acb182567bcb7b75bb5d9eedf3a3b5ec6c"
uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "0.9.0"
+version = "0.10.0"
[[StaticArrays]]
deps = ["LinearAlgebra", "Random", "Statistics"]
@@ -314,9 +320,9 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
[[StatsBase]]
deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"]
-git-tree-sha1 = "c53e809e63fe5cf5de13632090bc3520649c9950"
+git-tree-sha1 = "be5c7d45daa449d12868f4466dbf5882242cf2d9"
uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
-version = "0.32.0"
+version = "0.32.1"
[[Test]]
deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
@@ -343,21 +349,23 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
[[ZipFile]]
deps = ["Libdl", "Printf", "Zlib_jll"]
-git-tree-sha1 = "5de8320a46812da1a8ca98b16a8a4546d44efa62"
+git-tree-sha1 = "8748302cfdec02c4ae9c97b112cf10003f7f767f"
uuid = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea"
-version = "0.9.0"
+version = "0.9.1"
[[Zlib_jll]]
deps = ["Libdl", "Pkg"]
-git-tree-sha1 = "5618a43055eb09377edca21d19d0e99bce24a9c3"
+git-tree-sha1 = "fd36a6739e256527287c5444960d0266712cd49e"
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.11+7"
+version = "1.2.11+8"
[[Zygote]]
deps = ["DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"]
-git-tree-sha1 = "74382bcc4c1e8075e14554da67d75565f8fb7827"
+git-tree-sha1 = "3c65158c0aa0808cdfff8bca2a36430b038aad00"
+repo-rev = "master"
+repo-url = "https://github.com/FluxML/Zygote.jl.git"
uuid = "e88e6eb3-aa80-5325-afca-941959d7151f"
-version = "0.4.5"
+version = "0.4.7"
[[ZygoteRules]]
deps = ["MacroTools"]
diff --git a/Project.toml b/Project.toml
index 96d1d853..bd105730 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,6 +1,6 @@
name = "Flux"
uuid = "587475ba-b771-5e3f-ad9e-33799f191a9c"
-version = "0.10.1"
+version = "0.10.2"
[deps]
AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
@@ -27,9 +27,9 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f"
AbstractTrees = "0.2, 0.3"
Adapt = "1"
CodecZlib = "0.5, 0.6"
-Colors = "0.8, 0.9"
+Colors = "0.8, 0.9, 0.10, 0.11"
CuArrays = "1.6"
-Juno = "0.5, 0.6, 0.7"
+Juno = "0.5, 0.6, 0.7, 0.8"
MacroTools = "0.3, 0.4, 0.5"
NNlib = "0.6"
Reexport = "0.2"
@@ -40,7 +40,10 @@ julia = "1"
[extras]
Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+IterTools = "c8e1da08-722c-5040-9ed9-7db0dc04731e"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
[targets]
-test = ["Test", "Documenter"]
+test = ["Test", "Documenter", "IterTools", "LinearAlgebra"]
diff --git a/docs/make.jl b/docs/make.jl
index b950e959..0d597500 100644
--- a/docs/make.jl
+++ b/docs/make.jl
@@ -13,11 +13,14 @@ makedocs(modules=[Flux, NNlib],
["Basics" => "models/basics.md",
"Recurrence" => "models/recurrence.md",
"Regularisation" => "models/regularisation.md",
- "Model Reference" => "models/layers.md"],
+ "Model Reference" => "models/layers.md",
+ "NNlib" => "models/nnlib.md"],
+ "Handling Data" =>
+ ["One-Hot Encoding" => "data/onehot.md",
+ "DataLoader" => "data/dataloader.md"],
"Training Models" =>
["Optimisers" => "training/optimisers.md",
"Training" => "training/training.md"],
- "One-Hot Encoding" => "data/onehot.md",
"GPU Support" => "gpu.md",
"Saving & Loading" => "saving.md",
"Performance Tips" => "performance.md",
diff --git a/docs/src/data/dataloader.md b/docs/src/data/dataloader.md
new file mode 100644
index 00000000..70a883c9
--- /dev/null
+++ b/docs/src/data/dataloader.md
@@ -0,0 +1,6 @@
+# DataLoader
+Flux provides the `DataLoader` type in the `Flux.Data` module to handle iteration over mini-batches of data.
+
+```@docs
+Flux.Data.DataLoader
+```
\ No newline at end of file
diff --git a/docs/src/gpu.md b/docs/src/gpu.md
index bb13fdd1..19d0c8c6 100644
--- a/docs/src/gpu.md
+++ b/docs/src/gpu.md
@@ -30,7 +30,7 @@ If you define a structured model, like a `Dense` layer or `Chain`, you just need
```julia
d = Dense(10, 5, σ)
d = fmap(cu, d)
-d.W # Tracked CuArray
+d.W # CuArray
d(cu(rand(10))) # CuArray output
m = Chain(Dense(10, 5, σ), Dense(5, 2), softmax)
@@ -53,7 +53,7 @@ julia> x = rand(10) |> gpu
0.511655
julia> m(x)
-Tracked 5-element CuArray{Float32,1}:
+5-element CuArray{Float32,1}:
-0.30535
⋮
-0.618002
diff --git a/docs/src/models/basics.md b/docs/src/models/basics.md
index d83fc462..6e8d0b76 100644
--- a/docs/src/models/basics.md
+++ b/docs/src/models/basics.md
@@ -219,3 +219,24 @@ Flux.@functor Affine
```
This enables a useful extra set of functionality for our `Affine` layer, such as [collecting its parameters](../training/optimisers.md) or [moving it to the GPU](../gpu.md).
+
+## Utility functions
+
+Flux provides some utility functions to help you generate models in an automated fashion.
+
+`outdims` enables you to calculate the spatial output dimensions of layers like `Conv` when applied to input images of a given size.
+Currently limited to the following layers:
+- `Chain`
+- `Dense`
+- `Conv`
+- `Diagonal`
+- `Maxout`
+- `ConvTranspose`
+- `DepthwiseConv`
+- `CrossCor`
+- `MaxPool`
+- `MeanPool`
+
+```@docs
+outdims
+```
diff --git a/docs/src/models/layers.md b/docs/src/models/layers.md
index 5f2ab3ce..41e98f32 100644
--- a/docs/src/models/layers.md
+++ b/docs/src/models/layers.md
@@ -40,19 +40,6 @@ Maxout
SkipConnection
```
-## Activation Functions
-
-Non-linearities that go between layers of your model. Most of these functions are defined in [NNlib](https://github.com/FluxML/NNlib.jl) but are available by default in Flux.
-
-Note that, unless otherwise stated, activation functions operate on scalars. To apply them to an array you can call `σ.(xs)`, `relu.(xs)` and so on.
-
-```@docs
-σ
-relu
-leakyrelu
-elu
-swish
-```
## Normalisation & Regularisation
@@ -61,6 +48,7 @@ These layers don't affect the structure of the network but may improve training
```@docs
BatchNorm
Dropout
+Flux.dropout
AlphaDropout
LayerNorm
GroupNorm
@@ -68,12 +56,12 @@ GroupNorm
## Cost Functions
```@docs
-mse
-crossentropy
-logitcrossentropy
-binarycrossentropy
-logitbinarycrossentropy
-kldivergence
-poisson
-hinge
+Flux.mse
+Flux.crossentropy
+Flux.logitcrossentropy
+Flux.binarycrossentropy
+Flux.logitbinarycrossentropy
+Flux.kldivergence
+Flux.poisson
+Flux.hinge
```
diff --git a/docs/src/models/nnlib.md b/docs/src/models/nnlib.md
new file mode 100644
index 00000000..f5732574
--- /dev/null
+++ b/docs/src/models/nnlib.md
@@ -0,0 +1,37 @@
+## NNlib
+Flux re-exports all of the functions exported by the [NNlib](https://github.com/FluxML/NNlib.jl) package.
+
+## Activation Functions
+Non-linearities that go between layers of your model. Note that, unless otherwise stated, activation functions operate on scalars. To apply them to an array you can call `σ.(xs)`, `relu.(xs)` and so on.
+
+```@docs
+NNlib.elu
+NNlib.gelu
+NNlib.leakyrelu
+NNlib.logcosh
+NNlib.logsigmoid
+NNlib.sigmoid
+NNlib.relu
+NNlib.selu
+NNlib.softplus
+NNlib.softsign
+NNlib.swish
+```
+
+## Softmax
+```@docs
+NNlib.softmax
+NNlib.logsoftmax
+```
+
+## Pooling
+```@docs
+NNlib.maxpool
+NNlib.meanpool
+```
+
+## Convolution
+```@docs
+NNlib.conv
+NNlib.depthwiseconv
+```
\ No newline at end of file
diff --git a/docs/src/models/regularisation.md b/docs/src/models/regularisation.md
index e1d88d77..02aa3da8 100644
--- a/docs/src/models/regularisation.md
+++ b/docs/src/models/regularisation.md
@@ -31,7 +31,7 @@ julia> params(m)
param([0.0, 0.0, 0.0, 0.0, 0.0])
julia> sum(norm, params(m))
-26.01749952921026 (tracked)
+26.01749952921026
```
Here's a larger example with a multi-layer perceptron.
@@ -52,7 +52,7 @@ One can also easily add per-layer regularisation via the `activations` function:
```julia
julia> using Flux: activations
-julia> c = Chain(Dense(10,5,σ),Dense(5,2),softmax)
+julia> c = Chain(Dense(10, 5, σ), Dense(5, 2), softmax)
Chain(Dense(10, 5, σ), Dense(5, 2), softmax)
julia> activations(c, rand(10))
diff --git a/docs/src/training/optimisers.md b/docs/src/training/optimisers.md
index 5e8b95de..37288b5d 100644
--- a/docs/src/training/optimisers.md
+++ b/docs/src/training/optimisers.md
@@ -21,7 +21,7 @@ grads = gradient(() -> loss(x, y), θ)
We want to update each parameter, using the gradient, in order to improve (reduce) the loss. Here's one way to do that:
```julia
-using Flux: update!
+using Flux.Optimise: update!
η = 0.1 # Learning Rate
for p in (W, b)
@@ -46,6 +46,7 @@ An optimiser `update!` accepts a parameter and a gradient, and updates the param
All optimisers return an object that, when passed to `train!`, will update the parameters passed to it.
```@docs
+Flux.Optimise.update!
Descent
Momentum
Nesterov
diff --git a/docs/src/training/training.md b/docs/src/training/training.md
index b42db7c9..64b2b5e8 100644
--- a/docs/src/training/training.md
+++ b/docs/src/training/training.md
@@ -7,10 +7,10 @@ To actually train a model we need four things:
* A collection of data points that will be provided to the objective function.
* An [optimiser](optimisers.md) that will update the model parameters appropriately.
-With these we can call `Flux.train!`:
+With these we can call `train!`:
-```julia
-Flux.train!(objective, params, data, opt)
+```@docs
+Flux.Optimise.train!
```
There are plenty of examples in the [model zoo](https://github.com/FluxML/model-zoo).
@@ -56,7 +56,8 @@ data = [(x, y)]
```julia
data = [(x, y), (x, y), (x, y)]
# Or equivalently
-data = Iterators.repeated((x, y), 3)
+using IterTools: ncycle
+data = ncycle([(x, y)], 3)
```
It's common to load the `x`s and `y`s separately. In this case you can use `zip`:
@@ -67,6 +68,14 @@ ys = [rand( 10), rand( 10), rand( 10)]
data = zip(xs, ys)
```
+Training data can be conveniently partitioned for mini-batch training using the [`Flux.Data.DataLoader`](@ref) type:
+
+```julia
+X = rand(28, 28, 60000)
+Y = rand(0:9, 60000)
+data = DataLoader(X, Y, batchsize=128)
+```
+
Note that, by default, `train!` only loops over the data once (a single "epoch").
A convenient way to run multiple epochs from the REPL is provided by `@epochs`.
@@ -120,7 +129,7 @@ An example follows that works similar to the default `Flux.train` but with no ca
You don't need callbacks if you just code the calls to your functions directly into the loop.
E.g. in the places marked with comments.
-```
+```julia
function my_custom_train!(loss, ps, data, opt)
ps = Params(ps)
for d in data
diff --git a/src/Flux.jl b/src/Flux.jl
index 9969b323..c99e41a1 100644
--- a/src/Flux.jl
+++ b/src/Flux.jl
@@ -7,6 +7,7 @@ using Zygote, MacroTools, Juno, Reexport, Statistics, Random
using MacroTools: @forward
@reexport using NNlib
using Zygote: Params, @adjoint, gradient, pullback, @nograd
+
export gradient
export Chain, Dense, Maxout, RNN, LSTM, GRU, Conv, CrossCor, ConvTranspose, MaxPool, MeanPool,
diff --git a/src/data/Data.jl b/src/data/Data.jl
index d7cd0303..16a025a7 100644
--- a/src/data/Data.jl
+++ b/src/data/Data.jl
@@ -3,6 +3,9 @@ module Data
import ..Flux
import SHA
+using Random: shuffle!
+using Base: @propagate_inbounds
+
export CMUDict, cmudict
deps(path...) = joinpath(@__DIR__, "..", "..", "deps", path...)
@@ -26,6 +29,9 @@ function __init__()
mkpath(deps())
end
+include("dataloader.jl")
+export DataLoader
+
include("mnist.jl")
export MNIST
@@ -42,4 +48,7 @@ using .Sentiment
include("iris.jl")
export Iris
+include("housing.jl")
+export Housing
+
end
diff --git a/src/data/dataloader.jl b/src/data/dataloader.jl
new file mode 100644
index 00000000..baf32a83
--- /dev/null
+++ b/src/data/dataloader.jl
@@ -0,0 +1,88 @@
+# Adapted from Knet's src/data.jl (author: Deniz Yuret)
+
+struct DataLoader
+ data
+ batchsize::Int
+ nobs::Int
+ partial::Bool
+ imax::Int
+ indices::Vector{Int}
+ shuffle::Bool
+end
+
+"""
+ DataLoader(data...; batchsize=1, shuffle=false, partial=true)
+
+An object that iterates over mini-batches of `data`, each mini-batch containing `batchsize` observations
+(except possibly the last one).
+
+Takes as input one or more data tensors, e.g. X in unsupervised learning, X and Y in
+supervised learning. The last dimension in each tensor is considered to be the observation
+dimension.
+
+If `shuffle=true`, shuffles the observations each time iterations are re-started.
+If `partial=false`, drops the last mini-batch if it is smaller than the batchsize.
+
+Example usage:
+
+ Xtrain = rand(10, 100)
+ dtrain = DataLoader(Xtrain, batchsize=2)
+ # iterate over 50 mini-batches
+ for x in dtrain:
+ @assert size(x) == (10, 2)
+ ...
+ end
+
+ Xtrain = rand(10, 100)
+ Ytrain = rand(100)
+ dtrain = DataLoader(Xtrain, Ytrain, batchsize=2, shuffle=true)
+ for epoch in 1:100
+ for (x, y) in dtrain:
+ @assert size(x) == (10, 2)
+ @assert size(y) == (2,)
+ ...
+ end
+ end
+
+ # train for 10 epochs
+ using IterTools: ncycle
+ Flux.train!(loss, ps, ncycle(dtrain, 10), opt)
+"""
+function DataLoader(data...; batchsize=1, shuffle=false, partial=true)
+ length(data) > 0 || throw(ArgumentError("Need at least one data input"))
+ batchsize > 0 || throw(ArgumentError("Need positive batchsize"))
+
+ nx = size(data[1])[end]
+ for i=2:length(data)
+ nx != size(data[i])[end] && throw(DimensionMismatch("All data should contain same number of observations"))
+ end
+ if nx < batchsize
+ @warn "Number of data points less than batchsize, decreasing the batchsize to $nx"
+ batchsize = nx
+ end
+ imax = partial ? nx : nx - batchsize + 1
+ ids = 1:min(nx, batchsize)
+ DataLoader(data, batchsize, nx, partial, imax, [1:nx;], shuffle)
+end
+
+getdata(x::AbstractArray, ids) = x[(Base.Colon() for _=1:ndims(x)-1)..., ids]
+
+@propagate_inbounds function Base.iterate(d::DataLoader, i=0) # returns data in d.indices[i+1:i+batchsize]
+ i >= d.imax && return nothing
+ if d.shuffle && i == 0
+ shuffle!(d.indices)
+ end
+ nexti = min(i + d.batchsize, d.nobs)
+ ids = d.indices[i+1:nexti]
+ if length(d.data) == 1
+ batch = getdata(d.data[1], ids)
+ else
+ batch = ((getdata(x, ids) for x in d.data)...,)
+ end
+ return (batch, nexti)
+end
+
+function Base.length(d::DataLoader)
+ n = d.nobs / d.batchsize
+ d.partial ? ceil(Int,n) : floor(Int,n)
+end
\ No newline at end of file
diff --git a/src/data/housing.jl b/src/data/housing.jl
new file mode 100644
index 00000000..61391304
--- /dev/null
+++ b/src/data/housing.jl
@@ -0,0 +1,136 @@
+"""
+1. Title: Boston Housing Data
+
+2. Sources:
+ (a) Origin: This dataset was taken from the StatLib library which is
+ maintained at Carnegie Mellon University.
+ (b) Creator: Harrison, D. and Rubinfeld, D.L. 'Hedonic prices and the
+ demand for clean air', J. Environ. Economics & Management,
+ vol.5, 81-102, 1978.
+ (c) Date: July 7, 1993
+
+3. Number of Instances: 506
+
+4. Number of Attributes: 13 continuous attributes (including "class"
+ attribute "MEDV"), 1 binary-valued attribute.
+
+5. Attribute Information:
+
+ 1. CRIM per capita crime rate by town
+ 2. ZN proportion of residential land zoned for lots over
+ 25,000 sq.ft.
+ 3. INDUS proportion of non-retail business acres per town
+ 4. CHAS Charles River dummy variable (= 1 if tract bounds
+ river; 0 otherwise)
+ 5. NOX nitric oxides concentration (parts per 10 million)
+ 6. RM average number of rooms per dwelling
+ 7. AGE proportion of owner-occupied units built prior to 1940
+ 8. DIS weighted distances to five Boston employment centres
+ 9. RAD index of accessibility to radial highways
+ 10. TAX full-value property-tax rate per 10,000 dollars
+ 11. PTRATIO pupil-teacher ratio by town
+ 12. B 1000(Bk - 0.63)^2 where Bk is the proportion of blacks
+ by town
+ 13. LSTAT % lower status of the population
+ 14. MEDV Median value of owner-occupied homes in 1000's of dollars
+
+ Downloaded From: https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data
+
+"""
+module Housing
+
+using DelimitedFiles
+using ..Data: deps, download_and_verify
+
+#Uncomment if package exists
+#const cache_prefix = "https://cache.julialang.org/"
+const cache_prefix = ""
+
+function load()
+ isfile(deps("housing.data")) && return
+
+ @info "Downloading the Boston housing Dataset"
+ download_and_verify("$(cache_prefix)https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data",
+ deps("housing.data"),
+ "baadf72995725d76efe787b664e1f083388c79ba21ef9a7990d87f774184735a")
+
+ #@info "Download complete. Working on the files"
+ path = deps()
+ isfile(deps("housing.data")) && touch(joinpath(path, "tempfile.data"))
+ open(joinpath(path, "tempfile.data"), "a") do fout
+ open(deps("housing.data"), "r") do fin
+ for line in eachline(fin)
+ line = replace(lstrip(line), r" +" => s",")
+ println(fout, line)
+ end
+ end
+ end
+ mv(joinpath(path, "tempfile.data"), deps("housing.data"), force=true)
+end
+
+"""
+Gets the targets for the Boston housing dataset, a 506 element array listing the targets for each example
+
+```jldoctest
+julia> using Flux
+
+julia> target = Flux.Data.Housing.targets()
+
+julia> summary(target)
+506×1 Array{Float64,2}
+
+julia> target[1]
+24.0
+
+"""
+function targets()
+ load()
+ housing = readdlm(deps("housing.data"), ',')
+ reshape(Vector{Float64}(housing[1:end,end]), (506, 1))
+end
+
+
+"""
+Gets the names of the features provided in the dataset
+
+"""
+function feature_names()
+ ["crim","zn","indus","chas","nox","rm","age","dis","rad","tax","ptratio","b","lstat"]
+end
+
+
+"""
+Gets the features of the Boston Housing Dataset. This is a 506x13 Matrix of Float64 datatypes.
+The values are in the order ["crim","zn","indus","chas","nox","rm","age","dis","rad","tax","ptratio","b","lstat"].
+It has 506 examples.
+
+```jldoctest
+julia> using Flux
+
+julia> features = Flux.Data.Housing.features()
+
+julia> summary(features)
+506×13 Array{Float64,2}
+
+julia> features[1, :]
+13-element Array{Float64,1}:
+0.00632
+18.0
+2.31
+0.0
+0.538
+ ⋮
+296.0
+15.3
+396.9
+4.98
+
+"""
+function features()
+ load()
+ housing = readdlm(deps("housing.data"), ',')
+ Matrix{Float64}(housing[1:end, 1:13])
+end
+
+
+end
diff --git a/src/layers/basic.jl b/src/layers/basic.jl
index 2a465208..6f056429 100644
--- a/src/layers/basic.jl
+++ b/src/layers/basic.jl
@@ -39,6 +39,17 @@ function Base.show(io::IO, c::Chain)
print(io, ")")
end
+"""
+ outdims(c::Chain, isize)
+
+Calculate the output dimensions given the input dimensions, `isize`.
+
+```julia
+m = Chain(Conv((3, 3), 3 => 16), Conv((3, 3), 16 => 32))
+outdims(m, (10, 10)) == (6, 6)
+```
+"""
+outdims(c::Chain, isize) = foldl(∘, map(l -> (x -> outdims(l, x)), c.layers))(isize)
# This is a temporary and naive implementation
# it might be replaced in the future for better performance
@@ -116,6 +127,19 @@ end
(a::Dense{<:Any,W})(x::AbstractArray{<:AbstractFloat}) where {T <: Union{Float32,Float64}, W <: AbstractArray{T}} =
a(T.(x))
+"""
+ outdims(l::Dense, isize)
+
+Calculate the output dimensions given the input dimensions, `isize`.
+
+```julia
+m = Dense(10, 5)
+outdims(m, (5, 2)) == (5,)
+outdims(m, (10,)) == (5,)
+```
+"""
+outdims(l::Dense, isize) = (size(l.W)[1],)
+
"""
Diagonal(in::Integer)
@@ -145,6 +169,7 @@ function Base.show(io::IO, l::Diagonal)
print(io, "Diagonal(", length(l.α), ")")
end
+outdims(l::Diagonal, isize) = (length(l.α),)
"""
Maxout(over)
@@ -193,6 +218,8 @@ function (mo::Maxout)(input::AbstractArray)
mapreduce(f -> f(input), (acc, out) -> max.(acc, out), mo.over)
end
+outdims(l::Maxout, isize) = outdims(first(l.over), isize)
+
"""
SkipConnection(layers, connection)
diff --git a/src/layers/conv.jl b/src/layers/conv.jl
index f4de3ffc..ef167f71 100644
--- a/src/layers/conv.jl
+++ b/src/layers/conv.jl
@@ -1,4 +1,9 @@
-using NNlib: conv, ∇conv_data, depthwiseconv
+using NNlib: conv, ∇conv_data, depthwiseconv, output_size
+
+# pad dims of x with dims of y until ndims(x) == ndims(y)
+_paddims(x::Tuple, y::Tuple) = (x..., y[(end - (length(y) - length(x) - 1)):end]...)
+
+_convtransoutdims(isize, ksize, ssize, dsize, pad) = (isize .- 1).*ssize .+ 1 .+ (ksize .- 1).*dsize .- (pad[1:2:end] .+ pad[2:2:end])
expand(N, i::Tuple) = i
expand(N, i::Integer) = ntuple(_ -> i, N)
@@ -17,7 +22,7 @@ Example: Applying Conv layer to a 1-channel input using a 2x2 window size,
out = 16
Conv((2, 2), 1=>16, relu)
-Data should be stored in WHCN order (width, height, # channels, # batches).
+Data should be stored in WHCN order (width, height, # channels, batch size).
In other words, a 100×100 RGB image would be a `100×100×3×1` array,
and a batch of 50 would be a `100×100×3×50` array.
@@ -68,6 +73,21 @@ end
(a::Conv{<:Any,<:Any,W})(x::AbstractArray{<:Real}) where {T <: Union{Float32,Float64}, W <: AbstractArray{T}} =
a(T.(x))
+"""
+ outdims(l::Conv, isize::Tuple)
+
+Calculate the output dimensions given the input dimensions, `isize`.
+Batch size and channel size are ignored as per `NNlib.jl`.
+
+```julia
+m = Conv((3, 3), 3 => 16)
+outdims(m, (10, 10)) == (8, 8)
+outdims(m, (10, 10, 1, 3)) == (8, 8)
+```
+"""
+outdims(l::Conv, isize) =
+ output_size(DenseConvDims(_paddims(isize, size(l.weight)), size(l.weight); stride = l.stride, padding = l.pad, dilation = l.dilation))
+
"""
ConvTranspose(size, in=>out)
ConvTranspose(size, in=>out, relu)
@@ -140,6 +160,9 @@ end
(a::ConvTranspose{<:Any,<:Any,W})(x::AbstractArray{<:Real}) where {T <: Union{Float32,Float64}, W <: AbstractArray{T}} =
a(T.(x))
+
+outdims(l::ConvTranspose{N}, isize) where N = _convtransoutdims(isize[1:2], size(l.weight)[1:N], l.stride, l.dilation, l.pad)
+
"""
DepthwiseConv(size, in=>out)
DepthwiseConv(size, in=>out, relu)
@@ -204,6 +227,9 @@ end
(a::DepthwiseConv{<:Any,<:Any,W})(x::AbstractArray{<:Real}) where {T <: Union{Float32,Float64}, W <: AbstractArray{T}} =
a(T.(x))
+outdims(l::DepthwiseConv, isize) =
+ output_size(DepthwiseConvDims(_paddims(isize, (1, 1, size(l.weight)[end], 1)), size(l.weight); stride = l.stride, padding = l.pad, dilation = l.dilation))
+
"""
CrossCor(size, in=>out)
CrossCor(size, in=>out, relu)
@@ -275,6 +301,9 @@ end
(a::CrossCor{<:Any,<:Any,W})(x::AbstractArray{<:Real}) where {T <: Union{Float32,Float64}, W <: AbstractArray{T}} =
a(T.(x))
+outdims(l::CrossCor, isize) =
+ output_size(DenseConvDims(_paddims(isize, size(l.weight)), size(l.weight); stride = l.stride, padding = l.pad, dilation = l.dilation))
+
"""
MaxPool(k)
@@ -304,6 +333,8 @@ function Base.show(io::IO, m::MaxPool)
print(io, "MaxPool(", m.k, ", pad = ", m.pad, ", stride = ", m.stride, ")")
end
+outdims(l::MaxPool{N}, isize) where N = output_size(PoolDims(_paddims(isize, (l.k..., 1, 1)), l.k; stride = l.stride, padding = l.pad))
+
"""
MeanPool(k)
@@ -331,3 +362,5 @@ end
function Base.show(io::IO, m::MeanPool)
print(io, "MeanPool(", m.k, ", pad = ", m.pad, ", stride = ", m.stride, ")")
end
+
+outdims(l::MeanPool{N}, isize) where N = output_size(PoolDims(_paddims(isize, (l.k..., 1, 1)), l.k; stride = l.stride, padding = l.pad))
\ No newline at end of file
diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl
index b421d3e7..2268fdc0 100644
--- a/src/layers/normalise.jl
+++ b/src/layers/normalise.jl
@@ -7,6 +7,16 @@ _dropout_shape(s, dims) = tuple((i ∉ dims ? 1 : si for (i, si) ∈ enumerate(s
_dropout_kernel(y::T, p, q) where {T} = y > p ? T(1 / q) : T(0)
+"""
+ dropout(p, dims = :)
+
+Dropout function. For each input, either sets that input to `0` (with probability
+`p`) or scales it by `1/(1-p)`. The `dims` argument is to specify the unbroadcasted
+dimensions, i.e. `dims=1` does dropout along columns and `dims=2` along rows. This is
+used as a regularisation, i.e. it reduces overfitting during training.
+
+See also [`Dropout`](@ref).
+"""
dropout(x, p; dims = :) = x
@adjoint function dropout(x, p; dims = :)
@@ -18,10 +28,7 @@ end
"""
Dropout(p, dims = :)
-A Dropout layer. For each input, either sets that input to `0` (with probability
-`p`) or scales it by `1/(1-p)`. The `dims` argument is to specified the unbroadcasted
- dimensions, i.e. `dims=1` does dropout along columns and `dims=2` along rows. This is
- used as a regularisation, i.e. it reduces overfitting during training. see also [`dropout`](@ref).
+A Dropout layer. In the forward pass, applies the [`dropout`](@ref) function on the input.
"""
mutable struct Dropout{F,D}
p::F
@@ -43,6 +50,7 @@ end
"""
AlphaDropout(p)
+
A dropout layer. It is used in Self-Normalizing Neural Networks.
(https://papers.nips.cc/paper/6698-self-normalizing-neural-networks.pdf)
The AlphaDropout layer ensures that mean and variance of activations remains the same as before.
diff --git a/src/layers/recurrent.jl b/src/layers/recurrent.jl
index 499a21ab..647dda25 100644
--- a/src/layers/recurrent.jl
+++ b/src/layers/recurrent.jl
@@ -45,8 +45,7 @@ Base.show(io::IO, m::Recur) = print(io, "Recur(", m.cell, ")")
"""
reset!(rnn)
-Reset the hidden state of a recurrent layer back to its original value. See also
-`truncate!`.
+Reset the hidden state of a recurrent layer back to its original value.
Assuming you have a `Recur` layer `rnn`, this is roughly equivalent to
diff --git a/src/layers/stateless.jl b/src/layers/stateless.jl
index f05f19fc..9b6db037 100644
--- a/src/layers/stateless.jl
+++ b/src/layers/stateless.jl
@@ -1,17 +1,16 @@
-using CuArrays
-using NNlib: logsoftmax, logσ
-
# Cost functions
"""
mae(ŷ, y)
-L1 loss function. Computes the mean of absolute error between prediction and true values
+
+Return the mean of absolute error `sum(abs.(ŷ .- y)) * 1 / length(y)`
"""
mae(ŷ, y) = sum(abs.(ŷ .- y)) * 1 // length(y)
"""
mse(ŷ, y)
-L2 loss function. Computes the mean of the squared errors between prediction and true values
+
+Return the mean squared error `sum((ŷ .- y).^2) / length(y)`.
"""
mse(ŷ, y) = sum((ŷ .- y).^2) * 1 // length(y)
@@ -19,25 +18,8 @@ mse(ŷ, y) = sum((ŷ .- y).^2) * 1 // length(y)
"""
msle(ŷ, y;ϵ1=eps.(Float64.(ŷ)),ϵ2=eps.(Float64.(y)))
-Mean Squared Logarithmic Error,an L2 loss function. Returns the mean of the squared logarithmic errors of prediction ŷ, and true values y. The ϵ1 and ϵ2 terms provide numerical stability.
-(Computes mean of squared(log(predicted values)-log(true value)). This error penalizes an under-predicted estimate greater than an over-predicted estimate.
-
- ```julia
- julia> y=[14726,327378,74734]
- 3-element Array{Int64,1}:
- 14726
- 327378
- 74734
-
- julia> ŷ = [12466.1,16353.95,16367.98]
- 3-element Array{Float64,1}:
- 12466.1
- 16353.95
- 16367.98
-
- julia> msle(ŷ,y)
- 3.771271382334686
- ```
+Mean Squared Logarithmic Error. Returns the mean of the squared logarithmic errors `sum((log.(ŷ+ϵ1).-log.(y+ϵ2)).^2) * 1 / length(y)`.
+The ϵ1 and ϵ2 terms provide numerical stability. This error penalizes an under-predicted estimate greater than an over-predicted estimate.
"""
msle(ŷ, y;ϵ1=eps.(ŷ),ϵ2=eps.(eltype(ŷ).(y))) = sum((log.(ŷ+ϵ1).-log.(y+ϵ2)).^2) * 1 // length(y)
@@ -46,26 +28,12 @@ msle(ŷ, y;ϵ1=eps.(ŷ),ϵ2=eps.(eltype(ŷ).(y))) = sum((log.(ŷ+ϵ1).-log.(
"""
huber_loss(ŷ, y,delta=1.0)
-Computes the mean of the Huber loss between prediction ŷ and true values y. By default, delta is set to 1.0.
-[Huber Loss](https://en.wikipedia.org/wiki/Huber_loss).
-
- ```julia
- julia> y = [1.2636,1.25,1.73]
- 3-element Array{Float64,1}:
- 1.2636
- 1.25
- 1.73
-
- julia> y_= [-1.376,0,3.37]
- 3-element Array{Float64,1}:
- -1.376
- 0.0
- 3.37
-
- julia> huber_loss(y,y_)
- 0.7131999999999998
- ```
+Computes the mean of the Huber loss. By default, delta is set to 1.0.
+ | 0.5*|(ŷ-y)|, for |ŷ-y| ŷ = [0.33 .11 .98;0.11 0.34 0.11]
- 2×3 Array{Float64,2}:
- 0.33 0.11 0.98
- 0.11 0.34 0.11
-
- julia> y = [1 0 0;0 1 0]
- 2×3 Array{Int64,2}:
- 1 0 0
- 0 1 0
-
- julia> crossentropy(ŷ,y)
- 0.7291574286311803
- ```
-
-Note: If only two classes are there, better use binarycrossentropy(ŷ, y) function.
+See also [`logitcrossentropy`](@ref), [`binarycrossentropy`](@ref).
"""
crossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight=nothing) = _crossentropy(ŷ, y, weight)
+"""
+ logitcrossentropy(ŷ, y; weight=1)
-function logitcrossentropy(logŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight = 1)
- return -sum(y .* logsoftmax(logŷ) .* weight) * 1 // size(y, 2)
+Return the crossentropy computed after a [softmax](@ref) operation:
+
+ -sum(y .* logsoftmax(ŷ) .* weight) / size(y, 2)
+
+See also [`crossentropy`](@ref), [`binarycrossentropy`](@ref).
+"""
+function logitcrossentropy(ŷ::AbstractVecOrMat, y::AbstractVecOrMat; weight = 1)
+ return -sum(y .* logsoftmax(ŷ) .* weight) * 1 // size(y, 2)
end
"""
@@ -131,11 +89,7 @@ end
Return `-y*log(ŷ + ϵ) - (1-y)*log(1-ŷ + ϵ)`. The ϵ term provides numerical stability.
- julia> binarycrossentropy.(σ.([-1.1491, 0.8619, 0.3127]), [1, 1, 0.])
- 3-element Array{Float64,1}:
- 1.4244
- 0.352317
- 0.86167
+Typically, the prediction `ŷ` is given by the output of a [`sigmoid`](@ref) activation.
"""
binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -y*log(ŷ + ϵ) - (1 - y)*log(1 - ŷ + ϵ)
@@ -143,44 +97,42 @@ binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -y*log(ŷ + ϵ) - (1 - y)*log(1 - ŷ
CuArrays.@cufunc binarycrossentropy(ŷ, y; ϵ=eps(ŷ)) = -y*log(ŷ + ϵ) - (1 - y)*log(1 - ŷ + ϵ)
"""
- logitbinarycrossentropy(logŷ, y)
+ logitbinarycrossentropy(ŷ, y)
-`logitbinarycrossentropy(logŷ, y)` is mathematically equivalent to `binarycrossentropy(σ(logŷ), y)`
+`logitbinarycrossentropy(ŷ, y)` is mathematically equivalent to `binarycrossentropy(σ(ŷ), y)`
but it is more numerically stable.
- julia> logitbinarycrossentropy.([-1.1491, 0.8619, 0.3127], [1, 1, 0.])
- 3-element Array{Float64,1}:
- 1.4244
- 0.352317
- 0.86167
+See also [`binarycrossentropy`](@ref), [`sigmoid`](@ref), [`logsigmoid`](@ref).
"""
-logitbinarycrossentropy(logŷ, y) = (1 - y)*logŷ - logσ(logŷ)
+logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)
# Re-definition to fix interaction with CuArrays.
-CuArrays.@cufunc logitbinarycrossentropy(logŷ, y) = (1 - y)*logŷ - logσ(logŷ)
+CuArrays.@cufunc logitbinarycrossentropy(ŷ, y) = (1 - y)*ŷ - logσ(ŷ)
"""
- normalise(x::AbstractArray; dims=1)
+ normalise(x; dims=1)
Normalises `x` to mean 0 and standard deviation 1, across the dimensions given by `dims`. Defaults to normalising over columns.
- julia> a = reshape(collect(1:9), 3, 3)
- 3×3 Array{Int64,2}:
- 1 4 7
- 2 5 8
- 3 6 9
+```julia-repl
+julia> a = reshape(collect(1:9), 3, 3)
+3×3 Array{Int64,2}:
+ 1 4 7
+ 2 5 8
+ 3 6 9
- julia> normalise(a)
- 3×3 Array{Float64,2}:
- -1.22474 -1.22474 -1.22474
- 0.0 0.0 0.0
- 1.22474 1.22474 1.22474
+julia> normalise(a)
+3×3 Array{Float64,2}:
+ -1.22474 -1.22474 -1.22474
+ 0.0 0.0 0.0
+ 1.22474 1.22474 1.22474
- julia> normalise(a, dims=2)
- 3×3 Array{Float64,2}:
- -1.22474 0.0 1.22474
- -1.22474 0.0 1.22474
- -1.22474 0.0 1.22474
+julia> normalise(a, dims=2)
+3×3 Array{Float64,2}:
+ -1.22474 0.0 1.22474
+ -1.22474 0.0 1.22474
+ -1.22474 0.0 1.22474
+```
"""
function normalise(x::AbstractArray; dims=1)
μ′ = mean(x, dims = dims)
@@ -190,8 +142,10 @@ end
"""
kldivergence(ŷ, y)
+
KLDivergence is a measure of how much one probability distribution is different from the other.
It is always non-negative and zero only when both the distributions are equal everywhere.
+
[KL Divergence](https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence).
"""
function kldivergence(ŷ, y)
@@ -202,7 +156,9 @@ end
"""
poisson(ŷ, y)
+
Poisson loss function is a measure of how the predicted distribution diverges from the expected distribution.
+
[Poisson Loss](https://peltarion.com/knowledge-center/documentation/modeling-view/build-an-ai-model/loss-functions/poisson).
"""
poisson(ŷ, y) = sum(ŷ .- y .* log.(ŷ)) *1 // size(y,2)
@@ -210,15 +166,19 @@ poisson(ŷ, y) = sum(ŷ .- y .* log.(ŷ)) *1 // size(y,2)
"""
hinge(ŷ, y)
-L1 loss function. Measures the loss given the prediction ŷ and true labels y(containing 1 or -1).
-[Hinge Loss](https://en.wikipedia.org/wiki/Hinge_loss).
+Measures the loss given the prediction `ŷ` and true labels `y` (containing 1 or -1).
+
+[Hinge Loss](https://en.wikipedia.org/wiki/Hinge_loss)
+See also [`squared_hinge`](@ref)
"""
hinge(ŷ, y) = sum(max.(0, 1 .- ŷ .* y)) *1 // size(y,2)
"""
squared_hinge(ŷ, y)
-L2 loss function. Computes squared hinge loss over the prediction ŷ and true labels y(conatining 1 or -1)
+Computes squared hinge loss given the prediction `ŷ` and true labels `y` (conatining 1 or -1)
+
+See also [`hinge`](@ref)
"""
squared_hinge(ŷ, y) = sum((max.(0,1 .-ŷ .* y)).^2) *1//size(y,2)
@@ -241,6 +201,7 @@ end
Used with imbalanced data to give more weightage to False negatives. Larger β weigh recall higher than precision (by placing more emphasis on false negatives)
tversky_loss(ŷ,y,beta) = 1 - sum(|y.*ŷ| + 1) / (sum(y.*ŷ + beta*(1 .- y).*ŷ + (1 .- beta)*y.*(1 .- ŷ))+ 1)
+
Ref: [Tversky loss function for image segmentation using 3D fully convolutional deep networks](https://arxiv.org/pdf/1706.05721.pdf)
"""
function tversky_loss(y_pred,y_true,beta = eltype(y_pred)(0.7))
diff --git a/src/onehot.jl b/src/onehot.jl
index 754d0607..7a3123ec 100644
--- a/src/onehot.jl
+++ b/src/onehot.jl
@@ -125,6 +125,4 @@ onecold(y::AbstractMatrix, labels...) =
onecold(y::OneHotMatrix, labels...) =
mapreduce(x -> Flux.onecold(x, labels...), |, y.data, dims = 2, init = 0)
-# TODO probably still want this as a custom adjoint Zygote
-# onecold(x::TrackedVector, l...) = onecold(data(x), l...)
-# onecold(x::TrackedMatrix, l...) = onecold(data(x), l...)
+@nograd onecold, onehot, onehotbatch
diff --git a/src/optimise/Optimise.jl b/src/optimise/Optimise.jl
index 68c18a6f..28a1849d 100644
--- a/src/optimise/Optimise.jl
+++ b/src/optimise/Optimise.jl
@@ -1,6 +1,6 @@
module Optimise
-export train!,
+export train!, update!,
SGD, Descent, ADAM, Momentum, Nesterov, RMSProp,
ADAGrad, AdaMax, ADADelta, AMSGrad, NADAM, ADAMW,RADAM,
InvDecay, ExpDecay, WeightDecay, stop, Optimiser
diff --git a/src/optimise/optimisers.jl b/src/optimise/optimisers.jl
index fb3b9fc5..cf4496f4 100644
--- a/src/optimise/optimisers.jl
+++ b/src/optimise/optimisers.jl
@@ -1,5 +1,4 @@
using Flux
-using Base: @get!
using MacroTools: @forward
const ϵ = 1e-8
diff --git a/src/optimise/train.jl b/src/optimise/train.jl
index ae0f334c..79ebcc06 100644
--- a/src/optimise/train.jl
+++ b/src/optimise/train.jl
@@ -1,9 +1,22 @@
using Juno
import Zygote: Params, gradient
+
+"""
+ update!(opt, p, g)
+ update!(opt, ps::Params, gs)
+
+Perform an update step of the parameters `ps` (or the single parameter `p`)
+according to optimizer `opt` and the gradients `gs` (the gradient `g`).
+
+As a result, the parameters are mutated and the optimizer's internal state may change.
+
+ update!(x, x̄)
+
+Update the array `x` according to `x .-= x̄`.
+"""
function update!(x::AbstractArray, x̄)
- x .+= x̄
- return x
+ x .-= x̄
end
function update!(opt, x, x̄)
@@ -48,13 +61,14 @@ end
For each datapoint `d` in `data` computes the gradient of `loss(d...)` through
backpropagation and calls the optimizer `opt`.
+In case datapoints `d` are of numeric array type, assumes no splatting is needed
+and computes the gradient of `loss(d)`.
+
Takes a callback as keyword argument `cb`. For example, this will print "training"
every 10 seconds:
-```julia
-Flux.train!(loss, params, data, opt,
- cb = throttle(() -> println("training"), 10))
-```
+ train!(loss, params, data, opt,
+ cb = throttle(() -> println("training"), 10))
The callback can call `Flux.stop()` to interrupt the training loop.
@@ -65,8 +79,14 @@ function train!(loss, ps, data, opt; cb = () -> ())
cb = runall(cb)
@progress for d in data
try
- gs = gradient(ps) do
- loss(d...)
+ if d isa AbstractArray{<:Number}
+ gs = gradient(ps) do
+ loss(d)
+ end
+ else
+ gs = gradient(ps) do
+ loss(d...)
+ end
end
update!(opt, ps, gs)
cb()
diff --git a/test/cuda/cuda.jl b/test/cuda/cuda.jl
index c75cfb4e..128e5c7d 100644
--- a/test/cuda/cuda.jl
+++ b/test/cuda/cuda.jl
@@ -58,6 +58,13 @@ end
@test y[3,:] isa CuArray
end
+@testset "restructure gpu" begin
+ dudt = Dense(1,1) |> gpu
+ p,re = Flux.destructure(dudt)
+ foo(x) = sum(re(p)(x))
+ @test gradient(foo, cu(rand(1)))[1] isa CuArray
+end
+
if CuArrays.has_cudnn()
@info "Testing Flux/CUDNN"
include("cudnn.jl")
diff --git a/test/data.jl b/test/data.jl
index 6b777873..c7a8fdfd 100644
--- a/test/data.jl
+++ b/test/data.jl
@@ -1,22 +1,85 @@
-using Flux.Data
-using Test
+@testset "DataLoader" begin
+ X = reshape([1:10;], (2, 5))
+ Y = [1:5;]
-@test cmudict()["CATASTROPHE"] == :[K,AH0,T,AE1,S,T,R,AH0,F,IY0].args
+ d = DataLoader(X, batchsize=2)
+ batches = collect(d)
+ @test length(batches) == 3
+ @test batches[1] == X[:,1:2]
+ @test batches[2] == X[:,3:4]
+ @test batches[3] == X[:,5:5]
-@test length(CMUDict.phones()) == 39
+ d = DataLoader(X, batchsize=2, partial=false)
+ batches = collect(d)
+ @test length(batches) == 2
+ @test batches[1] == X[:,1:2]
+ @test batches[2] == X[:,3:4]
-@test length(CMUDict.symbols()) == 84
+ d = DataLoader(X, Y, batchsize=2)
+ batches = collect(d)
+ @test length(batches) == 3
+ @test length(batches[1]) == 2
+ @test length(batches[2]) == 2
+ @test length(batches[3]) == 2
+ @test batches[1][1] == X[:,1:2]
+ @test batches[1][2] == Y[1:2]
+ @test batches[2][1] == X[:,3:4]
+ @test batches[2][2] == Y[3:4]
+ @test batches[3][1] == X[:,5:5]
+ @test batches[3][2] == Y[5:5]
-@test MNIST.images()[1] isa Matrix
-@test MNIST.labels() isa Vector{Int64}
+ # test interaction with `train!`
+ θ = ones(2)
+ X = zeros(2, 10)
+ loss(x) = sum((x .- θ).^2)
+ d = DataLoader(X)
+ Flux.train!(loss, [θ], ncycle(d, 10), Descent(0.1))
+ @test norm(θ) < 1e-4
-@test FashionMNIST.images()[1] isa Matrix
-@test FashionMNIST.labels() isa Vector{Int64}
+ # test interaction with `train!`
+ θ = zeros(2)
+ X = ones(2, 10)
+ Y = fill(2, 10)
+ loss(x, y) = sum((y - x'*θ).^2)
+ d = DataLoader(X, Y)
+ Flux.train!(loss, [θ], ncycle(d, 10), Descent(0.1))
+ @test norm(θ .- 1) < 1e-10
+end
-@test Data.Sentiment.train() isa Vector{Data.Tree{Any}}
+@testset "CMUDict" begin
+ @test cmudict()["CATASTROPHE"] == :[K,AH0,T,AE1,S,T,R,AH0,F,IY0].args
-@test Iris.features() isa Matrix
-@test size(Iris.features()) == (4,150)
+ @test length(CMUDict.phones()) == 39
-@test Iris.labels() isa Vector{String}
-@test size(Iris.labels()) == (150,)
+ @test length(CMUDict.symbols()) == 84
+end
+
+@testset "MNIST" begin
+ @test MNIST.images()[1] isa Matrix
+ @test MNIST.labels() isa Vector{Int64}
+end
+
+@testset "FashionMNIST" begin
+ @test FashionMNIST.images()[1] isa Matrix
+ @test FashionMNIST.labels() isa Vector{Int64}
+end
+
+@testset "Sentiment" begin
+ @test Data.Sentiment.train() isa Vector{Data.Tree{Any}}
+end
+
+@testset "Iris" begin
+ @test Iris.features() isa Matrix
+ @test size(Iris.features()) == (4,150)
+
+ @test Iris.labels() isa Vector{String}
+ @test size(Iris.labels()) == (150,)
+end
+
+@testset "Housing" begin
+ @test Housing.features() isa Matrix
+ @test size(Housing.features()) == (506, 13)
+
+ @test Housing.targets() isa Array{Float64}
+ @test size(Housing.targets()) == (506, 1)
+end
diff --git a/test/layers/basic.jl b/test/layers/basic.jl
index 0ff1776d..421c7721 100644
--- a/test/layers/basic.jl
+++ b/test/layers/basic.jl
@@ -92,4 +92,19 @@ import Flux: activations
@test size(SkipConnection(Dense(10,10), (a,b) -> cat(a, b, dims = 2))(input)) == (10,4)
end
end
+
+ @testset "output dimensions" begin
+ m = Chain(Conv((3, 3), 3 => 16), Conv((3, 3), 16 => 32))
+ @test Flux.outdims(m, (10, 10)) == (6, 6)
+
+ m = Dense(10, 5)
+ @test Flux.outdims(m, (5, 2)) == (5,)
+ @test Flux.outdims(m, (10,)) == (5,)
+
+ m = Flux.Diagonal(10)
+ @test Flux.outdims(m, (10,)) == (10,)
+
+ m = Maxout(() -> Conv((3, 3), 3 => 16), 2)
+ @test Flux.outdims(m, (10, 10)) == (8, 8)
+ end
end
diff --git a/test/layers/conv.jl b/test/layers/conv.jl
index 1e7ede01..03a0d1a4 100644
--- a/test/layers/conv.jl
+++ b/test/layers/conv.jl
@@ -107,3 +107,55 @@ end
true
end
end
+
+@testset "conv output dimensions" begin
+ m = Conv((3, 3), 3 => 16)
+ @test Flux.outdims(m, (10, 10)) == (8, 8)
+ m = Conv((3, 3), 3 => 16; stride = 2)
+ @test Flux.outdims(m, (5, 5)) == (2, 2)
+ m = Conv((3, 3), 3 => 16; stride = 2, pad = 3)
+ @test Flux.outdims(m, (5, 5)) == (5, 5)
+ m = Conv((3, 3), 3 => 16; stride = 2, pad = 3, dilation = 2)
+ @test Flux.outdims(m, (5, 5)) == (4, 4)
+
+ m = ConvTranspose((3, 3), 3 => 16)
+ @test Flux.outdims(m, (8, 8)) == (10, 10)
+ m = ConvTranspose((3, 3), 3 => 16; stride = 2)
+ @test Flux.outdims(m, (2, 2)) == (5, 5)
+ m = ConvTranspose((3, 3), 3 => 16; stride = 2, pad = 3)
+ @test Flux.outdims(m, (5, 5)) == (5, 5)
+ m = ConvTranspose((3, 3), 3 => 16; stride = 2, pad = 3, dilation = 2)
+ @test Flux.outdims(m, (4, 4)) == (5, 5)
+
+ m = DepthwiseConv((3, 3), 3 => 6)
+ @test Flux.outdims(m, (10, 10)) == (8, 8)
+ m = DepthwiseConv((3, 3), 3 => 6; stride = 2)
+ @test Flux.outdims(m, (5, 5)) == (2, 2)
+ m = DepthwiseConv((3, 3), 3 => 6; stride = 2, pad = 3)
+ @test Flux.outdims(m, (5, 5)) == (5, 5)
+ m = DepthwiseConv((3, 3), 3 => 6; stride = 2, pad = 3, dilation = 2)
+ @test Flux.outdims(m, (5, 5)) == (4, 4)
+
+ m = CrossCor((3, 3), 3 => 16)
+ @test Flux.outdims(m, (10, 10)) == (8, 8)
+ m = CrossCor((3, 3), 3 => 16; stride = 2)
+ @test Flux.outdims(m, (5, 5)) == (2, 2)
+ m = CrossCor((3, 3), 3 => 16; stride = 2, pad = 3)
+ @test Flux.outdims(m, (5, 5)) == (5, 5)
+ m = CrossCor((3, 3), 3 => 16; stride = 2, pad = 3, dilation = 2)
+ @test Flux.outdims(m, (5, 5)) == (4, 4)
+
+ m = MaxPool((2, 2))
+ @test Flux.outdims(m, (10, 10)) == (5, 5)
+ m = MaxPool((2, 2); stride = 1)
+ @test Flux.outdims(m, (5, 5)) == (4, 4)
+ m = MaxPool((2, 2); stride = 2, pad = 3)
+ @test Flux.outdims(m, (5, 5)) == (5, 5)
+
+ m = MeanPool((2, 2))
+ @test Flux.outdims(m, (10, 10)) == (5, 5)
+ m = MeanPool((2, 2); stride = 1)
+ @test Flux.outdims(m, (5, 5)) == (4, 4)
+ m = MeanPool((2, 2); stride = 2, pad = 3)
+ @test Flux.outdims(m, (5, 5)) == (5, 5)
+end
\ No newline at end of file
diff --git a/test/runtests.jl b/test/runtests.jl
index 1505e96a..81182f0d 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,32 +1,49 @@
-using Flux, Test, Random, Statistics, Documenter
-using Random
+using Flux
+using Flux.Data
+using Test
+using Random, Statistics, LinearAlgebra
+using Documenter
+using IterTools: ncycle
Random.seed!(0)
@testset "Flux" begin
-@info "Testing Basics"
+ @testset "Utils" begin
+ include("utils.jl")
+ end
-include("utils.jl")
-include("onehot.jl")
-include("optimise.jl")
-include("data.jl")
+ @testset "Onehot" begin
+ include("onehot.jl")
+ end
-@info "Testing Layers"
+ @testset "Optimise" begin
+ include("optimise.jl")
+ end
-include("layers/basic.jl")
-include("layers/normalisation.jl")
-include("layers/stateless.jl")
-include("layers/conv.jl")
+ @testset "Data" begin
+ include("data.jl")
+ end
-if Flux.use_cuda[]
- include("cuda/cuda.jl")
-else
- @warn "CUDA unavailable, not testing GPU support"
-end
+ @testset "Layers" begin
+ include("layers/basic.jl")
+ include("layers/normalisation.jl")
+ include("layers/stateless.jl")
+ include("layers/conv.jl")
+ end
-if VERSION >= v"1.2"
- doctest(Flux)
-end
+ @testset "CUDA" begin
+ if Flux.use_cuda[]
+ include("cuda/cuda.jl")
+ else
+ @warn "CUDA unavailable, not testing GPU support"
+ end
+ end
-end
+ @testset "Docs" begin
+ if VERSION >= v"1.2"
+ doctest(Flux)
+ end
+ end
+
+end # testset Flux