From 5b37319289dbd0b439d6b53fbbaebba77448b87b Mon Sep 17 00:00:00 2001 From: Yueh-Hua Tu Date: Wed, 1 Aug 2018 00:10:53 +0800 Subject: [PATCH 1/4] Add Maxpool and Meanpool --- docs/src/models/layers.md | 2 ++ src/layers/conv.jl | 42 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/docs/src/models/layers.md b/docs/src/models/layers.md index c2056bb4..070f6737 100644 --- a/docs/src/models/layers.md +++ b/docs/src/models/layers.md @@ -6,6 +6,8 @@ These core layers form the foundation of almost all neural networks. Chain Dense Conv +Maxpool +Meanpool ``` ## Recurrent Layers diff --git a/src/layers/conv.jl b/src/layers/conv.jl index 38310aad..f074e77f 100644 --- a/src/layers/conv.jl +++ b/src/layers/conv.jl @@ -50,3 +50,45 @@ function Base.show(io::IO, l::Conv) l.σ == identity || print(io, ", ", l.σ) print(io, ")") end + + +""" + Maxpool(k) + +Maxpooling layer. `k` stands for the size of the window for each dimension of the input. + +Takes the keyword arguments `pad` and `stride`. +""" +struct Maxpool{N} + k::NTuple{N,Int} + pad::NTuple{N,Int} + stride::NTuple{N,Int} + Maxpool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) +end + +(m::Maxpool)(x) = maxpool(x, m.k; pad = m.pad, stride = m.stride) + +function Base.show(io::IO, m::Maxpool) + print(io, "Maxpool(", m.k, ", ", m.pad, ", ", m.stride, ")") +end + + +""" + Meanpool(k) + +Meanpooling layer. `k` stands for the size of the window for each dimension of the input. + +Takes the keyword arguments `pad` and `stride`. +""" +struct Meanpool{N} + k::NTuple{N,Int} + pad::NTuple{N,Int} + stride::NTuple{N,Int} + Meanpool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) +end + +(m::Meanpool)(x) = meanpool(x, m.k; pad = m.pad, stride = m.stride) + +function Base.show(io::IO, m::Meanpool) + print(io, "Meanpool(", m.k, ", ", m.pad, ", ", m.stride, ")") +end From 634d34686ee2278f61ac62cf7e93d21cfdf6980c Mon Sep 17 00:00:00 2001 From: Yueh-Hua Tu Date: Fri, 24 Aug 2018 10:31:13 +0800 Subject: [PATCH 2/4] Add new constructors and test --- docs/src/models/layers.md | 4 ++-- src/layers/conv.jl | 36 +++++++++++++++++++++++------------- test/layers/conv.jl | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 15 deletions(-) create mode 100644 test/layers/conv.jl diff --git a/docs/src/models/layers.md b/docs/src/models/layers.md index 070f6737..4bbb2ba0 100644 --- a/docs/src/models/layers.md +++ b/docs/src/models/layers.md @@ -6,8 +6,8 @@ These core layers form the foundation of almost all neural networks. Chain Dense Conv -Maxpool -Meanpool +MaxPool +MeanPool ``` ## Recurrent Layers diff --git a/src/layers/conv.jl b/src/layers/conv.jl index f074e77f..0f9243ef 100644 --- a/src/layers/conv.jl +++ b/src/layers/conv.jl @@ -53,42 +53,52 @@ end """ - Maxpool(k) + MaxPool(k) Maxpooling layer. `k` stands for the size of the window for each dimension of the input. Takes the keyword arguments `pad` and `stride`. """ -struct Maxpool{N} +struct MaxPool{N} k::NTuple{N,Int} pad::NTuple{N,Int} stride::NTuple{N,Int} - Maxpool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) + MaxPool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) end -(m::Maxpool)(x) = maxpool(x, m.k; pad = m.pad, stride = m.stride) +function MaxPool{N}(k::Int; pad = 0, stride = k) where N + k_ = Tuple(repeat([k, ], N)) + MaxPool(k_; pad = map(_->pad,k_), stride=map(_->stride,k_)) +end -function Base.show(io::IO, m::Maxpool) - print(io, "Maxpool(", m.k, ", ", m.pad, ", ", m.stride, ")") +(m::MaxPool)(x) = maxpool(x, m.k; pad = m.pad, stride = m.stride) + +function Base.show(io::IO, m::MaxPool) + print(io, "MaxPool(", m.k, ", ", m.pad, ", ", m.stride, ")") end """ - Meanpool(k) + MeanPool(k) Meanpooling layer. `k` stands for the size of the window for each dimension of the input. Takes the keyword arguments `pad` and `stride`. """ -struct Meanpool{N} +struct MeanPool{N} k::NTuple{N,Int} pad::NTuple{N,Int} stride::NTuple{N,Int} - Meanpool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) + MeanPool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) end -(m::Meanpool)(x) = meanpool(x, m.k; pad = m.pad, stride = m.stride) - -function Base.show(io::IO, m::Meanpool) - print(io, "Meanpool(", m.k, ", ", m.pad, ", ", m.stride, ")") +function MeanPool{N}(k::Int; pad = 0, stride = k) where N + k_ = Tuple(repeat([k, ], N)) + MeanPool(k_; pad = map(_->pad,k_), stride=map(_->stride,k_)) +end + +(m::MeanPool)(x) = meanpool(x, m.k; pad = m.pad, stride = m.stride) + +function Base.show(io::IO, m::MeanPool) + print(io, "MeanPool(", m.k, ", ", m.pad, ", ", m.stride, ")") end diff --git a/test/layers/conv.jl b/test/layers/conv.jl new file mode 100644 index 00000000..2e5e63dd --- /dev/null +++ b/test/layers/conv.jl @@ -0,0 +1,34 @@ +using Test +using Flux: Chain, Conv, MaxPool, MeanPool +using Base.conv + +@testset "pooling" begin + mp = MaxPool((2, 2)) + + @testset "maxpooling" begin + @test MaxPool{2}(2) == mp + @test MaxPool{2}(2; pad=1, stride=3) == MaxPool((2, 2); pad=(1, 1), stride=(3, 3)) + end + + mp = MeanPool((2, 2)) + + @testset "meanpooling" begin + @test MeanPool{2}(2) == mp + @test MeanPool{2}(2; pad=1, stride=3) == MeanPool((2, 2); pad=(1, 1), stride=(3, 3)) + end +end + +@testset "cnn" begin + r = zeros(28, 28) + m = Chain( + Conv((2, 2), 1=>16, relu), + MaxPool{2}(2), + Conv((2, 2), 16=>8, relu), + MaxPool{2}(2), + x -> reshape(x, :, size(x, 4)), + Dense(288, 10), softmax) + + @testset "inference" begin + @test size(m(r)) == (10, ) + end +end From 1e0fd07b097f36ff1d70675256ed6e7c7ed66287 Mon Sep 17 00:00:00 2001 From: Mike J Innes Date: Tue, 4 Sep 2018 14:30:02 +0100 Subject: [PATCH 3/4] use `expand` --- src/Flux.jl | 2 +- src/layers/conv.jl | 27 ++++++++++----------------- test/layers/conv.jl | 16 ++++------------ 3 files changed, 15 insertions(+), 30 deletions(-) diff --git a/src/Flux.jl b/src/Flux.jl index 614eeaf7..8c959fec 100644 --- a/src/Flux.jl +++ b/src/Flux.jl @@ -5,7 +5,7 @@ module Flux using MacroTools, Juno, Requires, Reexport, Statistics, Random using MacroTools: @forward -export Chain, Dense, RNN, LSTM, GRU, Conv, +export Chain, Dense, RNN, LSTM, GRU, Conv, MaxPool, MeanPool, Dropout, LayerNorm, BatchNorm, params, mapleaves, cpu, gpu diff --git a/src/layers/conv.jl b/src/layers/conv.jl index 5b239751..dbf8ccf9 100644 --- a/src/layers/conv.jl +++ b/src/layers/conv.jl @@ -1,6 +1,6 @@ using NNlib: conv -@generated sub2(::Type{Val{N}}) where N = :(Val($(N-2))) +@generated sub2(::Val{N}) where N = :(Val($(N-2))) expand(N, i::Tuple) = i expand(N, i::Integer) = ntuple(_ -> i, N) @@ -28,7 +28,7 @@ end Conv(w::AbstractArray{T,N}, b::AbstractVector{T}, σ = identity; stride = 1, pad = 0, dilation = 1) where {T,N} = - Conv(σ, w, b, expand.(sub2(Val{N}), (stride, pad, dilation))...) + Conv(σ, w, b, expand.(sub2(Val(N)), (stride, pad, dilation))...) Conv(k::NTuple{N,Integer}, ch::Pair{<:Integer,<:Integer}, σ = identity; init = initn, stride = 1, pad = 0, dilation = 1) where N = @@ -55,7 +55,7 @@ end """ MaxPool(k) -Maxpooling layer. `k` stands for the size of the window for each dimension of the input. +Max pooling layer. `k` stands for the size of the window for each dimension of the input. Takes the keyword arguments `pad` and `stride`. """ @@ -63,25 +63,21 @@ struct MaxPool{N} k::NTuple{N,Int} pad::NTuple{N,Int} stride::NTuple{N,Int} - MaxPool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) end -function MaxPool{N}(k::Int; pad = 0, stride = k) where N - k_ = Tuple(repeat([k, ], N)) - MaxPool(k_; pad = map(_->pad,k_), stride=map(_->stride,k_)) -end +MaxPool(k::NTuple{N,Integer}; pad = 0, stride = k) where N = + MaxPool(k, expand(Val(N), pad), expand(Val(N), stride)) (m::MaxPool)(x) = maxpool(x, m.k; pad = m.pad, stride = m.stride) function Base.show(io::IO, m::MaxPool) - print(io, "MaxPool(", m.k, ", ", m.pad, ", ", m.stride, ")") + print(io, "MaxPool(", m.k, ", pad = ", m.pad, ", stride = ", m.stride, ")") end - """ MeanPool(k) -Meanpooling layer. `k` stands for the size of the window for each dimension of the input. +Mean pooling layer. `k` stands for the size of the window for each dimension of the input. Takes the keyword arguments `pad` and `stride`. """ @@ -89,16 +85,13 @@ struct MeanPool{N} k::NTuple{N,Int} pad::NTuple{N,Int} stride::NTuple{N,Int} - MeanPool(k::NTuple{N,Int}; pad = map(_->0,k), stride = k) where N = new{N}(k, pad, stride) end -function MeanPool{N}(k::Int; pad = 0, stride = k) where N - k_ = Tuple(repeat([k, ], N)) - MeanPool(k_; pad = map(_->pad,k_), stride=map(_->stride,k_)) -end +MeanPool(k::NTuple{N,Integer}; pad = 0, stride = k) where N = + MeanPool(k, expand(Val(N), pad), expand(Val(N), stride)) (m::MeanPool)(x) = meanpool(x, m.k; pad = m.pad, stride = m.stride) function Base.show(io::IO, m::MeanPool) - print(io, "MeanPool(", m.k, ", ", m.pad, ", ", m.stride, ")") + print(io, "MeanPool(", m.k, ", pad = ", m.pad, ", stride = ", m.stride, ")") end diff --git a/test/layers/conv.jl b/test/layers/conv.jl index 2e5e63dd..07b8c290 100644 --- a/test/layers/conv.jl +++ b/test/layers/conv.jl @@ -1,21 +1,13 @@ using Test -using Flux: Chain, Conv, MaxPool, MeanPool +using Flux: Chain, Conv, MaxPool, MeanPool, maxpool, meanpool using Base.conv @testset "pooling" begin + x = randn(10, 10, 3, 2) mp = MaxPool((2, 2)) - - @testset "maxpooling" begin - @test MaxPool{2}(2) == mp - @test MaxPool{2}(2; pad=1, stride=3) == MaxPool((2, 2); pad=(1, 1), stride=(3, 3)) - end - + @test mp(x) == maxpool(x, (2,2)) mp = MeanPool((2, 2)) - - @testset "meanpooling" begin - @test MeanPool{2}(2) == mp - @test MeanPool{2}(2; pad=1, stride=3) == MeanPool((2, 2); pad=(1, 1), stride=(3, 3)) - end + @test mp(x) == meanpool(x, (2,2)) end @testset "cnn" begin From 1e90226077457249af527f69d9fb6018f21dc2e4 Mon Sep 17 00:00:00 2001 From: Mike J Innes Date: Tue, 4 Sep 2018 14:35:20 +0100 Subject: [PATCH 4/4] actually run tests --- test/layers/conv.jl | 29 +++++++++++++---------------- test/runtests.jl | 1 + 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/test/layers/conv.jl b/test/layers/conv.jl index 07b8c290..5928bd75 100644 --- a/test/layers/conv.jl +++ b/test/layers/conv.jl @@ -1,8 +1,7 @@ -using Test -using Flux: Chain, Conv, MaxPool, MeanPool, maxpool, meanpool -using Base.conv +using Flux, Test +using Flux: maxpool, meanpool -@testset "pooling" begin +@testset "Pooling" begin x = randn(10, 10, 3, 2) mp = MaxPool((2, 2)) @test mp(x) == maxpool(x, (2,2)) @@ -10,17 +9,15 @@ using Base.conv @test mp(x) == meanpool(x, (2,2)) end -@testset "cnn" begin - r = zeros(28, 28) - m = Chain( - Conv((2, 2), 1=>16, relu), - MaxPool{2}(2), - Conv((2, 2), 16=>8, relu), - MaxPool{2}(2), - x -> reshape(x, :, size(x, 4)), - Dense(288, 10), softmax) +@testset "CNN" begin + r = zeros(28, 28, 1, 5) + m = Chain( + Conv((2, 2), 1=>16, relu), + MaxPool((2,2)), + Conv((2, 2), 16=>8, relu), + MaxPool((2,2)), + x -> reshape(x, :, size(x, 4)), + Dense(288, 10), softmax) - @testset "inference" begin - @test size(m(r)) == (10, ) - end + @test size(m(r)) == (10, 5) end diff --git a/test/runtests.jl b/test/runtests.jl index fd48e547..70d929bf 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,6 +28,7 @@ include("onehot.jl") include("tracker.jl") include("layers/normalisation.jl") include("layers/stateless.jl") +include("layers/conv.jl") include("optimise.jl") include("data.jl")