From ccfe0f872059bdbb324f4a059f5da2be2edf76b8 Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Thu, 28 Mar 2019 17:07:04 +0800 Subject: [PATCH 01/10] naive implementation of activations --- src/layers/basic.jl | 18 +++++++++++++++++- test/layers/basic.jl | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 005915bb..e4b6096c 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -40,7 +40,23 @@ function Base.show(io::IO, c::Chain) print(io, ")") end -activations(c::Chain, x) = accumulate((x, m) -> m(x), c.layers, init = x) + +""" + activations(c::Union{Chain, Any}, x) +Calculate the forward results of each layers in Chain `c` +""" +activations(m, x) = activations(Chain(m), x) +function activations(c::Chain, x) + rst = [c[1](x), ] + if length(c) == 1 + return rst + end + for l in c[2:end] + push!(rst, l(rst[end])) + end + return rst +end + """ Dense(in::Integer, out::Integer, σ = identity) diff --git a/test/layers/basic.jl b/test/layers/basic.jl index 3c5229f4..04cf2b2b 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -1,4 +1,18 @@ using Test, Random +import Flux: activations + +@testset "helpers" begin + @testset "activations" begin + dummy_model = Chain(Dense(10,5,σ),Dense(5,2),softmax) + x = rand(10) + @test_nowarn activations(dummy_model[1], x) + @test_nowarn activations(dummy_model[1:end-1], x) + @test_nowarn activations(dummy_model, x) + + @test activations(dummy_model, x)[1] == dummy_model[1](x) + @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] + end +end @testset "basic" begin @testset "Chain" begin From 5c2a0717137a1ea520c41bc25373ed2722ce8bde Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Thu, 28 Mar 2019 17:20:41 +0800 Subject: [PATCH 02/10] add support for 0-element Chain --- src/layers/basic.jl | 3 +++ test/layers/basic.jl | 1 + 2 files changed, 4 insertions(+) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index e4b6096c..ec7422be 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -47,6 +47,9 @@ Calculate the forward results of each layers in Chain `c` """ activations(m, x) = activations(Chain(m), x) function activations(c::Chain, x) + if length(c) == 0 + return [] + end rst = [c[1](x), ] if length(c) == 1 return rst diff --git a/test/layers/basic.jl b/test/layers/basic.jl index 04cf2b2b..d98ed58c 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -9,6 +9,7 @@ import Flux: activations @test_nowarn activations(dummy_model[1:end-1], x) @test_nowarn activations(dummy_model, x) + @test activations(Chain(), x)[1] == [] @test activations(dummy_model, x)[1] == dummy_model[1](x) @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] end From 47728b1899f0a87426473900e63419cc31c31e69 Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Thu, 28 Mar 2019 17:45:12 +0800 Subject: [PATCH 03/10] fix test case error --- test/layers/basic.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/layers/basic.jl b/test/layers/basic.jl index d98ed58c..c16f61bf 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -9,7 +9,7 @@ import Flux: activations @test_nowarn activations(dummy_model[1:end-1], x) @test_nowarn activations(dummy_model, x) - @test activations(Chain(), x)[1] == [] + @test activations(Chain(), x) == [] @test activations(dummy_model, x)[1] == dummy_model[1](x) @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] end From c4ebd199dbe9457d2e786346b98702ae8196fab8 Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Thu, 28 Mar 2019 17:58:02 +0800 Subject: [PATCH 04/10] move test cases to "basic" testset --- test/layers/basic.jl | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/layers/basic.jl b/test/layers/basic.jl index c16f61bf..6ff12d8c 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -1,21 +1,21 @@ using Test, Random import Flux: activations -@testset "helpers" begin - @testset "activations" begin - dummy_model = Chain(Dense(10,5,σ),Dense(5,2),softmax) - x = rand(10) - @test_nowarn activations(dummy_model[1], x) - @test_nowarn activations(dummy_model[1:end-1], x) - @test_nowarn activations(dummy_model, x) - - @test activations(Chain(), x) == [] - @test activations(dummy_model, x)[1] == dummy_model[1](x) - @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] - end -end - @testset "basic" begin + @testset "helpers" begin + @testset "activations" begin + dummy_model = Chain(Dense(10,5,σ),Dense(5,2),softmax) + x = rand(10) + @test_nowarn activations(dummy_model[1], x) + @test_nowarn activations(dummy_model[1:end-1], x) + @test_nowarn activations(dummy_model, x) + + @test activations(Chain(), x) == [] + @test activations(dummy_model, x)[1] == dummy_model[1](x) + @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] + end + end + @testset "Chain" begin @test_nowarn Chain(Dense(10, 5, σ), Dense(5, 2))(randn(10)) @test_throws DimensionMismatch Chain(Dense(10, 5, σ),Dense(2, 1))(randn(10)) From 13c58494ec024c311703573925ba6ff52957c79f Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Thu, 28 Mar 2019 19:28:59 +0800 Subject: [PATCH 05/10] add x into results --- src/layers/basic.jl | 12 ++++-------- test/layers/basic.jl | 12 ++++-------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index ec7422be..ba6b5402 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -42,18 +42,14 @@ end """ - activations(c::Union{Chain, Any}, x) + activations(c::Chain, x) Calculate the forward results of each layers in Chain `c` """ -activations(m, x) = activations(Chain(m), x) function activations(c::Chain, x) - if length(c) == 0 - return [] - end - rst = [c[1](x), ] - if length(c) == 1 - return rst + if isempty(c) + return [x, ] end + rst = [x, c[1](x)] for l in c[2:end] push!(rst, l(rst[end])) end diff --git a/test/layers/basic.jl b/test/layers/basic.jl index 6ff12d8c..cfe3b031 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -6,16 +6,12 @@ import Flux: activations @testset "activations" begin dummy_model = Chain(Dense(10,5,σ),Dense(5,2),softmax) x = rand(10) - @test_nowarn activations(dummy_model[1], x) - @test_nowarn activations(dummy_model[1:end-1], x) - @test_nowarn activations(dummy_model, x) - - @test activations(Chain(), x) == [] - @test activations(dummy_model, x)[1] == dummy_model[1](x) - @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] + @test activations(Chain(), x) == [x, ] + @test activations(dummy_model, x)[2] == dummy_model[1](x) + @test activations(dummy_model, x)[3] == x |> dummy_model[1] |> dummy_model[2] end end - + @testset "Chain" begin @test_nowarn Chain(Dense(10, 5, σ), Dense(5, 2))(randn(10)) @test_throws DimensionMismatch Chain(Dense(10, 5, σ),Dense(2, 1))(randn(10)) From 82595648e28278704f8e92be37f002d3b1c52320 Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Thu, 28 Mar 2019 22:40:24 +0800 Subject: [PATCH 06/10] change 4-spaces tab to 2-spaces tab --- src/layers/basic.jl | 16 +++--- test/layers/basic.jl | 124 +++++++++++++++++++++---------------------- 2 files changed, 70 insertions(+), 70 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index ba6b5402..8f2d0c2a 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -46,14 +46,14 @@ end Calculate the forward results of each layers in Chain `c` """ function activations(c::Chain, x) - if isempty(c) - return [x, ] - end - rst = [x, c[1](x)] - for l in c[2:end] - push!(rst, l(rst[end])) - end - return rst + if isempty(c) + return [x, ] + end + rst = [x, c[1](x)] + for l in c[2:end] + push!(rst, l(rst[end])) + end + return rst end diff --git a/test/layers/basic.jl b/test/layers/basic.jl index cfe3b031..17702c95 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -2,73 +2,73 @@ using Test, Random import Flux: activations @testset "basic" begin - @testset "helpers" begin - @testset "activations" begin - dummy_model = Chain(Dense(10,5,σ),Dense(5,2),softmax) - x = rand(10) - @test activations(Chain(), x) == [x, ] - @test activations(dummy_model, x)[2] == dummy_model[1](x) - @test activations(dummy_model, x)[3] == x |> dummy_model[1] |> dummy_model[2] - end + @testset "helpers" begin + @testset "activations" begin + dummy_model = Chain(Dense(10,5,σ),Dense(5,2),softmax) + x = rand(10) + @test activations(Chain(), x) == [x, ] + @test activations(dummy_model, x)[2] == dummy_model[1](x) + @test activations(dummy_model, x)[3] == x |> dummy_model[1] |> dummy_model[2] + end + end + + @testset "Chain" begin + @test_nowarn Chain(Dense(10, 5, σ), Dense(5, 2))(randn(10)) + @test_throws DimensionMismatch Chain(Dense(10, 5, σ),Dense(2, 1))(randn(10)) + # numeric test should be put into testset of corresponding layer + end + + @testset "Dense" begin + @test length(Dense(10, 5)(randn(10))) == 5 + @test_throws DimensionMismatch Dense(10, 5)(randn(1)) + @test_throws MethodError Dense(10, 5)(1) # avoid broadcasting + @test_throws MethodError Dense(10, 5).(randn(10)) # avoid broadcasting + + @test Dense(10, 1, identity, initW = ones, initb = zeros)(ones(10,1)) == 10*ones(1, 1) + @test Dense(10, 1, identity, initW = ones, initb = zeros)(ones(10,2)) == 10*ones(1, 2) + @test Dense(10, 2, identity, initW = ones, initb = zeros)(ones(10,1)) == 10*ones(2, 1) + @test Dense(10, 2, identity, initW = ones, initb = zeros)([ones(10,1) 2*ones(10,1)]) == [10 20; 10 20] + + end + + @testset "Diagonal" begin + @test length(Flux.Diagonal(10)(randn(10))) == 10 + @test length(Flux.Diagonal(10)(1)) == 10 + @test length(Flux.Diagonal(10)(randn(1))) == 10 + @test_throws DimensionMismatch Flux.Diagonal(10)(randn(2)) + + @test Flux.Diagonal(2)([1 2]) == [1 2; 1 2] + @test Flux.Diagonal(2)([1,2]) == [1,2] + @test Flux.Diagonal(2)([1 2; 3 4]) == [1 2; 3 4] + end + + @testset "Maxout" begin + # Note that the normal common usage of Maxout is as per the docstring + # These are abnormal constructors used for testing purposes + + @testset "Constructor" begin + mo = Maxout(() -> identity, 4) + input = rand(40) + @test mo(input) == input end - @testset "Chain" begin - @test_nowarn Chain(Dense(10, 5, σ), Dense(5, 2))(randn(10)) - @test_throws DimensionMismatch Chain(Dense(10, 5, σ),Dense(2, 1))(randn(10)) - # numeric test should be put into testset of corresponding layer + @testset "simple alternatives" begin + mo = Maxout((x -> x, x -> 2x, x -> 0.5x)) + input = rand(40) + @test mo(input) == 2*input end - @testset "Dense" begin - @test length(Dense(10, 5)(randn(10))) == 5 - @test_throws DimensionMismatch Dense(10, 5)(randn(1)) - @test_throws MethodError Dense(10, 5)(1) # avoid broadcasting - @test_throws MethodError Dense(10, 5).(randn(10)) # avoid broadcasting - - @test Dense(10, 1, identity, initW = ones, initb = zeros)(ones(10,1)) == 10*ones(1, 1) - @test Dense(10, 1, identity, initW = ones, initb = zeros)(ones(10,2)) == 10*ones(1, 2) - @test Dense(10, 2, identity, initW = ones, initb = zeros)(ones(10,1)) == 10*ones(2, 1) - @test Dense(10, 2, identity, initW = ones, initb = zeros)([ones(10,1) 2*ones(10,1)]) == [10 20; 10 20] - + @testset "complex alternatives" begin + mo = Maxout((x -> [0.5; 0.1]*x, x -> [0.2; 0.7]*x)) + input = [3.0 2.0] + target = [0.5, 0.7].*input + @test mo(input) == target end - @testset "Diagonal" begin - @test length(Flux.Diagonal(10)(randn(10))) == 10 - @test length(Flux.Diagonal(10)(1)) == 10 - @test length(Flux.Diagonal(10)(randn(1))) == 10 - @test_throws DimensionMismatch Flux.Diagonal(10)(randn(2)) - - @test Flux.Diagonal(2)([1 2]) == [1 2; 1 2] - @test Flux.Diagonal(2)([1,2]) == [1,2] - @test Flux.Diagonal(2)([1 2; 3 4]) == [1 2; 3 4] - end - - @testset "Maxout" begin - # Note that the normal common usage of Maxout is as per the docstring - # These are abnormal constructors used for testing purposes - - @testset "Constructor" begin - mo = Maxout(() -> identity, 4) - input = rand(40) - @test mo(input) == input - end - - @testset "simple alternatives" begin - mo = Maxout((x -> x, x -> 2x, x -> 0.5x)) - input = rand(40) - @test mo(input) == 2*input - end - - @testset "complex alternatives" begin - mo = Maxout((x -> [0.5; 0.1]*x, x -> [0.2; 0.7]*x)) - input = [3.0 2.0] - target = [0.5, 0.7].*input - @test mo(input) == target - end - - @testset "params" begin - mo = Maxout(()->Dense(32, 64), 4) - ps = params(mo) - @test length(ps) == 8 #4 alts, each with weight and bias - end + @testset "params" begin + mo = Maxout(()->Dense(32, 64), 4) + ps = params(mo) + @test length(ps) == 8 #4 alts, each with weight and bias end + end end From de7a5f402488e40cf45124806ec63615c5363de3 Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Fri, 5 Apr 2019 18:16:44 +0800 Subject: [PATCH 07/10] correct the function behavior; support Any type --- src/layers/basic.jl | 7 +++++-- test/layers/basic.jl | 8 +++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 8f2d0c2a..094428d9 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -41,15 +41,18 @@ function Base.show(io::IO, c::Chain) end +# This is a temporary and naive inplementation +# see issue https://github.com/FluxML/Flux.jl/issues/702 """ activations(c::Chain, x) Calculate the forward results of each layers in Chain `c` """ function activations(c::Chain, x) + rst = Array{Any,1}() if isempty(c) - return [x, ] + return rst end - rst = [x, c[1](x)] + push!(rst, c[1](x)) for l in c[2:end] push!(rst, l(rst[end])) end diff --git a/test/layers/basic.jl b/test/layers/basic.jl index 17702c95..ea54e153 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -6,9 +6,11 @@ import Flux: activations @testset "activations" begin dummy_model = Chain(Dense(10,5,σ),Dense(5,2),softmax) x = rand(10) - @test activations(Chain(), x) == [x, ] - @test activations(dummy_model, x)[2] == dummy_model[1](x) - @test activations(dummy_model, x)[3] == x |> dummy_model[1] |> dummy_model[2] + @test activations(Chain(), x) == [] + @test activations(dummy_model, x)[1] == dummy_model[1](x) + @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] + + @test_nowarn activations(Chain(identity, x->:foo), 1) # different types end end From 3cafbbad02b2cd99fed9f0ed7db7201729781779 Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Fri, 5 Apr 2019 18:44:00 +0800 Subject: [PATCH 08/10] simplify the implementation --- src/layers/basic.jl | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 094428d9..3130634b 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -42,19 +42,18 @@ end # This is a temporary and naive inplementation +# it might be replaced in the future to get better performance # see issue https://github.com/FluxML/Flux.jl/issues/702 +# Johnny Chen -- @johnnychen94 """ - activations(c::Chain, x) -Calculate the forward results of each layers in Chain `c` + activations(c::Chain, input) +Calculate the forward results of each layers in Chain `c` with `input` as model input. """ -function activations(c::Chain, x) - rst = Array{Any,1}() - if isempty(c) - return rst - end - push!(rst, c[1](x)) - for l in c[2:end] - push!(rst, l(rst[end])) +function activations(c::Chain, input) + rst = [] + for l in c + x = get(rst, length(rst), input) + push!(rst, l(x)) end return rst end From 4626f7568c125c74d5d19c60dbc1260fe9db3b19 Mon Sep 17 00:00:00 2001 From: JohnnyChen Date: Fri, 5 Apr 2019 18:50:15 +0800 Subject: [PATCH 09/10] rewrite one test case --- test/layers/basic.jl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/layers/basic.jl b/test/layers/basic.jl index ea54e153..4deb545f 100644 --- a/test/layers/basic.jl +++ b/test/layers/basic.jl @@ -9,8 +9,7 @@ import Flux: activations @test activations(Chain(), x) == [] @test activations(dummy_model, x)[1] == dummy_model[1](x) @test activations(dummy_model, x)[2] == x |> dummy_model[1] |> dummy_model[2] - - @test_nowarn activations(Chain(identity, x->:foo), 1) # different types + @test activations(Chain(identity, x->:foo), x)[2] == :foo # results include `Any` type end end From a300376f71e42f56a7730b861149daaa9bd22f10 Mon Sep 17 00:00:00 2001 From: Johnny Chen Date: Fri, 5 Apr 2019 19:19:30 +0800 Subject: [PATCH 10/10] fix a typo in comment `inplementation` --> `implementation` --- src/layers/basic.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 3130634b..76704d0c 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -41,8 +41,8 @@ function Base.show(io::IO, c::Chain) end -# This is a temporary and naive inplementation -# it might be replaced in the future to get better performance +# This is a temporary and naive implementation +# it might be replaced in the future for better performance # see issue https://github.com/FluxML/Flux.jl/issues/702 # Johnny Chen -- @johnnychen94 """