From 7aad2242061d8ab0ea4605678c0fda1a414cf5e0 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Fri, 9 Jun 2017 22:13:25 +0200 Subject: [PATCH 01/12] add reshape --- src/Flux.jl | 2 +- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 5 +++++ test/backend/tensorflow.jl | 2 ++ 4 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 src/ops.jl diff --git a/src/Flux.jl b/src/Flux.jl index 9a508002..f8db5553 100644 --- a/src/Flux.jl +++ b/src/Flux.jl @@ -23,7 +23,7 @@ include("core.jl") import .FluxCore: back!, update!, graph include("utils.jl") - +include("ops.jl") include("params.jl") include("compiler/code.jl") diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index b34e23a7..9b73cfa7 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -31,6 +31,7 @@ graph(::typeof(svd), x) = svd(x) graph(::typeof(size), x, dim) = TensorFlow.size(x,convert(Tensor{Int32}, dim)) graph(::typeof(size), x) = TensorFlow.size(x) graph(::typeof(chol), args...) = TensorFlow.transpose(TensorFlow.cholesky(args...)) +graph(::typeof(reshape), x, dims) = TensorFlow.reshape(x,convert(Tensor{Int32},dims)) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl new file mode 100644 index 00000000..69329b00 --- /dev/null +++ b/src/ops.jl @@ -0,0 +1,5 @@ +export reshape + +import Base: reshape + +reshape(x::AbstractArray, dims::AbstractArray) = reshape(x,tuple(dims...)) diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 1dcfdf53..7d1940e6 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -47,6 +47,8 @@ end A = randn(6,5) A = A'*A @test tf(@net x -> chol(x))(A) ≈ chol(A) + A = randn(Float32,(6,3)) + @test transpose(tf(@net (x,y) -> reshape(x,y))(transpose(A),[2,9])) ≈ reshape(A,(9,2)) # Note: TF is row major and julia is not end end From 2fce3b195e4a90570e26144e1d932dc4f1cb9e89 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Fri, 9 Jun 2017 22:20:27 +0200 Subject: [PATCH 02/12] add tile --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 3 ++- test/backend/tensorflow.jl | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index 9b73cfa7..4235d047 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -32,6 +32,7 @@ graph(::typeof(size), x, dim) = TensorFlow.size(x,convert(Tensor{Int32}, dim)) graph(::typeof(size), x) = TensorFlow.size(x) graph(::typeof(chol), args...) = TensorFlow.transpose(TensorFlow.cholesky(args...)) graph(::typeof(reshape), x, dims) = TensorFlow.reshape(x,convert(Tensor{Int32},dims)) +graph(::typeof(Flux.tile), args...) = TensorFlow.tile(args...) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index 69329b00..7c25ba56 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,5 +1,6 @@ -export reshape +export reshape, tile import Base: reshape reshape(x::AbstractArray, dims::AbstractArray) = reshape(x,tuple(dims...)) +tile(x::AbstractArray, mult::AbstractArray) = repeat(x,outer=tuple(mult...)) diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 7d1940e6..0fdd5421 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -49,6 +49,8 @@ end @test tf(@net x -> chol(x))(A) ≈ chol(A) A = randn(Float32,(6,3)) @test transpose(tf(@net (x,y) -> reshape(x,y))(transpose(A),[2,9])) ≈ reshape(A,(9,2)) # Note: TF is row major and julia is not + A = randn(Float32,(4,3,1)) + @test tf(@net (x,y) -> Flux.tile(x,y))(A,[1,1,3]) ≈ repeat(A,outer=(1,1,3)) end end From f25f9851098cadc216f94c8f39af558df8657325 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Fri, 9 Jun 2017 22:34:07 +0200 Subject: [PATCH 03/12] add fill --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 5 +++-- test/backend/tensorflow.jl | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index 4235d047..562252d9 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -33,6 +33,7 @@ graph(::typeof(size), x) = TensorFlow.size(x) graph(::typeof(chol), args...) = TensorFlow.transpose(TensorFlow.cholesky(args...)) graph(::typeof(reshape), x, dims) = TensorFlow.reshape(x,convert(Tensor{Int32},dims)) graph(::typeof(Flux.tile), args...) = TensorFlow.tile(args...) +graph(::typeof(fill), x, dims) = Ops.fill(convert(Tensor{Int32}, dims), Tensor(x)) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index 7c25ba56..b76ef203 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,6 +1,7 @@ -export reshape, tile +export reshape, tile, fill -import Base: reshape +import Base: reshape, fill reshape(x::AbstractArray, dims::AbstractArray) = reshape(x,tuple(dims...)) tile(x::AbstractArray, mult::AbstractArray) = repeat(x,outer=tuple(mult...)) +fill{T}(x::T, dims::AbstractArray) = fill(x,tuple(dims...)) diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 0fdd5421..531bf38b 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -51,6 +51,7 @@ end @test transpose(tf(@net (x,y) -> reshape(x,y))(transpose(A),[2,9])) ≈ reshape(A,(9,2)) # Note: TF is row major and julia is not A = randn(Float32,(4,3,1)) @test tf(@net (x,y) -> Flux.tile(x,y))(A,[1,1,3]) ≈ repeat(A,outer=(1,1,3)) + @test tf(@net (x,y) -> fill(x,y))(3.2,[3,2]) ≈ convert(Array{Float32},3.2*ones(3,2)) end end From edbf698ed940ecce9d6d224e20fde3fb5e937f64 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Fri, 9 Jun 2017 23:45:16 +0200 Subject: [PATCH 04/12] add cast --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 3 ++- test/backend/tensorflow.jl | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index 562252d9..2b8f24bd 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -34,6 +34,7 @@ graph(::typeof(chol), args...) = TensorFlow.transpose(TensorFlow.cholesky(args.. graph(::typeof(reshape), x, dims) = TensorFlow.reshape(x,convert(Tensor{Int32},dims)) graph(::typeof(Flux.tile), args...) = TensorFlow.tile(args...) graph(::typeof(fill), x, dims) = Ops.fill(convert(Tensor{Int32}, dims), Tensor(x)) +graph(::typeof(Flux.cast), args...) = TensorFlow.cast(args...) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index b76ef203..a4c6c4e7 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,7 +1,8 @@ -export reshape, tile, fill +export reshape, tile, fill, cast import Base: reshape, fill reshape(x::AbstractArray, dims::AbstractArray) = reshape(x,tuple(dims...)) tile(x::AbstractArray, mult::AbstractArray) = repeat(x,outer=tuple(mult...)) fill{T}(x::T, dims::AbstractArray) = fill(x,tuple(dims...)) +cast{T}(x::AbstractArray, ::Type{T}) = convert(Array{T},x) diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 531bf38b..7d09eccd 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -52,6 +52,7 @@ end A = randn(Float32,(4,3,1)) @test tf(@net (x,y) -> Flux.tile(x,y))(A,[1,1,3]) ≈ repeat(A,outer=(1,1,3)) @test tf(@net (x,y) -> fill(x,y))(3.2,[3,2]) ≈ convert(Array{Float32},3.2*ones(3,2)) + @test typeof(tf(@net x -> Flux.cast(x,Int32))(A)) == Array{Int32,3} end end From 015e373a34def0a261eb06a8a34a0f5e8d0a1e30 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Sat, 10 Jun 2017 00:14:16 +0200 Subject: [PATCH 05/12] add solve --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 3 ++- test/backend/tensorflow.jl | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index 2b8f24bd..47113643 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -35,6 +35,7 @@ graph(::typeof(reshape), x, dims) = TensorFlow.reshape(x,convert(Tensor{Int32},d graph(::typeof(Flux.tile), args...) = TensorFlow.tile(args...) graph(::typeof(fill), x, dims) = Ops.fill(convert(Tensor{Int32}, dims), Tensor(x)) graph(::typeof(Flux.cast), args...) = TensorFlow.cast(args...) +graph(::typeof(solve), A, b) = TensorFlow.matrix_solve(A, b) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index a4c6c4e7..8abc6c41 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,4 +1,4 @@ -export reshape, tile, fill, cast +export reshape, tile, fill, cast, solve import Base: reshape, fill @@ -6,3 +6,4 @@ reshape(x::AbstractArray, dims::AbstractArray) = reshape(x,tuple(dims...)) tile(x::AbstractArray, mult::AbstractArray) = repeat(x,outer=tuple(mult...)) fill{T}(x::T, dims::AbstractArray) = fill(x,tuple(dims...)) cast{T}(x::AbstractArray, ::Type{T}) = convert(Array{T},x) +solve(A::AbstractArray, b::AbstractArray) = A\b diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 7d09eccd..3d9f56b0 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -53,6 +53,9 @@ end @test tf(@net (x,y) -> Flux.tile(x,y))(A,[1,1,3]) ≈ repeat(A,outer=(1,1,3)) @test tf(@net (x,y) -> fill(x,y))(3.2,[3,2]) ≈ convert(Array{Float32},3.2*ones(3,2)) @test typeof(tf(@net x -> Flux.cast(x,Int32))(A)) == Array{Int32,3} + A = randn(Float32,(5,5)) + b = randn(Float32,(5,1)) + @test tf(@net (x,y) -> solve(x,y))(A,b) ≈ A\b end end From c8d320233abe52129600716879be22c9967f7d53 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Sat, 10 Jun 2017 00:20:13 +0200 Subject: [PATCH 06/12] add triangular_solve --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 3 ++- test/backend/tensorflow.jl | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index 47113643..291da771 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -36,6 +36,7 @@ graph(::typeof(Flux.tile), args...) = TensorFlow.tile(args...) graph(::typeof(fill), x, dims) = Ops.fill(convert(Tensor{Int32}, dims), Tensor(x)) graph(::typeof(Flux.cast), args...) = TensorFlow.cast(args...) graph(::typeof(solve), A, b) = TensorFlow.matrix_solve(A, b) +graph(::typeof(triangular_solve), A, b) = TensorFlow.matrix_triangular_solve(A, b; lower=false) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index 8abc6c41..4d57e565 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,4 +1,4 @@ -export reshape, tile, fill, cast, solve +export reshape, tile, fill, cast, solve, triangular_solve import Base: reshape, fill @@ -7,3 +7,4 @@ tile(x::AbstractArray, mult::AbstractArray) = repeat(x,outer=tuple(mult...)) fill{T}(x::T, dims::AbstractArray) = fill(x,tuple(dims...)) cast{T}(x::AbstractArray, ::Type{T}) = convert(Array{T},x) solve(A::AbstractArray, b::AbstractArray) = A\b +triangular_solve(A::AbstractArray, b::AbstractArray) = A\b diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 3d9f56b0..4944cd7f 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -56,6 +56,8 @@ end A = randn(Float32,(5,5)) b = randn(Float32,(5,1)) @test tf(@net (x,y) -> solve(x,y))(A,b) ≈ A\b + _,A,_ = lu(A) + @test tf(@net (x,y) -> triangular_solve(x,y))(A,b) ≈ A\b end end From cd8c2c2f83d19321df02b08c7541f13cabb38a53 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Sat, 10 Jun 2017 00:33:30 +0200 Subject: [PATCH 07/12] add randu --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 3 ++- test/backend/tensorflow.jl | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index 291da771..bacf8652 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -37,6 +37,7 @@ graph(::typeof(fill), x, dims) = Ops.fill(convert(Tensor{Int32}, dims), Tensor(x graph(::typeof(Flux.cast), args...) = TensorFlow.cast(args...) graph(::typeof(solve), A, b) = TensorFlow.matrix_solve(A, b) graph(::typeof(triangular_solve), A, b) = TensorFlow.matrix_triangular_solve(A, b; lower=false) +graph(::typeof(randu), x) = Ops.random_uniform(convert(Tensor{Int32},x);dtype=Float32) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index 4d57e565..cac4805c 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,4 +1,4 @@ -export reshape, tile, fill, cast, solve, triangular_solve +export reshape, tile, fill, cast, solve, triangular_solve, randu import Base: reshape, fill @@ -8,3 +8,4 @@ fill{T}(x::T, dims::AbstractArray) = fill(x,tuple(dims...)) cast{T}(x::AbstractArray, ::Type{T}) = convert(Array{T},x) solve(A::AbstractArray, b::AbstractArray) = A\b triangular_solve(A::AbstractArray, b::AbstractArray) = A\b +randu(x::AbstractArray) = rand(tuple(x...)) diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 4944cd7f..8ca68c2e 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -58,6 +58,7 @@ end @test tf(@net (x,y) -> solve(x,y))(A,b) ≈ A\b _,A,_ = lu(A) @test tf(@net (x,y) -> triangular_solve(x,y))(A,b) ≈ A\b + @test size(tf(@net x -> randu(x))([2,3])) == (2,3) end end From 6d106c914d453cec6bdf2bdf235e6a9aeca4c585 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Sat, 10 Jun 2017 00:38:16 +0200 Subject: [PATCH 08/12] add randn --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 5 +++-- test/backend/tensorflow.jl | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index bacf8652..7707cf8f 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -38,6 +38,7 @@ graph(::typeof(Flux.cast), args...) = TensorFlow.cast(args...) graph(::typeof(solve), A, b) = TensorFlow.matrix_solve(A, b) graph(::typeof(triangular_solve), A, b) = TensorFlow.matrix_triangular_solve(A, b; lower=false) graph(::typeof(randu), x) = Ops.random_uniform(convert(Tensor{Int32},x);dtype=Float32) +graph(::typeof(randn), x) = TensorFlow.random_normal(convert(Tensor{Int32},x);dtype=Float32) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index cac4805c..31292501 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,6 +1,6 @@ -export reshape, tile, fill, cast, solve, triangular_solve, randu +export reshape, tile, fill, cast, solve, triangular_solve, randu, randn -import Base: reshape, fill +import Base: reshape, fill, randn reshape(x::AbstractArray, dims::AbstractArray) = reshape(x,tuple(dims...)) tile(x::AbstractArray, mult::AbstractArray) = repeat(x,outer=tuple(mult...)) @@ -9,3 +9,4 @@ cast{T}(x::AbstractArray, ::Type{T}) = convert(Array{T},x) solve(A::AbstractArray, b::AbstractArray) = A\b triangular_solve(A::AbstractArray, b::AbstractArray) = A\b randu(x::AbstractArray) = rand(tuple(x...)) +randn(x::AbstractArray) = randn(tuple(x...)) diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 8ca68c2e..77fdab19 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -59,6 +59,7 @@ end _,A,_ = lu(A) @test tf(@net (x,y) -> triangular_solve(x,y))(A,b) ≈ A\b @test size(tf(@net x -> randu(x))([2,3])) == (2,3) + @test size(tf(@net x -> randn(x))([2,3])) == (2,3) end end From ec7a0bd8f7e2d2c9cd3321c6ae300af705fded29 Mon Sep 17 00:00:00 2001 From: Ali Hamdi Date: Sat, 10 Jun 2017 00:49:34 +0200 Subject: [PATCH 09/12] add expand_dims --- src/backend/tensorflow/graph.jl | 1 + src/ops.jl | 8 +++++++- test/backend/tensorflow.jl | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/backend/tensorflow/graph.jl b/src/backend/tensorflow/graph.jl index 7707cf8f..cf01de74 100644 --- a/src/backend/tensorflow/graph.jl +++ b/src/backend/tensorflow/graph.jl @@ -39,6 +39,7 @@ graph(::typeof(solve), A, b) = TensorFlow.matrix_solve(A, b) graph(::typeof(triangular_solve), A, b) = TensorFlow.matrix_triangular_solve(A, b; lower=false) graph(::typeof(randu), x) = Ops.random_uniform(convert(Tensor{Int32},x);dtype=Float32) graph(::typeof(randn), x) = TensorFlow.random_normal(convert(Tensor{Int32},x);dtype=Float32) +graph(::typeof(Flux.expand_dims), x, dim) = TensorFlow.expand_dims(x,convert(Tensor{Int32},dim)) for op in (*, .*, .+, .^, log, exp, ceil, floor, sqrt, abs, cos, sin, tan, atan, asin, acos, tanh, lgamma, erf, erfc, real, imag, conj, diff --git a/src/ops.jl b/src/ops.jl index 31292501..e9cb75b1 100644 --- a/src/ops.jl +++ b/src/ops.jl @@ -1,4 +1,5 @@ -export reshape, tile, fill, cast, solve, triangular_solve, randu, randn +export reshape, tile, fill, cast, solve, triangular_solve, randu, randn, + expand_dims import Base: reshape, fill, randn @@ -10,3 +11,8 @@ solve(A::AbstractArray, b::AbstractArray) = A\b triangular_solve(A::AbstractArray, b::AbstractArray) = A\b randu(x::AbstractArray) = rand(tuple(x...)) randn(x::AbstractArray) = randn(tuple(x...)) + +function expand_dims(x,dim) + s = [size(x)...] + reshape(x,tuple(vcat(s[1:dim-1],1,s[dim:end])...)) +end diff --git a/test/backend/tensorflow.jl b/test/backend/tensorflow.jl index 77fdab19..872ef3ce 100644 --- a/test/backend/tensorflow.jl +++ b/test/backend/tensorflow.jl @@ -60,6 +60,11 @@ end @test tf(@net (x,y) -> triangular_solve(x,y))(A,b) ≈ A\b @test size(tf(@net x -> randu(x))([2,3])) == (2,3) @test size(tf(@net x -> randn(x))([2,3])) == (2,3) + m = tf(@net (x,y) -> Flux.expand_dims(x,y)) + A = randn(Float32,(3,2)) + @test m(A,1) ≈ Flux.expand_dims(A,1) + @test m(A,2) ≈ Flux.expand_dims(A,2) + @test m(A,3) ≈ Flux.expand_dims(A,3) end end From 88fa163c95231969a0898470a11c72dc65d4ca2e Mon Sep 17 00:00:00 2001 From: ylxdzsw Date: Fri, 21 Jul 2017 16:31:12 +0800 Subject: [PATCH 10/12] throttle --- src/training.jl | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) diff --git a/src/training.jl b/src/training.jl index 62a34c46..3632eb82 100644 --- a/src/training.jl +++ b/src/training.jl @@ -25,18 +25,59 @@ macro cb(ex, t, f) end) end +""" +Returns a function that when invoked, will only be triggered at most once +during `timeout` seconds. Normally, the throttled function will run +as much as it can, without ever going more than once per `wait` duration; +but if you'd like to disable the execution on the leading edge, pass +`leading=false`. To enable execution on the trailing edge, ditto. +""" +function throttle(f, timeout; leading=true, trailing=false) + cooldown = true + later = nothing + + function throttled(args...; kwargs...) + yield() + + if cooldown + if leading + f(args...; kwargs...) + else + later = () -> f(args...; kwargs...) + end + + cooldown = false + @schedule try + while (sleep(timeout); later != nothing) + later() + later = nothing + end + finally + cooldown = true + end + elseif trailing + later = () -> f(args...; kwargs...) + end + + nothing + end +end + function train!(m, train; cb = [], epoch = 1, η = 0.1, loss = mse) + callback = throttle(()->foreach(f -> f(), cb), 5) + @progress for e in 1:epoch info("Epoch $e") - @cb for (x, y) in train + for (x, y) in train x, y = mapt(tobatch, (x, y)) ŷ = m(x) any(isnan, ŷ) && error("NaN") Δ = back!(loss, 1, ŷ, y) back!(m, Δ, x) update!(m, η) - end 5 foreach(f -> f(), cb) + callback() + end end return m end From cce1a2a73e4155a6233822380c26d6c063f72de1 Mon Sep 17 00:00:00 2001 From: ylxdzsw Date: Wed, 26 Jul 2017 09:57:20 +0800 Subject: [PATCH 11/12] add tests for throttle --- test/runtests.jl | 1 + test/throttle.jl | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 test/throttle.jl diff --git a/test/runtests.jl b/test/runtests.jl index 655d924a..7128e13b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -18,6 +18,7 @@ include("backend/common.jl") include("basic.jl") include("recurrent.jl") include("optimizer.jl") +include("throttle.jl") @tfonly include("backend/tensorflow.jl") @mxonly include("backend/mxnet.jl") diff --git a/test/throttle.jl b/test/throttle.jl new file mode 100644 index 00000000..d3dcd925 --- /dev/null +++ b/test/throttle.jl @@ -0,0 +1,49 @@ +using Flux.throttle + +@testset "throttle" begin + @testset "default behaviour" begin + a = [] + f = throttle(()->push!(a, now()), 1, leading=true, trailing=false) + f() + f() + f() + sleep(1.01) + @test length(a) == 1 + end + + @testset "leading behaviour" begin + a = [] + f = throttle(()->push!(a, now()), 1, leading=true, trailing=false) + f() + @test length(a) == 1 + f() + @test length(a) == 1 + sleep(1.01) + f() + @test length(a) == 2 + end + + @testset "trailing behaviour" begin + a = [] + f = throttle(()->push!(a, now()), 1, leading=false, trailing=true) + f() + @test length(a) == 0 + f() + @test length(a) == 0 + sleep(1.01) + @test length(a) == 1 + end + + @testset "arguments" begin + a = [] + f = throttle((x)->push!(a, x), 1, leading=true, trailing=true) + f(1) + @test a == [1] + f(2) + @test a == [1] + f(3) + @test a == [1] + sleep(1.01) + @test a == [1, 3] + end +end From 0e325e0425606161ded20064ba0c5d929f497fad Mon Sep 17 00:00:00 2001 From: Mike J Innes Date: Thu, 27 Jul 2017 20:44:45 +0100 Subject: [PATCH 12/12] Revert "Merge pull request #51 from ylxdzsw/tf-train" This reverts commit b4a0841e9df6da38b19abcbdc8b5c1105a329dd2. --- src/backend/tensorflow/model.jl | 52 +++++++++++++--------------- src/backend/tensorflow/tensorflow.jl | 2 +- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/src/backend/tensorflow/model.jl b/src/backend/tensorflow/model.jl index cdb7f5e5..836861a0 100644 --- a/src/backend/tensorflow/model.jl +++ b/src/backend/tensorflow/model.jl @@ -4,22 +4,18 @@ struct Exec session ::Session input ::Any output ::Any - params ::Dict{Param,Param{Tensor}} + grads ::Any + params ::Dict{Flux.Param,Tensor} stacks ::Dict{Any,Any} end function makesession(model, inputs; session = Session(Graph())) inputs = mapt(_ -> placeholder(Float32), inputs) params, stacks, output = tograph(model, inputs...) - output = mapt(x->Param{Tensor}(x, placeholder(Float32)), output) - params = Dict(x=>Param{Tensor}(y, gradients(mapt(x->x.x, output), - y, mapt(x->x.Δx, output))) - for (x, y) in params) - inputs = mapt(x->Param{Tensor}(x, gradients(mapt(x->x.x, output), - x, mapt(x->x.Δx, output))), - inputs) + # grads = gradients(output, [collectt(inputs)..., values(params)...]) + grads = placeholder(Float32) run(session, global_variables_initializer()) - Exec(session, inputs, output, params, stacks) + Exec(session, inputs, output, grads, params, stacks) end retuple(xs) = xs @@ -27,33 +23,35 @@ retuple(xs::AbstractArray{<:AbstractArray}) = (retuple.(xs)...,) dictt(xs, ys) = Dict(zip(collectt(xs), collectt(ys))) -function (m::Exec)(args...) - dict = merge( - Dict(y.x=>x.x for (x, y) in m.params), - Dict(x.x=>y for (x, y) in zip(m.input, args)) - ) - retuple(run(m.session, mapt(x->x.x, m.output), dict)) +function params(m::Exec, args...) + shapecheckt(m.input, args) + idict = dictt(m.input, args) + pdict = Dict(t => p.x for (p, t) in m.params) + merge(idict, pdict) end +function (m::Exec)(args...) + retuple(run(m.session, m.output, params(m, args...))) +end + +pullt!(_, xs) = shift!(xs) +pullt!(x::Tuple, xs) = map(x -> pullt!(x, xs), x) + +# TODO: gradients don't work yet +# `gradients` lacks support for `grad_y`s and multiple `y`s + function Flux.back!(m::Exec, Δ, args...) - dict = merge( - Dict(y.x=>x.x for (x, y) in m.params), - Dict(x.x=>y for (x, y) in zip(m.input, args)), - Dict(x.Δx=>y for (x, y) in zip(collectt(m.output), collectt(Δ))) - ) - - Δin, Δps = run(m.session, (mapt(x->x.Δx, m.input), map(x->x.Δx, values(m.params))), dict) - + Δps = run(m.session, m.grads, params(m, args...)) + Δin = pullt!(m.input, Δps) for (p, Δ) in zip(keys(m.params), Δps) p.Δx .+= Δ end - Δin end function Flux.update!(m::Exec, η) for p in keys(m.params) - Flux.update!(p, η) + update!(p, η) end return m end @@ -72,8 +70,8 @@ function (m::Model)(args...) @tferr m.exec.stacks m.exec(args...) end -Flux.back!(m::Model, Δ, args...) = Flux.back!(m.exec, Δ, args...) -Flux.update!(m::Model, η) = (Flux.update!(m.exec, η); m) +Flux.back!(m::Model, Δ, args...) = back!(m.exec, Δ, args...) +Flux.update!(m::Model, η) = (update!(m.exec, η); m) # Recurrent Models diff --git a/src/backend/tensorflow/tensorflow.jl b/src/backend/tensorflow/tensorflow.jl index 7536bdbc..74c94012 100644 --- a/src/backend/tensorflow/tensorflow.jl +++ b/src/backend/tensorflow/tensorflow.jl @@ -1,7 +1,7 @@ module TF using ..Flux, DataFlow, TensorFlow, Juno -import Flux: accuracy, convertel, Param +import Flux: accuracy, convertel export tf