From 7ebb2cfac5cac9b011d09f20a633b736822dfbe3 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Wed, 11 Sep 2019 21:10:12 +0530 Subject: [PATCH 01/24] test on julia 1.2 --- .gitlab-ci.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 05217e81..9b39e5b7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,8 +13,6 @@ include: - julia -e 'using InteractiveUtils; versioninfo()' - mkdir $JULIA_DEPOT_PATH # Pkg3.jl#325 - - julia -e 'using Pkg; - Pkg.add("CuArrays");' - julia --project -e 'using Pkg; Pkg.instantiate(); Pkg.build(); @@ -35,3 +33,11 @@ test:v1.1: only: - staging - trying + +test:v1.2: + extends: .flux + variables: + CI_VERSION_TAG: 'v1.2' + only: + - staging + - trying From b8d872d842d873451b85b65090a1109e2191db98 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Wed, 11 Sep 2019 21:11:02 +0530 Subject: [PATCH 02/24] update to Flux 0.9+ --- test/cuda/cuda.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/cuda/cuda.jl b/test/cuda/cuda.jl index 3508e561..0bb7b2ef 100644 --- a/test/cuda/cuda.jl +++ b/test/cuda/cuda.jl @@ -1,4 +1,5 @@ -using Flux, CuArrays, Test +using Flux, Test +using Flux.CuArrays using Flux: gpu @info "Testing GPU Support" From 37fe91d54dcc8a3c16d20e865793a37264517e2e Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Wed, 18 Sep 2019 12:05:31 +0530 Subject: [PATCH 03/24] remove branch restrictions --- .gitlab-ci.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9b39e5b7..1e69dd3f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -22,22 +22,13 @@ test:v1.0: extends: .flux variables: CI_VERSION_TAG: 'v1.0' - only: - - staging - - trying test:v1.1: extends: .flux variables: CI_VERSION_TAG: 'v1.1' - only: - - staging - - trying test:v1.2: extends: .flux variables: CI_VERSION_TAG: 'v1.2' - only: - - staging - - trying From 99b6fe57e94d948a18894b67b62e3c565cd4a719 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Wed, 18 Sep 2019 12:32:11 +0530 Subject: [PATCH 04/24] extend test template --- .gitlab-ci.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1e69dd3f..8e0fa64e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,28 +7,17 @@ variables: include: - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v3/common.yml' -.flux: - extends: .test - script: - - julia -e 'using InteractiveUtils; - versioninfo()' - - mkdir $JULIA_DEPOT_PATH # Pkg3.jl#325 - - julia --project -e 'using Pkg; - Pkg.instantiate(); - Pkg.build(); - Pkg.test(; coverage=true);' - test:v1.0: - extends: .flux + extends: .test variables: CI_VERSION_TAG: 'v1.0' test:v1.1: - extends: .flux + extends: .test variables: CI_VERSION_TAG: 'v1.1' test:v1.2: - extends: .flux + extends: .test variables: CI_VERSION_TAG: 'v1.2' From f8d5d3b5fcd15ce5c4c150039bb15f4b6b512789 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 19 Sep 2019 14:12:11 +0100 Subject: [PATCH 05/24] broken normalisation layer params --- test/layers/normalisation.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/layers/normalisation.jl b/test/layers/normalisation.jl index cda0cc59..4bb46262 100644 --- a/test/layers/normalisation.jl +++ b/test/layers/normalisation.jl @@ -42,6 +42,8 @@ end let m = BatchNorm(2), x = [1.0 3.0 5.0; 2.0 4.0 6.0] + @test_broken length(params(m)) == 2 + @test m.β == [0, 0] # initβ(2) @test m.γ == [1, 1] # initγ(2) # initial m.σ is 1 @@ -109,7 +111,9 @@ end expand_inst = (x, as) -> reshape(repeat(x, outer=[1, as[length(as)]]), as...) # begin tests let m = InstanceNorm(2), sizes = (3, 2, 2), - x = reshape(collect(1:prod(sizes)), sizes) + x = reshape(collect(1:prod(sizes)), sizes) + + @test_broken length(params(m)) == 2 x = Float64.(x) @test m.β == [0, 0] # initβ(2) @test m.γ == [1, 1] # initγ(2) @@ -192,7 +196,9 @@ end squeeze(x) = dropdims(x, dims = tuple(findall(size(x) .== 1)...)) # To remove all singular dimensions let m = GroupNorm(4,2), sizes = (3,4,2), - x = reshape(collect(1:prod(sizes)), sizes) + x = reshape(collect(1:prod(sizes)), sizes) + + @test_broken length(params(m)) == 2 x = Float64.(x) @test m.β == [0, 0, 0, 0] # initβ(32) @test m.γ == [1, 1, 1, 1] # initγ(32) From 2c71fc282b9353efe9bb24f687e93911d6e0492a Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 19 Sep 2019 14:15:28 +0100 Subject: [PATCH 06/24] rename functor.jl --- src/Flux.jl | 2 +- src/{treelike.jl => functor.jl} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{treelike.jl => functor.jl} (100%) diff --git a/src/Flux.jl b/src/Flux.jl index 9d1fbfc5..7356832a 100644 --- a/src/Flux.jl +++ b/src/Flux.jl @@ -35,7 +35,7 @@ end include("utils.jl") include("onehot.jl") -include("treelike.jl") +include("functor.jl") include("layers/stateless.jl") include("layers/basic.jl") diff --git a/src/treelike.jl b/src/functor.jl similarity index 100% rename from src/treelike.jl rename to src/functor.jl From 6529dbcbe69d3a94b6edd131051ec0df7e26820d Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 19 Sep 2019 15:22:11 +0100 Subject: [PATCH 07/24] functor refactor --- src/functor.jl | 75 ++++++++++++++++++++++------------------- src/layers/basic.jl | 3 +- src/layers/normalise.jl | 18 ++-------- src/layers/recurrent.jl | 3 +- 4 files changed, 47 insertions(+), 52 deletions(-) diff --git a/src/functor.jl b/src/functor.jl index 42b10f23..2113d7e4 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -1,52 +1,67 @@ import Adapt: adapt, adapt_storage -import Zygote: IdSet +using Zygote: IdSet -children(x) = () -mapchildren(f, x) = x +functor(x) = (), _ -> x -children(x::Tuple) = x -children(x::NamedTuple) = x -mapchildren(f, x::Tuple) = map(f, x) -mapchildren(f, x::NamedTuple) = map(f, x) +functor(x::Tuple) = x, y -> y +functor(x::NamedTuple) = x, y -> y -function treelike(m::Module, T, fs = fieldnames(T)) +functor(x::AbstractArray) = x, y -> y +functor(x::AbstractArray{<:Number}) = (), _ -> x + +function makefunctor(m::Module, T, fs = fieldnames(T)) @eval m begin - Flux.children(x::$T) = ($([:(x.$f) for f in fs]...),) - Flux.mapchildren(f, x::$T) = $T(f.($children(x))...) + Flux.functor(x::$T) = ($([:($f=x.$f) for f in fs]...),), y -> $T(y...) end end -macro treelike(T, fs = nothing) +function functorm(T, fs = nothing) fs == nothing || isexpr(fs, :tuple) || error("@treelike T (a, b)") fs = fs == nothing ? [] : [:($(map(QuoteNode, fs.args)...),)] - :(treelike(@__MODULE__, $(esc(T)), $(fs...))) + :(makefunctor(@__MODULE__, $(esc(T)), $(fs...))) end -isleaf(x) = isempty(children(x)) +macro functor(args...) + functorm(args...) +end -function mapleaves(f, x; cache = IdDict()) +isleaf(x) = functor(x)[1] === () + +function fmap1(f, x) + func, re = functor(x) + re(map(f, func)) +end + +function fmap(f, x; cache = IdDict()) haskey(cache, x) && return cache[x] - cache[x] = isleaf(x) ? f(x) : mapchildren(x -> mapleaves(f, x, cache = cache), x) + cache[x] = isleaf(x) ? f(x) : fmap1(x -> fmap(f, x, cache = cache), x) end -function prefor(f, x; seen = IdSet()) - x ∈ seen && return +children(m) = functor(m)[1] + +params!(p::Params, x::AbstractArray{<:Real}, seen = IdSet()) = push!(p, x) + +function params!(p::Params, x, seen = IdSet()) + x in seen && return push!(seen, x) - f(x) - foreach(x -> prefor(f, x, seen = seen), children(x)) - return + for child in children(x) + params!(p, child, seen) + end end -function params(m) +function params(m...) ps = Params() - prefor(p -> - p isa AbstractArray{<:Real} && - !any(p′ -> p′ === p, ps) && push!(ps, p), - m) + params!(ps, m) return ps end -params(m...) = params(m) +# Deprecated stuff +macro treelike(args...) + functorm(args...) +end +mapleaves(f, x) = fmap(f, x) + +# function params function loadparams!(m, xs) for (p, x) in zip(params(m), xs) @@ -76,11 +91,3 @@ paramtype(T::Type{<:Real}, m) = mapleaves(x -> adapt(T, x), m) f32(m) = paramtype(Float32, m) f64(m) = paramtype(Float64, m) - -# General parameter map - -function mapparams(f, m) - mapleaves(m) do x - x isa Union{AbstractArray,Number} ? f(x) : x - end -end diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 0cebead1..1d885916 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -24,8 +24,7 @@ end @forward Chain.layers Base.getindex, Base.length, Base.first, Base.last, Base.iterate, Base.lastindex -children(c::Chain) = c.layers -mapchildren(f, c::Chain) = Chain(f.(c.layers)...) +functor(c::Chain) = c.layers, ls -> Chain(ls...) applychain(::Tuple{}, x) = x applychain(fs::Tuple, x) = applychain(tail(fs), first(fs)(x)) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index 61a62adf..7ea601f8 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -166,11 +166,7 @@ function (BN::BatchNorm)(x) end end -children(BN::BatchNorm) = - (BN.λ, BN.β, BN.γ, BN.μ, BN.σ², BN.ϵ, BN.momentum) - -mapchildren(f, BN::BatchNorm) = # e.g. mapchildren(cu, BN) - BatchNorm(BN.λ, f(BN.β), f(BN.γ), f(BN.μ), f(BN.σ²), BN.ϵ, BN.momentum) +@functor BatchNorm function Base.show(io::IO, l::BatchNorm) print(io, "BatchNorm($(join(size(l.β), ", "))") @@ -261,11 +257,7 @@ function (in::InstanceNorm)(x) end end -children(in::InstanceNorm) = - (in.λ, in.β, in.γ, in.μ, in.σ², in.ϵ, in.momentum) - -mapchildren(f, in::InstanceNorm) = # e.g. mapchildren(cu, in) - InstanceNorm(in.λ, f(in.β), f(in.γ), f(in.μ), f(in.σ²), in.ϵ, in.momentum) +@functor InstanceNorm function Base.show(io::IO, l::InstanceNorm) print(io, "InstanceNorm($(join(size(l.β), ", "))") @@ -360,11 +352,7 @@ function(gn::GroupNorm)(x) end end -children(gn::GroupNorm) = - (gn.λ, gn.β, gn.γ, gn.μ, gn.σ², gn.ϵ, gn.momentum) - -mapchildren(f, gn::GroupNorm) = # e.g. mapchildren(cu, BN) - GroupNorm(gn.G,gn.λ, f(gn.β), f(gn.γ), f(gn.μ), f(gn.σ²), gn.ϵ, gn.momentum) +@functor GroupNorm function Base.show(io::IO, l::GroupNorm) print(io, "GroupNorm($(join(size(l.β), ", "))") diff --git a/src/layers/recurrent.jl b/src/layers/recurrent.jl index b5eea4a4..ad8c6e80 100644 --- a/src/layers/recurrent.jl +++ b/src/layers/recurrent.jl @@ -52,7 +52,8 @@ Assuming you have a `Recur` layer `rnn`, this is roughly equivalent to rnn.state = hidden(rnn.cell) """ -reset!(m) = prefor(x -> x isa Recur && (x.state = x.init), m) +reset!(m::Recur) = (m.state = m.init) +reset!(m) = foreach(reset!, functor(m)[1]) flip(f, xs) = reverse(f.(reverse(xs))) From b951377426b210f11091c776b98b693672980b7a Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 19 Sep 2019 15:33:24 +0100 Subject: [PATCH 08/24] fix normalisation layer params --- src/functor.jl | 4 ++-- src/layers/normalise.jl | 6 ++++++ test/layers/normalisation.jl | 6 +++--- test/utils.jl | 1 - 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/functor.jl b/src/functor.jl index 2113d7e4..4e1aa247 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -37,14 +37,14 @@ function fmap(f, x; cache = IdDict()) cache[x] = isleaf(x) ? f(x) : fmap1(x -> fmap(f, x, cache = cache), x) end -children(m) = functor(m)[1] +trainable(m) = functor(m)[1] params!(p::Params, x::AbstractArray{<:Real}, seen = IdSet()) = push!(p, x) function params!(p::Params, x, seen = IdSet()) x in seen && return push!(seen, x) - for child in children(x) + for child in trainable(x) params!(p, child, seen) end end diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index 7ea601f8..7574272c 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -134,6 +134,8 @@ BatchNorm(chs::Integer, λ = identity; BatchNorm(λ, initβ(chs), initγ(chs), zeros(chs), ones(chs), ϵ, momentum) +trainable(bn::BatchNorm) = (bn.β, bn.γ) + function (BN::BatchNorm)(x) size(x, ndims(x)-1) == length(BN.β) || error("BatchNorm expected $(length(BN.β)) channels, got $(size(x, ndims(x)-1))") @@ -220,6 +222,8 @@ InstanceNorm(chs::Integer, λ = identity; InstanceNorm(λ, initβ(chs), initγ(chs), zeros(chs), ones(chs), ϵ, momentum) +trainable(in::InstanceNorm) = (in.β, in.γ) + function (in::InstanceNorm)(x) size(x, ndims(x)-1) == length(in.β) || error("InstanceNorm expected $(length(in.β)) channels, got $(size(x, ndims(x)-1))") @@ -303,6 +307,8 @@ GroupNorm(chs::Integer, G::Integer, λ = identity; GroupNorm(G, λ, initβ(chs), initγ(chs), zeros(G,1), ones(G,1), ϵ, momentum) +trainable(gn::GroupNorm) = (gn.β, gn.γ) + function(gn::GroupNorm)(x) size(x,ndims(x)-1) == length(gn.β) || error("Group Norm expected $(length(gn.β)) channels, but got $(size(x,ndims(x)-1)) channels") ndims(x) > 2 || error("Need to pass at least 3 channels for Group Norm to work") diff --git a/test/layers/normalisation.jl b/test/layers/normalisation.jl index 4bb46262..5b9e5a5e 100644 --- a/test/layers/normalisation.jl +++ b/test/layers/normalisation.jl @@ -42,7 +42,7 @@ end let m = BatchNorm(2), x = [1.0 3.0 5.0; 2.0 4.0 6.0] - @test_broken length(params(m)) == 2 + @test length(params(m)) == 2 @test m.β == [0, 0] # initβ(2) @test m.γ == [1, 1] # initγ(2) @@ -113,7 +113,7 @@ end let m = InstanceNorm(2), sizes = (3, 2, 2), x = reshape(collect(1:prod(sizes)), sizes) - @test_broken length(params(m)) == 2 + @test length(params(m)) == 2 x = Float64.(x) @test m.β == [0, 0] # initβ(2) @test m.γ == [1, 1] # initγ(2) @@ -198,7 +198,7 @@ end let m = GroupNorm(4,2), sizes = (3,4,2), x = reshape(collect(1:prod(sizes)), sizes) - @test_broken length(params(m)) == 2 + @test length(params(m)) == 2 x = Float64.(x) @test m.β == [0, 0, 0, 0] # initβ(32) @test m.γ == [1, 1, 1, 1] # initγ(32) diff --git a/test/utils.jl b/test/utils.jl index 3a840261..18a57139 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -83,7 +83,6 @@ end # Self-referential array. Just want params, no stack overflow pls. r = Any[nothing,m] - Flux.children(a::Vector{Any}) = Tuple(a) r[1] = r @test size.(params(r)) == [(5, 10), (5, 5), (5,), (5,)] end From cabb81e30b4bb1eaa1c43eb93636a1fa29ec9533 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 19 Sep 2019 15:53:31 +0100 Subject: [PATCH 09/24] internal rename --- docs/src/gpu.md | 6 +++--- docs/src/models/basics.md | 2 +- src/Flux.jl | 2 +- src/functor.jl | 10 ++++------ src/layers/basic.jl | 8 ++++---- src/layers/conv.jl | 8 ++++---- src/layers/normalise.jl | 2 +- src/layers/recurrent.jl | 8 ++++---- test/cuda/curnn.jl | 2 +- 9 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/src/gpu.md b/docs/src/gpu.md index aed33f4e..bb13fdd1 100644 --- a/docs/src/gpu.md +++ b/docs/src/gpu.md @@ -25,16 +25,16 @@ loss(x, y) # ~ 3 Note that we convert both the parameters (`W`, `b`) and the data set (`x`, `y`) to cuda arrays. Taking derivatives and training works exactly as before. -If you define a structured model, like a `Dense` layer or `Chain`, you just need to convert the internal parameters. Flux provides `mapleaves`, which allows you to alter all parameters of a model at once. +If you define a structured model, like a `Dense` layer or `Chain`, you just need to convert the internal parameters. Flux provides `fmap`, which allows you to alter all parameters of a model at once. ```julia d = Dense(10, 5, σ) -d = mapleaves(cu, d) +d = fmap(cu, d) d.W # Tracked CuArray d(cu(rand(10))) # CuArray output m = Chain(Dense(10, 5, σ), Dense(5, 2), softmax) -m = mapleaves(cu, m) +m = fmap(cu, m) d(cu(rand(10))) ``` diff --git a/docs/src/models/basics.md b/docs/src/models/basics.md index ddd81992..d83fc462 100644 --- a/docs/src/models/basics.md +++ b/docs/src/models/basics.md @@ -215,7 +215,7 @@ m(5) # => 26 Flux provides a set of helpers for custom layers, which you can enable by calling ```julia -Flux.@treelike Affine +Flux.@functor Affine ``` This enables a useful extra set of functionality for our `Affine` layer, such as [collecting its parameters](../training/optimisers.md) or [moving it to the GPU](../gpu.md). diff --git a/src/Flux.jl b/src/Flux.jl index 7356832a..84ffa29b 100644 --- a/src/Flux.jl +++ b/src/Flux.jl @@ -11,7 +11,7 @@ export gradient export Chain, Dense, Maxout, RNN, LSTM, GRU, Conv, CrossCor, ConvTranspose, MaxPool, MeanPool, DepthwiseConv, Dropout, AlphaDropout, LayerNorm, BatchNorm, InstanceNorm, GroupNorm, - SkipConnection, params, mapleaves, cpu, gpu, f32, f64 + SkipConnection, params, fmap, cpu, gpu, f32, f64 include("optimise/Optimise.jl") using .Optimise diff --git a/src/functor.jl b/src/functor.jl index 4e1aa247..73483ab9 100644 --- a/src/functor.jl +++ b/src/functor.jl @@ -16,7 +16,7 @@ function makefunctor(m::Module, T, fs = fieldnames(T)) end function functorm(T, fs = nothing) - fs == nothing || isexpr(fs, :tuple) || error("@treelike T (a, b)") + fs == nothing || isexpr(fs, :tuple) || error("@functor T (a, b)") fs = fs == nothing ? [] : [:($(map(QuoteNode, fs.args)...),)] :(makefunctor(@__MODULE__, $(esc(T)), $(fs...))) end @@ -61,8 +61,6 @@ macro treelike(args...) end mapleaves(f, x) = fmap(f, x) -# function params - function loadparams!(m, xs) for (p, x) in zip(params(m), xs) size(p) == size(x) || @@ -73,7 +71,7 @@ end # CPU/GPU movement conveniences -cpu(m) = mapleaves(x -> adapt(Array, x), m) +cpu(m) = fmap(x -> adapt(Array, x), m) const gpu_adaptor = if has_cuarrays() CuArrays.cu @@ -81,13 +79,13 @@ else identity end -gpu(x) = mapleaves(gpu_adaptor, x) +gpu(x) = fmap(gpu_adaptor, x) # Precision adapt_storage(T::Type{<:Real}, xs::AbstractArray{<:Real}) = convert.(T, xs) -paramtype(T::Type{<:Real}, m) = mapleaves(x -> adapt(T, x), m) +paramtype(T::Type{<:Real}, m) = fmap(x -> adapt(T, x), m) f32(m) = paramtype(Float32, m) f64(m) = paramtype(Float64, m) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 1d885916..67490472 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -91,7 +91,7 @@ function Dense(in::Integer, out::Integer, σ = identity; return Dense(initW(out, in), initb(out), σ) end -@treelike Dense +@functor Dense function (a::Dense)(x::AbstractArray) W, b, σ = a.W, a.b, a.σ @@ -130,7 +130,7 @@ end Diagonal(in::Integer; initα = ones, initβ = zeros) = Diagonal(initα(in), initβ(in)) -@treelike Diagonal +@functor Diagonal function (a::Diagonal)(x) α, β = a.α, a.β @@ -183,7 +183,7 @@ function Maxout(f, n_alts) return Maxout(over) end -@treelike Maxout +@functor Maxout function (mo::Maxout)(input::AbstractArray) mapreduce(f -> f(input), (acc, out) -> max.(acc, out), mo.over) @@ -208,7 +208,7 @@ struct SkipConnection connection #user can pass arbitrary connections here, such as (a,b) -> a + b end -@treelike SkipConnection +@functor SkipConnection function (skip::SkipConnection)(input) #We apply the layers to the input and return the result of the application of the layers and the original input diff --git a/src/layers/conv.jl b/src/layers/conv.jl index 4361a389..519f129f 100644 --- a/src/layers/conv.jl +++ b/src/layers/conv.jl @@ -45,7 +45,7 @@ Conv(k::NTuple{N,Integer}, ch::Pair{<:Integer,<:Integer}, σ = identity; Conv(init(k..., ch...), zeros(ch[2]), σ, stride = stride, pad = pad, dilation = dilation) -@treelike Conv +@functor Conv function (c::Conv)(x::AbstractArray) # TODO: breaks gpu broadcast :( @@ -102,7 +102,7 @@ ConvTranspose(k::NTuple{N,Integer}, ch::Pair{<:Integer,<:Integer}, σ = identity ConvTranspose(init(k..., reverse(ch)...), zeros(ch[2]), σ, stride = stride, pad = pad, dilation = dilation) -@treelike ConvTranspose +@functor ConvTranspose function conv_transpose_dims(c::ConvTranspose, x::AbstractArray) # Calculate size of "input", from ∇conv_data()'s perspective... @@ -180,7 +180,7 @@ function DepthwiseConv(k::NTuple{N,Integer}, ch::Pair{<:Integer,<:Integer}, σ = ) end -@treelike DepthwiseConv +@functor DepthwiseConv function (c::DepthwiseConv)(x) σ, b = c.σ, reshape(c.bias, map(_->1, c.stride)..., :, 1) @@ -244,7 +244,7 @@ CrossCor(k::NTuple{N,Integer}, ch::Pair{<:Integer,<:Integer}, σ = identity; CrossCor(init(k..., ch...), zeros(ch[2]), σ, stride = stride, pad = pad, dilation = dilation) -@treelike CrossCor +@functor CrossCor function crosscor(x, w, ddims::DenseConvDims) ddims = DenseConvDims(ddims, F=true) diff --git a/src/layers/normalise.jl b/src/layers/normalise.jl index 7574272c..b421d3e7 100644 --- a/src/layers/normalise.jl +++ b/src/layers/normalise.jl @@ -82,7 +82,7 @@ end LayerNorm(h::Integer) = LayerNorm(Diagonal(h)) -@treelike LayerNorm +@functor LayerNorm (a::LayerNorm)(x) = a.diag(normalise(x)) diff --git a/src/layers/recurrent.jl b/src/layers/recurrent.jl index ad8c6e80..f2344af8 100644 --- a/src/layers/recurrent.jl +++ b/src/layers/recurrent.jl @@ -38,7 +38,7 @@ function (m::Recur)(xs...) return y end -@treelike Recur cell, init +@functor Recur cell, init Base.show(io::IO, m::Recur) = print(io, "Recur(", m.cell, ")") @@ -80,7 +80,7 @@ end hidden(m::RNNCell) = m.h -@treelike RNNCell +@functor RNNCell function Base.show(io::IO, l::RNNCell) print(io, "RNNCell(", size(l.Wi, 2), ", ", size(l.Wi, 1)) @@ -128,7 +128,7 @@ end hidden(m::LSTMCell) = (m.h, m.c) -@treelike LSTMCell +@functor LSTMCell Base.show(io::IO, l::LSTMCell) = print(io, "LSTMCell(", size(l.Wi, 2), ", ", size(l.Wi, 1)÷4, ")") @@ -169,7 +169,7 @@ end hidden(m::GRUCell) = m.h -@treelike GRUCell +@functor GRUCell Base.show(io::IO, l::GRUCell) = print(io, "GRUCell(", size(l.Wi, 2), ", ", size(l.Wi, 1)÷3, ")") diff --git a/test/cuda/curnn.jl b/test/cuda/curnn.jl index 1e834d14..fa1a8567 100644 --- a/test/cuda/curnn.jl +++ b/test/cuda/curnn.jl @@ -13,7 +13,7 @@ end @testset "RNN" begin @testset for R in [RNN, GRU, LSTM], batch_size in (1, 5) rnn = R(10, 5) - curnn = mapleaves(gpu, rnn) + curnn = fmap(gpu, rnn) Flux.reset!(rnn) Flux.reset!(curnn) From b60df53ba1beb79be342c304787b6021c73a0883 Mon Sep 17 00:00:00 2001 From: Mike Innes Date: Thu, 19 Sep 2019 18:33:33 +0100 Subject: [PATCH 10/24] pkg up --- Manifest.toml | 20 +++++++++++--------- Project.toml | 1 + src/Flux.jl | 2 +- test/cuda/cudnn.jl | 10 +++++----- test/cuda/curnn.jl | 6 +++--- test/layers/normalisation.jl | 4 ++-- test/layers/stateless.jl | 2 +- 7 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 2d1af7e8..c524a684 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -46,9 +46,9 @@ version = "0.6.2" [[CUDAapi]] deps = ["Libdl", "Logging"] -git-tree-sha1 = "9b2b4b71d6b7f946c9689bb4dea03ff92e3c7091" +git-tree-sha1 = "e063efb91cfefd7e6afd92c435d01398107a500b" uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3" -version = "1.1.0" +version = "1.2.0" [[CUDAdrv]] deps = ["CUDAapi", "Libdl", "Printf"] @@ -147,9 +147,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[FFTW]] deps = ["AbstractFFTs", "BinaryProvider", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"] -git-tree-sha1 = "03f8776fbdae28c20c0d1d2ae4e090cd1dfcd247" +git-tree-sha1 = "6c5b420da0b8c12098048561b8d58f81adea506f" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.0.0" +version = "1.0.1" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays"] @@ -170,9 +170,9 @@ version = "0.10.3" [[GPUArrays]] deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "StaticArrays", "Test"] -git-tree-sha1 = "b5009ac44b141ded5e6f04c4db83807970f56e91" +git-tree-sha1 = "77e27264276fe97a7e7fb928bf8999a145abc018" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "1.0.2" +version = "1.0.3" [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] @@ -388,7 +388,7 @@ version = "0.8.3" [[Zygote]] deps = ["DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "ce6d7142d665b1e4c71c678fa7db4da3bbc6743f" +git-tree-sha1 = "38241b40ebd8748bcacad5e6c7ba3ab3cc7a15c9" repo-rev = "master" repo-url = "https://github.com/FluxML/Zygote.jl.git" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" @@ -396,6 +396,8 @@ version = "0.3.4" [[ZygoteRules]] deps = ["MacroTools"] -git-tree-sha1 = "def5f96ac2895fd9b48435f6b97020979ee0a4c6" +git-tree-sha1 = "c4c29b30b8ff3be13d4244e78be7df2a42bc54d0" +repo-rev = "master" +repo-url = "https://github.com/FluxML/ZygoteRules.jl.git" uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.1.0" +version = "0.2.0" diff --git a/Project.toml b/Project.toml index 2fcdc943..a55d1385 100644 --- a/Project.toml +++ b/Project.toml @@ -24,6 +24,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" [compat] CUDAapi = "1.1" diff --git a/src/Flux.jl b/src/Flux.jl index 9d1fbfc5..a46dc383 100644 --- a/src/Flux.jl +++ b/src/Flux.jl @@ -6,7 +6,7 @@ using Base: tail using Zygote, MacroTools, Juno, Reexport, Statistics, Random using MacroTools: @forward @reexport using NNlib -using Zygote: Params, @adjoint, gradient, forward +using Zygote: Params, @adjoint, gradient, pullback export gradient export Chain, Dense, Maxout, RNN, LSTM, GRU, Conv, CrossCor, ConvTranspose, MaxPool, MeanPool, diff --git a/test/cuda/cudnn.jl b/test/cuda/cudnn.jl index a7fc244e..881e0b39 100644 --- a/test/cuda/cudnn.jl +++ b/test/cuda/cudnn.jl @@ -1,5 +1,5 @@ using Flux, CuArrays, Test -using Flux: forward +using Flux: pullback @testset "CUDNN BatchNorm" begin @testset "4D Input" begin @@ -8,8 +8,8 @@ using Flux: forward cx = gpu(x) cm = gpu(m) - y, back = forward((m, x) -> m(x), m, x) - cy, cback = forward((m, x) -> m(x), cm, cx) + y, back = pullback((m, x) -> m(x), m, x) + cy, cback = pullback((m, x) -> m(x), cm, cx) @test cpu(cy) ≈ y @@ -28,8 +28,8 @@ using Flux: forward cx = gpu(x) cm = gpu(m) - y, back = forward((m, x) -> m(x), m, x) - cy, cback = forward((m, x) -> m(x), cm, cx) + y, back = pullback((m, x) -> m(x), m, x) + cy, cback = pullback((m, x) -> m(x), cm, cx) @test cpu(cy) ≈ y diff --git a/test/cuda/curnn.jl b/test/cuda/curnn.jl index 1e834d14..47ec8509 100644 --- a/test/cuda/curnn.jl +++ b/test/cuda/curnn.jl @@ -1,5 +1,5 @@ using Flux, CuArrays, Test -using Flux: forward +using Flux: pullback @testset for R in [RNN, GRU, LSTM] m = R(10, 5) |> gpu @@ -22,8 +22,8 @@ end rand(10, batch_size) cux = gpu(x) - y, back = forward((r, x) -> (r(x)), rnn, x) - cuy, cuback = forward((r, x) -> (r(x)), curnn, cux) + y, back = pullback((r, x) -> (r(x)), rnn, x) + cuy, cuback = pullback((r, x) -> (r(x)), curnn, cux) @test y ≈ collect(cuy) @test haskey(Flux.CUDA.descs, curnn.cell) diff --git a/test/layers/normalisation.jl b/test/layers/normalisation.jl index cda0cc59..d940b5ab 100644 --- a/test/layers/normalisation.jl +++ b/test/layers/normalisation.jl @@ -1,7 +1,7 @@ using Flux, Test, Statistics -using Zygote: forward +using Zygote: pullback -trainmode(f, x...) = forward(f, x...)[1] +trainmode(f, x...) = pullback(f, x...)[1] trainmode(f) = (x...) -> trainmode(f, x...) @testset "Dropout" begin diff --git a/test/layers/stateless.jl b/test/layers/stateless.jl index b853fc19..9e01af07 100644 --- a/test/layers/stateless.jl +++ b/test/layers/stateless.jl @@ -55,7 +55,7 @@ const ϵ = 1e-7 y = rand(T, 2) ŷ = rand(T, 2) for f in (mse, crossentropy, logitcrossentropy) - fwd, back = Flux.forward(f, ŷ, y) + fwd, back = Flux.pullback(f, ŷ, y) @test fwd isa T @test eltype(back(one(T))[1]) == T end From 787097f9ea0d34f27462bf80ad28399573ca1b2e Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Sat, 21 Sep 2019 00:20:54 +0530 Subject: [PATCH 11/24] use CuArrays#stable --- Manifest.toml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 17eb544e..4480dcfd 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -58,9 +58,9 @@ version = "3.1.0" [[CUDAnative]] deps = ["Adapt", "CUDAapi", "CUDAdrv", "DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Printf", "TimerOutputs"] -git-tree-sha1 = "0a00bef482b7c9127495c7f4a2a85e73b13b5af8" +git-tree-sha1 = "52ae1ce10ebfa686e227655c47b19add89308623" uuid = "be33ccc6-a3ff-5ff2-a52e-74243cff1e17" -version = "2.3.0" +version = "2.3.1" [[CodecZlib]] deps = ["BinaryProvider", "Libdl", "TranscodingStreams"] @@ -106,7 +106,9 @@ version = "4.0.0" [[CuArrays]] deps = ["AbstractFFTs", "Adapt", "CUDAapi", "CUDAdrv", "CUDAnative", "GPUArrays", "LinearAlgebra", "MacroTools", "NNlib", "Printf", "Random", "Requires", "SparseArrays", "TimerOutputs"] -git-tree-sha1 = "46b48742a84bb839e74215b7e468a4a1c6ba30f9" +git-tree-sha1 = "de756b0ed9ffe17890ce77b59bc76b10f96747e7" +repo-rev = "stable" +repo-url = "https://github.com/JuliaGPU/CuArrays.jl.git" uuid = "3a865a2d-5b23-5a0f-bc46-62713ec82fae" version = "1.2.1" @@ -147,9 +149,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[FFTW]] deps = ["AbstractFFTs", "BinaryProvider", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"] -git-tree-sha1 = "e1a479d3c972f20c9a70563eec740bbfc786f515" +git-tree-sha1 = "6c5b420da0b8c12098048561b8d58f81adea506f" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "0.3.0" +version = "1.0.1" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays"] @@ -170,9 +172,9 @@ version = "0.10.3" [[GPUArrays]] deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "StaticArrays", "Test"] -git-tree-sha1 = "dd169c636d1d3656a9faca772f5bd7c226a61254" +git-tree-sha1 = "77e27264276fe97a7e7fb928bf8999a145abc018" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "1.0.1" +version = "1.0.3" [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] @@ -198,9 +200,9 @@ version = "0.7.2" [[LLVM]] deps = ["CEnum", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "52cfea426bd248a427aace7d88eb5d45b84ea297" +git-tree-sha1 = "4a05f742837779a00bd8c9a18da6817367c4245d" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "1.2.0" +version = "1.3.0" [[LibGit2]] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" @@ -264,7 +266,7 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "0.3.7" [[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[Printf]] From 6846551f5756be70432c7bd89d107f7e690e59df Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Sun, 22 Sep 2019 22:02:05 +0530 Subject: [PATCH 12/24] fix cuda init --- Manifest.toml | 16 ++++++++-------- Project.toml | 1 + src/cuda/curnn.jl | 12 +++++++----- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 4480dcfd..c524a684 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -46,9 +46,9 @@ version = "0.6.2" [[CUDAapi]] deps = ["Libdl", "Logging"] -git-tree-sha1 = "9b2b4b71d6b7f946c9689bb4dea03ff92e3c7091" +git-tree-sha1 = "e063efb91cfefd7e6afd92c435d01398107a500b" uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3" -version = "1.1.0" +version = "1.2.0" [[CUDAdrv]] deps = ["CUDAapi", "Libdl", "Printf"] @@ -106,9 +106,7 @@ version = "4.0.0" [[CuArrays]] deps = ["AbstractFFTs", "Adapt", "CUDAapi", "CUDAdrv", "CUDAnative", "GPUArrays", "LinearAlgebra", "MacroTools", "NNlib", "Printf", "Random", "Requires", "SparseArrays", "TimerOutputs"] -git-tree-sha1 = "de756b0ed9ffe17890ce77b59bc76b10f96747e7" -repo-rev = "stable" -repo-url = "https://github.com/JuliaGPU/CuArrays.jl.git" +git-tree-sha1 = "46b48742a84bb839e74215b7e468a4a1c6ba30f9" uuid = "3a865a2d-5b23-5a0f-bc46-62713ec82fae" version = "1.2.1" @@ -390,7 +388,7 @@ version = "0.8.3" [[Zygote]] deps = ["DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "9186cb0b3b59219e4aba0840614d6a9d7282012e" +git-tree-sha1 = "38241b40ebd8748bcacad5e6c7ba3ab3cc7a15c9" repo-rev = "master" repo-url = "https://github.com/FluxML/Zygote.jl.git" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" @@ -398,6 +396,8 @@ version = "0.3.4" [[ZygoteRules]] deps = ["MacroTools"] -git-tree-sha1 = "def5f96ac2895fd9b48435f6b97020979ee0a4c6" +git-tree-sha1 = "c4c29b30b8ff3be13d4244e78be7df2a42bc54d0" +repo-rev = "master" +repo-url = "https://github.com/FluxML/ZygoteRules.jl.git" uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.1.0" +version = "0.2.0" diff --git a/Project.toml b/Project.toml index 2fcdc943..a55d1385 100644 --- a/Project.toml +++ b/Project.toml @@ -24,6 +24,7 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" +ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" [compat] CUDAapi = "1.1" diff --git a/src/cuda/curnn.jl b/src/cuda/curnn.jl index ca8b5140..2063b382 100644 --- a/src/cuda/curnn.jl +++ b/src/cuda/curnn.jl @@ -83,12 +83,14 @@ function rnnWorkspaceSize(r::RNNDesc, seqlen, xdesc) return Int(size[]) end -const workspace = [CuVector{UInt8}(undef, 1)] +const workspace = Ref{Union{Nothing,CuVector{UInt8}}}(nothing) -getworkspace(bytes) = - length(workspace[]) ≥ bytes ? - workspace[] : - (workspace[] = CuVector{UInt8}(undef, bytes)) +function getworkspace(bytes) + if workspace[] === nothing || length(workspace[]) < bytes + workspace[] = CuVector{UInt8}(undef, bytes) + end + workspace[] +end getworkspace(r::RNNDesc, seqlen, xdesc) = getworkspace(rnnWorkspaceSize(r, seqlen, xdesc)) From 783ae137e125a9636410b9a823a8efe72feb49dd Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 23 Sep 2019 16:51:11 +0530 Subject: [PATCH 13/24] remove targets and env --- .gitlab-ci.yml | 21 ++++++--------------- Manifest.toml | 30 ++++++++++++++---------------- Project.toml | 1 - 3 files changed, 20 insertions(+), 32 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e0fa64e..fffa1d7f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,19 +5,10 @@ variables: CI_IMAGE_TAG: 'cuda' include: - - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v3/common.yml' + - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.1.yml' + - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.2.yml' + - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.3.yml' + - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_dev.yml' -test:v1.0: - extends: .test - variables: - CI_VERSION_TAG: 'v1.0' - -test:v1.1: - extends: .test - variables: - CI_VERSION_TAG: 'v1.1' - -test:v1.2: - extends: .test - variables: - CI_VERSION_TAG: 'v1.2' +test:dev: + allow_failure: true diff --git a/Manifest.toml b/Manifest.toml index c524a684..17eb544e 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -46,9 +46,9 @@ version = "0.6.2" [[CUDAapi]] deps = ["Libdl", "Logging"] -git-tree-sha1 = "e063efb91cfefd7e6afd92c435d01398107a500b" +git-tree-sha1 = "9b2b4b71d6b7f946c9689bb4dea03ff92e3c7091" uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3" -version = "1.2.0" +version = "1.1.0" [[CUDAdrv]] deps = ["CUDAapi", "Libdl", "Printf"] @@ -58,9 +58,9 @@ version = "3.1.0" [[CUDAnative]] deps = ["Adapt", "CUDAapi", "CUDAdrv", "DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Printf", "TimerOutputs"] -git-tree-sha1 = "52ae1ce10ebfa686e227655c47b19add89308623" +git-tree-sha1 = "0a00bef482b7c9127495c7f4a2a85e73b13b5af8" uuid = "be33ccc6-a3ff-5ff2-a52e-74243cff1e17" -version = "2.3.1" +version = "2.3.0" [[CodecZlib]] deps = ["BinaryProvider", "Libdl", "TranscodingStreams"] @@ -147,9 +147,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[FFTW]] deps = ["AbstractFFTs", "BinaryProvider", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"] -git-tree-sha1 = "6c5b420da0b8c12098048561b8d58f81adea506f" +git-tree-sha1 = "e1a479d3c972f20c9a70563eec740bbfc786f515" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.0.1" +version = "0.3.0" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays"] @@ -170,9 +170,9 @@ version = "0.10.3" [[GPUArrays]] deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "StaticArrays", "Test"] -git-tree-sha1 = "77e27264276fe97a7e7fb928bf8999a145abc018" +git-tree-sha1 = "dd169c636d1d3656a9faca772f5bd7c226a61254" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "1.0.3" +version = "1.0.1" [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] @@ -198,9 +198,9 @@ version = "0.7.2" [[LLVM]] deps = ["CEnum", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "4a05f742837779a00bd8c9a18da6817367c4245d" +git-tree-sha1 = "52cfea426bd248a427aace7d88eb5d45b84ea297" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "1.3.0" +version = "1.2.0" [[LibGit2]] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" @@ -264,7 +264,7 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "0.3.7" [[Pkg]] -deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[Printf]] @@ -388,7 +388,7 @@ version = "0.8.3" [[Zygote]] deps = ["DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "38241b40ebd8748bcacad5e6c7ba3ab3cc7a15c9" +git-tree-sha1 = "9186cb0b3b59219e4aba0840614d6a9d7282012e" repo-rev = "master" repo-url = "https://github.com/FluxML/Zygote.jl.git" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" @@ -396,8 +396,6 @@ version = "0.3.4" [[ZygoteRules]] deps = ["MacroTools"] -git-tree-sha1 = "c4c29b30b8ff3be13d4244e78be7df2a42bc54d0" -repo-rev = "master" -repo-url = "https://github.com/FluxML/ZygoteRules.jl.git" +git-tree-sha1 = "def5f96ac2895fd9b48435f6b97020979ee0a4c6" uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.2.0" +version = "0.1.0" diff --git a/Project.toml b/Project.toml index a55d1385..2fcdc943 100644 --- a/Project.toml +++ b/Project.toml @@ -24,7 +24,6 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" ZipFile = "a5390f91-8eb1-5f08-bee0-b1d1ffed6cea" Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" -ZygoteRules = "700de1a5-db45-46bc-99cf-38207098b444" [compat] CUDAapi = "1.1" From 98308a85ea69a3be68bf4c69231b8bee43de014c Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Mon, 23 Sep 2019 16:55:53 +0530 Subject: [PATCH 14/24] add gitlab common yaml --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fffa1d7f..a7f471e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: CI_IMAGE_TAG: 'cuda' include: + - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/common.yml' - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.1.yml' - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.2.yml' - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.3.yml' From d8a069b3042d504a5eec25b5d439611358bfa234 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Tue, 24 Sep 2019 00:28:52 +0530 Subject: [PATCH 15/24] fix env --- Manifest.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 17eb544e..2d1af7e8 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -58,9 +58,9 @@ version = "3.1.0" [[CUDAnative]] deps = ["Adapt", "CUDAapi", "CUDAdrv", "DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Printf", "TimerOutputs"] -git-tree-sha1 = "0a00bef482b7c9127495c7f4a2a85e73b13b5af8" +git-tree-sha1 = "52ae1ce10ebfa686e227655c47b19add89308623" uuid = "be33ccc6-a3ff-5ff2-a52e-74243cff1e17" -version = "2.3.0" +version = "2.3.1" [[CodecZlib]] deps = ["BinaryProvider", "Libdl", "TranscodingStreams"] @@ -147,9 +147,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[FFTW]] deps = ["AbstractFFTs", "BinaryProvider", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"] -git-tree-sha1 = "e1a479d3c972f20c9a70563eec740bbfc786f515" +git-tree-sha1 = "03f8776fbdae28c20c0d1d2ae4e090cd1dfcd247" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "0.3.0" +version = "1.0.0" [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays"] @@ -170,9 +170,9 @@ version = "0.10.3" [[GPUArrays]] deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "StaticArrays", "Test"] -git-tree-sha1 = "dd169c636d1d3656a9faca772f5bd7c226a61254" +git-tree-sha1 = "b5009ac44b141ded5e6f04c4db83807970f56e91" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "1.0.1" +version = "1.0.2" [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] @@ -198,9 +198,9 @@ version = "0.7.2" [[LLVM]] deps = ["CEnum", "Libdl", "Printf", "Unicode"] -git-tree-sha1 = "52cfea426bd248a427aace7d88eb5d45b84ea297" +git-tree-sha1 = "4a05f742837779a00bd8c9a18da6817367c4245d" uuid = "929cbde3-209d-540e-8aea-75f648917ca0" -version = "1.2.0" +version = "1.3.0" [[LibGit2]] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" @@ -264,7 +264,7 @@ uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "0.3.7" [[Pkg]] -deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] +deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" [[Printf]] @@ -388,7 +388,7 @@ version = "0.8.3" [[Zygote]] deps = ["DiffRules", "FFTW", "FillArrays", "ForwardDiff", "IRTools", "InteractiveUtils", "LinearAlgebra", "MacroTools", "NNlib", "NaNMath", "Random", "Requires", "SpecialFunctions", "Statistics", "ZygoteRules"] -git-tree-sha1 = "9186cb0b3b59219e4aba0840614d6a9d7282012e" +git-tree-sha1 = "ce6d7142d665b1e4c71c678fa7db4da3bbc6743f" repo-rev = "master" repo-url = "https://github.com/FluxML/Zygote.jl.git" uuid = "e88e6eb3-aa80-5325-afca-941959d7151f" From 928b5dcc2a3623e0feefa7847477a699c1cfbf21 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Tue, 24 Sep 2019 00:51:35 +0530 Subject: [PATCH 16/24] fix Zygote --- Manifest.toml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index a7764294..c524a684 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -147,15 +147,9 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" [[FFTW]] deps = ["AbstractFFTs", "BinaryProvider", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"] -<<<<<<< HEAD -git-tree-sha1 = "03f8776fbdae28c20c0d1d2ae4e090cd1dfcd247" -uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" -version = "1.0.0" -======= git-tree-sha1 = "6c5b420da0b8c12098048561b8d58f81adea506f" uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" version = "1.0.1" ->>>>>>> upstream/master [[FillArrays]] deps = ["LinearAlgebra", "Random", "SparseArrays"] @@ -176,15 +170,9 @@ version = "0.10.3" [[GPUArrays]] deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "StaticArrays", "Test"] -<<<<<<< HEAD -git-tree-sha1 = "b5009ac44b141ded5e6f04c4db83807970f56e91" -uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" -version = "1.0.2" -======= git-tree-sha1 = "77e27264276fe97a7e7fb928bf8999a145abc018" uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7" version = "1.0.3" ->>>>>>> upstream/master [[IRTools]] deps = ["InteractiveUtils", "MacroTools", "Test"] From fe4ecc588035d2c4ec075cb589d139af67d8fc1e Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Tue, 24 Sep 2019 16:15:48 +0530 Subject: [PATCH 17/24] trying out extending directly --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a7f471e0..9173bd74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,19 @@ variables: include: - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/common.yml' + +.test: + extends: .test + script: + - julia -e 'using InteractiveUtils; + versioninfo()' + - mkdir $JULIA_DEPOT_PATH # Pkg3.jl#325 + - julia --project -e 'using Pkg; + Pkg.instantiate(); + Pkg.build(); + Pkg.test(; coverage=true);' + +include: - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.1.yml' - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.2.yml' - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.3.yml' From cf593a5744a06cd3809d138d27567de371221d02 Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Tue, 24 Sep 2019 16:43:48 +0530 Subject: [PATCH 18/24] revert to custom target --- .gitlab-ci.yml | 35 +++++++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9173bd74..9af14c6a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -7,7 +7,7 @@ variables: include: - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/common.yml' -.test: +.flux: extends: .test script: - julia -e 'using InteractiveUtils; @@ -18,11 +18,34 @@ include: Pkg.build(); Pkg.test(; coverage=true);' -include: - - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.1.yml' - - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.2.yml' - - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_v1.3.yml' - - 'https://raw.githubusercontent.com/JuliaGPU/gitlab-ci/master/templates/v4/test_dev.yml' +test:v1.0: + extends: .flux + variables: + CI_VERSION_TAG: 'v1.0' + +test:v1.1: + extends: .flux + variables: + CI_VERSION_TAG: 'v1.1' + +test:v1.2: + extends: .flux + variables: + CI_VERSION_TAG: 'v1.2' + +test:v1.3: + extends: .flux + variables: + CI_VERSION_TAG: 'v1.3' + +test:v1.0: + extends: .flux + variables: + CI_VERSION_TAG: 'v1.0' test:dev: + extends: .flux + variables: + CI_VERSION_TAG: 'dev' + allow_failure: true From ce910da948ee2ec33387fc34237fb2e0edb7231a Mon Sep 17 00:00:00 2001 From: Dhairya Gandhi Date: Tue, 24 Sep 2019 17:04:13 +0530 Subject: [PATCH 19/24] compat julia v1.0 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index a55d1385..609af949 100644 --- a/Project.toml +++ b/Project.toml @@ -31,7 +31,7 @@ CUDAapi = "1.1" CuArrays = "1.2" NNlib = "0.6" Zygote = "0.3" -julia = "1.1" +julia = "1" [extras] Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" From 19830c71b1d821efd6c6bf2d775f526637968dcd Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 25 Sep 2019 13:37:01 +0200 Subject: [PATCH 20/24] fix printing of SkipConnection --- src/layers/basic.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 67490472..66df1275 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -190,9 +190,9 @@ function (mo::Maxout)(input::AbstractArray) end """ - SkipConnection(layers...) + SkipConnection(layers, connection) -Creates a Skip Connection, which constitutes of a layer or Chain of consecutive layers +Creates a Skip Connection, which constitutes of a layer or `Chain` of consecutive layers and a shortcut connection linking the input to the block to the output through a user-supplied callable. @@ -200,7 +200,7 @@ output through a user-supplied callable. A 'ResNet'-type skip-connection with identity shortcut would simply be ```julia - SkipConnection(layer, (a,b) -> a + b) + SkipConnection(layer, +) ``` """ struct SkipConnection @@ -217,6 +217,7 @@ end function Base.show(io::IO, b::SkipConnection) print(io, "SkipConnection(") - join(io, b.layers, ", ") + b.layers isa Chain ? join(io, b.layers, ", ") : print(io, b.layers) + print(io, ",", b.connection) print(io, ")") end From 1a1a96571a80189d16880f944177f769c07ce911 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 25 Sep 2019 13:47:29 +0200 Subject: [PATCH 21/24] +Chain --- src/layers/basic.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 66df1275..e3fb605b 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -217,7 +217,7 @@ end function Base.show(io::IO, b::SkipConnection) print(io, "SkipConnection(") - b.layers isa Chain ? join(io, b.layers, ", ") : print(io, b.layers) - print(io, ",", b.connection) - print(io, ")") + b.layers isa Chain ? print(io, "Chain(", join(b.layers, ", "), "), ") : + print(io, b.layers, ", ") + print(io, b.connection, ")") end From 2de84ce79feb05cdcb12054b8c1b067489ae9230 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 25 Sep 2019 13:59:32 +0200 Subject: [PATCH 22/24] simplify --- src/layers/basic.jl | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index e3fb605b..39f0d759 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -211,13 +211,9 @@ end @functor SkipConnection function (skip::SkipConnection)(input) - #We apply the layers to the input and return the result of the application of the layers and the original input skip.connection(skip.layers(input), input) end function Base.show(io::IO, b::SkipConnection) - print(io, "SkipConnection(") - b.layers isa Chain ? print(io, "Chain(", join(b.layers, ", "), "), ") : - print(io, b.layers, ", ") - print(io, b.connection, ")") + print(io, "SkipConnection(", b.layers, ", ", b.connection, ")") end From 4245d9acad5210adc01448c48a2879fd4b8bf174 Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 25 Sep 2019 15:18:40 +0200 Subject: [PATCH 23/24] eg --- src/layers/basic.jl | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 39f0d759..060c8949 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -192,17 +192,23 @@ end """ SkipConnection(layers, connection) -Creates a Skip Connection, which constitutes of a layer or `Chain` of consecutive layers -and a shortcut connection linking the input to the block to the -output through a user-supplied callable. +Creates a Skip Connection, of a layer or `Chain` of consecutive layers +plus a shortcut connection. The connection function will combine the result of the layers +with the original input, to give the final output. -`SkipConnection` requires the output dimension to be the same as the input. +The simplest 'ResNet'-type connection is just `SkipConnection(layer, +)`, +and requires the output of the layers to be the same shape as the input. +Here is a more complicated example: +``` +m = Conv((3,3), 4=>7, pad=(1,1)) +x = ones(5,5,4,10); +size(m(x)) == (5, 5, 7, 10) -A 'ResNet'-type skip-connection with identity shortcut would simply be -```julia - SkipConnection(layer, +) +sm = SkipConnection(m, (mx, x) -> cat(mx, x, dims=3)) +size(sm(x)) == (5, 5, 11, 10) ``` """ +function SkipConnection end struct SkipConnection layers connection #user can pass arbitrary connections here, such as (a,b) -> a + b From 806e0c5c57f71cc97cdbb149f45c6f9473a57d5d Mon Sep 17 00:00:00 2001 From: Michael Abbott <32575566+mcabbott@users.noreply.github.com> Date: Wed, 25 Sep 2019 15:20:13 +0200 Subject: [PATCH 24/24] line --- src/layers/basic.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/layers/basic.jl b/src/layers/basic.jl index 060c8949..f42a9619 100644 --- a/src/layers/basic.jl +++ b/src/layers/basic.jl @@ -208,7 +208,6 @@ sm = SkipConnection(m, (mx, x) -> cat(mx, x, dims=3)) size(sm(x)) == (5, 5, 11, 10) ``` """ -function SkipConnection end struct SkipConnection layers connection #user can pass arbitrary connections here, such as (a,b) -> a + b