remove all
This commit is contained in:
Mike J Innes
parent
1010b18084
commit
dcbc77980e
5
.gitignore
vendored
5
.gitignore
vendored
@ -1,5 +0,0 @@
|
||||
*.jl.cov
|
||||
*.jl.*.cov
|
||||
*.jl.mem
|
||||
docs/build/
|
||||
docs/site/
|
||||
17
.travis.yml
17
.travis.yml
@ -1,17 +0,0 @@
|
||||
# Documentation: http://docs.travis-ci.com/user/languages/julia/
|
||||
language: julia
|
||||
os:
|
||||
- linux
|
||||
- osx
|
||||
julia:
|
||||
- 0.5
|
||||
notifications:
|
||||
email: false
|
||||
# uncomment the following lines to override the default test script
|
||||
script:
|
||||
- if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
|
||||
- julia -e 'Pkg.clone("https://github.com/MikeInnes/DataFlow.jl")'
|
||||
- julia -e 'Pkg.clone(pwd()); Pkg.build("Flux"); Pkg.test("Flux"; coverage=true)'
|
||||
# after_success:
|
||||
- julia -e 'Pkg.add("Documenter")'
|
||||
- julia -e 'cd(Pkg.dir("Flux")); include(joinpath("docs", "make.jl"))'
|
||||
22
LICENSE.md
22
LICENSE.md
@ -1,22 +0,0 @@
|
||||
The Flux.jl package is licensed under the MIT "Expat" License:
|
||||
|
||||
> Copyright (c) 2016: Mike Innes.
|
||||
>
|
||||
> Permission is hereby granted, free of charge, to any person obtaining
|
||||
> a copy of this software and associated documentation files (the
|
||||
> "Software"), to deal in the Software without restriction, including
|
||||
> without limitation the rights to use, copy, modify, merge, publish,
|
||||
> distribute, sublicense, and/or sell copies of the Software, and to
|
||||
> permit persons to whom the Software is furnished to do so, subject to
|
||||
> the following conditions:
|
||||
>
|
||||
> The above copyright notice and this permission notice shall be
|
||||
> included in all copies or substantial portions of the Software.
|
||||
>
|
||||
> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
> EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
> MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
> IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
||||
> CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
||||
> TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
||||
> SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
50
README.md
50
README.md
@ -1,50 +0,0 @@
|
||||
# Флукс
|
||||
|
||||
[](https://travis-ci.org/MikeInnes/Flux.jl)
|
||||
|
||||
Flux is a high-level API for machine learning, implemented in Julia.
|
||||
|
||||
Flux aims to provide a concise and expressive syntax for architectures that are hard to express within other frameworks. The notation should be familiar and extremely close to what you'd find in a paper or description of the model.
|
||||
|
||||
The current focus is on ANNs with TensorFlow or MXNet as a backend. While it's in a very early working-prototype stage, you can see what works so far in the [examples folder](/examples).
|
||||
|
||||
## Brief Examples
|
||||
|
||||
Simple multi-layer-perceptron for MNIST:
|
||||
|
||||
```julia
|
||||
Chain(
|
||||
Input(784),
|
||||
Affine(128), relu,
|
||||
Affine( 64), relu,
|
||||
Affine( 10), softmax)
|
||||
```
|
||||
|
||||
LSTM example:
|
||||
|
||||
```julia
|
||||
@net type LSTM
|
||||
Wxf; Wyf; bf
|
||||
Wxi; Wyi; bi
|
||||
Wxo; Wyo; bo
|
||||
Wxc; Wyc; bc
|
||||
y; state
|
||||
function (x)
|
||||
# Gates
|
||||
forget = σ( x * Wxf + y{-1} * Wyf + bf )
|
||||
input = σ( x * Wxi + y{-1} * Wyi + bi )
|
||||
output = σ( x * Wxo + y{-1} * Wyo + bo )
|
||||
# State update and output
|
||||
state′ = tanh( x * Wxc + y{-1} * Wyc + bc )
|
||||
state = forget .* state{-1} + input .* state′
|
||||
y = output .* tanh(state)
|
||||
end
|
||||
end
|
||||
|
||||
Chain(
|
||||
Input(N),
|
||||
LSTM(N, 256),
|
||||
LSTM(256, 256),
|
||||
Affine(256, N),
|
||||
softmax)
|
||||
```
|
||||
17
docs/make.jl
17
docs/make.jl
@ -1,17 +0,0 @@
|
||||
using Documenter, Flux
|
||||
|
||||
makedocs(modules=Module[Flux],
|
||||
doctest=false, clean=true,
|
||||
format = :html,
|
||||
sitename="Flux Documentation",
|
||||
pages = [
|
||||
"Home" => "index.md",
|
||||
])
|
||||
|
||||
deploydocs(
|
||||
repo = "github.com/MikeInnes/Flux.jl.git",
|
||||
target = "build",
|
||||
osname = "linux",
|
||||
julia = "0.5",
|
||||
deps = nothing,
|
||||
make = nothing)
|
||||
@ -1 +0,0 @@
|
||||
# Flux
|
||||
@ -1,22 +0,0 @@
|
||||
using Flux, MNIST
|
||||
|
||||
data = [(Vector{Float32}(trainfeatures(i)), onehot(Float32, trainlabel(i), 0:9)) for i = 1:60_000]
|
||||
train = data[1:50_000]
|
||||
test = data[50_001:60_000]
|
||||
|
||||
m = Chain(
|
||||
Input(784),
|
||||
Affine(128), relu,
|
||||
Affine( 64), relu,
|
||||
Affine( 10), softmax)
|
||||
|
||||
# Convert to TensorFlow
|
||||
model = tf(m)
|
||||
|
||||
# An example prediction pre-training
|
||||
model(data[1][1])
|
||||
|
||||
@time Flux.train!(model, train, test, η = 1e-3)
|
||||
|
||||
# An example prediction post-training
|
||||
model(data[1][1])
|
||||
@ -1,26 +0,0 @@
|
||||
using Flux
|
||||
|
||||
# Traditional Approach
|
||||
|
||||
# 100 samples of sequences of 15 28×28 3-colour images
|
||||
rand(100, 15, 28, 28, 3)
|
||||
|
||||
# Basic Batching
|
||||
|
||||
data = Batch([collect(reshape(9(i-1):9i-1, 3, 3)) for i = 1:10])
|
||||
|
||||
Batch(flatten.(data))
|
||||
|
||||
data |> structure
|
||||
|
||||
Batch(flatten.(data)) |> structure
|
||||
|
||||
# Nested Batching
|
||||
|
||||
# DNA seqence, encoded as a list of [A, T, G, C]
|
||||
x1 = Seq([[0,1,0,0], [1,0,0,0], [0,0,0,1]])
|
||||
x2 = Seq([[0,0,1,0], [0,0,0,1], [0,0,1,0]])
|
||||
|
||||
data = Batch([x1, x2])
|
||||
|
||||
data |> structure
|
||||
@ -1,38 +0,0 @@
|
||||
using Flux
|
||||
import StatsBase: wsample
|
||||
|
||||
nunroll = 50
|
||||
nbatch = 50
|
||||
|
||||
getseqs(chars, alphabet) = sequences((onehot(Float32, char, alphabet) for char in chars), nunroll)
|
||||
getbatches(chars, alphabet) = batches((getseqs(part, alphabet) for part in chunk(chars, nbatch))...)
|
||||
|
||||
input = readstring("$(homedir())/Downloads/shakespeare_input.txt")
|
||||
alphabet = unique(input)
|
||||
N = length(alphabet)
|
||||
|
||||
Xs, Ys = getbatches(input, alphabet), getbatches(input[2:end], alphabet)
|
||||
|
||||
model = Chain(
|
||||
Input(N),
|
||||
LSTM(N, 256),
|
||||
LSTM(256, 256),
|
||||
Affine(256, N),
|
||||
softmax)
|
||||
|
||||
m = tf(unroll(model, nunroll))
|
||||
|
||||
@time Flux.train!(m, Xs, Ys, η = 0.1, epoch = 1)
|
||||
|
||||
string(map(c -> onecold(c, alphabet), m(first(first(Xs))))...)
|
||||
|
||||
function sample(model, n, temp = 1)
|
||||
s = [rand(alphabet)]
|
||||
m = tf(unroll(model, 1))
|
||||
for i = 1:n
|
||||
push!(s, wsample(alphabet, softmax(m(Seq((onehot(Float32, s[end], alphabet),)))[1]./temp)))
|
||||
end
|
||||
return string(s...)
|
||||
end
|
||||
|
||||
sample(model, 100)
|
||||
@ -1,70 +0,0 @@
|
||||
using Flux, Juno
|
||||
|
||||
# Flux aims to provide high-level APIs that work well across backends, but in
|
||||
# some cases you may want to take advantage of features specific to a given
|
||||
# backend (or Flux may simply not have an implementation of that feature yet).
|
||||
# In these cases it's easy to "drop down" and use the backend's API directly,
|
||||
# where appropriate.
|
||||
|
||||
# In this example, both things are happening; firstly, Flux doesn't yet support
|
||||
# ConvNets in the pure-Julia backend, but this is invisible thanks to the use of
|
||||
# a simple "shim" type, `Conv2D`. This is provided by the library but could easily
|
||||
# have been user-defined.
|
||||
|
||||
# Secondly, we want to take advantage of TensorFlow.jl's training process and
|
||||
# optimisers. We can simply call `Tensor` exactly as we would on a regular
|
||||
# TensorFlow model, and the rest of the process trivially follows
|
||||
# TensorFlow.jl's usual API.
|
||||
|
||||
conv1 = Chain(
|
||||
Reshape(28,28,1),
|
||||
Conv2D((5,5), out = 20), tanh,
|
||||
MaxPool((2,2), stride = (2,2)))
|
||||
|
||||
conv2 = Chain(
|
||||
Input(12,12,20),
|
||||
Conv2D((5,5), in = 20, out = 50), tanh,
|
||||
MaxPool((2,2), stride = (2,2)))
|
||||
|
||||
lenet = Chain(
|
||||
conv1, conv2, flatten,
|
||||
Affine(500), tanh,
|
||||
Affine(10), softmax)
|
||||
|
||||
#--------------------------------------------------------------------------------
|
||||
|
||||
# Now we can continue exactly as in plain TensorFlow, following
|
||||
# https://github.com/malmaud/TensorFlow.jl/blob/master/examples/mnist_full.jl
|
||||
# (taking only the training and cost logic, not the graph building steps)
|
||||
|
||||
using TensorFlow, Distributions
|
||||
|
||||
include(Pkg.dir("TensorFlow", "examples", "mnist_loader.jl"))
|
||||
loader = DataLoader()
|
||||
|
||||
session = Session(Graph())
|
||||
|
||||
x = placeholder(Float32)
|
||||
y′ = placeholder(Float32)
|
||||
y = Tensor(lenet, x)
|
||||
|
||||
cross_entropy = reduce_mean(-reduce_sum(y′.*log(y), reduction_indices=[2]))
|
||||
|
||||
train_step = train.minimize(train.AdamOptimizer(1e-4), cross_entropy)
|
||||
|
||||
accuracy = reduce_mean(cast(indmax(y, 2) .== indmax(y′, 2), Float32))
|
||||
|
||||
run(session, initialize_all_variables())
|
||||
|
||||
@progress for i in 1:1000
|
||||
batch = next_batch(loader, 50)
|
||||
if i%100 == 1
|
||||
train_accuracy = run(session, accuracy, Dict(x=>batch[1], y′=>batch[2]))
|
||||
info("step $i, training accuracy $train_accuracy")
|
||||
end
|
||||
run(session, train_step, Dict(x=>batch[1], y′=>batch[2]))
|
||||
end
|
||||
|
||||
testx, testy = load_test_set()
|
||||
test_accuracy = run(session, accuracy, Dict(x=>testx, y′=>testy))
|
||||
info("test accuracy $test_accuracy")
|
||||
@ -1,52 +0,0 @@
|
||||
# Based on https://arxiv.org/abs/1409.0473
|
||||
|
||||
using Flux
|
||||
using Flux: flip
|
||||
|
||||
# A recurrent model which takes a token and returns a context-dependent
|
||||
# annotation.
|
||||
|
||||
@net type Encoder
|
||||
forward
|
||||
backward
|
||||
token -> hcat(forward(token), backward(token))
|
||||
end
|
||||
|
||||
Encoder(in::Integer, out::Integer) =
|
||||
Encoder(LSTM(in, out÷2), flip(LSTM(in, out÷2)))
|
||||
|
||||
# A recurrent model which takes a sequence of annotations, attends, and returns
|
||||
# a predicted output token.
|
||||
|
||||
@net type Decoder
|
||||
attend
|
||||
recur
|
||||
state; y; N
|
||||
function (anns)
|
||||
energies = map(ann -> exp(attend(hcat(state{-1}, ann))[1]), seq(anns, N))
|
||||
weights = energies./sum(energies)
|
||||
ctx = sum(map((α, ann) -> α .* ann, weights, anns))
|
||||
(_, state), y = recur((state{-1},y{-1}), ctx)
|
||||
y
|
||||
end
|
||||
end
|
||||
|
||||
Decoder(in::Integer, out::Integer; N = 1) =
|
||||
Decoder(Affine(in+out, 1),
|
||||
unroll1(LSTM(in, out)),
|
||||
param(zeros(1, out)), param(zeros(1, out)), N)
|
||||
|
||||
# The model
|
||||
|
||||
Nalpha = 5 # The size of the input token vector
|
||||
Nphrase = 7 # The length of (padded) phrases
|
||||
Nhidden = 12 # The size of the hidden state
|
||||
|
||||
encode = Encoder(Nalpha, Nhidden)
|
||||
decode = Chain(Decoder(Nhidden, Nhidden, N = Nphrase), Affine(Nhidden, Nalpha), softmax)
|
||||
|
||||
model = Chain(
|
||||
unroll(encode, Nphrase, stateful = false),
|
||||
unroll(decode, Nphrase, stateful = false, seq = false))
|
||||
|
||||
xs = Batch([Seq(rand(Float32, Nalpha) for _ = 1:Nphrase)])
|
||||
36
src/Flux.jl
36
src/Flux.jl
@ -1,36 +0,0 @@
|
||||
module Flux
|
||||
|
||||
using MacroTools, Lazy, DataFlow, Juno
|
||||
using DataFlow: graphm, syntax, prewalk!, postwalk!, prewalk, postwalk,
|
||||
iscyclic, Constant, constant, isconstant, group, Split, splitnode,
|
||||
detuple, value, inputs, thread!, value, inputs, Split, splitnode, inputnode,
|
||||
spliceinputs, bumpinputs, Frame
|
||||
using Juno: Tree, Row
|
||||
|
||||
# Zero Flux Given
|
||||
|
||||
include("model.jl")
|
||||
include("utils.jl")
|
||||
include("data.jl")
|
||||
|
||||
include("compiler/code.jl")
|
||||
include("compiler/loops.jl")
|
||||
include("compiler/interp.jl")
|
||||
include("compiler/shape.jl")
|
||||
|
||||
include("layers/affine.jl")
|
||||
include("layers/activation.jl")
|
||||
include("layers/recurrent.jl")
|
||||
include("layers/shape.jl")
|
||||
include("layers/chain.jl")
|
||||
include("layers/shims.jl")
|
||||
|
||||
include("dims/catmat.jl")
|
||||
include("dims/batching.jl")
|
||||
include("dims/seq.jl")
|
||||
|
||||
include("cost.jl")
|
||||
|
||||
include("backend/backend.jl")
|
||||
|
||||
end # module
|
||||
@ -1,11 +0,0 @@
|
||||
export tf
|
||||
|
||||
function loadtf()
|
||||
isdefined(Flux, :TF) && return
|
||||
@eval include(joinpath(dirname($@__FILE__), "tensorflow/tensorflow.jl"))
|
||||
end
|
||||
|
||||
function tf(args...)
|
||||
loadtf()
|
||||
TF.tf(args...)
|
||||
end
|
||||
@ -1,72 +0,0 @@
|
||||
using Base: @get!
|
||||
using DataFlow: Constant, constant, Context, interpret, Split,
|
||||
interpv, ituple, ilambda, iconst, iline, stack, mux
|
||||
using Flux: imap
|
||||
using TensorFlow: RawTensor
|
||||
|
||||
# TODO: implement Julia's type promotion rules
|
||||
|
||||
node(x::Tuple) = map(node, x)
|
||||
node(x::Tensor) = x
|
||||
node(x::Variable) = x
|
||||
node(x::Number) = TensorFlow.constant(Float32(x))
|
||||
|
||||
graph(::typeof(tuple), args...) = (args...,)
|
||||
graph(s::Split, t::Tuple) = t[s.n]
|
||||
graph(::typeof(softmax), x) = nn.softmax(x)
|
||||
graph(::typeof(relu), x) = nn.relu(x)
|
||||
graph(::typeof(σ), x) = nn.sigmoid(x)
|
||||
graph(::typeof(hcat), xs...) = concat(1, xs)
|
||||
graph(::typeof(seq), xs, n) = TensorFlow.unpack(xs, num = n, axis = 1)
|
||||
|
||||
for op in (tanh, *, .*, +, -)
|
||||
@eval graph(::typeof($op), args...) = $op(node(args)...)
|
||||
end
|
||||
|
||||
graph(::typeof(.-), args...) = -(node(args)...)
|
||||
|
||||
# reshape hack due to https://github.com/malmaud/TensorFlow.jl/issues/79
|
||||
batchsize(x::Tensor) = reduce_sum(slice(TensorFlow.shape(x), [0], [1]))
|
||||
graph(::typeof(flatten), x) = reshape(x, pack([batchsize(x), Int32(-1)]))
|
||||
graph(r::Reshape, x) = reshape(x, pack([batchsize(x), map(Int32, r.dims)...]))
|
||||
|
||||
graph(::Input, x) = x
|
||||
|
||||
graph(p::MaxPool, x) =
|
||||
nn.max_pool(x, [1, p.size..., 1], [1, p.stride..., 1], "VALID")
|
||||
|
||||
graph(op::Op, xs...) = op.f(xs...)
|
||||
|
||||
function graph(ctx::Context, model, args...)
|
||||
node = graph(model, interpv(ctx, args)...)
|
||||
isa(node, Tensor) && (ctx[:stacks][node.op.name] = stack(ctx))
|
||||
return node
|
||||
end
|
||||
|
||||
interp(ctx, c::Conv2D, x) =
|
||||
nn.conv2d(interpv(ctx, x), interp(ctx, Constant(c.filter)), [1,c.stride...,1], "VALID")
|
||||
|
||||
interp{T<:AArray}(ctx, p::Constant{Flux.Param{T}}) =
|
||||
haskey(ctx[:params], p.value) ?
|
||||
ctx[:params][p.value] :
|
||||
(ctx[:params][p.value] = Variable(p.value.x))
|
||||
|
||||
interp(ctx, p::Constant) = p.value
|
||||
|
||||
function interp(ctx, model, args...)
|
||||
g = Flux.graph(model)
|
||||
g == nothing && return graph(ctx, model, args...)
|
||||
DataFlow.iscyclic(g) && error("This model has a cycle; try unrolling it first.")
|
||||
interpret(ctx, g, interpv(ctx, args)...)
|
||||
end
|
||||
|
||||
function tograph(model, args...)
|
||||
ctx = Context(mux(iline, ilambda, ituple, imap, interp),
|
||||
params = ObjectIdDict(), stacks = Dict())
|
||||
out = interp(ctx, model, map(constant, args)...)
|
||||
return ctx[:params], ctx[:stacks], out
|
||||
end
|
||||
|
||||
TensorFlow.Tensor(m::Flux.Model, args...) = tograph(m, args...)[2]
|
||||
|
||||
RawTensor(data::Union{Batch,Seq}) = RawTensor(rawbatch(data))
|
||||
@ -1,98 +0,0 @@
|
||||
type Model
|
||||
model::Any
|
||||
session::Session
|
||||
params::Dict{Flux.Param,Tensor}
|
||||
stacks::Dict
|
||||
inputs::Vector{Tensor}
|
||||
output::Any
|
||||
end
|
||||
|
||||
function makesession(model, inputs; session = Session(Graph()))
|
||||
params, stacks, output = tograph(model, inputs...)
|
||||
run(session, initialize_all_variables())
|
||||
Model(model, session, params, stacks, inputs, output)
|
||||
end
|
||||
|
||||
function makesession(model, n::Integer; session = Session(Graph()))
|
||||
makesession(model, [placeholder(Float32) for _ = 1:n], session = session)
|
||||
end
|
||||
|
||||
tf(model) = makesession(model, 1)
|
||||
|
||||
function storeparams!(sess, params)
|
||||
for (p, t) in params
|
||||
p.x = run(sess, t)
|
||||
end
|
||||
end
|
||||
|
||||
storeparams!(m::Model) = storeparams!(m.session, m.params)
|
||||
|
||||
ismultioutput(m::Model) = !isa(m.output, Tensor)
|
||||
|
||||
function batch(xs)
|
||||
dims = ndims(xs)-1
|
||||
T = Array{eltype(xs),dims}
|
||||
B = Array{eltype(xs),dims+1}
|
||||
Batch{T,B}(xs)
|
||||
end
|
||||
|
||||
function tferr(model::Model, e)
|
||||
m = match(r"Node: ([\w\d]+) =", string(e.status))
|
||||
m == nothing && return
|
||||
node = m.captures[1]
|
||||
if haskey(model.stacks, node)
|
||||
stk = model.stacks[node]
|
||||
println("TensorFlow error occured at:")
|
||||
foreach(l -> println("$(l.file):$(l.line)"), stk)
|
||||
end
|
||||
end
|
||||
|
||||
function runmodel(m::Model, args...)
|
||||
@assert length(args) == length(m.inputs)
|
||||
try
|
||||
output = run(m.session, m.output, Dict(zip(m.inputs, args)))
|
||||
ismultioutput(m) ? (batch.(output)...,) : batch(output)
|
||||
catch e
|
||||
isa(e, TensorFlow.TFException) || rethrow(e)
|
||||
tferr(m, e)
|
||||
rethrow(e)
|
||||
end
|
||||
end
|
||||
|
||||
function (m::Model)(args::Batch...)
|
||||
runmodel(m, args...)
|
||||
end
|
||||
|
||||
function (m::Model)(args...)
|
||||
output = m(map(batchone, args)...)
|
||||
ismultioutput(m) ? map(first, output) : first(output)
|
||||
end
|
||||
|
||||
for f in :[back!, update!].args
|
||||
@eval function Flux.$f(m::Model, args...)
|
||||
error($(string(f)) * " is not yet supported on TensorFlow models")
|
||||
end
|
||||
end
|
||||
|
||||
import Juno: info
|
||||
|
||||
function Flux.train!(m::Model, train, test=[]; epoch = 1, η = 0.1,
|
||||
loss = (y, y′) -> reduce_sum((y - y′).^2)/2,
|
||||
opt = TensorFlow.train.GradientDescentOptimizer(η))
|
||||
i = 0
|
||||
Y = placeholder(Float32)
|
||||
Loss = loss(m.output, Y)
|
||||
minimize_op = TensorFlow.train.minimize(opt, Loss)
|
||||
for e in 1:epoch
|
||||
info("Epoch $e\n")
|
||||
@progress for (x, y) in train
|
||||
y, cur_loss, _ = run(m.session, vcat(m.output, Loss, minimize_op),
|
||||
Dict(m.inputs[1]=>batchone(x), Y=>batchone(y)))
|
||||
if i % 5000 == 0
|
||||
@show y
|
||||
@show accuracy(m, test)
|
||||
end
|
||||
i += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
@ -1,83 +0,0 @@
|
||||
# TODO: refactor, some of this is more general than just the TF backend
|
||||
|
||||
type SeqModel
|
||||
m::Model
|
||||
state::Any
|
||||
end
|
||||
|
||||
cgroup(xs...) = DataFlow.group(map(constant, xs)...)
|
||||
|
||||
function makesession(model::Flux.Unrolled)
|
||||
sess = Session(Graph())
|
||||
input = placeholder(Float32)
|
||||
inputs = TensorFlow.unpack(input, num = model.steps, axis = 1)
|
||||
let params, stacks, outputs, instates, outstates
|
||||
if model.stateful
|
||||
instates = [placeholder(Float32) for _ in model.state]
|
||||
params, stacks, (outstates, outputs) = tograph(model, cgroup(instates...), cgroup(inputs...))
|
||||
else
|
||||
params, stacks, outputs = tograph(model, cgroup(inputs...))
|
||||
end
|
||||
output = TensorFlow.pack(outputs, axis = 1)
|
||||
run(sess, initialize_all_variables())
|
||||
sess, params, stacks, (instates, input), (outstates, output)
|
||||
end
|
||||
end
|
||||
|
||||
function tf(model::Flux.Unrolled)
|
||||
sess, params, stacks, (instates, input), (outstates, output) = makesession(model)
|
||||
SeqModel(
|
||||
Model(model, sess, params, stacks,
|
||||
[instates..., input], [outstates..., output]),
|
||||
model.state)
|
||||
end
|
||||
|
||||
function batchseq(xs)
|
||||
dims = ndims(xs)-2
|
||||
T = Array{eltype(xs),dims}
|
||||
S = Array{eltype(xs),dims+1}
|
||||
B = Array{eltype(xs),dims+2}
|
||||
Batch{Seq{T,S},B}(xs)
|
||||
end
|
||||
|
||||
batchseq(xs::Batch) = batchseq(rawbatch(xs))
|
||||
|
||||
TensorFlow.get_tensors(x::Tuple) = TensorFlow.get_tensors(collect(x))
|
||||
|
||||
function (m::SeqModel)(x::BatchSeq)
|
||||
m.m.model.stateful || return batchseq(runmodel(m.m, x)[end])
|
||||
if isempty(m.state) || length(first(m.state)) ≠ length(x)
|
||||
m.state = batchone.(m.m.model.state)
|
||||
end
|
||||
output = runmodel(m.m, m.state..., x)
|
||||
m.state, output = output[1:end-1], output[end]
|
||||
return batchseq(output)
|
||||
end
|
||||
|
||||
(m::SeqModel)(x::Seq) = first(m(batchone(x)))
|
||||
|
||||
function Flux.train!(m::SeqModel, Xs, Ys; epoch = 1, η = 0.1,
|
||||
loss = (y, ŷ) -> -reduce_sum(y .* log(ŷ)),
|
||||
opt = () -> TensorFlow.train.GradientDescentOptimizer(η))
|
||||
batchlen, seqlen = length(first(Xs)), length(first(Xs)[1])
|
||||
state = batchone.(m.m.model.state)
|
||||
sess, params, stacks, (instates, input), (outstates, output) = makesession(m.m.model)
|
||||
Y = placeholder(Float32)
|
||||
Loss = loss(Y, output)/batchlen/seqlen
|
||||
minimize_op = TensorFlow.train.minimize(opt(), Loss)
|
||||
@progress "training" for e in 1:epoch
|
||||
info("Epoch $e\n")
|
||||
@progress "epoch" for (i, (x, y)) in enumerate(zip(Xs,Ys))
|
||||
out = run(sess, vcat(outstates..., output, Loss, minimize_op),
|
||||
merge(Dict(input=>batchone(x), Y=>batchone(y)),
|
||||
Dict(zip(instates, state))))
|
||||
state = out[1:length(state)]
|
||||
loss = out[end-1]
|
||||
isnan(loss) && error("Loss is NaN")
|
||||
isinf(loss) && error("Loss is Inf")
|
||||
(i-1) % 10 == 0 && @show loss
|
||||
end
|
||||
end
|
||||
storeparams!(sess, params)
|
||||
return
|
||||
end
|
||||
@ -1,21 +0,0 @@
|
||||
module TF
|
||||
|
||||
using ..Flux, DataFlow, TensorFlow, Juno
|
||||
import Flux: accuracy
|
||||
|
||||
export tf
|
||||
|
||||
type Op
|
||||
f
|
||||
shape
|
||||
end
|
||||
|
||||
Op(f) = Op(f, (d...) -> nothing)
|
||||
|
||||
Flux.shape(op::Op, d...) = op.shape(d...)
|
||||
|
||||
include("graph.jl")
|
||||
include("model.jl")
|
||||
include("recurrent.jl")
|
||||
|
||||
end
|
||||
@ -1,81 +0,0 @@
|
||||
import DataFlow: mapconst, cse
|
||||
|
||||
export @net, @ml
|
||||
|
||||
function process_func(ex, params = [])
|
||||
@capture(shortdef(ex), (args__,) -> body_)
|
||||
body = @> body MacroTools.flatten liftloops graphm DataFlow.il
|
||||
body = mapconst(x -> x in params ? :(self.$x) : x, body)
|
||||
return args, body
|
||||
end
|
||||
|
||||
function makegraph(graph, args)
|
||||
@assert length(args) == 1
|
||||
graph = prewalk(graph) do v
|
||||
isa(value(v), Constant) && value(v).value == args[1] ?
|
||||
inputnode(1) :
|
||||
v
|
||||
end
|
||||
graph = map(graph) do x
|
||||
isa(x, Offset) ?
|
||||
:(Flux.Offset($(Expr(:quote, x.name)), $(x.n), self.$(x.name))) :
|
||||
x
|
||||
end
|
||||
vertex(:(Flux.Frame(self)), graph)
|
||||
end
|
||||
|
||||
function build_type(T, params)
|
||||
@esc T
|
||||
ex = quote
|
||||
type $T <: Model
|
||||
$(params...)
|
||||
end
|
||||
end
|
||||
if any(x->isexpr(x, Symbol), params)
|
||||
push!(ex.args,
|
||||
:($T($(map(x->isexpr(x, Symbol) ? :($x::AArray) : x, params)...)) =
|
||||
$T($(map(x->isexpr(x, Symbol) ? :(param($x)) : namify(x), params)...))))
|
||||
end
|
||||
ex
|
||||
end
|
||||
|
||||
import Lazy: groupby
|
||||
|
||||
reifyparams(v::IVertex) = mapconst(x -> isa(x, Param) ? x.x : x, v)
|
||||
|
||||
function process_type(ex)
|
||||
@capture(ex, type T_ fs__ end)
|
||||
@destruct [params = false || [],
|
||||
funcs = true || []] = groupby(x->isexpr(x, :->, :function), fs)
|
||||
@assert length(funcs) == 1
|
||||
pnames = namify.(params)
|
||||
args, body = process_func(funcs[1], pnames)
|
||||
@assert length(args) == 1
|
||||
self = esc(:self)
|
||||
quote
|
||||
$(build_type(T, params))
|
||||
$(esc(:(self::$T)))($(args...),) = interpret(reifyparams(graph($self)), $(args...))
|
||||
$(esc(:(Flux.update!(self::$T, η)))) = ($(map(p -> :(update!($self.$p, η)), pnames)...);)
|
||||
$(esc(:(Flux.graph(self::$T)))) = $(DataFlow.constructor(mapconst(esc, makegraph(body, args))))
|
||||
nothing
|
||||
end
|
||||
end
|
||||
|
||||
macro net(ex)
|
||||
isexpr(ex, :type) ? process_type(ex) :
|
||||
isexpr(ex, :->, :function) ? error("@net functions not implemented") :
|
||||
error("Unsupported model expression $ex")
|
||||
end
|
||||
|
||||
function process_anon(ex)
|
||||
args, body = process_func(ex)
|
||||
@assert length(args) == 1
|
||||
:(Flux.Capacitor($(DataFlow.constructor(mapconst(esc, makegraph(body, args))))))
|
||||
end
|
||||
|
||||
macro ml(ex)
|
||||
@capture(shortdef(ex), ((xs__,) -> body_ ) | (f_(xs__,) = body_)) ||
|
||||
error("@ml requires a function definition")
|
||||
ex = process_anon(:($(xs...,) -> $body))
|
||||
f == nothing ? ex : :($(esc(f)) = $ex)
|
||||
end
|
||||
@ -1,26 +0,0 @@
|
||||
using DataFlow: mux, interpret, interpv, ituple, ilambda, iconst, Context
|
||||
|
||||
function astuple(xs::Vertex)
|
||||
isconstant(xs) && isa(value(xs).value, Tuple) ? value(xs).value :
|
||||
isa(xs, Vertex) && value(xs) == tuple ? inputs(xs) :
|
||||
nothing
|
||||
end
|
||||
|
||||
astuple(xs::Tuple) = xs
|
||||
|
||||
astuple(xs) = nothing
|
||||
|
||||
function astuples(xs)
|
||||
xs = [astuple(x) for x in xs]
|
||||
all(x->!(x==nothing), xs) ? xs : nothing
|
||||
end
|
||||
|
||||
function imap(cb, ctx, ::typeof(map), f, xs...)
|
||||
f, xs = interpv(ctx, (f, xs))
|
||||
xs′ = astuples(xs)
|
||||
xs′ ≠ nothing ?
|
||||
group(map(f, xs′...)...) :
|
||||
cb(ctx, map, constant(f), xs...)
|
||||
end
|
||||
|
||||
imap(f, args...) = f(args...)
|
||||
@ -1,134 +0,0 @@
|
||||
export unroll, unroll1
|
||||
|
||||
type Offset
|
||||
name::Symbol
|
||||
n::Int
|
||||
default::Nullable{Param}
|
||||
end
|
||||
|
||||
Offset(name, n) = Offset(name, n, nothing)
|
||||
|
||||
Base.:-(o::Offset) = Offset(o.name, -o.n, o.default)
|
||||
|
||||
function liftloops(ex)
|
||||
ex = DataFlow.normedges(ex)
|
||||
decls = Dict()
|
||||
ex = MacroTools.postwalk(ex) do ex
|
||||
@capture(ex, x_{n_}) || return ex
|
||||
haskey(decls, (x,n)) && return namify(decls[(x,n)])
|
||||
@gensym edge
|
||||
decls[(x,n)] = :($edge = $(Offset(x,n))($x))
|
||||
edge
|
||||
end
|
||||
prepend!(ex.args, collect(values(decls)))
|
||||
ex
|
||||
end
|
||||
|
||||
function hasloops(model)
|
||||
g = graph(model)
|
||||
g == nothing && return false
|
||||
iscyclic(g) && return true
|
||||
result = false
|
||||
map(m -> hasloops(m) && (result = true), g)
|
||||
return result
|
||||
end
|
||||
|
||||
function atomise(model)
|
||||
postwalk(graph(model)) do v
|
||||
hasloops(value(v)) || return v
|
||||
spliceinputs(atomise(value(v)), inputs(v)...)
|
||||
end
|
||||
end
|
||||
|
||||
function collect_state(v::IVertex)
|
||||
state = typeof(v)[]
|
||||
offset = Int[]
|
||||
default = Param[]
|
||||
prewalk!(v) do v
|
||||
isa(value(v), Offset) || return v
|
||||
if (i = findfirst(state, v[1])) == 0
|
||||
push!(state, v[1])
|
||||
push!(offset, max(0, -value(v).n))
|
||||
push!(default, get(value(v).default))
|
||||
else
|
||||
offset[i] = max(offset[i], -value(v).n)
|
||||
end
|
||||
v
|
||||
end
|
||||
return state, offset, default
|
||||
end
|
||||
|
||||
hiddeninput(n) = vertex(Split(n), inputnode(1))
|
||||
|
||||
function create_steps(v::IVertex, n; seq = true, stateful = true)
|
||||
[(stateful ? bumpinputs : copy)(seq ? spliceinputs(v, hiddeninput(i)) : v) for i = 1:n]
|
||||
end
|
||||
|
||||
function getvar(n, step, steps, offset, default; stateful = true)
|
||||
if stateful && step < 1
|
||||
hiddeninput(sum(offset[1:n-1]) + 1 - step)
|
||||
elseif step ∉ 1:length(steps)
|
||||
constant(default[n])
|
||||
else
|
||||
steps[step][1,n]
|
||||
end
|
||||
end
|
||||
|
||||
function stateout(steps, offset, default)
|
||||
outs = []
|
||||
defaults = []
|
||||
for i = 1:length(offset), j = 1:offset[i]
|
||||
push!(outs, getvar(i, length(steps)-j+1, steps, offset, default))
|
||||
push!(defaults, default[i])
|
||||
end
|
||||
group(outs...), defaults
|
||||
end
|
||||
|
||||
function unrollgraph(v::IVertex, n; seq = true, stateful = true)
|
||||
state, offset, default = collect_state(v)
|
||||
v = group(group(state...), v)
|
||||
steps = create_steps(v, n, seq = seq, stateful = stateful)
|
||||
for i = 1:n
|
||||
vars = inputs(steps[i][1])
|
||||
postwalk!(steps[i]) do v
|
||||
isa(value(v), Offset) || return v
|
||||
varid = findfirst(vars,v[1])
|
||||
getvar(varid, value(v).n + i, steps, offset, default, stateful = stateful)
|
||||
end
|
||||
end
|
||||
out = group(map(x->x[2], steps)...)
|
||||
if stateful
|
||||
state, defaults = stateout(steps, offset, default)
|
||||
group(state,out), map(Flux.state, defaults)
|
||||
else
|
||||
out, []
|
||||
end
|
||||
end
|
||||
|
||||
unrollgraph(m, n; kws...) = unrollgraph(atomise(m), n; kws...)
|
||||
|
||||
# TODO: perhaps split into SeqModel + StatefulModel
|
||||
type Unrolled <: Model
|
||||
model
|
||||
graph::IVertex{Any}
|
||||
state::Vector{Any}
|
||||
stateful::Bool
|
||||
steps::Int
|
||||
end
|
||||
|
||||
(m::Unrolled)(xs...) = interpret(reifyparams(m.graph), xs...)
|
||||
|
||||
graph(u::Unrolled) = u.graph
|
||||
|
||||
function unroll(model, n; seq = true, stateful = true)
|
||||
graph, state = unrollgraph(model, n; seq = seq, stateful = stateful)
|
||||
seq || stateful ? Unrolled(model, graph, state, stateful, n) : Capacitor(graph)
|
||||
end
|
||||
|
||||
function unroll1(model)
|
||||
graph, state = unrollgraph(model, 1; seq = false)
|
||||
graph = group(graph[1], map(x->x[1], inputs(graph)[2:end])...)
|
||||
Unrolled(model, graph, state, false, 1)
|
||||
end
|
||||
|
||||
flip(model) = Capacitor(map(x -> isa(x, Offset) ? -x : x, atomise(model)))
|
||||
@ -1,48 +0,0 @@
|
||||
using DataFlow: ilinev, iargs, applylines, Line
|
||||
|
||||
type Hint
|
||||
typ
|
||||
end
|
||||
|
||||
DataFlow.tocall(h::Hint, x) = :($x::$(h.typ))
|
||||
|
||||
function gethint(v::IVertex)
|
||||
while isa(value(v), Union{Line,Frame}) v = v[1] end
|
||||
isa(value(v), Hint) && return value(v).typ
|
||||
return
|
||||
end
|
||||
|
||||
ihint(f, ctx::Context, h::Hint, x) = vertex(h, x)
|
||||
ihint(f, args...) = f(args...)
|
||||
|
||||
hintify(c::Constant) = hintify(state(c.value))
|
||||
hintify(xs::AbstractArray) = vertex(Hint(size(xs)), constant(:_))
|
||||
|
||||
interpshape = mux(ilinev, ihint, iargs, ituple, hintify)
|
||||
|
||||
function hintify(f, xs...)
|
||||
sh = infer(f, map(gethint, xs)...)
|
||||
sh ≠ nothing ? vertex(Hint(sh), vertex(f, xs...)) :
|
||||
!any(x->x==nothing, xs) && graph(f) ≠ nothing ? interpret(Context(interpshape), graph(f), xs...) :
|
||||
vertex(f, xs...)
|
||||
end
|
||||
|
||||
function shapesv(f, args...)
|
||||
(g = graph(f)) == nothing && return
|
||||
ins = [vertex(Hint(d), inputnode(i)) for (i,d) in enumerate(args)]
|
||||
interpret(Context(interpshape), g, ins...)
|
||||
end
|
||||
|
||||
shapes(args...) = shapesv(args...) |> syntax |> applylines |> (x->prettify(x, lines=true))
|
||||
|
||||
# Inference primitives
|
||||
|
||||
infer(f, args...) = graph(f) == nothing ? nothing : gethint(shapesv(f, args...))
|
||||
|
||||
function infer(::typeof(*), a::NTuple{2}, b::NTuple{2})
|
||||
a[2] == b[1] || return nothing
|
||||
(a[1], b[2])
|
||||
end
|
||||
|
||||
# TODO: make correct
|
||||
infer(::typeof(+), a, b) = a
|
||||
@ -1,8 +0,0 @@
|
||||
export mse, mse!
|
||||
|
||||
function mse!(Δ, pred, target)
|
||||
map!(-, Δ, pred, target)
|
||||
sumabs2(Δ)/2
|
||||
end
|
||||
|
||||
mse(pred, target) = mse(similar(pred), pred, target)
|
||||
36
src/data.jl
36
src/data.jl
@ -1,36 +0,0 @@
|
||||
export onehot, onecold, chunk, partition, batches, sequences
|
||||
|
||||
"""
|
||||
onehot('b', ['a', 'b', 'c', 'd']) => [false, true, false, false]
|
||||
|
||||
onehot(Float32, 'c', ['a', 'b', 'c', 'd']) => [0., 0., 1., 0.]
|
||||
|
||||
Produce a one-hot-encoded version of an item, given a list of possible values
|
||||
for the item.
|
||||
"""
|
||||
onehot(T::Type, label, labels) = T[i == label for i in labels]
|
||||
onehot(label, labels) = onehot(Int, label, labels)
|
||||
|
||||
"""
|
||||
onecold([0.0, 1.0, 0.0, ...],
|
||||
['a', 'b', 'c', ...]) => 'b'
|
||||
|
||||
The inverse of `onehot`; takes an output prediction vector and a list of
|
||||
possible values, and produces the appropriate value.
|
||||
"""
|
||||
onecold(pred, labels = 1:length(pred)) = labels[findfirst(pred, maximum(pred))]
|
||||
|
||||
using Iterators
|
||||
import Iterators: partition
|
||||
|
||||
export partition
|
||||
|
||||
Base.length(l::Iterators.Partition) = length(l.xs) ÷ l.step
|
||||
|
||||
_partition(r::UnitRange, step::Integer) = (step*(i-1)+1:step*i for i in 1:(r.stop÷step))
|
||||
_partition(xs, step) = (xs[i] for i in _partition(1:length(xs), step))
|
||||
|
||||
chunk(xs, n) = _partition(xs, length(xs)÷n)
|
||||
|
||||
batches(xs...) = (Batch(x) for x in zip(xs...))
|
||||
sequences(xs, len) = (Seq(x) for x in partition(xs, len))
|
||||
@ -1,21 +0,0 @@
|
||||
export Batch, batchone
|
||||
|
||||
immutable Batch{T,S} <: AbstractVector{T}
|
||||
data::CatMat{T,S}
|
||||
end
|
||||
|
||||
@forward Batch.data size, eltype, getindex, setindex!, rawbatch
|
||||
|
||||
Batch(xs) = Batch(CatMat(xs))
|
||||
|
||||
convert{T,S}(::Type{Batch{T,S}},storage::S) =
|
||||
Batch{T,S}(storage)
|
||||
|
||||
batchone(x) = Batch((x,))
|
||||
batchone(x::Batch) = x
|
||||
|
||||
@render Juno.Inline b::Batch begin
|
||||
Tree(Row(Text("Batch of "), eltype(b),
|
||||
Juno.fade("[$(length(b))]")),
|
||||
Juno.trim(collect(b)))
|
||||
end
|
||||
@ -1,50 +0,0 @@
|
||||
import Base: eltype, size, getindex, setindex!, convert
|
||||
|
||||
export CatMat, rawbatch
|
||||
|
||||
immutable CatMat{T,S} <: AbstractVector{T}
|
||||
data::S
|
||||
end
|
||||
|
||||
convert{T,S}(::Type{CatMat{T,S}},storage::S) =
|
||||
CatMat{T,S}(storage)
|
||||
|
||||
eltype{T}(::CatMat{T}) = T
|
||||
|
||||
size(b::CatMat) = (size(b.data, 1),)
|
||||
|
||||
getindex(b::CatMat, i)::eltype(b) = slicedim(b.data, 1, i)
|
||||
|
||||
setindex!(b::CatMat, v, i) = b[i, :] = v
|
||||
|
||||
allequal(xs) = all(x -> x == first(xs), xs)
|
||||
|
||||
function (::Type{CatMat{T,S}}){T,S}(xs, storage::S)
|
||||
@assert @>> xs map(size) allequal
|
||||
@assert size(storage) == (length(xs), size(first(xs))...)
|
||||
for i = 1:length(xs)
|
||||
storage[i, :] = xs[i]
|
||||
end
|
||||
return CatMat{T,S}(storage)
|
||||
end
|
||||
|
||||
function (::Type{CatMat{T}}){T}(xs)
|
||||
xs′ = map(rawbatch, xs)
|
||||
storage = similar(first(xs′), (length(xs′), size(first(xs′))...))
|
||||
CatMat{T,typeof(storage)}(xs′, storage)
|
||||
end
|
||||
|
||||
function CatMat(xs)
|
||||
xs = promote(xs...)
|
||||
CatMat{eltype(xs)}(xs)
|
||||
end
|
||||
|
||||
@render Juno.Inline b::CatMat begin
|
||||
Tree(Row(Text("CatMat of "), eltype(b),
|
||||
Juno.fade("[$(length(b))]")),
|
||||
Juno.trim(collect(b)))
|
||||
end
|
||||
|
||||
rawbatch(xs) = xs
|
||||
|
||||
rawbatch(xs::CatMat) = xs.data
|
||||
@ -1,20 +0,0 @@
|
||||
export seq, Seq, BatchSeq
|
||||
|
||||
immutable Seq{T,S} <: AbstractVector{T}
|
||||
data::CatMat{T,S}
|
||||
end
|
||||
|
||||
@forward Seq.data size, eltype, getindex, setindex!, rawbatch
|
||||
|
||||
Seq(xs) = Seq(CatMat(xs))
|
||||
|
||||
convert{T,S}(::Type{Seq{T,S}},storage::S) =
|
||||
Seq{T,S}(storage)
|
||||
|
||||
@render Juno.Inline b::Seq begin
|
||||
Tree(Row(Text("Seq of "), eltype(b),
|
||||
Juno.fade("[$(length(b))]")),
|
||||
Juno.trim(collect(b)))
|
||||
end
|
||||
|
||||
typealias BatchSeq{T<:Seq} Batch{T}
|
||||
@ -1,18 +0,0 @@
|
||||
export σ, relu, softmax, flatten
|
||||
|
||||
σ(x) = 1 ./ (1 + exp.(-x))
|
||||
|
||||
back!(::typeof(σ), Δ, x) = Δ .* σ(x)./(1.-σ(x))
|
||||
|
||||
relu(x) = max(0, x)
|
||||
|
||||
back!(::typeof(relu), Δ, x) = Δ .* (x .< 0)
|
||||
|
||||
softmax(xs) = exp.(xs) ./ sum(exp.(xs))
|
||||
|
||||
flatten(xs) = reshape(xs, length(xs))
|
||||
|
||||
shape(::typeof(flatten), in) = prod(in)
|
||||
|
||||
infer(::typeof(softmax), x) = x
|
||||
infer(::typeof(σ), x) = x
|
||||
@ -1,20 +0,0 @@
|
||||
export Affine
|
||||
|
||||
# TODO: type hints for parameters
|
||||
|
||||
@net type Affine
|
||||
W
|
||||
b
|
||||
x -> x*W + b
|
||||
end
|
||||
|
||||
Affine(in::Integer, out::Integer; init = initn) =
|
||||
Affine(init(in, out), init(1, out))
|
||||
|
||||
@net type Sigmoid
|
||||
layer::Model
|
||||
x -> σ(layer(x))
|
||||
end
|
||||
|
||||
Sigmoid(in::Integer, out::Integer; init = randn) =
|
||||
Sigmoid(Affine(in, out, init = init))
|
||||
@ -1,32 +0,0 @@
|
||||
export Chain
|
||||
|
||||
function inferchain(ms)
|
||||
chain = []
|
||||
sh = nothing
|
||||
for m in ms
|
||||
m = init(m, single(sh))
|
||||
sh = shape(m, sh)
|
||||
push!(chain, m)
|
||||
end
|
||||
return chain, sh
|
||||
end
|
||||
|
||||
type Chain <: Model
|
||||
layers::Vector{Any}
|
||||
shape
|
||||
function Chain(ms...)
|
||||
ms, shape = inferchain(ms)
|
||||
return new(ms, shape)
|
||||
end
|
||||
end
|
||||
|
||||
@forward Chain.layers Base.getindex, Base.first, Base.last
|
||||
|
||||
(s::Chain)(x) = foldl((x, m) -> m(x), x, s.layers)
|
||||
back!(s::Chain, Δ) = foldr((m, Δ) -> back!(m, Δ), Δ, s.layers)
|
||||
update!(s::Chain, η) = foreach(l -> update!(l, η), s.layers)
|
||||
|
||||
graph(s::Chain) =
|
||||
foldl((v, m) -> vertex(m, v), constant(inputnode(1)), s.layers)
|
||||
|
||||
shape(c::Chain, in) = c.shape
|
||||
@ -1,51 +0,0 @@
|
||||
export Recurrent, GatedRecurrent, LSTM
|
||||
|
||||
@net type Recurrent
|
||||
Wxy; Wyy; by
|
||||
y
|
||||
function (x)
|
||||
y = tanh( x * Wxy + y{-1} * Wyy + by )
|
||||
end
|
||||
end
|
||||
|
||||
Recurrent(in, out; init = initn) =
|
||||
Recurrent(init((in, out)), init((out, out)), init(out), init(out))
|
||||
|
||||
@net type GatedRecurrent
|
||||
Wxr; Wyr; br
|
||||
Wxu; Wyu; bu
|
||||
Wxh; Wyh; bh
|
||||
y
|
||||
function (x)
|
||||
reset = σ( x * Wxr + y{-1} * Wyr + br )
|
||||
update = σ( x * Wxu + y{-1} * Wyu + bu )
|
||||
y′ = tanh( x * Wxh + (reset .* y{-1}) * Wyh + bh )
|
||||
y = (1 .- update) .* y′ + update .* y{-1}
|
||||
end
|
||||
end
|
||||
|
||||
GatedRecurrent(in, out; init = initn) =
|
||||
GatedRecurrent(vcat([[init((in, out)), init((out, out)), init(out)] for _ = 1:3]...)...,
|
||||
zeros(Float32, out))
|
||||
|
||||
@net type LSTM
|
||||
Wxf; Wyf; bf
|
||||
Wxi; Wyi; bi
|
||||
Wxo; Wyo; bo
|
||||
Wxc; Wyc; bc
|
||||
y; state
|
||||
function (x)
|
||||
# Gates
|
||||
forget = σ( x * Wxf + y{-1} * Wyf + bf )
|
||||
input = σ( x * Wxi + y{-1} * Wyi + bi )
|
||||
output = σ( x * Wxo + y{-1} * Wyo + bo )
|
||||
# State update and output
|
||||
state′ = tanh( x * Wxc + y{-1} * Wyc + bc )
|
||||
state = forget .* state{-1} + input .* state′
|
||||
y = output .* tanh(state)
|
||||
end
|
||||
end
|
||||
|
||||
LSTM(in, out; init = initn) =
|
||||
LSTM(vcat([[init((in, out)), init((out, out)), init((1, out))] for _ = 1:4]...)...,
|
||||
zeros(Float32, out), zeros(Float32, out))
|
||||
@ -1,47 +0,0 @@
|
||||
export Input
|
||||
|
||||
typealias Dims{N} NTuple{N,Int}
|
||||
|
||||
dims(d::Dims) = d
|
||||
|
||||
dims(i...) = (i...,)
|
||||
|
||||
single(i) = i
|
||||
single(i::Dims) = length(i) == 1 ? first(i) : i
|
||||
|
||||
# Shim for kicking off shape inference
|
||||
|
||||
type ShapeError <: Exception
|
||||
layer
|
||||
shape
|
||||
end
|
||||
|
||||
type Input{N} <: Model
|
||||
dims::Dims{N}
|
||||
end
|
||||
|
||||
Input(i...) = Input(dims(i...))
|
||||
|
||||
(::Input)(x) = x
|
||||
back!(::Input, Δ, x) = Δ
|
||||
|
||||
# Initialise placeholder
|
||||
|
||||
type Init{F}
|
||||
f::F
|
||||
end
|
||||
|
||||
init(i::Init, input...) = i.f(input...)
|
||||
init(m, input...) = m
|
||||
|
||||
# Shape inference API
|
||||
|
||||
shape(x, in) = in
|
||||
|
||||
shape(i::Input, _) = i.dims
|
||||
|
||||
# Implementation for bundled layers
|
||||
|
||||
shape(d::Affine, _) = length(state(d.b)) # TODO: could perhaps infer this
|
||||
|
||||
Affine(out::Integer) = Init(in::Integer -> Affine(in, out))
|
||||
@ -1,44 +0,0 @@
|
||||
export Conv2D, MaxPool, Reshape
|
||||
|
||||
type Conv2D <: Model
|
||||
filter::Param{Array{Float32,4}} # [height, width, inchans, outchans]
|
||||
stride::Dims{2}
|
||||
end
|
||||
|
||||
Conv2D(size; in = 1, out = 1, stride = (1,1), init = initn) =
|
||||
Conv2D(param(initn(size..., in, out)), stride)
|
||||
|
||||
shape(c::Conv2D, in::Dims{2}) =
|
||||
(map(i -> (in[i]-size(c.filter,i))÷c.stride[i]+1, (1,2))..., size(c.filter, 4))
|
||||
|
||||
shape(c::Conv2D, in::Dims{3}) =
|
||||
shape(c, (in[1],in[2]))
|
||||
|
||||
type MaxPool <: Model
|
||||
size::Dims{2}
|
||||
stride::Dims{2}
|
||||
end
|
||||
|
||||
MaxPool(size; stride = (1,1)) =
|
||||
MaxPool(size, stride)
|
||||
|
||||
shape(c::MaxPool, in::Dims{2}) =
|
||||
map(i -> (in[i]-c.size[i])÷c.stride[i]+1, (1,2))
|
||||
|
||||
shape(c::MaxPool, in::Dims{3}) =
|
||||
(shape(c, (in[1],in[2]))..., in[3])
|
||||
|
||||
shape(c::MaxPool, in) = throw(ShapeError(c, in))
|
||||
|
||||
immutable Reshape{N}
|
||||
dims::Dims{N}
|
||||
end
|
||||
|
||||
Reshape(dims::Integer...) = Reshape(dims)
|
||||
|
||||
function shape(r::Reshape, dims)
|
||||
prod(dims) == prod(r.dims) || throw(ShapeError(r, dims))
|
||||
return r.dims
|
||||
end
|
||||
|
||||
shape(r::Reshape, ::Void) = r.dims
|
||||
111
src/model.jl
111
src/model.jl
@ -1,111 +0,0 @@
|
||||
export Model, back!, update!, param
|
||||
|
||||
# Basic model API
|
||||
|
||||
"""
|
||||
(m::Model)(X...) => Y
|
||||
|
||||
A "model" is a function with state. For example, a logistic regression is the
|
||||
function
|
||||
|
||||
x -> σ(x * W + b)
|
||||
|
||||
where `W` and `b` are a trainable matrix and vector of weights repectively. The
|
||||
`Model` abstract type is used loosely; in general the concept of a model is
|
||||
closer to a protocol, and models don't need to inherit from this type. Normal
|
||||
Julia functions are models with 0 parameters, for example.
|
||||
"""
|
||||
abstract Model
|
||||
|
||||
"""
|
||||
back!(m::Model, ΔY, X...) => ΔX
|
||||
|
||||
Backpropagate the gradient `ΔY` through the model `m`, accumulating the
|
||||
gradients of any parameters. Returns the gradient of the input `X`. Gradients
|
||||
may be arrays or tuples of arrays (for multiple inputs/outputs).
|
||||
"""
|
||||
back!(m::Model, Δ, xs...) = error("Backprop not implemented for $(typeof(m))")
|
||||
|
||||
"""
|
||||
update!(m::Model, η) => m
|
||||
|
||||
Update the parameters of the model `m` using the accumulated gradients from
|
||||
`back!`, using the learning rate `η`.
|
||||
"""
|
||||
update!(m, η) = m
|
||||
|
||||
"""
|
||||
graph(m::Model) => ::IVertex{Any} | nothing
|
||||
|
||||
Returns the graph representation of the model, if any. Most models are built
|
||||
from lower-level components and can simply implement this method to get most of
|
||||
Flux's functionality. If this method isn't available, functionality like
|
||||
backpropagation or conversion for backend must be implemented on a case-by-case
|
||||
basis. Alternatively, one can implement this method and override individual
|
||||
methods as necessary.
|
||||
"""
|
||||
graph(m) = nothing
|
||||
|
||||
# Model parameters
|
||||
|
||||
"""
|
||||
A `Param` object stores a parameter array along with an accumulated delta to
|
||||
that array. When converting to backends like TensorFlow, identical `Param`s will
|
||||
result in identical variable objects, making model reuse trivial.
|
||||
"""
|
||||
type Param{T}
|
||||
x::T
|
||||
Δx::T
|
||||
end
|
||||
|
||||
"""
|
||||
param(x::T) => ::Param{T}
|
||||
|
||||
Convenience method for creating a `Param` object for a given array.
|
||||
"""
|
||||
param(x) = Param(x, zero(x))
|
||||
|
||||
state(p::Param) = p.x
|
||||
|
||||
"""
|
||||
accumulate!(p::Param, Δ) => p
|
||||
|
||||
Accumulates the update `Δ` on `p`. The value of `p` won't change until
|
||||
`update!`.
|
||||
"""
|
||||
function accumulate!(p::Param, Δ)
|
||||
p.Δx += Δ
|
||||
return p
|
||||
end
|
||||
|
||||
"""
|
||||
update!(p::Param)
|
||||
|
||||
Apply the accumulated updates to the value of the parameter.
|
||||
"""
|
||||
function update!(p::Param, η)
|
||||
p.x .-= p.Δx .* η
|
||||
p.Δx[:] = 0
|
||||
return p
|
||||
end
|
||||
|
||||
state(x) = x
|
||||
accumulate!(x, Δ) = x
|
||||
|
||||
@forward Param.x Base.size
|
||||
|
||||
function Base.show(io::IO, p::Param)
|
||||
print(io, "Param", size(p.x))
|
||||
end
|
||||
|
||||
# Anonymous models
|
||||
|
||||
export Capacitor
|
||||
|
||||
type Capacitor <: Model
|
||||
graph::IVertex{Any}
|
||||
end
|
||||
|
||||
(m::Capacitor)(xs...) = interpret(reifyparams(m.graph), xs...)
|
||||
|
||||
graph(cap::Capacitor) = cap.graph
|
||||
30
src/utils.jl
30
src/utils.jl
@ -1,30 +0,0 @@
|
||||
export AArray
|
||||
|
||||
const AArray = AbstractArray
|
||||
|
||||
initn(dims...) = randn(Float32, dims...)/10
|
||||
|
||||
function train!(m, train, test = []; epoch = 1, batch = 10, η = 0.1)
|
||||
i = 0
|
||||
Δ = zeros(length(train[1][2]))
|
||||
for _ in 1:epoch
|
||||
@progress for (x, y) in train
|
||||
i += 1
|
||||
pred = m(x)
|
||||
any(isnan, pred) && error("NaN")
|
||||
err = mse!(Δ, pred, y)
|
||||
back!(m, Δ, x)
|
||||
i % batch == 0 && update!(m, η)
|
||||
i % 1000 == 0 && @show accuracy(m, test)
|
||||
end
|
||||
end
|
||||
return m
|
||||
end
|
||||
|
||||
function accuracy(m, data)
|
||||
correct = 0
|
||||
for (x, y) in data
|
||||
onecold(m(x)) == onecold(y) && (correct += 1)
|
||||
end
|
||||
return correct/length(data)
|
||||
end
|
||||
@ -1,10 +0,0 @@
|
||||
xs = randn(10)' # TODO: batching semantics
|
||||
|
||||
d = Affine(10, 20)
|
||||
|
||||
@test d(xs) == xs*d.W.x + d.b.x
|
||||
|
||||
let
|
||||
@capture(syntax(d), _Frame(_Line(x_[1] * W_ + b_)))
|
||||
@test isa(x, Input) && isa(W, Param) && isa(b, Param)
|
||||
end
|
||||
@ -1,10 +0,0 @@
|
||||
using Flux, DataFlow, MacroTools, Base.Test
|
||||
using Flux: graph, Param
|
||||
using DataFlow: Input, Line, Frame
|
||||
|
||||
syntax(v::Vertex) = prettify(DataFlow.syntax(v))
|
||||
syntax(x) = syntax(graph(x))
|
||||
|
||||
include("basic.jl")
|
||||
include("recurrent.jl")
|
||||
include("backend.jl")
|
||||
Loading…
Reference in New Issue
Block a user