will sort these out later
This commit is contained in:
Mike J Innes
parent
b17eb78e0a
commit
4604e9f515
@ -1,24 +0,0 @@
|
||||
using Flux, MNIST
|
||||
using Flux: accuracy, onehot, tobatch
|
||||
|
||||
data = [(trainfeatures(i), onehot(trainlabel(i), 0:9)) for i = 1:60_000]
|
||||
train = data[1:50_000]
|
||||
test = data[50_001:60_000]
|
||||
|
||||
m = @Chain(
|
||||
Input(784),
|
||||
Affine(128), relu,
|
||||
Affine( 64), relu,
|
||||
Affine( 10), softmax)
|
||||
|
||||
# Convert to MXNet
|
||||
model = mxnet(m)
|
||||
|
||||
# An example prediction pre-training
|
||||
model(tobatch(data[1][1]))
|
||||
|
||||
Flux.train!(model, train, η = 1e-3,
|
||||
cb = [()->@show accuracy(m, test)])
|
||||
|
||||
# An example prediction post-training
|
||||
model(tobatch(data[1][1]))
|
||||
@ -1,41 +0,0 @@
|
||||
using Flux
|
||||
using Flux: onehot, logloss, unsqueeze
|
||||
using Flux.Batches: Batch, tobatch, seqs, chunk
|
||||
import StatsBase: wsample
|
||||
|
||||
nunroll = 50
|
||||
nbatch = 50
|
||||
|
||||
encode(input) = seqs((onehot(ch, alphabet) for ch in input), nunroll)
|
||||
|
||||
cd(@__DIR__)
|
||||
input = readstring("shakespeare_input.txt");
|
||||
alphabet = unique(input)
|
||||
N = length(alphabet)
|
||||
|
||||
Xs = (Batch(ss) for ss in zip(encode.(chunk(input, 50))...))
|
||||
Ys = (Batch(ss) for ss in zip(encode.(chunk(input[2:end], 50))...))
|
||||
|
||||
model = Chain(
|
||||
LSTM(N, 256),
|
||||
LSTM(256, 256),
|
||||
Affine(256, N),
|
||||
softmax)
|
||||
|
||||
m = mxnet(unroll(model, nunroll))
|
||||
|
||||
eval = tobatch.(first.(drop.((Xs, Ys), 5)))
|
||||
evalcb = () -> @show logloss(m(eval[1]), eval[2])
|
||||
|
||||
# @time Flux.train!(m, zip(Xs, Ys), η = 0.001, loss = logloss, cb = [evalcb], epoch = 10)
|
||||
|
||||
function sample(model, n, temp = 1)
|
||||
s = [rand(alphabet)]
|
||||
m = unroll1(model)
|
||||
for i = 1:n-1
|
||||
push!(s, wsample(alphabet, softmax(m(unsqueeze(onehot(s[end], alphabet)))./temp)[1,:]))
|
||||
end
|
||||
return string(s...)
|
||||
end
|
||||
|
||||
# s = sample(model[1:end-1], 100)
|
||||
@ -1,35 +0,0 @@
|
||||
using Flux, MXNet
|
||||
|
||||
Flux.loadmx()
|
||||
|
||||
conv1 = Chain(
|
||||
Conv2D((5,5), out = 20), tanh,
|
||||
MaxPool((2,2), stride = (2,2)))
|
||||
|
||||
conv2 = Chain(
|
||||
Conv2D((5,5), in = 20, out = 50), tanh,
|
||||
MaxPool((2,2), stride = (2,2)))
|
||||
|
||||
lenet = @Chain(
|
||||
Input(28,28,1),
|
||||
conv1, conv2,
|
||||
flatten,
|
||||
Affine(500), tanh,
|
||||
Affine(10), softmax)
|
||||
|
||||
#--------------------------------------------------------------------------------
|
||||
|
||||
# Now we can continue exactly as in plain MXNet, following
|
||||
# https://github.com/dmlc/MXNet.jl/blob/master/examples/mnist/lenet.jl
|
||||
|
||||
batch_size = 100
|
||||
include(Pkg.dir("MXNet", "examples", "mnist", "mnist-data.jl"))
|
||||
train_provider, eval_provider = get_mnist_providers(batch_size; flat=false)
|
||||
|
||||
model = mx.FeedForward(lenet)
|
||||
|
||||
mx.infer_shape(model.arch, data = (28, 28, 1, 100))
|
||||
|
||||
optimizer = mx.SGD(lr=0.05, momentum=0.9, weight_decay=0.00001)
|
||||
|
||||
mx.fit(model, optimizer, train_provider, n_epoch=1, eval_data=eval_provider)
|
||||
@ -1,54 +0,0 @@
|
||||
using Flux, Juno
|
||||
|
||||
conv1 = Chain(
|
||||
Conv2D((5,5), out = 20), tanh,
|
||||
MaxPool((2,2), stride = (2,2)))
|
||||
|
||||
conv2 = Chain(
|
||||
Conv2D((5,5), in = 20, out = 50), tanh,
|
||||
MaxPool((2,2), stride = (2,2)))
|
||||
|
||||
lenet = @Chain(
|
||||
Input(28,28,1),
|
||||
conv1, conv2,
|
||||
flatten,
|
||||
Affine(500), tanh,
|
||||
Affine(10), softmax)
|
||||
|
||||
#--------------------------------------------------------------------------------
|
||||
|
||||
# Now we can continue exactly as in plain TensorFlow, following
|
||||
# https://github.com/malmaud/TensorFlow.jl/blob/master/examples/mnist_full.jl
|
||||
# (taking only the training and cost logic, not the graph building steps)
|
||||
|
||||
using TensorFlow, Distributions
|
||||
|
||||
include(Pkg.dir("TensorFlow", "examples", "mnist_loader.jl"))
|
||||
loader = DataLoader()
|
||||
|
||||
session = Session(Graph())
|
||||
|
||||
x = placeholder(Float32)
|
||||
y′ = placeholder(Float32)
|
||||
y = Tensor(lenet, x)
|
||||
|
||||
cross_entropy = reduce_mean(-reduce_sum(y′.*log(y), reduction_indices=[2]))
|
||||
|
||||
train_step = train.minimize(train.AdamOptimizer(1e-4), cross_entropy)
|
||||
|
||||
accuracy = reduce_mean(cast(indmax(y, 2) .== indmax(y′, 2), Float32))
|
||||
|
||||
run(session, global_variables_initializer())
|
||||
|
||||
@progress for i in 1:1000
|
||||
batch = next_batch(loader, 50)
|
||||
if i%100 == 1
|
||||
train_accuracy = run(session, accuracy, Dict(x=>batch[1], y′=>batch[2]))
|
||||
info("step $i, training accuracy $train_accuracy")
|
||||
end
|
||||
run(session, train_step, Dict(x=>batch[1], y′=>batch[2]))
|
||||
end
|
||||
|
||||
testx, testy = load_test_set()
|
||||
test_accuracy = run(session, accuracy, Dict(x=>testx, y′=>testy))
|
||||
info("test accuracy $test_accuracy")
|
||||
@ -1,50 +0,0 @@
|
||||
# Based on https://arxiv.org/abs/1409.0473
|
||||
|
||||
using Flux
|
||||
using Flux: Batch, Seq, param, flip, stateless, broadcastto, ∘
|
||||
|
||||
Nbatch = 3 # Number of phrases to batch together
|
||||
Nphrase = 5 # The length of (padded) phrases
|
||||
Nalpha = 7 # The size of the token vector
|
||||
Nhidden = 10 # The size of the hidden state
|
||||
|
||||
# A recurrent model which takes a token and returns a context-dependent
|
||||
# annotation.
|
||||
|
||||
forward = LSTM(Nalpha, Nhidden÷2)
|
||||
backward = flip(LSTM(Nalpha, Nhidden÷2))
|
||||
encoder = @net token -> hcat(forward(token), backward(token))
|
||||
|
||||
alignnet = Affine(2Nhidden, 1)
|
||||
align = @net (s, t) -> alignnet(hcat(broadcastto(s, (Nbatch, 1)), t))
|
||||
|
||||
# A recurrent model which takes a sequence of annotations, attends, and returns
|
||||
# a predicted output token.
|
||||
|
||||
recur = unroll1(LSTM(Nhidden, Nhidden)).model
|
||||
state = param(zeros(1, Nhidden))
|
||||
y = param(zeros(1, Nhidden))
|
||||
toalpha = Affine(Nhidden, Nalpha)
|
||||
|
||||
decoder = @net function (tokens)
|
||||
energies = map(token -> exp.(align(state{-1}, token)), tokens)
|
||||
weights = map(e -> e ./ sum(energies), energies)
|
||||
context = sum(map(∘, weights, tokens))
|
||||
(y, state), _ = recur((y{-1},state{-1}), context)
|
||||
return softmax(toalpha(y))
|
||||
end
|
||||
|
||||
# Building the full model
|
||||
|
||||
a, b = rand(Nbatch, Nalpha), rand(Nbatch, Nalpha)
|
||||
|
||||
model = @Chain(
|
||||
stateless(unroll(encoder, Nphrase)),
|
||||
@net(x -> repeated(x, Nphrase)),
|
||||
stateless(unroll(decoder, Nphrase)))
|
||||
|
||||
model = mxnet(Flux.SeqModel(model, Nphrase))
|
||||
|
||||
xs = Batch([Seq(rand(Nalpha) for i = 1:Nphrase) for i = 1:Nbatch])
|
||||
|
||||
model(xs)
|
||||
Loading…
Reference in New Issue
Block a user