93 lines
2.8 KiB
Julia
93 lines
2.8 KiB
Julia
|
using Flux
|
|||
|
|
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
# Logistic Regression from scratch #
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
|
|||
|
|
W = param(zeros(10,784))
|
|||
|
|
b = param(zeros(10))
|
|||
|
|
|
|||
|
|
pred(x) = softmax(W*x .+ b)
|
|||
|
|
|
|||
|
|
cost(x, y) = mean(sum(log.(pred(x)).*y, 1))
|
|||
|
|
|
|||
|
|
# See an example predction
|
|||
|
|
pred(rand(784))
|
|||
|
|
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
# Custom Layer: Dense #
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
|
|||
|
|
struct Dense
|
|||
|
|
σ
|
|||
|
|
W
|
|||
|
|
b
|
|||
|
|
end
|
|||
|
|
|
|||
|
|
function Dense(in::Integer, out::Integer, σ = σ)
|
|||
|
|
W = param(randn(out, in))
|
|||
|
|
b = param(zeros(out))
|
|||
|
|
return Dense(σ, W, b)
|
|||
|
|
end
|
|||
|
|
|
|||
|
|
# Note that Julia compiles:
|
|||
|
|
# * Specialised code for the forward pass (wrt activation function,
|
|||
|
|
# number types, ...)
|
|||
|
|
# * Including a single GPU/CPU kernel for the broadcast call
|
|||
|
|
function (m::Dense)(x)
|
|||
|
|
σ, W, b = m.σ, m.W, m.b
|
|||
|
|
σ.(W*x .+ b)
|
|||
|
|
end
|
|||
|
|
|
|||
|
|
d = Dense(10, 5, relu)
|
|||
|
|
d(rand(10))
|
|||
|
|
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
# RNN from scratch #
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
|
|||
|
|
in = 10
|
|||
|
|
out = 5
|
|||
|
|
|
|||
|
|
Wi = param(randn(out, in))
|
|||
|
|
Wh = param(randn(out, out))
|
|||
|
|
b = param(zeros(out))
|
|||
|
|
|
|||
|
|
function rnn(h, x)
|
|||
|
|
h = tanh.(Wi*x .+ Wh*h .+ b)
|
|||
|
|
return h, h
|
|||
|
|
end
|
|||
|
|
|
|||
|
|
h = rand(out)
|
|||
|
|
xs = [rand(in) for i = 1:13] # Sequence of length 13
|
|||
|
|
ys = []
|
|||
|
|
|
|||
|
|
for x in xs
|
|||
|
|
h, y = rnn(h, x)
|
|||
|
|
push!(ys, y)
|
|||
|
|
end
|
|||
|
|
|
|||
|
|
# Output hidden state and sequence
|
|||
|
|
h, ys
|
|||
|
|
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
# Recursive Net #
|
|||
|
|
# ––––––––––––––––––––––––––––––––––––––––––––––––––––––– #
|
|||
|
|
|
|||
|
|
N = 10
|
|||
|
|
|
|||
|
|
# Generate dummy data
|
|||
|
|
tree() = rand() < 0.5 ? rand(N) : (tree(), tree())
|
|||
|
|
|
|||
|
|
# Model
|
|||
|
|
|
|||
|
|
shrink = Dense(2N, N)
|
|||
|
|
combine(a, b) = shrink([a; b])
|
|||
|
|
|
|||
|
|
model(x) = x
|
|||
|
|
model(x::Tuple) = combine(model(x[1]), model(x[2]))
|
|||
|
|
|
|||
|
|
# The model is able to compress an arbitrary tree into
|
|||
|
|
# a single length N representation.
|
|||
|
|
model(tree())
|