bayesiancnn-data-parsing/aux_func.jl

using StatsBase: countmap
using PyCall



function converttobyte(array)
    temp = []
    for l in array
        if l[2] == "Mbyte"
            append!(temp, parse(Float64, l[3]) * 1048576.0)
        elseif l[2] == "Kbyte"
            append!(temp, parse(Float64, l[3]) * 1024.0)
        else
            append!(temp, parse(Float64, l[3]))
        end

    end
    return temp
end


function approx_δ(array)
    temp = []
    for i in 1:(length(array)-1)
        push!(temp,array[i+1]-array[i])
    end
    return temp
end


py"""
import pickle

def load_pickle(fpath):
    with open(fpath, "rb") as f:
        data = pickle.load(f)
    return data
"""

load_pickle = py"load_pickle"

py"""
import pickle

def save_pickle(pickle_name, data_dump):
  with open(pickle_name, 'wb') as f:
    pickle.dump(data_dump, f)
"""

save_pickle = py"save_pickle"


function movavg(X::Vector, numofele::Int)
    Back = div(numofele, 2)
    Forward = isodd(numofele) ? div(numofele, 2) : div(numofele, 2) - 1
    len = length(X)
    Y = similar(X)
    for n = 1:len
        lo = max(1, n - Back)
        hi = min(len, n + Forward)
        Y[n] = mean(X[lo:hi])
    end
    return Y
end


function removewatt(X::Vector{String})
    temp = match.(r"[0-9]+.[0-9]+|[0-9]+", X)
    temp = [x.match for x in temp]
    return parse.(Float64, temp)
end


function removewatt(X::SubString{String})
    return parse.(Float64, match.(r"[0-9]+.[0-9]+|[0-9]+", X).match)
end


function removewatt(X::String)
    return parse(Float64, match(r"[0-9]+.[0-9]+|[0-9]+", X).match)
end


function incwzeros(X::Vector, newsize::Int)
    temp = X[:]
    l = length(temp)
    for i = 1:newsize
        push!(temp, 28)
    end
    return temp
end


function readcpudata(dat_fo, typ, dat_fl, exp)
    folder = "$(typ)_$(dat_fo)_cpu_data_$(exp)/"
    files_tmp = Dict()
    for s = 1:5
        file = "$(folder)$(typ)_$(s)_$(dat_fl)"
        #println("File being loaded: $(file)")
        io = open(file, "r")
        t = read(io, String)
        close(io)
        files_tmp[s] = t
    end
    return files_tmp
end


function getimes(file_name::String)
    io = open(file_name, "r")
    times_bayes = read(io, String)
    close(io)

    times_bayes = split(times_bayes, "\n")
    times_bayes = [s for s in times_bayes if !all(isempty.(s))]
    times_bayes = [split(r, "|") for r in times_bayes]

    temp = Vector()
    for vec in times_bayes
        t = [strip(s) for s in vec if !all(isempty.(s))]
        push!(temp, t)
    end
    times_bayes = temp

    times_bayes = [s for s in times_bayes if s[1] != "------"]

    l = length(times_bayes)

    times_bayes_mnist = times_bayes[1:(l÷2)]
    popfirst!(times_bayes_mnist)
    times_bayes_cifar = times_bayes[(l÷2)+1:end]
    popfirst!(times_bayes_cifar)

    temp = Vector()
    for (times, i) in zip(times_bayes_mnist, 1:length(times_bayes_mnist))
        if i > 1
            t = map(x -> Time(x), times[2:end])
            push!(temp, t)
        end
    end
    times_bayes_mnist = temp

    temp = Vector()
    for (times, i) in zip(times_bayes_cifar, 1:length(times_bayes_cifar))
        if i > 1
            t = map(x -> Time(x), times[2:end])
            push!(temp, t)
        end
    end
    times_bayes_cifar = temp

    temp = Vector()
    for (p, o) in zip(2:2:10, 1:2:10)
        t = convert.(Dates.Second, times_bayes_mnist[p] .- times_bayes_mnist[o])
        push!(temp, t)
    end
    times_bayes_mnist = temp

    temp = Vector()
    for (p, o) in zip(2:2:10, 1:2:10)
        t = convert.(Dates.Second, times_bayes_cifar[p] .- times_bayes_cifar[o])
        push!(temp, t)
    end
    times_bayes_cifar = temp

    if file_name == "times_frequentist"
        typ = "Frequentist"
    elseif file_name == "times_bayesian"
        typ = "Bayesian"
    end

    to_file = Vector()
    push!(to_file, "\nMNIST $(typ) Results: \n")
    push!(to_file, "          acc > 99   ES       W Budget   100 Epochs \n")
    for i = 1:5
        push!(
            to_file,
            "size $(i): $(times_bayes_mnist[i][1]) $(times_bayes_mnist[i][2]) $(times_bayes_mnist[i][3]) $(times_bayes_mnist[i][4]) \n",
        )
    end

    push!(to_file, "\nCIFAR $(typ) Results: \n")
    push!(to_file, "          acc > 99   ES       W Budget   100 Epochs\n")
    for i = 1:5
        push!(
            to_file,
            "size $(i): $(times_bayes_cifar[i][1]) $(times_bayes_cifar[i][2]) $(times_bayes_cifar[i][3]) $(times_bayes_cifar[i][4]) \n",
        )
    end
    return to_file
end


function getcpuwatt(watt_vector)
    test = watt_vector
    test = split(test, "\n")
    test = [s for s in test if !all(isempty.(s))]
    test = [split(r, " ") for r in test]

    temp = Vector()
    for vec in test
        if occursin(r"\d\d:\d\d:\d\d", vec[1])
            push!(temp, vec)
        end
    end
    test = temp

    temp = Vector()
    for vec in test
        t = [strip(s) for s in vec if !all(isempty.(s))]
        push!(temp, t)
    end
    test = temp

    temp = Vector()
    for vec in test
        push!(temp, vec[end])
    end
    test = temp
    test = map(x -> parse(Float32, x), test)
    return test
end


function getramuse(ram_vector)
    test = ram_vector
    test = split(test, "\n")
    test = [s for s in test if !all(isempty.(s))]
    test = [split(r, " ") for r in test]

    temp = Vector()
    for vec in test
        t = [strip(s) for s in vec if !all(isempty.(s))]
        push!(temp, t)
    end
    test = temp


    temp = Vector()
    for vec in test
        if vec[1] == "Mem:"
            push!(temp, vec)
        end
    end
    test = temp

    temp = Vector()
    regex = r"[^0-9]+$"
    for vec in test
        unit = match(regex, vec[3]).match
        qty = removewatt(vec[3])
        push!(temp, (qty, unit))
    end
    test = temp

    temp = Vector()
    for u in test
        if u[2] == "Gi"
            push!(temp, u[1] * 1024.0)
        elseif u[2] == "Mi"
            println(u[1])
        end
    end
    test = temp
    return test
end


function concat(arr::Vector)
    temp = ""
    for s in arr
        temp = temp * s
    end
    return temp
end


function getgpudata(folder, model, type, save = "d")
    temp = Dict()
    for s = 1:5
        if type == "watt"
            path = "$(folder)$(model)_$(type)data_$(s).pkl"
            watt_data = removewatt(load_pickle(path)[:, 1])
            mem_data = removewatt(load_pickle(path)[:, 2])
            if save == "d"
                temp[s] = Dict("Ene" => watt_data, "Mem" => mem_data)
            elseif save == "eo"
                temp[s] = Dict("Ene" => watt_data, "Mem" => nothing)
            elseif save == "mo"
                temp[s] = Dict("Ene" => nothing, "Mem" => mem_data)
            end
        elseif type == "exp"
            path = "$(folder)$(model)_$(type)_data_$(s).pkl"
            temp[s] = load_pickle(path)
        end
    end
    return temp
end


function getuniquevalues(vector::Vector)::Tuple
    dict = countmap(vector)
    uniq = unique(vector)
    vals = Vector()
    for u in uniq
        push!(vals, dict[u])
    end
    return (uniq, vals)
end


function expdatatodict(matrix::Matrix{Real})::Dict
    return Dict(
        "tloss" => matrix[:, 2],
        "acc" => matrix[:, 3],
        "vloss" => matrix[:, 4],
        "pre" => matrix[:, 5],
    )
end

@views function makechunks(X::AbstractVector, n::Integer)
    c = length(X) ÷ n
    return [X[1+c*k:(k == n-1 ? end : c*k+c)] for k = 0:n-1]
end

function resize1(vect)
    temp = []
    size = length(vect[end])
    for v in vect
        temp_size = length(v)
        if temp_size == size
            push!(temp, Array(v))
        elseif temp_size < size
            app = zeros(size - temp_size)
            x = Array(deepcopy(v))
            new = vcat(x,app)
            push!(temp, new)
        elseif temp_size > size
            resize!(Array(v),size)
            push!(temp, v)
        end
    end
    return temp
end

function join_vectors(vect)
    result = []
    temp = []
    for v in vect
        temp = vcat(temp,v)
        push!(result,temp)
    end
    return result
end