TastyPancakes/PkgTemplates.jl
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove Docker plugin for now

Browse Source
This commit is contained in:
Chris de Graaf 2017-08-14 11:36:24 -05:00
parent 3866caffc9
commit 9d725f4316
4 changed files with 1 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
defaults/Dockerfile
View File

@ -1,105 +0,0 @@
FROM {{BASE_IMAGE}}
LABEL maintainer "{{MAINTAINER}}"
ENV PKG_PATH "$JULIA_PKGDIR/$JULIA_PKGVER/{{PKGNAME}}"
# Note: The "/etc/yum/protected.d/*.conf" contains the names of packages which are
# protected by yum and should not be removed. Protecting packages is necessary as
# this avoids the problem of accidentally removing a dependency of a package which
# is a runtime requirement.
# Get security updates
RUN yum -y update-minimal && \
yum -y clean all
# Install the current repo as a Julia package
COPY . $PKG_PATH
WORKDIR $PKG_PATH
# Record SHA of the revision the git repo. Note that the repo may not have been in
# a clean repo which makes the SHA the last known commit.
RUN REF=$(cat .git/HEAD | cut -d' ' -f2) && \
SHA=$(cat .git/$REF 2>/dev/null || echo $REF) && \
echo $SHA > REVISION && echo "Revision: $SHA" && \
rm -rf .git
# Install system requirements.
{{#!system}}# {{/!system}}ENV PKGS $(cat ./system-pkgs.txt)
{{#!system}}# {{/!system}}RUN yum -y install $PKGS && \
{{#!system}}# {{/!system}} echo $PKGS | tr -s '\t ' '\n' > /etc/yum/protected.d/system-pkgs.conf && \
{{#!system}}# {{/!system}} yum -y clean all
# Install Python requirements. Using the latest version of pip to avoid issues with
# outdated system versions. Note most pip installations will use wheels which don't require
# any compilation but if we need to compile a package from source additional requirements
# may be needed. For example "pandas" requires: python27-devel, python27-Cython, gcc,
# gcc-c++, and gcc-gfortran to be compiled from source.
{{#!python}}# {{/!python}}ENV PKGS \
{{#!python}}# {{/!python}} python27-devel \
{{#!python}}# {{/!python}} python27-Cython \
{{#!python}}# {{/!python}} gcc \
{{#!python}}# {{/!python}} gcc-c++ \
{{#!python}}# {{/!python}} gcc-gfortran
{{#!python}}# {{/!python}}ENV PINNED_PKGS \
{{#!python}}# {{/!python}} python27
{{#!python}}# {{/!python}}RUN yum -y install $PKGS $PINNED_PKGS && \
{{#!python}}# {{/!python}} echo $PINNED_PKGS | tr -s '\t ' '\n' > /etc/yum/protected.d/python.conf && \
{{#!python}}# {{/!python}} curl https://bootstrap.pypa.io/get-pip.py | python && \
{{#!python}}# {{/!python}} pip install -r ./requirements.txt && \
{{#!python}}# {{/!python}} yum -y autoremove $PKGS && \
{{#!python}}# {{/!python}} yum -y clean all
# Add and build the all of the required Julia packages. In order to allow the use of
# BinDeps.jl we need to temporarily install additional system packages.
#
# - BinDeps.jl runtime requirements: sudo, make, gcc, unzip, bzip2, xz, unzip
#
# BinDeps runtime requirements are only used while other packages are being built which
# makes them safe to only be temporarily installed. When installing system libraries
# BinDeps always uses "sudo" to install system packages and waits for user confirmation
# before installing a package. We'll both install "sudo" and always supply the `-y` flag
# to ensure that BinDeps installations work automatically. A good test to make sure
# BinDeps is setup correctly is to run `Pkg.add("Cairo"); Pkg.test("Cairo")`
#
# - HDF5.jl requires the EPEL repo to install hdf5 automatically through BinDeps
# (https://aws.amazon.com/premiumsupport/knowledge-center/ec2-enable-epel/)
# - yum-config-manager is installed by: yum-utils
ENV PKGS \
sudo \
make \
gcc \
gcc-c++ \
bzip2 \
xz \
unzip \
epel-release \
yum-utils
RUN yum -y install $PKGS && \
yum-config-manager --setopt=assumeyes=1 --save > /dev/null && \
yum-config-manager --enable epel > /dev/null && \
yum list installed | tr -s ' ' | cut -d' ' -f1 | sort > /tmp/pre_state && \
update-metadata && julia -e 'Pkg.update(); Pkg.resolve(); Pkg.build("{{PKGNAME}}")' && \
yum list installed | tr -s ' ' | cut -d' ' -f1 | sort > /tmp/post_state && \
comm -3 /tmp/pre_state /tmp/post_state | grep $'\t' | sed 's/\t//' | sed 's/\..*//' > /etc/yum/protected.d/julia-pkgs.conf && \
rm /tmp/pre_state /tmp/post_state && \
yum-config-manager --disable epel > /dev/null && \
for p in $PKGS; do yum -y autoremove $p &>/dev/null && echo "Removed $p" || echo "Skipping removal of $p"; done && \
yum -y clean all
# Improve the startup time of packages by pre-compiling {{PKGNAME}} and its dependencies
# into the default system image.
# Note: Need to have libc to avoid "/usr/bin/ld: cannot find crti.o: No such file or directory"
{{#!userimg}}# {{/!userimg}}ENV PKGS \
{{#!userimg}}# {{/!userimg}} gcc
{{#!userimg}}# {{/!userimg}}ENV PINNED_PKGS \
{{#!userimg}}# {{/!userimg}} glibc
{{#!userimg}}# {{/!userimg}}RUN yum -y install $PKGS $PINNED_PKGS && \
{{#!userimg}}# {{/!userimg}} cd $JULIA_PATH/base && \
{{#!userimg}}# {{/!userimg}} source $JULIA_PATH/Make.user && \
{{#!userimg}}# {{/!userimg}} $JULIA_PATH/julia -C $MARCH --output-o $JULIA_PATH/userimg.o --sysimage $JULIA_PATH/usr/lib/julia/sys.so --startup-file=no -e "using {{PKGNAME}}" && \
{{#!userimg}}# {{/!userimg}} cc -shared -o $JULIA_PATH/userimg.so $JULIA_PATH/userimg.o -ljulia -L$JULIA_PATH/usr/lib && \
{{#!userimg}}# {{/!userimg}} mv $JULIA_PATH/userimg.o $JULIA_PATH/usr/lib/julia/sys.o && \
{{#!userimg}}# {{/!userimg}} mv $JULIA_PATH/userimg.so $JULIA_PATH/usr/lib/julia/sys.so && \
{{#!userimg}}# {{/!userimg}} yum -y autoremove $PKGS && \
{{#!userimg}}# {{/!userimg}} yum -y clean all

22
defaults/dockerignore
View File

@ -1,22 +0,0 @@
# When this Julia package is copied into the Docker image if the resulting package is a
# clean git repo on an untracked branch (a branch with no associated remote branch) then
# Julia will error when trying to run `Pkg.update`. This problem can be solved by either
# making a tracked branch (hard) or making the git repo dirty (easy).
#
# If we exclude any committed file then this repo will be dirty inside of the Docker image.
Dockerfile
.dockerignore
# Ignore dependencies that are built/retrieved. Note that copying binary dependencies is
# brittle as these packages may break due to differences in operating system versions,
# build environments, and/or absolute path dependencies. Be sure to add required build
# files within "deps" directory to the exception list below.
deps/
!deps/build.jl
# Avoid copying the .git content into the image. The exception being a couple of files to
# identify the revision of this repo which is embedded into the image.
.git*
!.git/HEAD
!.git/refs/heads/

3
src/PkgTemplates.jl
View File

@ -4,7 +4,7 @@ import Base.==
using Mustache
using URIParser
export generate, show_license, Template, GitHubPages, Docker, AppVeyor, TravisCI, CodeCov
export generate, show_license, Template, GitHubPages, AppVeyor, TravisCI, CodeCov
abstract type Plugin end
@ -15,7 +15,6 @@ include(joinpath("plugins", "documenter.jl"))
include(joinpath("plugins", "appveyor.jl"))
include(joinpath("plugins", "codecov.jl"))
include(joinpath("plugins", "travis.jl"))
include(joinpath("plugins", "docker.jl"))
include(joinpath("plugins", "githubpages.jl"))

122
src/plugins/docker.jl
View File

@ -1,122 +0,0 @@
"""
Docker(; kwargs...) -> Docker
Add Docker to a template's plugins to create a Dockerfile suitable for building a
Docker image containing the package.
# Keyword Arguments
* `base_image::AbstractString="julia:latest"`: The base image used in the Dockerfile.
* `dockerfile_file::Union{AbstractString, Void}=""`: The path to the Dockerfile template
to use. If `nothing` is supplied, then no file will be generated.
* `dockerignore_file::Union{AbstractString, Void}=""`: The path to the .dockerignore
template to use. If `nothing` is supplied, then no file will be generated.
* `system_pkgs::Vector{AbstractString}=String[]`: Linux system packages to install.
* `python_pkgs::Vector{AbstractString}=String[]`: Python packages to install with `pip`.
* `user_image::Bool=true`: Allows the Dockerfile to build a Julia system image which
includes the package.
# Notes
## Ordering
Linux packages will be installed first, followed by Python packages. Julia will then use
`Pkg.resolve()` to install the packages in this package's REQUIRE file.
## Modification
Package developers are encouraged to modify `requirements.txt` and `system-pkgs.txt` to
reflect new requirements as they would `REQUIRE`.
"""
struct Docker <: Plugin
base_image::AbstractString
dockerfile_file::AbstractString
dockerignore_file::AbstractString
user_image::Bool
system_pkgs::Vector{AbstractString}
python_pkgs::Vector{AbstractString}
gitignore_files::Vector{AbstractString}
function Docker(;
base_image="julia:latest",
dockerfile_file::Union{AbstractString, Void}="",
dockerignore_file::Union{AbstractString, Void}="",
user_image::Bool=true,
system_pkgs::Vector{AbstractString}=String[],
python_pkgs::Vector{AbstractString}=String[],
)
if dockerfile_file != nothing
if isempty(dockerfile_file)
dockerfile_file = joinpath(DEFAULTS_DIR, "Dockerfile")
end
if !isfile(dockerfile_file)
throw(ArgumentError("File $dockerfile_file does not exist"))
end
end
if dockerignore_file != nothing
if isempty(dockerignore_file)
dockerignore_file = joinpath(DEFAULTS_DIR, "dockerignore")
end
if !isfile(dockerignore_file)
throw(ArgumentError("File $dockerignore_file does not exist"))
end
end
new(
base_image, dockerfile_file, dockerignore_file,
user_image, system_pkgs, python_pkgs, String[],
)
end
end
"""
gen_plugin(plugin::Docker, template::Template, pkg_name::AbstractString) -> Vector{String}
Generate a Dockerfile for running an app-style package and generate dependency files of
different sorts for installation within a Docker container.
# Arguments
* `plugin::Docker`: Plugin whose files are being generated.
* `template::Template`: Template configuration and plugins.
* `pkg_name::AbstractString`: Name of the package.
Returns an array of generated files.
"""
function gen_plugin(plugin::Docker, template::Template, pkg_name::AbstractString)
pkg_dir = joinpath(template.path, pkg_name)
return_files = String[]
if plugin.dockerignore_file != nothing
push!(return_files, ".dockerignore")
text = substitute(readstring(plugin.dockerignore_file), pkg_name, template)
gen_file(joinpath(pkg_dir, ".dockerignore"), text)
end
if plugin.dockerfile_file != nothing
push!(return_files, "Dockerfile")
view = Dict(
"BASE_IMAGE" => plugin.base_image,
"MAINTAINER" => template.authors,
"!system" => !isempty(plugin.system_pkgs),
"!python" => !isempty(plugin.python_pkgs),
"!userimg" => !plugin.user_image,
)
text = substitute(readstring(plugin.dockerfile_file), pkg_name, template, view)
gen_file(joinpath(pkg_dir, "Dockerfile"), text)
end
pkg_lists = Dict(
"system-pkgs.txt" => plugin.system_pkgs,
"requirements.txt" => plugin.python_pkgs,
)
for (file_name, pkg_list) in pkg_lists
isempty(pkg_list) && continue
open(joinpath(pkg_dir, file_name), "w") do fp
for pkg in pkg_list
println(fp, pkg)
end
end
push!(return_files, file_name)
end
return return_files
end