From 9d725f4316ec6bfb1f46159ac8d5f8f72070cfa8 Mon Sep 17 00:00:00 2001 From: Chris de Graaf Date: Mon, 14 Aug 2017 11:36:24 -0500 Subject: [PATCH] Remove Docker plugin for now --- defaults/Dockerfile | 105 ------------------------------------ defaults/dockerignore | 22 -------- src/PkgTemplates.jl | 3 +- src/plugins/docker.jl | 122 ------------------------------------------ 4 files changed, 1 insertion(+), 251 deletions(-) delete mode 100644 defaults/Dockerfile delete mode 100644 defaults/dockerignore delete mode 100644 src/plugins/docker.jl diff --git a/defaults/Dockerfile b/defaults/Dockerfile deleted file mode 100644 index e9f7257..0000000 --- a/defaults/Dockerfile +++ /dev/null @@ -1,105 +0,0 @@ -FROM {{BASE_IMAGE}} -LABEL maintainer "{{MAINTAINER}}" - -ENV PKG_PATH "$JULIA_PKGDIR/$JULIA_PKGVER/{{PKGNAME}}" - -# Note: The "/etc/yum/protected.d/*.conf" contains the names of packages which are -# protected by yum and should not be removed. Protecting packages is necessary as -# this avoids the problem of accidentally removing a dependency of a package which -# is a runtime requirement. - -# Get security updates -RUN yum -y update-minimal && \ - yum -y clean all - -# Install the current repo as a Julia package -COPY . $PKG_PATH -WORKDIR $PKG_PATH - -# Record SHA of the revision the git repo. Note that the repo may not have been in -# a clean repo which makes the SHA the last known commit. -RUN REF=$(cat .git/HEAD | cut -d' ' -f2) && \ - SHA=$(cat .git/$REF 2>/dev/null || echo $REF) && \ - echo $SHA > REVISION && echo "Revision: $SHA" && \ - rm -rf .git - -# Install system requirements. -{{#!system}}# {{/!system}}ENV PKGS $(cat ./system-pkgs.txt) -{{#!system}}# {{/!system}}RUN yum -y install $PKGS && \ -{{#!system}}# {{/!system}} echo $PKGS | tr -s '\t ' '\n' > /etc/yum/protected.d/system-pkgs.conf && \ -{{#!system}}# {{/!system}} yum -y clean all - -# Install Python requirements. Using the latest version of pip to avoid issues with -# outdated system versions. Note most pip installations will use wheels which don't require -# any compilation but if we need to compile a package from source additional requirements -# may be needed. For example "pandas" requires: python27-devel, python27-Cython, gcc, -# gcc-c++, and gcc-gfortran to be compiled from source. -{{#!python}}# {{/!python}}ENV PKGS \ -{{#!python}}# {{/!python}} python27-devel \ -{{#!python}}# {{/!python}} python27-Cython \ -{{#!python}}# {{/!python}} gcc \ -{{#!python}}# {{/!python}} gcc-c++ \ -{{#!python}}# {{/!python}} gcc-gfortran -{{#!python}}# {{/!python}}ENV PINNED_PKGS \ -{{#!python}}# {{/!python}} python27 -{{#!python}}# {{/!python}}RUN yum -y install $PKGS $PINNED_PKGS && \ -{{#!python}}# {{/!python}} echo $PINNED_PKGS | tr -s '\t ' '\n' > /etc/yum/protected.d/python.conf && \ -{{#!python}}# {{/!python}} curl https://bootstrap.pypa.io/get-pip.py | python && \ -{{#!python}}# {{/!python}} pip install -r ./requirements.txt && \ -{{#!python}}# {{/!python}} yum -y autoremove $PKGS && \ -{{#!python}}# {{/!python}} yum -y clean all - - -# Add and build the all of the required Julia packages. In order to allow the use of -# BinDeps.jl we need to temporarily install additional system packages. -# -# - BinDeps.jl runtime requirements: sudo, make, gcc, unzip, bzip2, xz, unzip -# -# BinDeps runtime requirements are only used while other packages are being built which -# makes them safe to only be temporarily installed. When installing system libraries -# BinDeps always uses "sudo" to install system packages and waits for user confirmation -# before installing a package. We'll both install "sudo" and always supply the `-y` flag -# to ensure that BinDeps installations work automatically. A good test to make sure -# BinDeps is setup correctly is to run `Pkg.add("Cairo"); Pkg.test("Cairo")` -# -# - HDF5.jl requires the EPEL repo to install hdf5 automatically through BinDeps -# (https://aws.amazon.com/premiumsupport/knowledge-center/ec2-enable-epel/) -# - yum-config-manager is installed by: yum-utils -ENV PKGS \ - sudo \ - make \ - gcc \ - gcc-c++ \ - bzip2 \ - xz \ - unzip \ - epel-release \ - yum-utils -RUN yum -y install $PKGS && \ - yum-config-manager --setopt=assumeyes=1 --save > /dev/null && \ - yum-config-manager --enable epel > /dev/null && \ - yum list installed | tr -s ' ' | cut -d' ' -f1 | sort > /tmp/pre_state && \ - update-metadata && julia -e 'Pkg.update(); Pkg.resolve(); Pkg.build("{{PKGNAME}}")' && \ - yum list installed | tr -s ' ' | cut -d' ' -f1 | sort > /tmp/post_state && \ - comm -3 /tmp/pre_state /tmp/post_state | grep $'\t' | sed 's/\t//' | sed 's/\..*//' > /etc/yum/protected.d/julia-pkgs.conf && \ - rm /tmp/pre_state /tmp/post_state && \ - yum-config-manager --disable epel > /dev/null && \ - for p in $PKGS; do yum -y autoremove $p &>/dev/null && echo "Removed $p" || echo "Skipping removal of $p"; done && \ - yum -y clean all - -# Improve the startup time of packages by pre-compiling {{PKGNAME}} and its dependencies -# into the default system image. -# Note: Need to have libc to avoid "/usr/bin/ld: cannot find crti.o: No such file or directory" -{{#!userimg}}# {{/!userimg}}ENV PKGS \ -{{#!userimg}}# {{/!userimg}} gcc -{{#!userimg}}# {{/!userimg}}ENV PINNED_PKGS \ -{{#!userimg}}# {{/!userimg}} glibc -{{#!userimg}}# {{/!userimg}}RUN yum -y install $PKGS $PINNED_PKGS && \ -{{#!userimg}}# {{/!userimg}} cd $JULIA_PATH/base && \ -{{#!userimg}}# {{/!userimg}} source $JULIA_PATH/Make.user && \ -{{#!userimg}}# {{/!userimg}} $JULIA_PATH/julia -C $MARCH --output-o $JULIA_PATH/userimg.o --sysimage $JULIA_PATH/usr/lib/julia/sys.so --startup-file=no -e "using {{PKGNAME}}" && \ -{{#!userimg}}# {{/!userimg}} cc -shared -o $JULIA_PATH/userimg.so $JULIA_PATH/userimg.o -ljulia -L$JULIA_PATH/usr/lib && \ -{{#!userimg}}# {{/!userimg}} mv $JULIA_PATH/userimg.o $JULIA_PATH/usr/lib/julia/sys.o && \ -{{#!userimg}}# {{/!userimg}} mv $JULIA_PATH/userimg.so $JULIA_PATH/usr/lib/julia/sys.so && \ -{{#!userimg}}# {{/!userimg}} yum -y autoremove $PKGS && \ -{{#!userimg}}# {{/!userimg}} yum -y clean all diff --git a/defaults/dockerignore b/defaults/dockerignore deleted file mode 100644 index c137883..0000000 --- a/defaults/dockerignore +++ /dev/null @@ -1,22 +0,0 @@ -# When this Julia package is copied into the Docker image if the resulting package is a -# clean git repo on an untracked branch (a branch with no associated remote branch) then -# Julia will error when trying to run `Pkg.update`. This problem can be solved by either -# making a tracked branch (hard) or making the git repo dirty (easy). -# -# If we exclude any committed file then this repo will be dirty inside of the Docker image. - -Dockerfile -.dockerignore - -# Ignore dependencies that are built/retrieved. Note that copying binary dependencies is -# brittle as these packages may break due to differences in operating system versions, -# build environments, and/or absolute path dependencies. Be sure to add required build -# files within "deps" directory to the exception list below. -deps/ -!deps/build.jl - -# Avoid copying the .git content into the image. The exception being a couple of files to -# identify the revision of this repo which is embedded into the image. -.git* -!.git/HEAD -!.git/refs/heads/ diff --git a/src/PkgTemplates.jl b/src/PkgTemplates.jl index 84acb70..948b0c6 100644 --- a/src/PkgTemplates.jl +++ b/src/PkgTemplates.jl @@ -4,7 +4,7 @@ import Base.== using Mustache using URIParser -export generate, show_license, Template, GitHubPages, Docker, AppVeyor, TravisCI, CodeCov +export generate, show_license, Template, GitHubPages, AppVeyor, TravisCI, CodeCov abstract type Plugin end @@ -15,7 +15,6 @@ include(joinpath("plugins", "documenter.jl")) include(joinpath("plugins", "appveyor.jl")) include(joinpath("plugins", "codecov.jl")) include(joinpath("plugins", "travis.jl")) -include(joinpath("plugins", "docker.jl")) include(joinpath("plugins", "githubpages.jl")) diff --git a/src/plugins/docker.jl b/src/plugins/docker.jl deleted file mode 100644 index 8c1b076..0000000 --- a/src/plugins/docker.jl +++ /dev/null @@ -1,122 +0,0 @@ -""" - Docker(; kwargs...) -> Docker - -Add Docker to a template's plugins to create a Dockerfile suitable for building a -Docker image containing the package. - -# Keyword Arguments -* `base_image::AbstractString="julia:latest"`: The base image used in the Dockerfile. -* `dockerfile_file::Union{AbstractString, Void}=""`: The path to the Dockerfile template - to use. If `nothing` is supplied, then no file will be generated. -* `dockerignore_file::Union{AbstractString, Void}=""`: The path to the .dockerignore - template to use. If `nothing` is supplied, then no file will be generated. -* `system_pkgs::Vector{AbstractString}=String[]`: Linux system packages to install. -* `python_pkgs::Vector{AbstractString}=String[]`: Python packages to install with `pip`. -* `user_image::Bool=true`: Allows the Dockerfile to build a Julia system image which - includes the package. - -# Notes -## Ordering -Linux packages will be installed first, followed by Python packages. Julia will then use -`Pkg.resolve()` to install the packages in this package's REQUIRE file. - -## Modification -Package developers are encouraged to modify `requirements.txt` and `system-pkgs.txt` to -reflect new requirements as they would `REQUIRE`. -""" -struct Docker <: Plugin - base_image::AbstractString - dockerfile_file::AbstractString - dockerignore_file::AbstractString - user_image::Bool - system_pkgs::Vector{AbstractString} - python_pkgs::Vector{AbstractString} - gitignore_files::Vector{AbstractString} - - function Docker(; - base_image="julia:latest", - dockerfile_file::Union{AbstractString, Void}="", - dockerignore_file::Union{AbstractString, Void}="", - user_image::Bool=true, - system_pkgs::Vector{AbstractString}=String[], - python_pkgs::Vector{AbstractString}=String[], - ) - if dockerfile_file != nothing - if isempty(dockerfile_file) - dockerfile_file = joinpath(DEFAULTS_DIR, "Dockerfile") - end - if !isfile(dockerfile_file) - throw(ArgumentError("File $dockerfile_file does not exist")) - end - end - - if dockerignore_file != nothing - if isempty(dockerignore_file) - dockerignore_file = joinpath(DEFAULTS_DIR, "dockerignore") - end - if !isfile(dockerignore_file) - throw(ArgumentError("File $dockerignore_file does not exist")) - end - end - - new( - base_image, dockerfile_file, dockerignore_file, - user_image, system_pkgs, python_pkgs, String[], - ) - end -end - -""" - gen_plugin(plugin::Docker, template::Template, pkg_name::AbstractString) -> Vector{String} - -Generate a Dockerfile for running an app-style package and generate dependency files of -different sorts for installation within a Docker container. - -# Arguments - -* `plugin::Docker`: Plugin whose files are being generated. -* `template::Template`: Template configuration and plugins. -* `pkg_name::AbstractString`: Name of the package. - -Returns an array of generated files. -""" -function gen_plugin(plugin::Docker, template::Template, pkg_name::AbstractString) - pkg_dir = joinpath(template.path, pkg_name) - return_files = String[] - - if plugin.dockerignore_file != nothing - push!(return_files, ".dockerignore") - text = substitute(readstring(plugin.dockerignore_file), pkg_name, template) - gen_file(joinpath(pkg_dir, ".dockerignore"), text) - end - - if plugin.dockerfile_file != nothing - push!(return_files, "Dockerfile") - view = Dict( - "BASE_IMAGE" => plugin.base_image, - "MAINTAINER" => template.authors, - "!system" => !isempty(plugin.system_pkgs), - "!python" => !isempty(plugin.python_pkgs), - "!userimg" => !plugin.user_image, - ) - text = substitute(readstring(plugin.dockerfile_file), pkg_name, template, view) - gen_file(joinpath(pkg_dir, "Dockerfile"), text) - end - - pkg_lists = Dict( - "system-pkgs.txt" => plugin.system_pkgs, - "requirements.txt" => plugin.python_pkgs, - ) - - for (file_name, pkg_list) in pkg_lists - isempty(pkg_list) && continue - open(joinpath(pkg_dir, file_name), "w") do fp - for pkg in pkg_list - println(fp, pkg) - end - end - push!(return_files, file_name) - end - - return return_files -end