diff --git a/packaging/python_wheel/Dockerfile b/packaging/python_wheel/Dockerfile new file mode 100644 index 0000000..56dff55 --- /dev/null +++ b/packaging/python_wheel/Dockerfile @@ -0,0 +1,45 @@ +FROM quay.io/pypa/manylinux1_x86_64 +RUN yum install -y zlib-devel bzip2-devel flex + +############ +# CMake +############ + +# Need to build this from source, since the binaries from Kitware don't +# run with the old version of the libraries in this Centos version +RUN wget --no-check-certificate http://cmake.org/files/v3.7/cmake-3.7.2.tar.gz && tar xfz cmake-3.7.2.tar.gz && cd cmake-3.7.2 && ./configure && make -j2 install +RUN rm -rf cmake-3.7.2.tar.gz cmake-3.7.2 + + +############ +# HDF5 +############ + +RUN wget --no-check-certificate http://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.17/src/hdf5-1.8.17.tar.bz2 && tar xfj hdf5-1.8.17.tar.bz2 +RUN cd hdf5-1.8.17 && ./configure CFLAGS=-fPIC CXXFLAGS=-fPIC --prefix=/usr/local --enable-cxx --with-pic --disable-shared --with-zlib=/usr/include/,/usr/lib64/ && make -j2 install + +# --disable-shared doesn't add libz to the archive, so it +# must be done manually at the end, there aren't any collisions in +# the .a file, so currently safe +RUN cd hdf5-1.8.17 && ar x /usr/lib64/libz.a && ar r /usr/local/lib/libhdf5.a *.o +RUN rm -rf hdf5-1.8.17.tar.bz2 hdf5-1.8.17 + + +############ +# Boost +############ + +RUN wget --no-check-certificate http://downloads.sourceforge.net/project/boost/boost/1.59.0/boost_1_59_0.tar.gz && tar xfz boost_1_59_0.tar.gz +RUN cd boost_1_59_0 && ./bootstrap.sh --with-libraries=date_time,filesystem,program_options,regex,serialization,system,test && ./b2 -j2 -q cxxflags=-fPIC cflags=-fPIC threading=multi link=static --build-type=minimal install +RUN cd boost_1_59_0 && ./b2 --clean +RUN rm -rf boost_1_59_0.tar.gz # don't delete build folder, needed to build boost.python for different python versions on-the-fly + +############ +# Doxygen +############ + +# Needed for docstrings generation and system doxygen is too old +# after 1.8.6, we can't use anymore system flex, see https://stackoverflow.com/questions/20844635 +RUN wget --no-check-certificate https://github.com/doxygen/doxygen/archive/Release_1_8_5.tar.gz && tar xfz Release_1_8_5 && cd doxygen-Release_1_8_5 && ./configure && make -j2 && make install +RUN rm -rf Release_1_8_5 doxygen-Release_1_8_5 + diff --git a/packaging/python_wheel/build_docker_image.sh b/packaging/python_wheel/build_docker_image.sh new file mode 100755 index 0000000..47d750a --- /dev/null +++ b/packaging/python_wheel/build_docker_image.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +BASE=$(git rev-parse --show-toplevel) + +docker build -t brain_wheel \ + --build-arg http_proxy=$http_proxy \ + --build-arg https_proxy=$https_proxy \ + $BASE/packaging/python_wheel | tee build.log + +ID=$(tail -1 build.log | awk '{print $3;}') +upload2repo -t docker -i $ID -g latest -n brain_wheel diff --git a/packaging/python_wheel/build_wheel.sh b/packaging/python_wheel/build_wheel.sh index 55ce9a7..297cc69 100755 --- a/packaging/python_wheel/build_wheel.sh +++ b/packaging/python_wheel/build_wheel.sh @@ -1,21 +1,15 @@ #!/usr/bin/env bash BASE=$(git rev-parse --show-toplevel) -SOURCE="http://cmake.org/files/v3.7/cmake-3.7.2.tar.gz \ - http://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.8.17/src/hdf5-1.8.17.tar.bz2 \ - http://downloads.sourceforge.net/project/boost/boost/1.59.0/boost_1_59_0.tar.gz \ - https://github.com/doxygen/doxygen/archive/Release_1_8_5.tar.gz" -pushd $BASE/build -cmake .. -DCLONE_SUBPROJECTS=ON -rm -rf * -popd - -mkdir -p $BASE/ext_src -for src in $SOURCE; do - output=$(basename "$src") - [[ -e $basename ]] || wget --continue -O $BASE/ext_src/$output $src -done +export PIPPROXY="-i https://bbpteam.epfl.ch/repository/devpi/simple" +export http_proxy=$HTTP_PROXY +export https_proxy=$HTTPS_PROXY docker run --rm \ - -v $BASE:/io quay.io/pypa/manylinux1_x86_64 \ + -e http_proxy=$http_proxy \ + -e https_proxy=$https_proxy \ + -e PIPPROXY="$PIPPROXY" \ + -e UID=$UID \ + -v $BASE:/io:Z \ + bbpdocker.epfl.ch/brain_wheel \ /bin/bash /io/packaging/python_wheel/docker_build_wheel.sh diff --git a/packaging/python_wheel/docker_build_wheel.sh b/packaging/python_wheel/docker_build_wheel.sh index fbe6e71..f8066aa 100755 --- a/packaging/python_wheel/docker_build_wheel.sh +++ b/packaging/python_wheel/docker_build_wheel.sh @@ -5,7 +5,6 @@ PYTHON_VERSIONS="cp27-cp27mu cp35-cp35m cp36-cp36m" NUMPY_VERSION=1.12.0 PACKAGING_DIR=/io/packaging/python_wheel/ WHEELHOUSE=/io/packaging/python_wheel/wheelhouse/ -EXT_SRC=/io/ext_src get_python_include() @@ -14,82 +13,25 @@ get_python_include() $PYTHON -c 'import distutils.sysconfig as s; print(s.get_python_inc())' } -install_reqs() -{ - yum install -y zlib-devel bzip2-devel flex -} - -# Need to build this from source, since the binaries from Kitware don't -# run with the old version of the libraries in this Centos version -build_cmake() -{ - [[ -e /usr/local/bin/cmake ]] && return - - [[ -e /tmp/cmake-3.7.2 ]] || tar xfz $EXT_SRC/cmake-3.7.2.tar.gz -C /tmp - cd /tmp/cmake-3.7.2 - ./configure - make -j8 install -} - -# h5 morphologies depend on hdf5, so need to build it -build_hdf() -{ - [[ -e /usr/local/lib/libhdf5.a ]] && return - - [[ -e /tmp/hdf5-1.8.17 ]] || tar xjf $EXT_SRC/hdf5-1.8.17.tar.bz2 -C /tmp - cd /tmp/hdf5-1.8.17 - ./configure CFLAGS=-fPIC CXXFLAGS=-fPIC \ - --prefix=/usr/local \ - --enable-cxx \ - --with-pic \ - --disable-shared \ - --with-zlib=/usr/include/,/usr/lib64/ - make -j8 install - - # --disable-shared doesn't add libz to the archive, so it - # must be done manually at the end, there aren't any collisions in - # the .a file, so currently safe - ar x /usr/lib64/libz.a - ar r /usr/local/lib/libhdf5.a *.o -} - -# Needed for boost_python/boost_regex/boost_filesystem -build_boost() +build_boost_python() { local PYTHON=$1; shift local PYTHON_INC=$1; shift - tar xfz $EXT_SRC/boost_1_59_0.tar.gz -C /tmp - cd /tmp/boost_1_59_0 - - rm -f project-config.jam - ./bootstrap.sh --with-python=$PYTHON - ./b2 clean - - # HACK: b2 doesn't take into account the python ABIFLAGS, - # so it doesn't find the correct header files - perl -i -pe "s#(using python [^;]+);#\1 : $PYTHON_INC ;#" project-config.jam - - ./b2 -j8 -q --without-mpi \ - cxxflags=-fPIC \ - cflags=-fPIC \ - threading=multi \ - link=static \ - install -} -# Needed for docstrings generation and system doxygen is too old -build_doxygen() -{ - [[ -e /usr/local/bin/doxygen ]] && return + pushd /boost_1_59_0 - # after 1.8.6, we can't use anymore system flex - # https://stackoverflow.com/questions/20844635 - tar xfz $EXT_SRC/Release_1_8_5.tar.gz -C /tmp - cd /tmp/doxygen-Release_1_8_5 + ./b2 --clean + ./bootstrap.sh --with-libraries=python --with-python=$PYTHON - ./configure - make -j8 - make install + ./b2 -j2 -q \ + --build-type=minimal \ + cxxflags=-fPIC \ + cflags=-fPIC \ + threading=multi \ + link=static \ + include=$PYTHON_INC \ + install + popd } build_brain() @@ -102,16 +44,18 @@ build_brain() # for each python version the ABI may have changed, so rebuild # the boost python bindings - build_boost $PYTHON $PYTHON_INC + build_boost_python $PYTHON $PYTHON_INC /opt/python/$version/bin/pip install "numpy==$NUMPY_VERSION" /opt/python/$version/bin/pip install "sphinx==1.3.6" lxml + mkdir -p /io/build cd /io/build + rm -rf * - # disabling OMP so we don't have to link in libgomp cmake .. \ -DCMAKE_BUILD_TYPE=Release \ + -DCLONE_SUBPROJECTS=ON \ -DUSE_PYTHON_VERSION=$MAJOR_VERSION \ -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON \ -DPYTHON_INCLUDE_DIR=$PYTHON_INC \ @@ -119,20 +63,20 @@ build_brain() -DCOMMON_LIBRARY_TYPE=STATIC \ -DHDF5_USE_STATIC_LIBRARIES=ON \ -DSPHINX_ROOT=/opt/python/$version - make -j8 brain_python + make -j2 brain_python cp packaging/python_wheel/setup.py $PACKAGING_DIR/setup.cfg lib pushd lib - $PYTHON setup.py bdist_wheel + $PYTHON setup.py bdist_wheel --bdist-dir=/home auditwheel repair -w $WHEELHOUSE dist/*${version}* popd -} -install_reqs -build_cmake -build_hdf -build_doxygen + cd .. + rm -rf build +} for version in $PYTHON_VERSIONS; do build_brain $version done + +chown -R $UID.$UID $WHEELHOUSE diff --git a/packaging/python_wheel/upload_wheel.sh b/packaging/python_wheel/upload_wheel.sh new file mode 100755 index 0000000..6c8bbe3 --- /dev/null +++ b/packaging/python_wheel/upload_wheel.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +BASE=$(git rev-parse --show-toplevel) + +for wheel in `ls $BASE/packaging/python_wheel/wheelhouse/brain*.whl`; do + upload2repo -t python -r dev -f $wheel +done + +# cleanup afterwards +docker run --rm \ + -v $BASE:/io:Z \ + bbpdocker.epfl.ch/brain_wheel \ + /bin/rm -rf /io/packaging/python_wheel/wheelhouse