From 63ed6c034768eec736fa6f06853f3c085e61d6d4 Mon Sep 17 00:00:00 2001 From: Bertrand NOEL Date: Wed, 27 Apr 2016 15:59:43 +0200 Subject: [PATCH 1/6] Dockerfile --- Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0f8a8fc --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM jupyter/jupyterhub:0.5.0 + +COPY . /srv/everware +WORKDIR /srv/everware/ +RUN make install + +EXPOSE 8000 +EXPOSE 8081 + +ENTRYPOINT ["/srv/everware/scripts/everware-server", "-f"] + From 03ddcef7d6d87bce2385582a26b19a8cf4f33bb7 Mon Sep 17 00:00:00 2001 From: Bertrand NOEL Date: Wed, 27 Apr 2016 15:59:56 +0200 Subject: [PATCH 2/6] Config and env file to run on local or remote Docker --- env.docker-local.orig | 4 ++++ env.docker-remote.orig | 5 +++++ etc/container_config.py | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+) create mode 100644 env.docker-local.orig create mode 100644 env.docker-remote.orig create mode 100644 etc/container_config.py diff --git a/env.docker-local.orig b/env.docker-local.orig new file mode 100644 index 0000000..3b59e7e --- /dev/null +++ b/env.docker-local.orig @@ -0,0 +1,4 @@ +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= +OAUTH_CALLBACK_URL=http://xxxxxxxx:8000/hub/oauth_callback +EVERWARE_WHITELIST=whitelist.txt diff --git a/env.docker-remote.orig b/env.docker-remote.orig new file mode 100644 index 0000000..62ce772 --- /dev/null +++ b/env.docker-remote.orig @@ -0,0 +1,5 @@ +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= +OAUTH_CALLBACK_URL=http://xxxxxxxx:8000/hub/oauth_callback +EVERWARE_WHITELIST=whitelist.txt +DOCKER_HOST=tcp://xxx.xxx.xxx.xxx:2376 diff --git a/etc/container_config.py b/etc/container_config.py new file mode 100644 index 0000000..7730a12 --- /dev/null +++ b/etc/container_config.py @@ -0,0 +1,18 @@ +# Use this config file to run everware in a container, +# and create user containers in local or remote Docker service +# +# In case of local, don't forget to mount /var/run/docker.sock +# In case of remote, don't forget the DOCKER_HOST environment variable + +c = get_config() +load_subconfig('etc/base_config.py') +load_subconfig('etc/github_auth.py') + +c.JupyterHub.hub_ip = '0.0.0.0' +c.JupyterHub.proxy_api_ip = '0.0.0.0' + +# Change these two next settings: +# IP of the machine where the Everware can be contacted +c.DockerSpawner.hub_ip_connect = 'xxx.xxx.xxx.xxx' +# IP of the machine running Docker service +c.DockerSpawner.container_ip = 'xxx.xxx.xxx.xxx' From 8508be46914a1ea043cfb1c50f1a762edab9b397 Mon Sep 17 00:00:00 2001 From: Bertrand NOEL Date: Wed, 27 Apr 2016 16:00:08 +0200 Subject: [PATCH 3/6] Config and env file to run on Docker Swarm --- env.docker-swarm.orig | 7 +++++++ etc/container_swarm_config.py | 21 +++++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 env.docker-swarm.orig create mode 100644 etc/container_swarm_config.py diff --git a/env.docker-swarm.orig b/env.docker-swarm.orig new file mode 100644 index 0000000..8495b76 --- /dev/null +++ b/env.docker-swarm.orig @@ -0,0 +1,7 @@ +GITHUB_CLIENT_ID= +GITHUB_CLIENT_SECRET= +OAUTH_CALLBACK_URL=http://xxxxxxxx:8000/hub/oauth_callback +EVERWARE_WHITELIST=whitelist.txt +DOCKER_CERT_PATH=/etc/docker +DOCKER_HOST=tcp://xxx.xxx.xxx.xxx:2376 +DOCKER_TLS_VERIFY=1 diff --git a/etc/container_swarm_config.py b/etc/container_swarm_config.py new file mode 100644 index 0000000..8db0926 --- /dev/null +++ b/etc/container_swarm_config.py @@ -0,0 +1,21 @@ +# Use this config file to run everware in a container, +# and create the user container on a Docker Swarm cluster +# +# No need to mount /var/run/docker.sock, +# but don't forget to set Docker Swarm environment +# variables in the env file + +c = get_config() +load_subconfig('etc/base_config.py') +load_subconfig('etc/github_auth.py') + +c.JupyterHub.hub_ip = '0.0.0.0' +c.JupyterHub.proxy_api_ip = '0.0.0.0' + +c.JupyterHub.spawner_class = 'everware.CustomSwarmSpawner' +c.Spawner.tls = True +c.DockerSpawner.tls = True + +# Change this setting: +# IP of the machine where the Everware can be contacted +c.DockerSpawner.hub_ip_connect = 'xxx.xxx.xxx.xxx' From 48ea1bb6c79af704caf975997eac1ab564339bf2 Mon Sep 17 00:00:00 2001 From: Bertrand NOEL Date: Wed, 27 Apr 2016 16:20:18 +0200 Subject: [PATCH 4/6] Documentation --- README.md | 2 ++ docker.md | 73 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 docker.md diff --git a/README.md b/README.md index 81cc305..5d5f53f 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ In order to deploy your own `everware` instance, you have to: everware-server -f etc/local_config.py --debug --no-ssl ``` +Everware can also be deployed in a Docker container. See [Everware in Docker container](docker.md) + ## Development diff --git a/docker.md b/docker.md new file mode 100644 index 0000000..6f8fd2d --- /dev/null +++ b/docker.md @@ -0,0 +1,73 @@ + +# Run Everware in a Docker container + +This section explains how to run Everware in a container, and the different possibilities to run users container + +## Build Everware container + +[Dockerfile](Dockerfile) + +``` + docker build -t everware . +``` + +JupyterHub Dockerfile needs a ["jupyterhub_config.py"](jupyterhub_config.py) file to be present at the root of the project. + +## Create user containers on same machine as the Everware one + +Fill the file which will contains environment variables for Everware. +``` + cp env.docker-local.orig env.docker-local +``` +Define the `GITHUB_CLIENT_ID`, `GITHUB_CLIENT_SECRET`, and `OAUTH_CALLBACK_URL` + +Edit `etc/container_config.py` file to set `c.DockerSpawner.hub_ip_connect` and `c.DockerSpawner.container_ip` to the IP of your machine running the Everware container. + +``` + docker run -d --name everware \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v $PWD/etc/container_config.py:/srv/everware/etc/container_config.py \ + --env-file=env.docker-local \ + -p 8000:8000 \ + -p 8081:8081 \ + everware /srv/everware/etc/container_config.py --no-ssl --debug +``` + +Note that you can skip `-v $PWD/etc/container_config.py:/srv/everware/etc/container_config.py` part if you build the container after having modified the config file `etc/container_config.py` + +## Create user containers on a remote Docker machine + +``` + cp env.docker-remote.orig env.docker-remote +``` +Same as before, and define `DOCKER_HOST`, the IP of your remote Docker machine. + +``` +docker run -d --name everware \ + -v $PWD/etc/container_config.py:/srv/everware/etc/container_config.py \ + --env-file=env.docker-remote \ + -p 8000:8000 \ + -p 8081:8081 \ + everware /srv/everware/etc/container_config.py --no-ssl --debug +``` + +## Create user containers on a Docker Swarm cluster + +This allows to create users container on a Docker Swarm cluster. + +``` + cp env.docker-swarm.orig env.docker-swarm +``` +Define the variable, and set `DOCKER_HOST` to the IP of your Swarm master. + +Edit `etc/container_swarm.py` file to set `c.DockerSpawner.hub_ip_connect` to the IP of your machine hosting Everware container. + +``` + docker run --rm -it --name everware \ + -v $PWD/etc/container_swarm_config.py:/srv/everware/etc/container_swarm_config.py \ + -v /home/ubuntu/docker:/etc/docker \ + --env-file=env.docker-swarm \ + -p 8000:8000 \ + -p 8081:8081 \ + everware /srv/everware/etc/container_swarm_config.py --no-ssl --debug +``` From d72a0c44b0a4a3a459ea869a61fe2b16a56dc88d Mon Sep 17 00:00:00 2001 From: anaderi Date: Sun, 2 Apr 2017 05:15:30 +0000 Subject: [PATCH 5/6] fix Dockerfile --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0f8a8fc..bd687d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM jupyter/jupyterhub:0.5.0 +FROM jupyterhub/jupyterhub:0.7.2 +RUN apt-get update && apt-get install make COPY . /srv/everware WORKDIR /srv/everware/ From 16583e7cbf355c5ebefbfdff3661621b400c819c Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 4 Apr 2017 21:38:41 +0300 Subject: [PATCH 6/6] updated configs and makefile to deploy everware as docker image --- Makefile | 36 +---------------------- docker.md => docs/docker.md | 2 -- env.docker-remote.orig | 2 ++ env.docker-swarm.orig | 2 ++ etc/container_config.py | 9 +++--- etc/container_swarm_config.py | 2 +- run.makefile | 67 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 77 insertions(+), 43 deletions(-) rename docker.md => docs/docker.md (94%) create mode 100644 run.makefile diff --git a/Makefile b/Makefile index 639b5e9..be7784a 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,6 @@ SHELL := /bin/bash TEST_OPTIONS := -s tests -N 2 TESTS := test_happy_mp -LOG := everware.log -PIDFILE := everware.pid IP = $(shell python -c 'from IPython.utils.localinterfaces import public_ips; print (public_ips()[0])' 2>/dev/null) OPTIONS = --debug --port 8000 --no-ssl --JupyterHub.hub_ip=${IP} UPLOADDIR ?= ~/upload_screens @@ -27,14 +25,13 @@ else $(error Unable to find python) endif -EXECUTOR = everware-server - ifeq ($(shell uname -s),Linux) SPAWNER_IP = "127.0.0.1" else SPAWNER_IP = "192.168.99.100" endif +include run.makefile .PHONY: install reload clean run run-daemon stop test tail @@ -56,37 +53,6 @@ install: ## install everware if [ ! -f env.sh ] ; then cp env.sh.orig env.sh ; fi -reload: ## reload everware whitelist - PID=`pgrep '${EXECUTOR}'` ;\ - if [ -z "$${PID}" ] ; then echo "Cannot find running ${EXECUTOR}" ; exit 1 ; fi - pkill -1 '${EXECUTOR}' - -clean: ## clean user base - if [ -f ${PIDFILE} ] ; then echo "${PIDFILE} exists, cannot continute" ; exit 1; fi - rm -f jupyterhub.sqlite - -run-linux: clean ## run everware server on linux - source ./env.sh && \ - ${EXECUTOR} -f etc/local_config.py --no-ssl 2>&1 | tee ${LOG} - -run-dockermachine: clean ## run everware server on MacOS - source ./env.sh && \ - ${EXECUTOR} -f etc/local_dockermachine_config.py --no-ssl 2>&1 | tee ${LOG} - -run-daemon: clean ## run everware in daemon mode, linux only, SSL required - source ./env.sh && \ - ${EXECUTOR} -f etc/local_config.py >> ${LOG} 2>&1 & - pgrep ${EXECUTOR} > ${PIDFILE} || ( tail ${LOG} && exit 1 ) - echo "Started. Log saved to ${LOG}" - -stop: - -rm ${PIDFILE} - -pkill -9 ${EXECUTOR} - -pkill -9 node - -logs: ${LOG} ## watch log file - tail -f ${LOG} - test: ## run all tests export UPLOADDIR=${UPLOADDIR}; \ py.test everware/ ; \ diff --git a/docker.md b/docs/docker.md similarity index 94% rename from docker.md rename to docs/docker.md index 6f8fd2d..60b0548 100644 --- a/docker.md +++ b/docs/docker.md @@ -11,8 +11,6 @@ This section explains how to run Everware in a container, and the different poss docker build -t everware . ``` -JupyterHub Dockerfile needs a ["jupyterhub_config.py"](jupyterhub_config.py) file to be present at the root of the project. - ## Create user containers on same machine as the Everware one Fill the file which will contains environment variables for Everware. diff --git a/env.docker-remote.orig b/env.docker-remote.orig index 62ce772..75eef6e 100644 --- a/env.docker-remote.orig +++ b/env.docker-remote.orig @@ -3,3 +3,5 @@ GITHUB_CLIENT_SECRET= OAUTH_CALLBACK_URL=http://xxxxxxxx:8000/hub/oauth_callback EVERWARE_WHITELIST=whitelist.txt DOCKER_HOST=tcp://xxx.xxx.xxx.xxx:2376 +# most likely the same IP as DOCKER_HOST +DOCKER_PUBLIC_IP=xxx.xxx.xxx.xxx diff --git a/env.docker-swarm.orig b/env.docker-swarm.orig index 8495b76..7178d7c 100644 --- a/env.docker-swarm.orig +++ b/env.docker-swarm.orig @@ -5,3 +5,5 @@ EVERWARE_WHITELIST=whitelist.txt DOCKER_CERT_PATH=/etc/docker DOCKER_HOST=tcp://xxx.xxx.xxx.xxx:2376 DOCKER_TLS_VERIFY=1 +# most likely the same IP as DOCKER_HOST +DOCKER_PUBLIC_IP=xxx.xxx.xxx.xxx diff --git a/etc/container_config.py b/etc/container_config.py index 7730a12..06620cb 100644 --- a/etc/container_config.py +++ b/etc/container_config.py @@ -11,8 +11,7 @@ c.JupyterHub.hub_ip = '0.0.0.0' c.JupyterHub.proxy_api_ip = '0.0.0.0' -# Change these two next settings: -# IP of the machine where the Everware can be contacted -c.DockerSpawner.hub_ip_connect = 'xxx.xxx.xxx.xxx' -# IP of the machine running Docker service -c.DockerSpawner.container_ip = 'xxx.xxx.xxx.xxx' +# IP of the machine where the Everware (run as docker container) can be contacted +c.DockerSpawner.hub_ip_connect = os.environ['DOCKER_PUBLIC_IP'] +# IP of the machine running Docker service, normally the same as above +c.DockerSpawner.container_ip = c.DockerSpawner.hub_ip_connect diff --git a/etc/container_swarm_config.py b/etc/container_swarm_config.py index 8db0926..e9a7262 100644 --- a/etc/container_swarm_config.py +++ b/etc/container_swarm_config.py @@ -18,4 +18,4 @@ # Change this setting: # IP of the machine where the Everware can be contacted -c.DockerSpawner.hub_ip_connect = 'xxx.xxx.xxx.xxx' +c.DockerSpawner.hub_ip_connect = os.environ['DOCKER_PUBLIC_IP'] diff --git a/run.makefile b/run.makefile new file mode 100644 index 0000000..06bf43b --- /dev/null +++ b/run.makefile @@ -0,0 +1,67 @@ +HERE=$(shell pwd) +LOG := everware.log +PIDFILE := everware.pid +EXECUTOR = everware-server + +reload: ## reload everware whitelist + PID=`pgrep '${EXECUTOR}'` ;\ + if [ -z "$${PID}" ] ; then echo "Cannot find running ${EXECUTOR}" ; exit 1 ; fi + pkill -1 '${EXECUTOR}' + +clean: ## clean user base + if [ -f ${PIDFILE} ] ; then echo "${PIDFILE} exists, cannot continute" ; exit 1; fi + rm -f jupyterhub.sqlite + +run-linux: clean ## run everware server on linux + source ./env.sh && \ + ${EXECUTOR} -f etc/local_config.py --no-ssl 2>&1 | tee ${LOG} + +run-dockermachine: clean ## run everware server on MacOS + source ./env.sh && \ + ${EXECUTOR} -f etc/local_dockermachine_config.py --no-ssl 2>&1 | tee ${LOG} + +run-daemon: clean ## run everware in daemon mode, linux only, SSL required + source ./env.sh && \ + ${EXECUTOR} -f etc/local_config.py >> ${LOG} 2>&1 & + pgrep ${EXECUTOR} > ${PIDFILE} || ( tail ${LOG} && exit 1 ) + echo "Started. Log saved to ${LOG}" + +stop: + -rm ${PIDFILE} + -pkill -9 ${EXECUTOR} + -pkill -9 node +run-docker-local: + docker run -d --name everware \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v ${HERE}/etc/container_config.py:/srv/everware/etc/container_config.py \ + --env-file=env.docker-local \ + -p 8000:8000 \ + -p 8081:8081 \ + everware/everware:latest /srv/everware/etc/container_config.py --no-ssl --debug + +run-docker-remote: + docker run -d --name everware \ + -v ${HERE}/etc/container_config.py:/srv/everware/etc/container_config.py \ + --env-file=env.docker-remote \ + -p 8000:8000 \ + -p 8081:8081 \ + everware/everware:latest /srv/everware/etc/container_config.py --no-ssl --debug + +run-docker-swarm: + docker run -d --name everware \ + -v ${HERE}/etc/container_swarm_config.py:/srv/everware/etc/container_swarm_config.py \ + --env-file=env.docker-swarm \ + -p 8000:8000 \ + -p 8081:8081 \ + everware/everware:latest /srv/everware/etc/container_swarm_config.py --no-ssl --debug + +stop-docker: + docker stop everware + docker rm everware + +stop-docker-swarm: + bash -c "source env.docker-swarm && docker stop everware && docker rm everware" + +logs: ${LOG} ## watch log file + tail -f ${LOG} +