diff --git a/.travis.yml b/.travis.yml index 0e48d58..5ca35c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,20 +4,24 @@ language: python sudo: required services: - docker + addons: apt: packages: - build-essential + python: - 3.4 - 3.5 + env: + global: + # GIST_TOKEN=1234 for screen uploading + - secure: "ZY7fEEgp4/dlz7LlD4YgzsZ8NscP/J6CXWxshFhHISMZ3Fdk6bUGfsIhEIEPVE9h2wwXyMeRUsmXivqC92wHx0SHJilr5cUbby9YTsKSj5bCF5EWz+JDAEaooTcL0QHyP4YB8TbQ5UsVW6H4cSrJI/WmHKllFnt+83ZOT1r8vXHxiFTTTshnZV11F0CqAfsbjCZiOCyX0s8vxUEdzpCEU5d1ky6JH1SFqEckaxWPItZoeQ+iG3W3AfMKKqJXFLJJ/YIFfuMQiEyW4HqPfeoG23ac1J4QimMKOdAABI+HGzagoB1yYc47XuMpIeO4yhNNRSnk9+KSqKIdZDRnVB+/GuClYNTlBWDZfTuzhYCMaU4KQb1X/15Hpiy26fzjgz12ypXgygFqCP2YlU6sNCyYESusuOanwMc1C03r4Uqebn6XPPwhDTQ/UjbigNyjsaSJgiFeqRvqV1iX4Ug2iGO1k6hI8lkc/nqBXQ9p1QrrDKQ8GmZCK+765B0WQiF7ubjK+0L0/ZijEk7hqjaVY4tZr+qXsfTVGplbz1warncGolHV0OLZhAEaGQDNdZUH+MDBId7PbhVyJc7ebGgmqXEL8tfVU9xT5eWvkN8YXf4L/JP7qik6Xp39IpJJvMDX7RUNNuwhfCn5IKl7H8QtdS7VNysyx5oAraHWPAuVM572gaU=" + matrix: - JHUB_VERSION=master - JHUB_VERSION=latest # latest released version -# GIST_TOKEN=1234 for screen uploading -secure: "ZY7fEEgp4/dlz7LlD4YgzsZ8NscP/J6CXWxshFhHISMZ3Fdk6bUGfsIhEIEPVE9h2wwXyMeRUsmXivqC92wHx0SHJilr5cUbby9YTsKSj5bCF5EWz+JDAEaooTcL0QHyP4YB8TbQ5UsVW6H4cSrJI/WmHKllFnt+83ZOT1r8vXHxiFTTTshnZV11F0CqAfsbjCZiOCyX0s8vxUEdzpCEU5d1ky6JH1SFqEckaxWPItZoeQ+iG3W3AfMKKqJXFLJJ/YIFfuMQiEyW4HqPfeoG23ac1J4QimMKOdAABI+HGzagoB1yYc47XuMpIeO4yhNNRSnk9+KSqKIdZDRnVB+/GuClYNTlBWDZfTuzhYCMaU4KQb1X/15Hpiy26fzjgz12ypXgygFqCP2YlU6sNCyYESusuOanwMc1C03r4Uqebn6XPPwhDTQ/UjbigNyjsaSJgiFeqRvqV1iX4Ug2iGO1k6hI8lkc/nqBXQ9p1QrrDKQ8GmZCK+765B0WQiF7ubjK+0L0/ZijEk7hqjaVY4tZr+qXsfTVGplbz1warncGolHV0OLZhAEaGQDNdZUH+MDBId7PbhVyJc7ebGgmqXEL8tfVU9xT5eWvkN8YXf4L/JP7qik6Xp39IpJJvMDX7RUNNuwhfCn5IKl7H8QtdS7VNysyx5oAraHWPAuVM572gaU=" - before_install: # XXX remove IPv6 entry via https://github.com/travis-ci/travis-ci/issues/4978 - sudo [ $(ip addr show | grep "inet6 ::1" | wc -l) -lt "1" ] && sudo sed -i '/^::1/d' /etc/hosts @@ -35,6 +39,7 @@ before_install: - pip freeze - npm list - if [ -d $HOME/frontend-test-screenshots/ ] ; then ls -alR $HOME/frontend-test-screenshots/ ; fi + - python -c "import multiprocessing; print('CPU cores - %d' % multiprocessing.cpu_count())" install: - npm install -g configurable-http-proxy @@ -44,8 +49,7 @@ install: script: - nose2 -v --start-dir everware # unit tests live in everware/ - ./build_tools/test_frontend.sh - - prefix="build $TRAVIS_JOB_NUMBER" - - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M="travis-CI screens ($JHUB_VERSION v$TRAVIS_PYTHON_VERSION)" ; fi + - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M=travis-${TRAVIS_JOB_NUMBER}_${JHUB_VERSION}_v${TRAVIS_PYTHON_VERSION} ; fi cache: apt: true diff --git a/Makefile b/Makefile index 8bc4c6e..345a320 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,8 @@ TEST_OPTIONS := -s tests -N 2 TESTS := test_happy_mp LOG := everware.log PIDFILE := everware.pid -IP = $(shell python -c 'from IPython.utils.localinterfaces import public_ips; print (public_ips()[0])' 2>/dev/null) -OPTIONS = --debug --port 8000 --no-ssl --JupyterHub.hub_ip=${IP} +IP ?= $(shell python -c 'from IPython.utils.localinterfaces import public_ips; print (public_ips()[0])' 2>/dev/null) +OPTIONS = --debug --port 8000 --no-ssl --JupyterHub.hub_ip=$${IP} IS_DOCKER_MACHINE := $(shell which docker-machine > /dev/null ; echo $$?) UPLOADDIR ?= ~/upload_screens ifeq (0, $(IS_DOCKER_MACHINE)) @@ -34,7 +34,7 @@ help: install: ## install everware npm install - npm install -g configurable-http-proxy + npm install configurable-http-proxy PYTHON_MAJOR=`python -c 'import sys; print(sys.version_info[0])'` ;\ if [ $${PYTHON_MAJOR} -eq 3 ] ; then \ PYTHON=python ;\ @@ -66,7 +66,7 @@ clean: ## clean user base run: clean ## run everware server source ./env.sh && \ - jupyterhub ${OPTIONS} + jupyterhub ${OPTIONS} | tee ${LOG} run-daemon: clean source ./env.sh && \ @@ -76,9 +76,11 @@ run-daemon: clean echo "Started. Log saved to ${LOG}" stop: ${PIDFILE} - kill -9 `cat ${PIDFILE}` - pkill -9 -f configurable-http-proxy rm ${PIDFILE} + kill -9 `cat ${PIDFILE}` || pkill -9 -f configurable-http-proxy + +stop-zombie: + pkill -9 -f jupyterhub || pkill -9 -f configurable-http-proxy run-test-server: clean ## run everware instance for testing (no auth) cat jupyterhub_config.py <(echo c.JupyterHub.authenticator_class = 'dummyauthenticator.DummyAuthenticator') \ @@ -94,6 +96,9 @@ run-test-server: clean ## run everware instance for testing (no auth) logs: ${LOG} ## watch log file tail -f ${LOG} +test: ## run tests + export UPLOADDIR=${UPLOADDIR} && build_tools/test_frontend.sh + test-client: ## run tests export UPLOADDIR=${UPLOADDIR} ; \ nose2 ${TEST_OPTIONS} ${TESTS} @@ -117,11 +122,12 @@ upload_screens: ## upload screenshots of failed tests fi ; \ fi ;\ OPTIONS="--no-open" ; \ - if [ "${M}" != "" ] ; then OPTIONS+=" --description ${M}" ; fi ;\ + if [ "${M}" != "" ] ; then OPTIONS+=" --description '${M}'" ; fi ;\ gistup $${OPTIONS} ; \ else \ git add * ;\ git commit -am "${M}" ;\ git push ;\ fi ;\ - fi \ No newline at end of file + fi + diff --git a/build_tools/test_frontend.sh b/build_tools/test_frontend.sh index a3e094f..fb4249e 100755 --- a/build_tools/test_frontend.sh +++ b/build_tools/test_frontend.sh @@ -18,6 +18,7 @@ HUB_PID=$! sleep 3 echo "Start running frontend tests" +[ -z "$UPLOADDIR" ] && echo "no UPLOADDIR defined" && exit 1 [ -d $UPLOADDIR ] && rm -rf $UPLOADDIR/* nose2 -v -N $NPROC --start-dir frontend_tests || FAIL=1 @@ -25,7 +26,8 @@ if [ -f $LOG ]; then echo ">>> Frontend test hub log:" cat $LOG echo "<<< Frontend test hub log:" + docker ps -a fi kill ${HUB_PID} -exit $FAIL \ No newline at end of file +exit $FAIL diff --git a/everware/authenticator.py b/everware/authenticator.py index ed06494..399de4b 100644 --- a/everware/authenticator.py +++ b/everware/authenticator.py @@ -116,7 +116,7 @@ def get(self): self.authorize_redirect( redirect_uri=redirect_uri, client_id=self.authenticator.client_id, - scope=[], + scope=['repo'], response_type='code', extra_params={'state': self.create_signed_value('state', repr(state))}) @@ -142,10 +142,12 @@ def get(self): self.log.debug('State dict: %s', state) state.pop('unique') - username = yield self.authenticator.authenticate(self) + username, token = yield self.authenticator.authenticate(self) if username: user = self.user_from_username(username) + user.token = token self.set_login_cookie(user) + user.login_service = "github" if 'repourl' in state: self.log.debug("Redirect with %s", state) self.redirect(self.hub.server.base_url +'/home?'+urllib.parse.urlencode(state)) @@ -227,7 +229,7 @@ def authenticate(self, handler): username = resp_json["login"] if self.whitelist and username not in self.whitelist: username = None - raise gen.Return(username) + raise gen.Return((username, access_token)) class BitbucketOAuthenticator(Authenticator): diff --git a/everware/home_handler.py b/everware/home_handler.py index 534c7cd..20f2d49 100644 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -1,17 +1,124 @@ from tornado import web, gen +from docker.errors import NotFound from jupyterhub.handlers.base import BaseHandler from IPython.html.utils import url_path_join from tornado.httputil import url_concat +from tornado.httpclient import HTTPRequest, AsyncHTTPClient +import json + +import re + +@gen.coroutine +def _fork_github_repo(url, token): + http_client = AsyncHTTPClient() + + headers={"User-Agent": "JupyterHub", + "Authorization": "token {}".format(token) + } + + result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url) + if not result: + raise ValueError('URL is not a github URL') + + owner, repo = result[0] + + api_url = "https://api.github.com/repos/%s/%s/forks" % (owner, repo) + + req = HTTPRequest(api_url, + method="POST", + headers=headers, + body='', + ) + + resp = yield http_client.fetch(req) + return json.loads(resp.body.decode('utf8', 'replace')) + +@gen.coroutine +def _github_fork_exists(username, url, token): + http_client = AsyncHTTPClient() + + headers={"User-Agent": "JupyterHub", + "Authorization": "token {}".format(token) + } + + result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url) + if not result: + raise ValueError('URL is not a github URL') + + owner, repo = result[0] + api_url = "https://api.github.com/repos/%s/%s" % (username, repo) + + req = HTTPRequest(api_url, + method="GET", + headers=headers, + ) + + try: + resp = yield http_client.fetch(req) + return True + except: + return False + +@gen.coroutine +def _repository_changed(user): + try: + setup = yield user.spawner.docker( + 'exec_create', + container=user.spawner.container_id, + cmd="bash -c 'cd analysis/ && \ + (git fetch --unshallow > /dev/null 2>&1; true) && \ + git diff --name-only'", + ) + out = yield user.spawner.docker( + 'exec_start', + exec_id=setup['Id'], + ) + except NotFound: + return False + if out: + return True + else: + return False + +@gen.coroutine +def _push_github_repo(user, url, token): + result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url) + if not result: + raise ValueError('URL is not a github URL') + + owner, repo = result[0] + fork_url = "https://{}@github.com/{}/{}".format(token, user.name, repo) + + out = yield user.spawner.docker( + 'exec_create', + container=user.spawner.container_id, + cmd="bash -c 'cd analysis/ && \ + git config --global user.email \"everware@everware.xyz\" && \ + git config --global user.name \"Everware\" && \ + (git fetch --unshallow; true) && \ + git add . && \ + git commit -m \"Update through everware\" && \ + (git remote add everware-fork {}; true) && \ + (git checkout -b everware; true) && \ + git push everware-fork everware'".format(fork_url), + ) + response = yield user.spawner.docker( + 'exec_start', + exec_id=out['Id'], + ) class HomeHandler(BaseHandler): """Render the user's home page.""" @web.authenticated + @gen.coroutine def get(self): user = self.get_current_user() repourl = self.get_argument('repourl', '') + do_fork = self.get_argument('do_fork', False) + do_push = self.get_argument('do_push', False) if repourl: self.log.info('Got %s in home' % repourl) self.redirect(url_concat( @@ -20,10 +127,52 @@ def get(self): } )) return + + stat = yield user.spawner.poll() + self.log.debug("The container is {}".format(repr(stat))) + if user.running and hasattr(user, "login_service") and user.login_service == "github": + if do_fork: + self.log.info('Will fork %s' % user.spawner.repo_url) + yield _fork_github_repo( + user.spawner.repo_url, + user.token, + ) + self.redirect('/hub/home') + return + if do_push: + self.log.info('Will push to fork') + yield _push_github_repo( + user, + user.spawner.repo_url, + user.token, + ) + self.redirect('/hub/home') + return + repo_url = user.spawner.repo_url + fork_exists = yield _github_fork_exists( + user.name, + user.spawner.repo_url, + user.token, + ) + repository_changed = yield _repository_changed(user) + else: + repo_url = '' + fork_exists = False + repository_changed = False + + + if hasattr(user, 'login_service'): + loginservice = user.login_service + else: + loginservice = 'none' html = self.render_template('home.html', - user=user + user=user, + repourl=repo_url, + login_service=loginservice, + fork_exists=fork_exists, + repository_changed=repository_changed, ) - self.finish(html) + self.finish(html) diff --git a/everware/image_handler.py b/everware/image_handler.py index c2a66ed..82ba345 100644 --- a/everware/image_handler.py +++ b/everware/image_handler.py @@ -49,7 +49,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self._building_log = [] if isinstance(exc_value, Exception): - self._exception = exc_type(exc_value) + self._exception = exc_value self._mutex.set() def add_to_log(self, message, level=1): diff --git a/everware/spawner.py b/everware/spawner.py index 307e1a6..07739a6 100644 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -25,6 +25,9 @@ from .image_handler import ImageHandler +import ssl + +ssl._create_default_https_context = ssl._create_unverified_context class CustomDockerSpawner(DockerSpawner): def __init__(self, **kwargs): @@ -59,12 +62,15 @@ def _docker(self, method, *args, **kwargs): if method in generator_methods: def lister(mm): ret = [] - for l in mm: - ret.append(str(l)) - # include only high-level docker's log - if 'stream' in l and not l['stream'].startswith(' --->'): - # self._add_to_log(l['stream'], 2) - self._cur_waiter.add_to_log(l['stream'], 2) + try: + for l in mm: + ret.append(str(l)) + # include only high-level docker's log + if 'stream' in l and not l['stream'].startswith(' --->'): + # self._add_to_log(l['stream'], 2) + self._cur_waiter.add_to_log(l['stream'], 2) + except JSONDecodeError as e: + self.warn("Error parsing docker output (%s)" % repr(e)) return ret return lister(m(*args, **kwargs)) else: @@ -132,7 +138,7 @@ def options_from_form(self, formdata): @property def repo_url(self): - return self.user_options['repo_url'] + return self.user_options.get('repo_url', '') _escaped_repo_url = None @property diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 620e16a..6723a1c 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -17,15 +17,17 @@ # repo = "docker:everware/https_github_com_everware_everware_dimuon_example-9bec6770485eb6b245648bc251d045a204973cc9" # REPO = "docker:yandex/rep-tutorial" -DRIVER = "phantomjs" -# DRIVER = "firefox" +if os.environ['TRAVIS'] == 'true': + DRIVER = "phantomjs" +else: + DRIVER = "firefox" # Test matrix -# SCENARIOS = ["scenario_short", "scenario_full"] +SCENARIOS = ["scenario_full", "scenario_short"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] -SCENARIOS = ["scenario_short"] -USERS = ["an1", "an2"] -TIMEOUT = 30 +# USERS = ["user_1", "an2"] +USERS = ["user1", "user2"] +TIMEOUT = 250 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): @@ -49,7 +51,7 @@ def __init__(self, login=None, repo=REPO, driver_type=DRIVER): def get_driver(self): if self.driver_type == "phantomjs": - self.driver = webdriver.PhantomJS('/usr/local/bin/phantomjs') + self.driver = webdriver.PhantomJS() self.driver.set_window_size(1024, 768) if self.driver_type == "firefox": self.driver = webdriver.Firefox() @@ -65,7 +67,7 @@ def log(self, message): print("{}: {}".format(self.login, message)) - def wait_for_element_present(self, how, what, displayed=True, timeout=30): + def wait_for_element_present(self, how, what, displayed=True, timeout=TIMEOUT): for i in range(timeout): element = self.driver.find_element(by=how, value=what) if element is not None and element.is_displayed() == displayed: break @@ -73,7 +75,7 @@ def wait_for_element_present(self, how, what, displayed=True, timeout=30): else: assert False, "time out waiting for (%s, %s)" % (how, what) - def wait_for_element_id_is_gone(self, value, timeout=30): + def wait_for_element_id_is_gone(self, value, timeout=TIMEOUT): for i in range(timeout): try: element = self.driver.find_element_by_id(value) @@ -81,7 +83,7 @@ def wait_for_element_id_is_gone(self, value, timeout=30): break time.sleep(1) # self.log("waiting for %s to go %d" % (value, i)) - else: self.fail("time out wairing for (%s) to disappear" % (what)) + else: assert False, "time out waiting for (%s) to disappear" % (what) self.log("gone finally (%d)" % i) @@ -102,12 +104,12 @@ def run_scenario(scenario, username): try: globals()[scenario](user) except NoSuchElementException as e: - make_screenshot(user.driver, "./{}-{}.png".format(scenario, username)) assert False, "Cannot find element {}\n{}".format(e.msg, ''.join(traceback.format_stack())) except Exception as e: print("oops: %s" % repr(e)) assert False, traceback.format_stack() finally: + make_screenshot(user.driver, "{}-{}.png".format(scenario, username)) user.tearDown() @@ -153,15 +155,13 @@ def scenario_full(user): driver.find_element_by_id("repository_input").clear() driver.find_element_by_id("repository_input").send_keys(user.repo) driver.find_element_by_xpath("//input[@value='Spawn']").click() - user.log("start clicked") + user.log("spawn clicked") user.wait_for_element_present(By.LINK_TEXT, "Control Panel") driver.find_element_by_link_text("Control Panel").click() user.wait_for_element_present(By.ID, "stop") driver.find_element_by_id("stop").click() user.log("stop clicked") - user.wait_for_element_present(By.ID, "wait") - user.log("waiting to stop") - user.wait_for_element_id_is_gone("wait") + user.wait_for_element_id_is_gone("stop") driver.find_element_by_id("logout").click() user.log("logout clicked") diff --git a/share/static/html/home.html b/share/static/html/home.html index a2886e9..8ac2b23 100644 --- a/share/static/html/home.html +++ b/share/static/html/home.html @@ -15,6 +15,24 @@
+

You are currently logged in through {{ login_service }}.

+ {% if user.running %} +

Currently checked out repository: {{ repourl }}

+ {% if fork_exists %} +

You have a repository with the same name, which we can push to!

+ {% if repository_changed %} +

You have made changes. Do you want to push?

+ {% else %} +

But you have to make changes to the checked out repository before you can push.

+ {% endif %} + {% else %} +

You don't have a repository with the same name. Do you want to fork it?

+ {% endif %} + {% endif %} +
+
+
+
{% if user.running %} Stop My Server {% endif %}