From add52d34e8109daa2d92743ec5443f4e4dfa79a6 Mon Sep 17 00:00:00 2001 From: Igor Babuschkin Date: Sat, 5 Mar 2016 21:45:58 +0000 Subject: [PATCH 01/17] Add basic push to Github functionality --- everware/authenticator.py | 8 ++- everware/home_handler.py | 148 +++++++++++++++++++++++++++++++++++++++++++- everware/spawner.py | 2 +- share/static/html/home.html | 18 ++++++ 4 files changed, 170 insertions(+), 6 deletions(-) diff --git a/everware/authenticator.py b/everware/authenticator.py index ed06494..399de4b 100644 --- a/everware/authenticator.py +++ b/everware/authenticator.py @@ -116,7 +116,7 @@ def get(self): self.authorize_redirect( redirect_uri=redirect_uri, client_id=self.authenticator.client_id, - scope=[], + scope=['repo'], response_type='code', extra_params={'state': self.create_signed_value('state', repr(state))}) @@ -142,10 +142,12 @@ def get(self): self.log.debug('State dict: %s', state) state.pop('unique') - username = yield self.authenticator.authenticate(self) + username, token = yield self.authenticator.authenticate(self) if username: user = self.user_from_username(username) + user.token = token self.set_login_cookie(user) + user.login_service = "github" if 'repourl' in state: self.log.debug("Redirect with %s", state) self.redirect(self.hub.server.base_url +'/home?'+urllib.parse.urlencode(state)) @@ -227,7 +229,7 @@ def authenticate(self, handler): username = resp_json["login"] if self.whitelist and username not in self.whitelist: username = None - raise gen.Return(username) + raise gen.Return((username, access_token)) class BitbucketOAuthenticator(Authenticator): diff --git a/everware/home_handler.py b/everware/home_handler.py index 534c7cd..740d332 100644 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -1,17 +1,124 @@ from tornado import web, gen +from docker.errors import NotFound from jupyterhub.handlers.base import BaseHandler from IPython.html.utils import url_path_join from tornado.httputil import url_concat +from tornado.httpclient import HTTPRequest, AsyncHTTPClient +import json + +import re + +@gen.coroutine +def _fork_github_repo(url, token): + http_client = AsyncHTTPClient() + + headers={"User-Agent": "JupyterHub", + "Authorization": "token {}".format(token) + } + + result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url) + if not result: + raise ValueError('URL is not a github URL') + + owner, repo = result[0] + + api_url = "https://api.github.com/repos/%s/%s/forks" % (owner, repo) + + req = HTTPRequest(api_url, + method="POST", + headers=headers, + body='', + ) + + resp = yield http_client.fetch(req) + return json.loads(resp.body.decode('utf8', 'replace')) + +@gen.coroutine +def _github_fork_exists(username, url, token): + http_client = AsyncHTTPClient() + + headers={"User-Agent": "JupyterHub", + "Authorization": "token {}".format(token) + } + + result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url) + if not result: + raise ValueError('URL is not a github URL') + + owner, repo = result[0] + api_url = "https://api.github.com/repos/%s/%s" % (username, repo) + + req = HTTPRequest(api_url, + method="GET", + headers=headers, + ) + + try: + resp = yield http_client.fetch(req) + return True + except: + return False + +@gen.coroutine +def _repository_changed(user): + try: + setup = yield user.spawner.docker( + 'exec_create', + container=user.spawner.container_id, + cmd="bash -c 'cd analysis/ && \ + (git fetch --unshallow > /dev/null 2>&1; true) && \ + git diff --name-only'", + ) + out = yield user.spawner.docker( + 'exec_start', + exec_id=setup['Id'], + ) + except NotFound: + return False + if out: + return True + else: + return False + +@gen.coroutine +def _push_github_repo(user, url, token): + result = re.findall('^https://github.com/([^/]+)/([^/]+).*', url) + if not result: + raise ValueError('URL is not a github URL') + + owner, repo = result[0] + fork_url = "https://{}@github.com/{}/{}".format(token, user.name, repo) + + out = yield user.spawner.docker( + 'exec_create', + container=user.spawner.container_id, + cmd="bash -c 'cd analysis/ && \ + git config --global user.email \"everware@everware.xyz\" && \ + git config --global user.name \"Everware\" && \ + (git fetch --unshallow; true) && \ + git add . && \ + git commit -m \"Update through everware\" && \ + (git remote add everware-fork {}; true) && \ + (git checkout -b everware; true) && \ + git push everware-fork everware'".format(fork_url), + ) + response = yield user.spawner.docker( + 'exec_start', + exec_id=out['Id'], + ) class HomeHandler(BaseHandler): """Render the user's home page.""" @web.authenticated + @gen.coroutine def get(self): user = self.get_current_user() repourl = self.get_argument('repourl', '') + do_fork = self.get_argument('do_fork', False) + do_push = self.get_argument('do_push', False) if repourl: self.log.info('Got %s in home' % repourl) self.redirect(url_concat( @@ -20,10 +127,47 @@ def get(self): } )) return + + stat = yield user.spawner.poll() + self.log.debug("The container is {}".format(repr(stat))) + if user.running: + if do_fork: + self.log.info('Will fork %s' % user.spawner.repo_url) + yield _fork_github_repo( + user.spawner.repo_url, + user.token, + ) + self.redirect('/hub/home') + return + if do_push: + self.log.info('Will push to fork') + yield _push_github_repo( + user, + user.spawner.repo_url, + user.token, + ) + self.redirect('/hub/home') + return + repo_url = user.spawner.repo_url + fork_exists = yield _github_fork_exists( + user.name, + user.spawner.repo_url, + user.token, + ) + repository_changed = yield _repository_changed(user) + else: + repo_url = '' + fork_exists = False + repository_changed = False + html = self.render_template('home.html', - user=user + user=user, + repourl=repo_url, + login_service=user.login_service, + fork_exists=fork_exists, + repository_changed=repository_changed, ) - self.finish(html) + self.finish(html) diff --git a/everware/spawner.py b/everware/spawner.py index 307e1a6..b3f4e85 100644 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -132,7 +132,7 @@ def options_from_form(self, formdata): @property def repo_url(self): - return self.user_options['repo_url'] + return self.user_options.get('repo_url', '') _escaped_repo_url = None @property diff --git a/share/static/html/home.html b/share/static/html/home.html index a2886e9..8ac2b23 100644 --- a/share/static/html/home.html +++ b/share/static/html/home.html @@ -15,6 +15,24 @@
+

You are currently logged in through {{ login_service }}.

+ {% if user.running %} +

Currently checked out repository: {{ repourl }}

+ {% if fork_exists %} +

You have a repository with the same name, which we can push to!

+ {% if repository_changed %} +

You have made changes. Do you want to push?

+ {% else %} +

But you have to make changes to the checked out repository before you can push.

+ {% endif %} + {% else %} +

You don't have a repository with the same name. Do you want to fork it?

+ {% endif %} + {% endif %} +
+
+
+
{% if user.running %} Stop My Server {% endif %} From a7fb710d94b654b7849789346de15fdf87eae37b Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Sun, 13 Mar 2016 23:31:03 +0100 Subject: [PATCH 02/17] added full happy scenario --- frontend_tests/test_happy_mp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 620e16a..878ca57 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -21,9 +21,8 @@ # DRIVER = "firefox" # Test matrix -# SCENARIOS = ["scenario_short", "scenario_full"] +SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] -SCENARIOS = ["scenario_short"] USERS = ["an1", "an2"] TIMEOUT = 30 UPLOADDIR = os.environ['UPLOADDIR'] From 0d31e1f43e08cd5eabcc4bb542617494789d2792 Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Mon, 14 Mar 2016 02:06:58 +0100 Subject: [PATCH 03/17] increased timeout to 60 --- everware/image_handler.py | 2 +- frontend_tests/test_happy_mp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/everware/image_handler.py b/everware/image_handler.py index c2a66ed..82ba345 100644 --- a/everware/image_handler.py +++ b/everware/image_handler.py @@ -49,7 +49,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self._building_log = [] if isinstance(exc_value, Exception): - self._exception = exc_type(exc_value) + self._exception = exc_value self._mutex.set() def add_to_log(self, message, level=1): diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 878ca57..c2808f1 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -24,7 +24,7 @@ SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] USERS = ["an1", "an2"] -TIMEOUT = 30 +TIMEOUT = 60 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): From a2ee27c52fcff95088e33b31fe3f4ab9d8c6950a Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Mon, 14 Mar 2016 02:38:43 +0100 Subject: [PATCH 04/17] to 100 --- frontend_tests/test_happy_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index c2808f1..050011f 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -24,7 +24,7 @@ SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] USERS = ["an1", "an2"] -TIMEOUT = 60 +TIMEOUT = 100 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): From a27a56c4551ff390cb72c533f1a4d49872931be7 Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Mon, 14 Mar 2016 10:38:06 +0100 Subject: [PATCH 05/17] reduce hardcoding --- frontend_tests/test_happy_mp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 050011f..263a8e7 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -24,7 +24,7 @@ SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] USERS = ["an1", "an2"] -TIMEOUT = 100 +TIMEOUT = 60 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): @@ -64,7 +64,7 @@ def log(self, message): print("{}: {}".format(self.login, message)) - def wait_for_element_present(self, how, what, displayed=True, timeout=30): + def wait_for_element_present(self, how, what, displayed=True, timeout=TIMEOUT): for i in range(timeout): element = self.driver.find_element(by=how, value=what) if element is not None and element.is_displayed() == displayed: break @@ -72,7 +72,7 @@ def wait_for_element_present(self, how, what, displayed=True, timeout=30): else: assert False, "time out waiting for (%s, %s)" % (how, what) - def wait_for_element_id_is_gone(self, value, timeout=30): + def wait_for_element_id_is_gone(self, value, timeout=TIMEOUT): for i in range(timeout): try: element = self.driver.find_element_by_id(value) From 0c9ce9c3348fd607c90951326d3ef86e0c8709e7 Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 15 Mar 2016 02:56:31 +0300 Subject: [PATCH 06/17] fix Makefile --- Makefile | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 8bc4c6e..a1b9083 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,8 @@ TEST_OPTIONS := -s tests -N 2 TESTS := test_happy_mp LOG := everware.log PIDFILE := everware.pid -IP = $(shell python -c 'from IPython.utils.localinterfaces import public_ips; print (public_ips()[0])' 2>/dev/null) -OPTIONS = --debug --port 8000 --no-ssl --JupyterHub.hub_ip=${IP} +IP ?= $(shell python -c 'from IPython.utils.localinterfaces import public_ips; print (public_ips()[0])' 2>/dev/null) +OPTIONS = --debug --port 8000 --no-ssl --JupyterHub.hub_ip=$${IP} IS_DOCKER_MACHINE := $(shell which docker-machine > /dev/null ; echo $$?) UPLOADDIR ?= ~/upload_screens ifeq (0, $(IS_DOCKER_MACHINE)) @@ -34,7 +34,7 @@ help: install: ## install everware npm install - npm install -g configurable-http-proxy + npm install configurable-http-proxy PYTHON_MAJOR=`python -c 'import sys; print(sys.version_info[0])'` ;\ if [ $${PYTHON_MAJOR} -eq 3 ] ; then \ PYTHON=python ;\ @@ -80,6 +80,10 @@ stop: ${PIDFILE} pkill -9 -f configurable-http-proxy rm ${PIDFILE} +stop-zombie: + pkill -9 -f jupyterhub + pkill -9 -f configurable-http-proxy + run-test-server: clean ## run everware instance for testing (no auth) cat jupyterhub_config.py <(echo c.JupyterHub.authenticator_class = 'dummyauthenticator.DummyAuthenticator') \ <(echo c.Spawner.container_ip = \'${SPAWNER_IP}\') \ @@ -124,4 +128,4 @@ upload_screens: ## upload screenshots of failed tests git commit -am "${M}" ;\ git push ;\ fi ;\ - fi \ No newline at end of file + fi From 34c3fa1284e7c24385aad2ead04541a0d240bcff Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 15 Mar 2016 03:22:45 +0300 Subject: [PATCH 07/17] fix @ibab's push&fork --- Makefile | 1 + build_tools/test_frontend.sh | 3 ++- everware/home_handler.py | 7 ++++++- frontend_tests/test_happy_mp.py | 4 ++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a1b9083..6c86460 100644 --- a/Makefile +++ b/Makefile @@ -129,3 +129,4 @@ upload_screens: ## upload screenshots of failed tests git push ;\ fi ;\ fi + diff --git a/build_tools/test_frontend.sh b/build_tools/test_frontend.sh index a3e094f..ea191de 100755 --- a/build_tools/test_frontend.sh +++ b/build_tools/test_frontend.sh @@ -18,6 +18,7 @@ HUB_PID=$! sleep 3 echo "Start running frontend tests" +[ -z "$UPLOADDIR" ] && echo "no UPLOADDIR defined" && exit 1 [ -d $UPLOADDIR ] && rm -rf $UPLOADDIR/* nose2 -v -N $NPROC --start-dir frontend_tests || FAIL=1 @@ -28,4 +29,4 @@ if [ -f $LOG ]; then fi kill ${HUB_PID} -exit $FAIL \ No newline at end of file +exit $FAIL diff --git a/everware/home_handler.py b/everware/home_handler.py index 740d332..53bd143 100644 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -160,10 +160,15 @@ def get(self): fork_exists = False repository_changed = False + + if hasattr(user, 'login_service'): + loginservice = user.login_service + else: + loginservice = 'none' html = self.render_template('home.html', user=user, repourl=repo_url, - login_service=user.login_service, + login_service=loginservice, fork_exists=fork_exists, repository_changed=repository_changed, ) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 263a8e7..9291386 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -24,7 +24,7 @@ SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] USERS = ["an1", "an2"] -TIMEOUT = 60 +TIMEOUT = 100 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): @@ -48,7 +48,7 @@ def __init__(self, login=None, repo=REPO, driver_type=DRIVER): def get_driver(self): if self.driver_type == "phantomjs": - self.driver = webdriver.PhantomJS('/usr/local/bin/phantomjs') + self.driver = webdriver.PhantomJS() self.driver.set_window_size(1024, 768) if self.driver_type == "firefox": self.driver = webdriver.Firefox() From f763e99cdb36a2694b779f27f6eb7551e598eceb Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 15 Mar 2016 03:34:33 +0300 Subject: [PATCH 08/17] one more check --- everware/home_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/everware/home_handler.py b/everware/home_handler.py index 53bd143..20f2d49 100644 --- a/everware/home_handler.py +++ b/everware/home_handler.py @@ -130,7 +130,7 @@ def get(self): stat = yield user.spawner.poll() self.log.debug("The container is {}".format(repr(stat))) - if user.running: + if user.running and hasattr(user, "login_service") and user.login_service == "github": if do_fork: self.log.info('Will fork %s' % user.spawner.repo_url) yield _fork_github_repo( From f493c4fcf4a249f77431a0bbb78296ee2aa9da06 Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 15 Mar 2016 03:49:01 +0300 Subject: [PATCH 09/17] +20sec waiting --- frontend_tests/test_happy_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 9291386..29959ed 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -24,7 +24,7 @@ SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] USERS = ["an1", "an2"] -TIMEOUT = 100 +TIMEOUT = 120 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): From d2f1396c09dc04d4aa6dd8cdd551c97d9a877013 Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 15 Mar 2016 04:19:42 +0300 Subject: [PATCH 10/17] smile --- frontend_tests/test_happy_mp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 29959ed..23a1fc7 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -24,7 +24,7 @@ SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] USERS = ["an1", "an2"] -TIMEOUT = 120 +TIMEOUT = 180 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): @@ -101,12 +101,12 @@ def run_scenario(scenario, username): try: globals()[scenario](user) except NoSuchElementException as e: - make_screenshot(user.driver, "./{}-{}.png".format(scenario, username)) assert False, "Cannot find element {}\n{}".format(e.msg, ''.join(traceback.format_stack())) except Exception as e: print("oops: %s" % repr(e)) assert False, traceback.format_stack() finally: + make_screenshot(user.driver, "{}-{}.png".format(scenario, username)) user.tearDown() From b8a6e45ce7b804d0c1f1779adf9b9aa9ea9eac13 Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 15 Mar 2016 04:42:15 +0300 Subject: [PATCH 11/17] one proc, quoted --- Makefile | 2 +- build_tools/test_frontend.sh | 3 ++- frontend_tests/test_happy_mp.py | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6c86460..1471c71 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ upload_screens: ## upload screenshots of failed tests fi ; \ fi ;\ OPTIONS="--no-open" ; \ - if [ "${M}" != "" ] ; then OPTIONS+=" --description ${M}" ; fi ;\ + if [ "${M}" != "" ] ; then OPTIONS+=" --description '${M}'" ; fi ;\ gistup $${OPTIONS} ; \ else \ git add * ;\ diff --git a/build_tools/test_frontend.sh b/build_tools/test_frontend.sh index ea191de..8090abb 100755 --- a/build_tools/test_frontend.sh +++ b/build_tools/test_frontend.sh @@ -5,7 +5,7 @@ LOG="/tmp/frontend_test_hub.log" FAIL=0 -NPROC=2 +NPROC=1 echo "In" `pwd` @@ -26,6 +26,7 @@ if [ -f $LOG ]; then echo ">>> Frontend test hub log:" cat $LOG echo "<<< Frontend test hub log:" + docker ps -a fi kill ${HUB_PID} diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 23a1fc7..9dbcbd8 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -23,8 +23,9 @@ # Test matrix SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] -USERS = ["an1", "an2"] -TIMEOUT = 180 +# USERS = ["user_1", "an2"] +USERS = ["user_1"] +TIMEOUT = 250 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): From f74e6994e48e796b677f5f3e18eddeb5d40c91ed Mon Sep 17 00:00:00 2001 From: Andrey U Date: Tue, 15 Mar 2016 05:16:17 +0300 Subject: [PATCH 12/17] hm --- .travis.yml | 3 +-- frontend_tests/test_happy_mp.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0e48d58..50720c6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,8 +44,7 @@ install: script: - nose2 -v --start-dir everware # unit tests live in everware/ - ./build_tools/test_frontend.sh - - prefix="build $TRAVIS_JOB_NUMBER" - - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M="travis-CI screens ($JHUB_VERSION v$TRAVIS_PYTHON_VERSION)" ; fi + - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M="travis-${TRAVIS_JOB_NUMBER}_${JHUB_VERSION}_v${TRAVIS_PYTHON_VERSION}" ; fi cache: apt: true diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 9dbcbd8..8487816 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -24,7 +24,7 @@ SCENARIOS = ["scenario_short", "scenario_full"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] # USERS = ["user_1", "an2"] -USERS = ["user_1"] +USERS = ["user1"] TIMEOUT = 250 UPLOADDIR = os.environ['UPLOADDIR'] From 2c843453c544535550b99d25dd3c679a36644aaa Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Tue, 15 Mar 2016 05:11:55 +0100 Subject: [PATCH 13/17] hmm --- Makefile | 11 ++++++----- everware/spawner.py | 18 ++++++++++++------ frontend_tests/test_happy_mp.py | 12 +++++------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 1471c71..345a320 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ clean: ## clean user base run: clean ## run everware server source ./env.sh && \ - jupyterhub ${OPTIONS} + jupyterhub ${OPTIONS} | tee ${LOG} run-daemon: clean source ./env.sh && \ @@ -76,13 +76,11 @@ run-daemon: clean echo "Started. Log saved to ${LOG}" stop: ${PIDFILE} - kill -9 `cat ${PIDFILE}` - pkill -9 -f configurable-http-proxy rm ${PIDFILE} + kill -9 `cat ${PIDFILE}` || pkill -9 -f configurable-http-proxy stop-zombie: - pkill -9 -f jupyterhub - pkill -9 -f configurable-http-proxy + pkill -9 -f jupyterhub || pkill -9 -f configurable-http-proxy run-test-server: clean ## run everware instance for testing (no auth) cat jupyterhub_config.py <(echo c.JupyterHub.authenticator_class = 'dummyauthenticator.DummyAuthenticator') \ @@ -98,6 +96,9 @@ run-test-server: clean ## run everware instance for testing (no auth) logs: ${LOG} ## watch log file tail -f ${LOG} +test: ## run tests + export UPLOADDIR=${UPLOADDIR} && build_tools/test_frontend.sh + test-client: ## run tests export UPLOADDIR=${UPLOADDIR} ; \ nose2 ${TEST_OPTIONS} ${TESTS} diff --git a/everware/spawner.py b/everware/spawner.py index b3f4e85..07739a6 100644 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -25,6 +25,9 @@ from .image_handler import ImageHandler +import ssl + +ssl._create_default_https_context = ssl._create_unverified_context class CustomDockerSpawner(DockerSpawner): def __init__(self, **kwargs): @@ -59,12 +62,15 @@ def _docker(self, method, *args, **kwargs): if method in generator_methods: def lister(mm): ret = [] - for l in mm: - ret.append(str(l)) - # include only high-level docker's log - if 'stream' in l and not l['stream'].startswith(' --->'): - # self._add_to_log(l['stream'], 2) - self._cur_waiter.add_to_log(l['stream'], 2) + try: + for l in mm: + ret.append(str(l)) + # include only high-level docker's log + if 'stream' in l and not l['stream'].startswith(' --->'): + # self._add_to_log(l['stream'], 2) + self._cur_waiter.add_to_log(l['stream'], 2) + except JSONDecodeError as e: + self.warn("Error parsing docker output (%s)" % repr(e)) return ret return lister(m(*args, **kwargs)) else: diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 8487816..a0a0592 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -18,14 +18,14 @@ # REPO = "docker:yandex/rep-tutorial" DRIVER = "phantomjs" -# DRIVER = "firefox" +DRIVER = "firefox" # Test matrix -SCENARIOS = ["scenario_short", "scenario_full"] +SCENARIOS = ["scenario_full", "scenario_short"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] # USERS = ["user_1", "an2"] USERS = ["user1"] -TIMEOUT = 250 +TIMEOUT = 120 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): @@ -153,15 +153,13 @@ def scenario_full(user): driver.find_element_by_id("repository_input").clear() driver.find_element_by_id("repository_input").send_keys(user.repo) driver.find_element_by_xpath("//input[@value='Spawn']").click() - user.log("start clicked") + user.log("spawn clicked") user.wait_for_element_present(By.LINK_TEXT, "Control Panel") driver.find_element_by_link_text("Control Panel").click() user.wait_for_element_present(By.ID, "stop") driver.find_element_by_id("stop").click() user.log("stop clicked") - user.wait_for_element_present(By.ID, "wait") - user.log("waiting to stop") - user.wait_for_element_id_is_gone("wait") + user.wait_for_element_id_is_gone("stop") driver.find_element_by_id("logout").click() user.log("logout clicked") From f05c2ab3e0be8e10902d84af6df7907bbe784d7b Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Tue, 15 Mar 2016 05:23:39 +0100 Subject: [PATCH 14/17] chose webdriver --- frontend_tests/test_happy_mp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index a0a0592..621c1d1 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -17,8 +17,10 @@ # repo = "docker:everware/https_github_com_everware_everware_dimuon_example-9bec6770485eb6b245648bc251d045a204973cc9" # REPO = "docker:yandex/rep-tutorial" -DRIVER = "phantomjs" -DRIVER = "firefox" +if os.environ['TRAVIS'] == 'true': + DRIVER = "phantomjs" +else: + DRIVER = "firefox" # Test matrix SCENARIOS = ["scenario_full", "scenario_short"] From 9ac0704e5e54c494a0fb684d96e6770f8d400a53 Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Tue, 15 Mar 2016 09:46:23 +0100 Subject: [PATCH 15/17] get back 2 users & 2 processes --- build_tools/test_frontend.sh | 2 +- frontend_tests/test_happy_mp.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build_tools/test_frontend.sh b/build_tools/test_frontend.sh index 8090abb..fb4249e 100755 --- a/build_tools/test_frontend.sh +++ b/build_tools/test_frontend.sh @@ -5,7 +5,7 @@ LOG="/tmp/frontend_test_hub.log" FAIL=0 -NPROC=1 +NPROC=2 echo "In" `pwd` diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 621c1d1..94e28c0 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -26,7 +26,7 @@ SCENARIOS = ["scenario_full", "scenario_short"] # SCENARIOS = ["scenario_short", "scenario_short_bad"] # USERS = ["user_1", "an2"] -USERS = ["user1"] +USERS = ["user1", "user2"] TIMEOUT = 120 UPLOADDIR = os.environ['UPLOADDIR'] From 07a6909174883f9b30c35c96b9d53c1852999ab4 Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Tue, 15 Mar 2016 10:32:45 +0100 Subject: [PATCH 16/17] timeout 180 --- frontend_tests/test_happy_mp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 94e28c0..815a04d 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -27,7 +27,7 @@ # SCENARIOS = ["scenario_short", "scenario_short_bad"] # USERS = ["user_1", "an2"] USERS = ["user1", "user2"] -TIMEOUT = 120 +TIMEOUT = 180 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): From b35744900437dc8b7fe4dded6747bd5c7f2d9bdd Mon Sep 17 00:00:00 2001 From: Andrey Ustyuzhanin Date: Tue, 15 Mar 2016 15:59:49 +0100 Subject: [PATCH 17/17] fix user.fail error --- .travis.yml | 13 +++++++++---- frontend_tests/test_happy_mp.py | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 50720c6..5ca35c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,20 +4,24 @@ language: python sudo: required services: - docker + addons: apt: packages: - build-essential + python: - 3.4 - 3.5 + env: + global: + # GIST_TOKEN=1234 for screen uploading + - secure: "ZY7fEEgp4/dlz7LlD4YgzsZ8NscP/J6CXWxshFhHISMZ3Fdk6bUGfsIhEIEPVE9h2wwXyMeRUsmXivqC92wHx0SHJilr5cUbby9YTsKSj5bCF5EWz+JDAEaooTcL0QHyP4YB8TbQ5UsVW6H4cSrJI/WmHKllFnt+83ZOT1r8vXHxiFTTTshnZV11F0CqAfsbjCZiOCyX0s8vxUEdzpCEU5d1ky6JH1SFqEckaxWPItZoeQ+iG3W3AfMKKqJXFLJJ/YIFfuMQiEyW4HqPfeoG23ac1J4QimMKOdAABI+HGzagoB1yYc47XuMpIeO4yhNNRSnk9+KSqKIdZDRnVB+/GuClYNTlBWDZfTuzhYCMaU4KQb1X/15Hpiy26fzjgz12ypXgygFqCP2YlU6sNCyYESusuOanwMc1C03r4Uqebn6XPPwhDTQ/UjbigNyjsaSJgiFeqRvqV1iX4Ug2iGO1k6hI8lkc/nqBXQ9p1QrrDKQ8GmZCK+765B0WQiF7ubjK+0L0/ZijEk7hqjaVY4tZr+qXsfTVGplbz1warncGolHV0OLZhAEaGQDNdZUH+MDBId7PbhVyJc7ebGgmqXEL8tfVU9xT5eWvkN8YXf4L/JP7qik6Xp39IpJJvMDX7RUNNuwhfCn5IKl7H8QtdS7VNysyx5oAraHWPAuVM572gaU=" + matrix: - JHUB_VERSION=master - JHUB_VERSION=latest # latest released version -# GIST_TOKEN=1234 for screen uploading -secure: "ZY7fEEgp4/dlz7LlD4YgzsZ8NscP/J6CXWxshFhHISMZ3Fdk6bUGfsIhEIEPVE9h2wwXyMeRUsmXivqC92wHx0SHJilr5cUbby9YTsKSj5bCF5EWz+JDAEaooTcL0QHyP4YB8TbQ5UsVW6H4cSrJI/WmHKllFnt+83ZOT1r8vXHxiFTTTshnZV11F0CqAfsbjCZiOCyX0s8vxUEdzpCEU5d1ky6JH1SFqEckaxWPItZoeQ+iG3W3AfMKKqJXFLJJ/YIFfuMQiEyW4HqPfeoG23ac1J4QimMKOdAABI+HGzagoB1yYc47XuMpIeO4yhNNRSnk9+KSqKIdZDRnVB+/GuClYNTlBWDZfTuzhYCMaU4KQb1X/15Hpiy26fzjgz12ypXgygFqCP2YlU6sNCyYESusuOanwMc1C03r4Uqebn6XPPwhDTQ/UjbigNyjsaSJgiFeqRvqV1iX4Ug2iGO1k6hI8lkc/nqBXQ9p1QrrDKQ8GmZCK+765B0WQiF7ubjK+0L0/ZijEk7hqjaVY4tZr+qXsfTVGplbz1warncGolHV0OLZhAEaGQDNdZUH+MDBId7PbhVyJc7ebGgmqXEL8tfVU9xT5eWvkN8YXf4L/JP7qik6Xp39IpJJvMDX7RUNNuwhfCn5IKl7H8QtdS7VNysyx5oAraHWPAuVM572gaU=" - before_install: # XXX remove IPv6 entry via https://github.com/travis-ci/travis-ci/issues/4978 - sudo [ $(ip addr show | grep "inet6 ::1" | wc -l) -lt "1" ] && sudo sed -i '/^::1/d' /etc/hosts @@ -35,6 +39,7 @@ before_install: - pip freeze - npm list - if [ -d $HOME/frontend-test-screenshots/ ] ; then ls -alR $HOME/frontend-test-screenshots/ ; fi + - python -c "import multiprocessing; print('CPU cores - %d' % multiprocessing.cpu_count())" install: - npm install -g configurable-http-proxy @@ -44,7 +49,7 @@ install: script: - nose2 -v --start-dir everware # unit tests live in everware/ - ./build_tools/test_frontend.sh - - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M="travis-${TRAVIS_JOB_NUMBER}_${JHUB_VERSION}_v${TRAVIS_PYTHON_VERSION}" ; fi + - if [ "$TRAVIS_PULL_REQUEST" == "false" ] ; then make upload_screens -e M=travis-${TRAVIS_JOB_NUMBER}_${JHUB_VERSION}_v${TRAVIS_PYTHON_VERSION} ; fi cache: apt: true diff --git a/frontend_tests/test_happy_mp.py b/frontend_tests/test_happy_mp.py index 815a04d..6723a1c 100644 --- a/frontend_tests/test_happy_mp.py +++ b/frontend_tests/test_happy_mp.py @@ -27,7 +27,7 @@ # SCENARIOS = ["scenario_short", "scenario_short_bad"] # USERS = ["user_1", "an2"] USERS = ["user1", "user2"] -TIMEOUT = 180 +TIMEOUT = 250 UPLOADDIR = os.environ['UPLOADDIR'] def make_screenshot(driver, name): @@ -83,7 +83,7 @@ def wait_for_element_id_is_gone(self, value, timeout=TIMEOUT): break time.sleep(1) # self.log("waiting for %s to go %d" % (value, i)) - else: self.fail("time out wairing for (%s) to disappear" % (what)) + else: assert False, "time out waiting for (%s) to disappear" % (what) self.log("gone finally (%d)" % i)