diff --git a/everware/git_processor.py b/everware/git_processor.py index 5731f83..dcf6700 100644 --- a/everware/git_processor.py +++ b/everware/git_processor.py @@ -1,9 +1,13 @@ import re import git from concurrent.futures import ThreadPoolExecutor -from tornado import gen +from tornado import gen,concurrent +from tornado.httpclient import AsyncHTTPClient import os import os.path +from urllib.parse import urlparse +import yaml +import socket class GitMixin: @@ -118,6 +122,34 @@ def prepare_local_repo(self): else: return True + @gen.coroutine + def prepare_everware_yml(self, _user_log): + self.directory_data = self._repo_dir + "-data" + self._user_log = _user_log + os.mkdir(self.directory_data) + self.everware_yml_param["directory_data"] = self.directory_data + everware_yml_path = os.path.join(self._repo_dir, 'everware.yml') + if os.path.isfile(everware_yml_path): + with open(everware_yml_path) as file: + text = yaml.load(file) + all_data = [] + for element in text: + all_data += text.get(element).get("data") + for data in all_data: + if type(data) == dict: + url = data.get("url") + ssl = data.get("ssl") + else: + url = data + url_struct = urlparse(url) + self._user_log.append(({'text': "Start download from %s" % url_struct.geturl(), 'level': 2})) + if url_struct.scheme == "root": + yield self.download_xrootd(url_struct, ssl) + else: + yield self.download_http(url_struct, ssl) + + self.everware_yml_param["user_log"] = self._user_log + return self.everware_yml_param @property def escaped_repo_url(self): @@ -183,4 +215,52 @@ def get_state(self): def load_state(self, state): for key in self.STATE_VARS: if key in state: - setattr(self, key, state[key]) \ No newline at end of file + setattr(self, key, state[key]) + + def download_file(self, url): + url_struct = urlparse(url) + if url_struct.scheme == "root": + self.download_xrootd(self, url_struct) + else: + self.download_http(self, url_struct) + + @concurrent.run_on_executor(executor='_git_executor') + def download_xrootd(self, url_struct, ssl): + try: + from sh import xrdcp + except: + self.log.info("Sorry, but everware doesn't support xrootd now") + return + try: + n_check_connection = 1 + self.log.info("Downloading from xrootd server %s" % url_struct.hostname) + xrdcp("-DIConnectionRetry", n_check_connection,"-r", url_struct.geturl(), self.directory_data) + self.log.info("Downloaded") + self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2})) + except: + self.log.info("Cannot download") + self._user_log.append(({'text': "Fail to download from %s" % url_struct.geturl(), 'level': 2})) + return + + @gen.coroutine + def download_http(self, url_struct, ssl): + http_client = AsyncHTTPClient() + filename = url_struct.path.split("/")[-1] + if len(filename) == 0: + filename = url_struct.path.split("/")[-2] + #folder with data files is deleted after container removing + if ssl: + pass + try: + self.log.info("Downloading from http server %s" % url_struct.hostname) + response = yield http_client.fetch(url_struct.geturl()) + with open(self.directory_data + '/' + filename, "ab") as f: + f.write(response.body) + self.log.info("Downloaded") + self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2})) + http_client.close() + except: + self.log.info("Something went wrong during downloading from %s " % url_struct.hostname) + pass + + diff --git a/everware/spawner.py b/everware/spawner.py index c929241..0fd235b 100755 --- a/everware/spawner.py +++ b/everware/spawner.py @@ -39,6 +39,7 @@ def __init__(self, **kwargs): self._image_handler = ImageHandler() self._cur_waiter = None self._is_empty = False + self.everware_yml_param = {} DockerSpawner.__init__(self, **kwargs) EmailNotificator.__init__(self) @@ -268,6 +269,8 @@ def build_image(self): )) self.log.info('Cloning repo %s' % self.repo_url) dockerfile_exists = yield self.prepare_local_repo() + self.everware_yml_param = yield self.prepare_everware_yml(self._user_log) + self._user_log = self._user_log and self.everware_yml_param["user_log"] if not dockerfile_exists: self._add_to_log('No dockerfile. Use the default one %s' % os.environ['DEFAULT_DOCKER_IMAGE']) @@ -371,6 +374,9 @@ def start(self, image=None): yield self.remove_old_container() self.log.info("Starting container from image: %s" % image_name) self._add_to_log('Creating container') + #can't mount into /notebooks folder - it deletes all git files + if self.everware_yml_param.get("directory_data"): + self.volumes = {self.everware_yml_param.get("directory_data") : "/data"} yield super(CustomDockerSpawner, self).start( image=image_name ) @@ -425,6 +431,7 @@ def stop(self, now=False): self.container_name, self.container_id[:7]) # remove the container, as well as any associated volumes yield self.docker('remove_container', self.container_id, v=True) + rmtree(self.everware_yml_param.get("directory_data")) self.clear_state() diff --git a/requirements.txt b/requirements.txt index 0c80c20..5f6f92b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,11 @@ ipython[notebook] coveralls nose2 +sh +pyyaml pytest>=2.8 GitPython==2.1.1 selenium==2.52.0 jupyterhub==0.7.2 docker-py==1.10.6 --e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner \ No newline at end of file +-e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner