From 52d3fa4333d676b323ac6712130c40bc0405561f Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Thu, 30 Mar 2017 22:08:59 +0300
Subject: [PATCH 01/12] everware.yml with volumes

mount files downloaded from <url> into /notebook/data
can't mount in /notebooks (it deletes git files)
---
 everware/git_processor.py | 28 ++++++++++++++++++++++++++++
 everware/spawner.py       |  3 +++
 2 files changed, 31 insertions(+)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index bd7fc9d..9a1ea97 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -118,6 +118,12 @@ def prepare_local_repo(self):
         else:
             return True
 
+    @gen.coroutine
+    def prepare_everware_yml(self):
+        everware_yml_path = os.path.join(self._repo_dir, 'everware.yml')
+        if os.path.isfile(everware_yml_path):
+            self.parse(everware_yml_path)
+
 
     @property
     def escaped_repo_url(self):
@@ -177,3 +183,25 @@ def load_state(self, state):
             if key in state:
                 setattr(self, key, state[key])
 
+    def parse(self, everware_yml):
+        import yaml
+        with open(everware_yml) as file:
+            text = yaml.load(file)
+        volumes = []
+        for element in text:
+            volumes += text.get(element).get("volumes")
+        for volume in volumes:
+            #check url
+            self.download_file(volume)
+
+    def download_file(self, url):
+        import urllib
+        filename = url.split("/")[-1]
+        if len(filename) == 0:
+            filename = url.split("/")[-2]
+        #should think about deleting this folder someday
+        self.directory_volume = self._repo_dir + "-volume"
+        os.mkdir(self.directory_volume)
+        urllib.request.urlretrieve(url, self.directory_volume + '/' + filename)
+
+
diff --git a/everware/spawner.py b/everware/spawner.py
index c929241..17a5ab1 100755
--- a/everware/spawner.py
+++ b/everware/spawner.py
@@ -268,6 +268,7 @@ def build_image(self):
             ))
             self.log.info('Cloning repo %s' % self.repo_url)
             dockerfile_exists = yield self.prepare_local_repo()
+            self.prepare_everware_yml()
             if not dockerfile_exists:
                 self._add_to_log('No dockerfile. Use the default one %s' % os.environ['DEFAULT_DOCKER_IMAGE'])
 
@@ -371,6 +372,8 @@ def start(self, image=None):
                 yield self.remove_old_container()
             self.log.info("Starting container from image: %s" % image_name)
             self._add_to_log('Creating container')
+            #can't mount into /notebooks folder - it deletes all git files
+            self.volumes = {self.directory_volume: "/notebook/data"}
             yield super(CustomDockerSpawner, self).start(
                 image=image_name
             )

From 1733e6b84998a64856c6b7156e74f3bf7eaf8f60 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Fri, 31 Mar 2017 18:30:40 +0300
Subject: [PATCH 02/12] Test

---
 everware/spawner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/everware/spawner.py b/everware/spawner.py
index 17a5ab1..5c103fc 100755
--- a/everware/spawner.py
+++ b/everware/spawner.py
@@ -373,7 +373,7 @@ def start(self, image=None):
             self.log.info("Starting container from image: %s" % image_name)
             self._add_to_log('Creating container')
             #can't mount into /notebooks folder - it deletes all git files
-            self.volumes = {self.directory_volume: "/notebook/data"}
+            self.volumes = {self._repo_dir + "-volume": "/notebook/data"}
             yield super(CustomDockerSpawner, self).start(
                 image=image_name
             )

From cc6c6c0ad17d7be1f38bac2032768457283a0f90 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Fri, 31 Mar 2017 19:27:13 +0300
Subject: [PATCH 03/12] Create special dict for everware.yml parameters

---
 everware/git_processor.py | 14 ++++++++++----
 everware/spawner.py       |  6 ++++--
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index 9a1ea97..c16bb37 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -120,10 +120,11 @@ def prepare_local_repo(self):
 
     @gen.coroutine
     def prepare_everware_yml(self):
+        self.everware_yml_param = {}
         everware_yml_path = os.path.join(self._repo_dir, 'everware.yml')
         if os.path.isfile(everware_yml_path):
             self.parse(everware_yml_path)
-
+        return self.everware_yml_param
 
     @property
     def escaped_repo_url(self):
@@ -200,8 +201,13 @@ def download_file(self, url):
         if len(filename) == 0:
             filename = url.split("/")[-2]
         #should think about deleting this folder someday
-        self.directory_volume = self._repo_dir + "-volume"
-        os.mkdir(self.directory_volume)
-        urllib.request.urlretrieve(url, self.directory_volume + '/' + filename)
+        directory_volume = self._repo_dir + "-volume"
+        os.mkdir(directory_volume)
+        try:
+            urllib.request.urlretrieve(url, directory_volume + '/' + filename)
+            self.everware_yml_param["directory_volume"] = directory_volume
+        except:
+            #some work
+            pass
 
 
diff --git a/everware/spawner.py b/everware/spawner.py
index 5c103fc..3210b6e 100755
--- a/everware/spawner.py
+++ b/everware/spawner.py
@@ -39,6 +39,7 @@ def __init__(self, **kwargs):
         self._image_handler = ImageHandler()
         self._cur_waiter = None
         self._is_empty = False
+        self.everware_yml_param = {}
         DockerSpawner.__init__(self, **kwargs)
         EmailNotificator.__init__(self)
 
@@ -268,7 +269,7 @@ def build_image(self):
             ))
             self.log.info('Cloning repo %s' % self.repo_url)
             dockerfile_exists = yield self.prepare_local_repo()
-            self.prepare_everware_yml()
+            self.everware_yml_param = yield self.prepare_everware_yml()
             if not dockerfile_exists:
                 self._add_to_log('No dockerfile. Use the default one %s' % os.environ['DEFAULT_DOCKER_IMAGE'])
 
@@ -373,7 +374,8 @@ def start(self, image=None):
             self.log.info("Starting container from image: %s" % image_name)
             self._add_to_log('Creating container')
             #can't mount into /notebooks folder - it deletes all git files
-            self.volumes = {self._repo_dir + "-volume": "/notebook/data"}
+            if self.everware_yml_param.get("directory_volume"):
+                self.volumes = {self.everware_yml_param.get("directory_volume") : "/notebook/data"}
             yield super(CustomDockerSpawner, self).start(
                 image=image_name
             )

From 9d8ee5c1d7ae55ae893153d6b9aa6021cd8e6f94 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Sat, 8 Apr 2017 23:05:17 +0300
Subject: [PATCH 04/12] AsyncHTTP

---
 everware/git_processor.py | 27 ++++++++++++++++-----------
 everware/spawner.py       |  4 ++--
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index 37cec5a..a65b7ac 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -2,6 +2,7 @@
 import git
 from concurrent.futures import ThreadPoolExecutor
 from tornado import gen
+from  tornado.httpclient import AsyncHTTPClient
 import os
 import os.path
 
@@ -123,7 +124,7 @@ def prepare_everware_yml(self):
         self.everware_yml_param = {}
         everware_yml_path = os.path.join(self._repo_dir, 'everware.yml')
         if os.path.isfile(everware_yml_path):
-            self.parse(everware_yml_path)
+            self.parse_everware_yml(everware_yml_path)
         return self.everware_yml_param
 
     @property
@@ -192,28 +193,32 @@ def load_state(self, state):
             if key in state:
                 setattr(self, key, state[key])
 
-    def parse(self, everware_yml):
+    def parse_everware_yml(self, everware_yml):
         import yaml
         with open(everware_yml) as file:
             text = yaml.load(file)
-        volumes = []
+        all_data = []
         for element in text:
-            volumes += text.get(element).get("volumes")
-        for volume in volumes:
+            all_data += text.get(element).get("data")
+        for data in all_data:
             #check url
-            self.download_file(volume)
+            self.download_file(data)
 
+    @gen.coroutine
     def download_file(self, url):
-        import urllib
+        http_client = AsyncHTTPClient()
         filename = url.split("/")[-1]
         if len(filename) == 0:
             filename = url.split("/")[-2]
         #should think about deleting this folder someday
-        directory_volume = self._repo_dir + "-volume"
-        os.mkdir(directory_volume)
+        directory_data = self._repo_dir + "-data"
+        os.mkdir(directory_data)
         try:
-            urllib.request.urlretrieve(url, directory_volume + '/' + filename)
-            self.everware_yml_param["directory_volume"] = directory_volume
+            response = yield http_client.fetch(url)
+            with open(directory_data + '/' + filename, "ab") as f:
+                f.write(response.body)
+            self.everware_yml_param["directory_data"] = directory_data
+            http_client.close()
         except:
             #some work
             pass
diff --git a/everware/spawner.py b/everware/spawner.py
index 3210b6e..af05598 100755
--- a/everware/spawner.py
+++ b/everware/spawner.py
@@ -374,8 +374,8 @@ def start(self, image=None):
             self.log.info("Starting container from image: %s" % image_name)
             self._add_to_log('Creating container')
             #can't mount into /notebooks folder - it deletes all git files
-            if self.everware_yml_param.get("directory_volume"):
-                self.volumes = {self.everware_yml_param.get("directory_volume") : "/notebook/data"}
+            if self.everware_yml_param.get("directory_data"):
+                self.volumes = {self.everware_yml_param.get("directory_data") : "/notebook/data"}
             yield super(CustomDockerSpawner, self).start(
                 image=image_name
             )

From 5a388e9a1b0dae5b8b421db3a8ee1dddb5e5bd1d Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Mon, 10 Apr 2017 08:12:22 +0300
Subject: [PATCH 05/12] xrdcp

---
 everware/git_processor.py | 52 +++++++++++++++++++++++++++++------------------
 1 file changed, 32 insertions(+), 20 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index a65b7ac..73d6668 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -5,6 +5,9 @@
 from  tornado.httpclient import AsyncHTTPClient
 import os
 import os.path
+from sh import xrdcp
+from urllib.parse import urlparse
+import yaml
 
 
 class GitMixin:
@@ -121,10 +124,22 @@ def prepare_local_repo(self):
 
     @gen.coroutine
     def prepare_everware_yml(self):
-        self.everware_yml_param = {}
+        self.directory_data = self._repo_dir + "-data"
+        os.mkdir(self.directory_data)
+        self.everware_yml_param["directory_data"] = self.directory_data
         everware_yml_path = os.path.join(self._repo_dir, 'everware.yml')
         if os.path.isfile(everware_yml_path):
-            self.parse_everware_yml(everware_yml_path)
+            with open(everware_yml_path) as file:
+                text = yaml.load(file)
+            all_data = []
+            for element in text:
+                all_data += text.get(element).get("data")
+            for url in all_data:
+                url_struct = urlparse(url)
+                if url_struct.scheme == "root":
+                    yield self.download_xrootd(url_struct)
+                else:
+                    yield self.download_http(url_struct)
         return self.everware_yml_param
 
     @property
@@ -193,31 +208,28 @@ def load_state(self, state):
             if key in state:
                 setattr(self, key, state[key])
 
-    def parse_everware_yml(self, everware_yml):
-        import yaml
-        with open(everware_yml) as file:
-            text = yaml.load(file)
-        all_data = []
-        for element in text:
-            all_data += text.get(element).get("data")
-        for data in all_data:
-            #check url
-            self.download_file(data)
+    def download_file(self, url):
+        url_struct = urlparse(url)
+        if url_struct.scheme == "root":
+            self.download_xrootd(self, url_struct)
+        else:
+            self.download_http(self, url_struct)
 
     @gen.coroutine
-    def download_file(self, url):
+    def download_xrootd(self, url_struct):
+        xrdcp("-r", url_struct.geturl(), self.directory_data)
+
+    @gen.coroutine
+    def download_http(self, url_struct):
         http_client = AsyncHTTPClient()
-        filename = url.split("/")[-1]
+        filename = url_struct.path.split("/")[-1]
         if len(filename) == 0:
-            filename = url.split("/")[-2]
+            filename = url_struct.path.split("/")[-2]
         #should think about deleting this folder someday
-        directory_data = self._repo_dir + "-data"
-        os.mkdir(directory_data)
         try:
-            response = yield http_client.fetch(url)
-            with open(directory_data + '/' + filename, "ab") as f:
+            response = yield http_client.fetch(url_struct.geturl())
+            with open(self.directory_data + '/' + filename, "ab") as f:
                 f.write(response.body)
-            self.everware_yml_param["directory_data"] = directory_data
             http_client.close()
         except:
             #some work

From 3d80342fa48f598c84fa9905d9b067a899c70eb8 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Mon, 10 Apr 2017 08:27:57 +0300
Subject: [PATCH 06/12] add "sh" package and  remove directory_data after
 container removing

---
 everware/spawner.py | 1 +
 requirements.txt    | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/everware/spawner.py b/everware/spawner.py
index af05598..6e8b8b1 100755
--- a/everware/spawner.py
+++ b/everware/spawner.py
@@ -430,6 +430,7 @@ def stop(self, now=False):
                     self.container_name, self.container_id[:7])
                 # remove the container, as well as any associated volumes
                 yield self.docker('remove_container', self.container_id, v=True)
+                rmtree(self.everware_yml_param.get("directory_data"))
 
         self.clear_state()
 
diff --git a/requirements.txt b/requirements.txt
index 0c80c20..070e8f9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,10 @@
 ipython[notebook]
 coveralls
 nose2
+sh
 pytest>=2.8
 GitPython==2.1.1
 selenium==2.52.0
 jupyterhub==0.7.2
 docker-py==1.10.6
--e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner
\ No newline at end of file
+-e git+https://github.com/jupyter/dockerspawner.git@fdf5577ca2b4087a064507cca8b250e4ad792e02#egg=dockerspawner

From 89f6ba2a1390842e552b6725299c114873f689c4 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Mon, 10 Apr 2017 20:19:06 +0300
Subject: [PATCH 07/12] add run_on_executor

---
 everware/git_processor.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index 73d6668..39bc1ef 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -1,7 +1,7 @@
 import re
 import git
 from concurrent.futures import ThreadPoolExecutor
-from tornado import gen
+from tornado import gen,concurrent
 from  tornado.httpclient import AsyncHTTPClient
 import os
 import os.path
@@ -215,9 +215,10 @@ def download_file(self, url):
         else:
             self.download_http(self, url_struct)
 
-    @gen.coroutine
+    @concurrent.run_on_executor(executor='_git_executor')
     def download_xrootd(self, url_struct):
         xrdcp("-r", url_struct.geturl(), self.directory_data)
+        return
 
     @gen.coroutine
     def download_http(self, url_struct):

From ed6421a45ded7c1422e73057555fc6a0af1aca02 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Mon, 10 Apr 2017 22:00:11 +0300
Subject: [PATCH 08/12] xrtood timeout, log.info + exception

xrdcp doesn't die with timeout if connection doesn't exist \
---
 everware/git_processor.py | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index 39bc1ef..9810fd3 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -5,9 +5,9 @@
 from  tornado.httpclient import AsyncHTTPClient
 import os
 import os.path
-from sh import xrdcp
 from urllib.parse import urlparse
 import yaml
+import socket
 
 
 class GitMixin:
@@ -217,7 +217,24 @@ def download_file(self, url):
 
     @concurrent.run_on_executor(executor='_git_executor')
     def download_xrootd(self, url_struct):
+        try:
+            from sh import xrdcp
+        except:
+            self.log.info("Sorry, but everware doesn't support xrootd now")
+            return
+        with socket.socket() as s:
+            s.settimeout(3)
+            try:
+                port = url_struct.port
+                if not port:
+                    port = 1094
+                s.connect((url_struct.hostname, port))
+            except:
+                self.log.info("Server %s doesn't respond" % url_struct.hostname)
+                return
+        self.log.info("Downloading from xrootd server %s" % url_struct.hostname)
         xrdcp("-r", url_struct.geturl(), self.directory_data)
+        self.log.info("Downloaded")
         return
 
     @gen.coroutine
@@ -228,12 +245,14 @@ def download_http(self, url_struct):
             filename = url_struct.path.split("/")[-2]
         #should think about deleting this folder someday
         try:
+            self.log.info("Downloading from http server %s" % url_struct.hostname)
             response = yield http_client.fetch(url_struct.geturl())
             with open(self.directory_data + '/' + filename, "ab") as f:
                 f.write(response.body)
+            self.log.info("Downloaded")
             http_client.close()
         except:
-            #some work
+            self.log.info("Something went wrong during downloading from %s " % url_struct.hostname)
             pass
 
 

From e30f7e2daf6432431833aac6114b882aa99ef484 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Mon, 10 Apr 2017 22:35:34 +0300
Subject: [PATCH 09/12] user log

---
 everware/git_processor.py | 9 ++++++++-
 everware/spawner.py       | 3 ++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index 9810fd3..812a377 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -123,8 +123,9 @@ def prepare_local_repo(self):
             return True
 
     @gen.coroutine
-    def prepare_everware_yml(self):
+    def prepare_everware_yml(self, _user_log):
         self.directory_data = self._repo_dir + "-data"
+        self._user_log = _user_log
         os.mkdir(self.directory_data)
         self.everware_yml_param["directory_data"] = self.directory_data
         everware_yml_path = os.path.join(self._repo_dir, 'everware.yml')
@@ -136,10 +137,14 @@ def prepare_everware_yml(self):
                 all_data += text.get(element).get("data")
             for url in all_data:
                 url_struct = urlparse(url)
+                self._user_log.append(({'text': "Start download from %s" % url_struct.geturl(), 'level': 2}))
                 if url_struct.scheme == "root":
+
                     yield self.download_xrootd(url_struct)
                 else:
                     yield self.download_http(url_struct)
+
+        self.everware_yml_param["user_log"] = self._user_log
         return self.everware_yml_param
 
     @property
@@ -235,6 +240,7 @@ def download_xrootd(self, url_struct):
         self.log.info("Downloading from xrootd server %s" % url_struct.hostname)
         xrdcp("-r", url_struct.geturl(), self.directory_data)
         self.log.info("Downloaded")
+        self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2}))
         return
 
     @gen.coroutine
@@ -250,6 +256,7 @@ def download_http(self, url_struct):
             with open(self.directory_data + '/' + filename, "ab") as f:
                 f.write(response.body)
             self.log.info("Downloaded")
+            self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2}))
             http_client.close()
         except:
             self.log.info("Something went wrong during downloading from %s " % url_struct.hostname)
diff --git a/everware/spawner.py b/everware/spawner.py
index 6e8b8b1..22f57f7 100755
--- a/everware/spawner.py
+++ b/everware/spawner.py
@@ -269,7 +269,8 @@ def build_image(self):
             ))
             self.log.info('Cloning repo %s' % self.repo_url)
             dockerfile_exists = yield self.prepare_local_repo()
-            self.everware_yml_param = yield self.prepare_everware_yml()
+            self.everware_yml_param = yield self.prepare_everware_yml(self._user_log)
+            self._user_log = self._user_log and self.everware_yml_param["user_log"]
             if not dockerfile_exists:
                 self._add_to_log('No dockerfile. Use the default one %s' % os.environ['DEFAULT_DOCKER_IMAGE'])
 

From 7806687bb10b38488b54b8c902145780568ae847 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Tue, 11 Apr 2017 01:16:09 +0300
Subject: [PATCH 10/12] yaml + ssl parameter

---
 everware/git_processor.py | 20 +++++++++++++-------
 requirements.txt          |  1 +
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index 812a377..0d9486e 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -135,14 +135,18 @@ def prepare_everware_yml(self, _user_log):
             all_data = []
             for element in text:
                 all_data += text.get(element).get("data")
-            for url in all_data:
+            for data in all_data:
+                if type(data) == dict:
+                    url = data.get("url")
+                    ssl = data.get("ssl")
+                else:
+                    url = data
                 url_struct = urlparse(url)
                 self._user_log.append(({'text': "Start download from %s" % url_struct.geturl(), 'level': 2}))
                 if url_struct.scheme == "root":
-
-                    yield self.download_xrootd(url_struct)
+                    yield self.download_xrootd(url_struct, ssl)
                 else:
-                    yield self.download_http(url_struct)
+                    yield self.download_http(url_struct, ssl)
 
         self.everware_yml_param["user_log"] = self._user_log
         return self.everware_yml_param
@@ -221,7 +225,7 @@ def download_file(self, url):
             self.download_http(self, url_struct)
 
     @concurrent.run_on_executor(executor='_git_executor')
-    def download_xrootd(self, url_struct):
+    def download_xrootd(self, url_struct, ssl):
         try:
             from sh import xrdcp
         except:
@@ -244,12 +248,14 @@ def download_xrootd(self, url_struct):
         return
 
     @gen.coroutine
-    def download_http(self, url_struct):
+    def download_http(self, url_struct, ssl):
         http_client = AsyncHTTPClient()
         filename = url_struct.path.split("/")[-1]
         if len(filename) == 0:
             filename = url_struct.path.split("/")[-2]
-        #should think about deleting this folder someday
+        #folder with data files is deleted after container removing
+        if ssl:
+            pass
         try:
             self.log.info("Downloading from http server %s" % url_struct.hostname)
             response = yield http_client.fetch(url_struct.geturl())
diff --git a/requirements.txt b/requirements.txt
index 070e8f9..3cad4fe 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,6 +2,7 @@ ipython[notebook]
 coveralls
 nose2
 sh
+yaml
 pytest>=2.8
 GitPython==2.1.1
 selenium==2.52.0

From dea1a2078d24581d4a4f654e78c6e9934fb220d7 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Tue, 11 Apr 2017 12:42:59 +0300
Subject: [PATCH 11/12] pyyaml; without socket; mount into /data

---
 everware/git_processor.py | 22 ++++++++--------------
 everware/spawner.py       |  2 +-
 requirements.txt          |  2 +-
 3 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index 0d9486e..babb34e 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -231,20 +231,14 @@ def download_xrootd(self, url_struct, ssl):
         except:
             self.log.info("Sorry, but everware doesn't support xrootd now")
             return
-        with socket.socket() as s:
-            s.settimeout(3)
-            try:
-                port = url_struct.port
-                if not port:
-                    port = 1094
-                s.connect((url_struct.hostname, port))
-            except:
-                self.log.info("Server %s doesn't respond" % url_struct.hostname)
-                return
-        self.log.info("Downloading from xrootd server %s" % url_struct.hostname)
-        xrdcp("-r", url_struct.geturl(), self.directory_data)
-        self.log.info("Downloaded")
-        self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2}))
+        try:
+            self.log.info("Downloading from xrootd server %s" % url_struct.hostname)
+            xrdcp("-DIConnectionRetry 1", "-r", url_struct.geturl(), self.directory_data)
+            self.log.info("Downloaded")
+            self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2}))
+        except:
+            self.log.info("Cannot download")
+            self._user_log.append(({'text': "Fail to download from %s" % url_struct.geturl(), 'level': 2}))
         return
 
     @gen.coroutine
diff --git a/everware/spawner.py b/everware/spawner.py
index 22f57f7..0fd235b 100755
--- a/everware/spawner.py
+++ b/everware/spawner.py
@@ -376,7 +376,7 @@ def start(self, image=None):
             self._add_to_log('Creating container')
             #can't mount into /notebooks folder - it deletes all git files
             if self.everware_yml_param.get("directory_data"):
-                self.volumes = {self.everware_yml_param.get("directory_data") : "/notebook/data"}
+                self.volumes = {self.everware_yml_param.get("directory_data") : "/data"}
             yield super(CustomDockerSpawner, self).start(
                 image=image_name
             )
diff --git a/requirements.txt b/requirements.txt
index 3cad4fe..5f6f92b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@ ipython[notebook]
 coveralls
 nose2
 sh
-yaml
+pyyaml
 pytest>=2.8
 GitPython==2.1.1
 selenium==2.52.0

From a64152029042a2d0d2cbe03ff3cdcb34fef11ba5 Mon Sep 17 00:00:00 2001
From: IgorMusinov <igor.musinov@phystech.edu>
Date: Tue, 11 Apr 2017 13:04:38 +0300
Subject: [PATCH 12/12] connection retry constant

---
 everware/git_processor.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/everware/git_processor.py b/everware/git_processor.py
index babb34e..dcf6700 100644
--- a/everware/git_processor.py
+++ b/everware/git_processor.py
@@ -232,8 +232,9 @@ def download_xrootd(self, url_struct, ssl):
             self.log.info("Sorry, but everware doesn't support xrootd now")
             return
         try:
+            n_check_connection = 1
             self.log.info("Downloading from xrootd server %s" % url_struct.hostname)
-            xrdcp("-DIConnectionRetry 1", "-r", url_struct.geturl(), self.directory_data)
+            xrdcp("-DIConnectionRetry", n_check_connection,"-r", url_struct.geturl(), self.directory_data)
             self.log.info("Downloaded")
             self._user_log.append(({'text': "Successfully downloaded from %s" % url_struct.geturl(), 'level': 2}))
         except: