diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py
index 2619827c226b..0dc553fab00f 100644
--- a/python/servo/devenv_commands.py
+++ b/python/servo/devenv_commands.py
@@ -143,25 +143,28 @@ def grep(self, params):
["git"] + ["grep"] + params + ['--'] + grep_paths + [':(exclude)*.min.js'],
env=self.build_env())
- @Command('wpt-upgrade',
+ @Command('wptrunner-upgrade',
description='upgrade wptrunner.',
category='devenv')
def upgrade_wpt_runner(self):
+ env = self.build_env()
with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')):
- code = call(["git", "init"], env=self.build_env())
+ code = call(["git", "init"], env=env)
if code:
return code
+ # No need to report an error if this fails, as it will for the first use
+ call(["git", "remote", "rm", "upstream"], env=env)
code = call(
- ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env())
+ ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=env)
if code:
return code
- code = call(["git", "fetch", "upstream"], env=self.build_env())
+ code = call(["git", "fetch", "upstream"], env=env)
if code:
return code
- code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=self.build_env())
+ code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=env)
if code:
return code
- code = call(["rm", "-rf", ".git"], env=self.build_env())
+ code = call(["rm", "-rf", ".git"], env=env)
if code:
return code
return 0
diff --git a/tests/wpt/harness/.travis.yml b/tests/wpt/harness/.travis.yml
new file mode 100644
index 000000000000..b5ec736a3844
--- /dev/null
+++ b/tests/wpt/harness/.travis.yml
@@ -0,0 +1,26 @@
+language: python
+python: 2.7
+
+sudo: false
+
+cache:
+ directories:
+ - $HOME/.cache/pip
+
+env:
+ - TOXENV=py27-base
+ - TOXENV=py27-b2g
+ - TOXENV=py27-chrome
+ - TOXENV=py27-firefox
+ - TOXENV=py27-servo
+ - TOXENV=pypy-base
+ - TOXENV=pypy-b2g
+ - TOXENV=pypy-chrome
+ - TOXENV=pypy-firefox
+ - TOXENV=pypy-servo
+
+install:
+ - pip install -U tox
+
+script:
+ - tox
diff --git a/tests/wpt/harness/tox.ini b/tests/wpt/harness/tox.ini
new file mode 100644
index 000000000000..30a28e3cddb4
--- /dev/null
+++ b/tests/wpt/harness/tox.ini
@@ -0,0 +1,16 @@
+[pytest]
+xfail_strict=true
+
+[tox]
+envlist = {py27,pypy}-{base,b2g,chrome,firefox,servo}
+
+[testenv]
+deps =
+ pytest>=2.9
+ -r{toxinidir}/requirements.txt
+ b2g: -r{toxinidir}/requirements_b2g.txt
+ chrome: -r{toxinidir}/requirements_chrome.txt
+ firefox: -r{toxinidir}/requirements_firefox.txt
+ servo: -r{toxinidir}/requirements_servo.txt
+
+commands = py.test []
diff --git a/tests/wpt/harness/wptrunner/browsers/firefox.py b/tests/wpt/harness/wptrunner/browsers/firefox.py
index dcd7a4f1d17d..a3c6f3f5753f 100644
--- a/tests/wpt/harness/wptrunner/browsers/firefox.py
+++ b/tests/wpt/harness/wptrunner/browsers/firefox.py
@@ -3,6 +3,7 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
+import platform
import subprocess
import sys
@@ -128,10 +129,16 @@ def start(self):
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False,
- "network.dns.localDomains": ",".join(hostnames)})
+ "network.dns.localDomains": ",".join(hostnames),
+ "places.history.enabled": False})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
+ # Bug 1262954: winxp + e10s, disable hwaccel
+ if (self.e10s and platform.system() in ("Windows", "Microsoft") and
+ '5.1' in platform.version()):
+ self.profile.set_preferences({"layers.acceleration.disabled": True})
+
if self.ca_certificate_path is not None:
self.setup_ssl()
diff --git a/tests/wpt/harness/wptrunner/browsers/servo.py b/tests/wpt/harness/wptrunner/browsers/servo.py
index bc90cefcfc9e..4e4bc20128c4 100644
--- a/tests/wpt/harness/wptrunner/browsers/servo.py
+++ b/tests/wpt/harness/wptrunner/browsers/servo.py
@@ -6,7 +6,7 @@
from .base import NullBrowser, ExecutorBrowser, require_arg
from ..executors import executor_kwargs as base_executor_kwargs
-from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor
+from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor, ServoWdspecExecutor
here = os.path.join(os.path.split(__file__)[0])
@@ -14,7 +14,8 @@
"check_args": "check_args",
"browser": "ServoBrowser",
"executor": {"testharness": "ServoTestharnessExecutor",
- "reftest": "ServoRefTestExecutor"},
+ "reftest": "ServoRefTestExecutor",
+ "wdspec": "ServoWdspecExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options",
diff --git a/tests/wpt/harness/wptrunner/browsers/webdriver.py b/tests/wpt/harness/wptrunner/browsers/webdriver.py
deleted file mode 100644
index 54b1c1b5ef93..000000000000
--- a/tests/wpt/harness/wptrunner/browsers/webdriver.py
+++ /dev/null
@@ -1,137 +0,0 @@
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
-
-import errno
-import socket
-import time
-import traceback
-import urlparse
-
-import mozprocess
-
-from .base import get_free_port, cmd_arg
-
-
-__all__ = ["SeleniumLocalServer", "ChromedriverLocalServer"]
-
-
-class LocalServer(object):
- used_ports = set()
- default_endpoint = "/"
-
- def __init__(self, logger, binary, port=None, endpoint=None):
- self.logger = logger
- self.binary = binary
- self.port = port
- self.endpoint = endpoint or self.default_endpoint
-
- if self.port is None:
- self.port = get_free_port(4444, exclude=self.used_ports)
- self.used_ports.add(self.port)
- self.url = "http://127.0.0.1:%i%s" % (self.port, self.endpoint)
-
- self.proc, self.cmd = None, None
-
- def start(self):
- self.proc = mozprocess.ProcessHandler(
- self.cmd, processOutputLine=self.on_output)
- try:
- self.proc.run()
- except OSError as e:
- if e.errno == errno.ENOENT:
- raise IOError(
- "chromedriver executable not found: %s" % self.binary)
- raise
-
- self.logger.debug(
- "Waiting for server to become accessible: %s" % self.url)
- surl = urlparse.urlparse(self.url)
- addr = (surl.hostname, surl.port)
- try:
- wait_service(addr)
- except:
- self.logger.error(
- "Server was not accessible within the timeout:\n%s" % traceback.format_exc())
- raise
- else:
- self.logger.info("Server listening on port %i" % self.port)
-
- def stop(self):
- if hasattr(self.proc, "proc"):
- self.proc.kill()
-
- def is_alive(self):
- if hasattr(self.proc, "proc"):
- exitcode = self.proc.poll()
- return exitcode is None
- return False
-
- def on_output(self, line):
- self.logger.process_output(self.pid,
- line.decode("utf8", "replace"),
- command=" ".join(self.cmd))
-
- @property
- def pid(self):
- if hasattr(self.proc, "proc"):
- return self.proc.pid
-
-
-class SeleniumLocalServer(LocalServer):
- default_endpoint = "/wd/hub"
-
- def __init__(self, logger, binary, port=None):
- LocalServer.__init__(self, logger, binary, port=port)
- self.cmd = ["java",
- "-jar", self.binary,
- "-port", str(self.port)]
-
- def start(self):
- self.logger.debug("Starting local Selenium server")
- LocalServer.start(self)
-
- def stop(self):
- LocalServer.stop(self)
- self.logger.info("Selenium server stopped listening")
-
-
-class ChromedriverLocalServer(LocalServer):
- default_endpoint = "/wd/hub"
-
- def __init__(self, logger, binary="chromedriver", port=None, endpoint=None):
- LocalServer.__init__(self, logger, binary, port=port, endpoint=endpoint)
- # TODO: verbose logging
- self.cmd = [self.binary,
- cmd_arg("port", str(self.port)) if self.port else "",
- cmd_arg("url-base", self.endpoint) if self.endpoint else ""]
-
- def start(self):
- self.logger.debug("Starting local chromedriver server")
- LocalServer.start(self)
-
- def stop(self):
- LocalServer.stop(self)
- self.logger.info("chromedriver server stopped listening")
-
-
-def wait_service(addr, timeout=15):
- """Waits until network service given as a tuple of (host, port) becomes
- available or the `timeout` duration is reached, at which point
- ``socket.error`` is raised."""
- end = time.time() + timeout
- while end > time.time():
- so = socket.socket()
- try:
- so.connect(addr)
- except socket.timeout:
- pass
- except socket.error as e:
- if e[0] != errno.ECONNREFUSED:
- raise
- else:
- return True
- finally:
- so.close()
- time.sleep(0.5)
- raise socket.error("Service is unavailable: %s:%i" % addr)
diff --git a/tests/wpt/harness/wptrunner/executors/executormarionette.py b/tests/wpt/harness/wptrunner/executors/executormarionette.py
index c4b1cba689af..50200d74677f 100644
--- a/tests/wpt/harness/wptrunner/executors/executormarionette.py
+++ b/tests/wpt/harness/wptrunner/executors/executormarionette.py
@@ -292,7 +292,7 @@ def is_alive(self):
class ExecuteAsyncScriptRun(object):
def __init__(self, logger, func, marionette, url, timeout):
self.logger = logger
- self.result = None
+ self.result = (None, None)
self.marionette = marionette
self.func = func
self.url = url
@@ -323,11 +323,9 @@ def run(self):
wait_timeout = None
flag = self.result_flag.wait(wait_timeout)
- if self.result is None:
+ if self.result[1] is None:
self.logger.debug("Timed out waiting for a result")
- assert not flag
self.result = False, ("EXTERNAL-TIMEOUT", None)
-
return self.result
def _run(self):
@@ -409,7 +407,8 @@ def do_testharness(self, marionette, url, timeout):
"timeout": timeout_ms,
"explicit_timeout": timeout is None}
- return marionette.execute_async_script(script, new_sandbox=False)
+ rv = marionette.execute_async_script(script, new_sandbox=False)
+ return rv
class MarionetteRefTestExecutor(RefTestExecutor):
@@ -487,7 +486,7 @@ def _screenshot(self, marionette, url, timeout):
class WdspecRun(object):
def __init__(self, func, session, path, timeout):
self.func = func
- self.result = None
+ self.result = (None, None)
self.session = session
self.path = path
self.timeout = timeout
@@ -504,8 +503,7 @@ def run(self):
executor.start()
flag = self.result_flag.wait(self.timeout)
- if self.result is None:
- assert not flag
+ if self.result[1] is None:
self.result = False, ("EXTERNAL-TIMEOUT", None)
return self.result
diff --git a/tests/wpt/harness/wptrunner/executors/executorservo.py b/tests/wpt/harness/wptrunner/executors/executorservo.py
index 7f5e90673197..2d9c8c1e865f 100644
--- a/tests/wpt/harness/wptrunner/executors/executorservo.py
+++ b/tests/wpt/harness/wptrunner/executors/executorservo.py
@@ -4,11 +4,13 @@
import base64
import hashlib
+import httplib
import json
import os
import subprocess
import tempfile
import threading
+import traceback
import urlparse
import uuid
from collections import defaultdict
@@ -19,15 +21,23 @@
Protocol,
RefTestImplementation,
testharness_result_converter,
- reftest_result_converter)
+ reftest_result_converter,
+ WdspecExecutor)
from .process import ProcessTestExecutor
from ..browsers.base import browser_command
-render_arg = None
+from ..wpttest import WdspecResult, WdspecSubtestResult
+from ..webdriver_server import ServoDriverServer
+from .executormarionette import WdspecRun
+from . import pytestrunner
+render_arg = None
+webdriver = None
+extra_timeout = 5 # seconds
def do_delayed_imports():
- global render_arg
+ global render_arg, webdriver
from ..browsers.servo import render_arg
+ import webdriver
hosts_text = """127.0.0.1 web-platform.test
127.0.0.1 www.web-platform.test
@@ -214,10 +224,7 @@ def screenshot(self, test, viewport_size, dpi):
for pref in test.environment.get('prefs', {}):
command += ["--pref", pref]
- if viewport_size:
- command += ["--resolution", viewport_size]
- else:
- command += ["--resolution", "800x600"]
+ command += ["--resolution", viewport_size or "800x600"]
if dpi:
command += ["--device-pixel-ratio", dpi]
@@ -275,3 +282,77 @@ def on_output(self, line):
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))
+
+class ServoWdspecProtocol(Protocol):
+ def __init__(self, executor, browser):
+ do_delayed_imports()
+ Protocol.__init__(self, executor, browser)
+ self.session = None
+ self.server = None
+
+ def setup(self, runner):
+ try:
+ self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args, render_backend=self.browser.render_backend)
+ self.server.start(block=False)
+ self.logger.info(
+ "WebDriver HTTP server listening at %s" % self.server.url)
+
+ self.logger.info(
+ "Establishing new WebDriver session with %s" % self.server.url)
+ self.session = webdriver.Session(
+ self.server.host, self.server.port, self.server.base_path)
+ except Exception:
+ self.logger.error(traceback.format_exc())
+ self.executor.runner.send_message("init_failed")
+ else:
+ self.executor.runner.send_message("init_succeeded")
+
+ def teardown(self):
+ if self.server is not None:
+ try:
+ if self.session.session_id is not None:
+ self.session.end()
+ except Exception:
+ pass
+ if self.server.is_alive:
+ self.server.stop()
+
+ @property
+ def is_alive(self):
+ conn = httplib.HTTPConnection(self.server.host, self.server.port)
+ conn.request("HEAD", self.server.base_path + "invalid")
+ res = conn.getresponse()
+ return res.status == 404
+
+class ServoWdspecExecutor(WdspecExecutor):
+ def __init__(self, browser, server_config,
+ timeout_multiplier=1, close_after_done=True, debug_info=None,
+ **kwargs):
+ WdspecExecutor.__init__(self, browser, server_config,
+ timeout_multiplier=timeout_multiplier,
+ debug_info=debug_info)
+ self.protocol = ServoWdspecProtocol(self, browser)
+
+ def is_alive(self):
+ return self.protocol.is_alive
+
+ def on_environment_change(self, new_environment):
+ pass
+
+ def do_test(self, test):
+ timeout = test.timeout * self.timeout_multiplier + extra_timeout
+
+ success, data = WdspecRun(self.do_wdspec,
+ self.protocol.session,
+ test.path,
+ timeout).run()
+
+ if success:
+ return self.convert_result(test, data)
+
+ return (test.result_cls(*data), [])
+
+ def do_wdspec(self, session, path, timeout):
+ harness_result = ("OK", None)
+ subtest_results = pytestrunner.run(path, session, timeout=timeout)
+ return (harness_result, subtest_results)
diff --git a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py
index 77afb4a36842..b2b9913bf487 100644
--- a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py
+++ b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py
@@ -2,8 +2,12 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
-import pytest
+from tools import pytest
+@pytest.fixture
+def webdriver():
+ import webdriver
+ return webdriver
"""pytest fixtures for use in Python-based WPT tests.
@@ -17,7 +21,7 @@ class Session(object):
in tests.
The session is not created by default to enable testing of session
- creation. However, a module-scoped session will be implicitly created
+ creation. However, a function-scoped session will be implicitly created
at the first call to a WebDriver command. This means methods such as
`session.send_command` and `session.session_id` are possible to use
without having a session.
@@ -45,14 +49,62 @@ def setup(request, session):
def test_something(setup, session):
assert session.url == "https://example.org"
- The session is closed when the test module goes out of scope by an
- implicit call to `session.end`.
+ When the test function goes out of scope, any remaining user prompts
+ and opened windows are closed, and the current browsing context is
+ switched back to the top-level browsing context.
"""
def __init__(self, client):
self.client = client
- @pytest.fixture(scope="module")
+ @pytest.fixture(scope="function")
def session(self, request):
- request.addfinalizer(self.client.end)
+ # finalisers are popped off a stack,
+ # making their ordering reverse
+ request.addfinalizer(self.switch_to_top_level_browsing_context)
+ request.addfinalizer(self.restore_windows)
+ request.addfinalizer(self.dismiss_user_prompts)
+
return self.client
+
+ def dismiss_user_prompts(self):
+ """Dismisses any open user prompts in windows."""
+ current_window = self.client.window_handle
+
+ for window in self.windows():
+ self.client.window_handle = window
+ try:
+ self.client.alert.dismiss()
+ except webdriver.NoSuchAlertException:
+ pass
+
+ self.client.window_handle = current_window
+
+ def restore_windows(self):
+ """Closes superfluous windows opened by the test without ending
+ the session implicitly by closing the last window.
+ """
+ current_window = self.client.window_handle
+
+ for window in self.windows(exclude=[current_window]):
+ self.client.window_handle = window
+ if len(self.client.window_handles) > 1:
+ self.client.close()
+
+ self.client.window_handle = current_window
+
+ def switch_to_top_level_browsing_context(self):
+ """If the current browsing context selected by WebDriver is a
+ `` or an `