diff --git a/python/servo/devenv_commands.py b/python/servo/devenv_commands.py index 2619827c226b..0dc553fab00f 100644 --- a/python/servo/devenv_commands.py +++ b/python/servo/devenv_commands.py @@ -143,25 +143,28 @@ def grep(self, params): ["git"] + ["grep"] + params + ['--'] + grep_paths + [':(exclude)*.min.js'], env=self.build_env()) - @Command('wpt-upgrade', + @Command('wptrunner-upgrade', description='upgrade wptrunner.', category='devenv') def upgrade_wpt_runner(self): + env = self.build_env() with cd(path.join(self.context.topdir, 'tests', 'wpt', 'harness')): - code = call(["git", "init"], env=self.build_env()) + code = call(["git", "init"], env=env) if code: return code + # No need to report an error if this fails, as it will for the first use + call(["git", "remote", "rm", "upstream"], env=env) code = call( - ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=self.build_env()) + ["git", "remote", "add", "upstream", "https://github.com/w3c/wptrunner.git"], env=env) if code: return code - code = call(["git", "fetch", "upstream"], env=self.build_env()) + code = call(["git", "fetch", "upstream"], env=env) if code: return code - code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=self.build_env()) + code = call(["git", "reset", "--hard", "remotes/upstream/master"], env=env) if code: return code - code = call(["rm", "-rf", ".git"], env=self.build_env()) + code = call(["rm", "-rf", ".git"], env=env) if code: return code return 0 diff --git a/tests/wpt/harness/.travis.yml b/tests/wpt/harness/.travis.yml new file mode 100644 index 000000000000..b5ec736a3844 --- /dev/null +++ b/tests/wpt/harness/.travis.yml @@ -0,0 +1,26 @@ +language: python +python: 2.7 + +sudo: false + +cache: + directories: + - $HOME/.cache/pip + +env: + - TOXENV=py27-base + - TOXENV=py27-b2g + - TOXENV=py27-chrome + - TOXENV=py27-firefox + - TOXENV=py27-servo + - TOXENV=pypy-base + - TOXENV=pypy-b2g + - TOXENV=pypy-chrome + - TOXENV=pypy-firefox + - TOXENV=pypy-servo + +install: + - pip install -U tox + +script: + - tox diff --git a/tests/wpt/harness/tox.ini b/tests/wpt/harness/tox.ini new file mode 100644 index 000000000000..30a28e3cddb4 --- /dev/null +++ b/tests/wpt/harness/tox.ini @@ -0,0 +1,16 @@ +[pytest] +xfail_strict=true + +[tox] +envlist = {py27,pypy}-{base,b2g,chrome,firefox,servo} + +[testenv] +deps = + pytest>=2.9 + -r{toxinidir}/requirements.txt + b2g: -r{toxinidir}/requirements_b2g.txt + chrome: -r{toxinidir}/requirements_chrome.txt + firefox: -r{toxinidir}/requirements_firefox.txt + servo: -r{toxinidir}/requirements_servo.txt + +commands = py.test [] diff --git a/tests/wpt/harness/wptrunner/browsers/firefox.py b/tests/wpt/harness/wptrunner/browsers/firefox.py index dcd7a4f1d17d..a3c6f3f5753f 100644 --- a/tests/wpt/harness/wptrunner/browsers/firefox.py +++ b/tests/wpt/harness/wptrunner/browsers/firefox.py @@ -3,6 +3,7 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. import os +import platform import subprocess import sys @@ -128,10 +129,16 @@ def start(self): self.profile.set_preferences({"marionette.defaultPrefs.enabled": True, "marionette.defaultPrefs.port": self.marionette_port, "dom.disable_open_during_load": False, - "network.dns.localDomains": ",".join(hostnames)}) + "network.dns.localDomains": ",".join(hostnames), + "places.history.enabled": False}) if self.e10s: self.profile.set_preferences({"browser.tabs.remote.autostart": True}) + # Bug 1262954: winxp + e10s, disable hwaccel + if (self.e10s and platform.system() in ("Windows", "Microsoft") and + '5.1' in platform.version()): + self.profile.set_preferences({"layers.acceleration.disabled": True}) + if self.ca_certificate_path is not None: self.setup_ssl() diff --git a/tests/wpt/harness/wptrunner/browsers/servo.py b/tests/wpt/harness/wptrunner/browsers/servo.py index bc90cefcfc9e..4e4bc20128c4 100644 --- a/tests/wpt/harness/wptrunner/browsers/servo.py +++ b/tests/wpt/harness/wptrunner/browsers/servo.py @@ -6,7 +6,7 @@ from .base import NullBrowser, ExecutorBrowser, require_arg from ..executors import executor_kwargs as base_executor_kwargs -from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor +from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor, ServoWdspecExecutor here = os.path.join(os.path.split(__file__)[0]) @@ -14,7 +14,8 @@ "check_args": "check_args", "browser": "ServoBrowser", "executor": {"testharness": "ServoTestharnessExecutor", - "reftest": "ServoRefTestExecutor"}, + "reftest": "ServoRefTestExecutor", + "wdspec": "ServoWdspecExecutor"}, "browser_kwargs": "browser_kwargs", "executor_kwargs": "executor_kwargs", "env_options": "env_options", diff --git a/tests/wpt/harness/wptrunner/browsers/webdriver.py b/tests/wpt/harness/wptrunner/browsers/webdriver.py deleted file mode 100644 index 54b1c1b5ef93..000000000000 --- a/tests/wpt/harness/wptrunner/browsers/webdriver.py +++ /dev/null @@ -1,137 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. - -import errno -import socket -import time -import traceback -import urlparse - -import mozprocess - -from .base import get_free_port, cmd_arg - - -__all__ = ["SeleniumLocalServer", "ChromedriverLocalServer"] - - -class LocalServer(object): - used_ports = set() - default_endpoint = "/" - - def __init__(self, logger, binary, port=None, endpoint=None): - self.logger = logger - self.binary = binary - self.port = port - self.endpoint = endpoint or self.default_endpoint - - if self.port is None: - self.port = get_free_port(4444, exclude=self.used_ports) - self.used_ports.add(self.port) - self.url = "http://127.0.0.1:%i%s" % (self.port, self.endpoint) - - self.proc, self.cmd = None, None - - def start(self): - self.proc = mozprocess.ProcessHandler( - self.cmd, processOutputLine=self.on_output) - try: - self.proc.run() - except OSError as e: - if e.errno == errno.ENOENT: - raise IOError( - "chromedriver executable not found: %s" % self.binary) - raise - - self.logger.debug( - "Waiting for server to become accessible: %s" % self.url) - surl = urlparse.urlparse(self.url) - addr = (surl.hostname, surl.port) - try: - wait_service(addr) - except: - self.logger.error( - "Server was not accessible within the timeout:\n%s" % traceback.format_exc()) - raise - else: - self.logger.info("Server listening on port %i" % self.port) - - def stop(self): - if hasattr(self.proc, "proc"): - self.proc.kill() - - def is_alive(self): - if hasattr(self.proc, "proc"): - exitcode = self.proc.poll() - return exitcode is None - return False - - def on_output(self, line): - self.logger.process_output(self.pid, - line.decode("utf8", "replace"), - command=" ".join(self.cmd)) - - @property - def pid(self): - if hasattr(self.proc, "proc"): - return self.proc.pid - - -class SeleniumLocalServer(LocalServer): - default_endpoint = "/wd/hub" - - def __init__(self, logger, binary, port=None): - LocalServer.__init__(self, logger, binary, port=port) - self.cmd = ["java", - "-jar", self.binary, - "-port", str(self.port)] - - def start(self): - self.logger.debug("Starting local Selenium server") - LocalServer.start(self) - - def stop(self): - LocalServer.stop(self) - self.logger.info("Selenium server stopped listening") - - -class ChromedriverLocalServer(LocalServer): - default_endpoint = "/wd/hub" - - def __init__(self, logger, binary="chromedriver", port=None, endpoint=None): - LocalServer.__init__(self, logger, binary, port=port, endpoint=endpoint) - # TODO: verbose logging - self.cmd = [self.binary, - cmd_arg("port", str(self.port)) if self.port else "", - cmd_arg("url-base", self.endpoint) if self.endpoint else ""] - - def start(self): - self.logger.debug("Starting local chromedriver server") - LocalServer.start(self) - - def stop(self): - LocalServer.stop(self) - self.logger.info("chromedriver server stopped listening") - - -def wait_service(addr, timeout=15): - """Waits until network service given as a tuple of (host, port) becomes - available or the `timeout` duration is reached, at which point - ``socket.error`` is raised.""" - end = time.time() + timeout - while end > time.time(): - so = socket.socket() - try: - so.connect(addr) - except socket.timeout: - pass - except socket.error as e: - if e[0] != errno.ECONNREFUSED: - raise - else: - return True - finally: - so.close() - time.sleep(0.5) - raise socket.error("Service is unavailable: %s:%i" % addr) diff --git a/tests/wpt/harness/wptrunner/executors/executormarionette.py b/tests/wpt/harness/wptrunner/executors/executormarionette.py index c4b1cba689af..50200d74677f 100644 --- a/tests/wpt/harness/wptrunner/executors/executormarionette.py +++ b/tests/wpt/harness/wptrunner/executors/executormarionette.py @@ -292,7 +292,7 @@ def is_alive(self): class ExecuteAsyncScriptRun(object): def __init__(self, logger, func, marionette, url, timeout): self.logger = logger - self.result = None + self.result = (None, None) self.marionette = marionette self.func = func self.url = url @@ -323,11 +323,9 @@ def run(self): wait_timeout = None flag = self.result_flag.wait(wait_timeout) - if self.result is None: + if self.result[1] is None: self.logger.debug("Timed out waiting for a result") - assert not flag self.result = False, ("EXTERNAL-TIMEOUT", None) - return self.result def _run(self): @@ -409,7 +407,8 @@ def do_testharness(self, marionette, url, timeout): "timeout": timeout_ms, "explicit_timeout": timeout is None} - return marionette.execute_async_script(script, new_sandbox=False) + rv = marionette.execute_async_script(script, new_sandbox=False) + return rv class MarionetteRefTestExecutor(RefTestExecutor): @@ -487,7 +486,7 @@ def _screenshot(self, marionette, url, timeout): class WdspecRun(object): def __init__(self, func, session, path, timeout): self.func = func - self.result = None + self.result = (None, None) self.session = session self.path = path self.timeout = timeout @@ -504,8 +503,7 @@ def run(self): executor.start() flag = self.result_flag.wait(self.timeout) - if self.result is None: - assert not flag + if self.result[1] is None: self.result = False, ("EXTERNAL-TIMEOUT", None) return self.result diff --git a/tests/wpt/harness/wptrunner/executors/executorservo.py b/tests/wpt/harness/wptrunner/executors/executorservo.py index 7f5e90673197..2d9c8c1e865f 100644 --- a/tests/wpt/harness/wptrunner/executors/executorservo.py +++ b/tests/wpt/harness/wptrunner/executors/executorservo.py @@ -4,11 +4,13 @@ import base64 import hashlib +import httplib import json import os import subprocess import tempfile import threading +import traceback import urlparse import uuid from collections import defaultdict @@ -19,15 +21,23 @@ Protocol, RefTestImplementation, testharness_result_converter, - reftest_result_converter) + reftest_result_converter, + WdspecExecutor) from .process import ProcessTestExecutor from ..browsers.base import browser_command -render_arg = None +from ..wpttest import WdspecResult, WdspecSubtestResult +from ..webdriver_server import ServoDriverServer +from .executormarionette import WdspecRun +from . import pytestrunner +render_arg = None +webdriver = None +extra_timeout = 5 # seconds def do_delayed_imports(): - global render_arg + global render_arg, webdriver from ..browsers.servo import render_arg + import webdriver hosts_text = """127.0.0.1 web-platform.test 127.0.0.1 www.web-platform.test @@ -214,10 +224,7 @@ def screenshot(self, test, viewport_size, dpi): for pref in test.environment.get('prefs', {}): command += ["--pref", pref] - if viewport_size: - command += ["--resolution", viewport_size] - else: - command += ["--resolution", "800x600"] + command += ["--resolution", viewport_size or "800x600"] if dpi: command += ["--device-pixel-ratio", dpi] @@ -275,3 +282,77 @@ def on_output(self, line): self.logger.process_output(self.proc.pid, line, " ".join(self.command)) + +class ServoWdspecProtocol(Protocol): + def __init__(self, executor, browser): + do_delayed_imports() + Protocol.__init__(self, executor, browser) + self.session = None + self.server = None + + def setup(self, runner): + try: + self.server = ServoDriverServer(self.logger, binary=self.browser.binary, binary_args=self.browser.binary_args, render_backend=self.browser.render_backend) + self.server.start(block=False) + self.logger.info( + "WebDriver HTTP server listening at %s" % self.server.url) + + self.logger.info( + "Establishing new WebDriver session with %s" % self.server.url) + self.session = webdriver.Session( + self.server.host, self.server.port, self.server.base_path) + except Exception: + self.logger.error(traceback.format_exc()) + self.executor.runner.send_message("init_failed") + else: + self.executor.runner.send_message("init_succeeded") + + def teardown(self): + if self.server is not None: + try: + if self.session.session_id is not None: + self.session.end() + except Exception: + pass + if self.server.is_alive: + self.server.stop() + + @property + def is_alive(self): + conn = httplib.HTTPConnection(self.server.host, self.server.port) + conn.request("HEAD", self.server.base_path + "invalid") + res = conn.getresponse() + return res.status == 404 + +class ServoWdspecExecutor(WdspecExecutor): + def __init__(self, browser, server_config, + timeout_multiplier=1, close_after_done=True, debug_info=None, + **kwargs): + WdspecExecutor.__init__(self, browser, server_config, + timeout_multiplier=timeout_multiplier, + debug_info=debug_info) + self.protocol = ServoWdspecProtocol(self, browser) + + def is_alive(self): + return self.protocol.is_alive + + def on_environment_change(self, new_environment): + pass + + def do_test(self, test): + timeout = test.timeout * self.timeout_multiplier + extra_timeout + + success, data = WdspecRun(self.do_wdspec, + self.protocol.session, + test.path, + timeout).run() + + if success: + return self.convert_result(test, data) + + return (test.result_cls(*data), []) + + def do_wdspec(self, session, path, timeout): + harness_result = ("OK", None) + subtest_results = pytestrunner.run(path, session, timeout=timeout) + return (harness_result, subtest_results) diff --git a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py index 77afb4a36842..b2b9913bf487 100644 --- a/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py +++ b/tests/wpt/harness/wptrunner/executors/pytestrunner/fixtures.py @@ -2,8 +2,12 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this file, # You can obtain one at http://mozilla.org/MPL/2.0/. -import pytest +from tools import pytest +@pytest.fixture +def webdriver(): + import webdriver + return webdriver """pytest fixtures for use in Python-based WPT tests. @@ -17,7 +21,7 @@ class Session(object): in tests. The session is not created by default to enable testing of session - creation. However, a module-scoped session will be implicitly created + creation. However, a function-scoped session will be implicitly created at the first call to a WebDriver command. This means methods such as `session.send_command` and `session.session_id` are possible to use without having a session. @@ -45,14 +49,62 @@ def setup(request, session): def test_something(setup, session): assert session.url == "https://example.org" - The session is closed when the test module goes out of scope by an - implicit call to `session.end`. + When the test function goes out of scope, any remaining user prompts + and opened windows are closed, and the current browsing context is + switched back to the top-level browsing context. """ def __init__(self, client): self.client = client - @pytest.fixture(scope="module") + @pytest.fixture(scope="function") def session(self, request): - request.addfinalizer(self.client.end) + # finalisers are popped off a stack, + # making their ordering reverse + request.addfinalizer(self.switch_to_top_level_browsing_context) + request.addfinalizer(self.restore_windows) + request.addfinalizer(self.dismiss_user_prompts) + return self.client + + def dismiss_user_prompts(self): + """Dismisses any open user prompts in windows.""" + current_window = self.client.window_handle + + for window in self.windows(): + self.client.window_handle = window + try: + self.client.alert.dismiss() + except webdriver.NoSuchAlertException: + pass + + self.client.window_handle = current_window + + def restore_windows(self): + """Closes superfluous windows opened by the test without ending + the session implicitly by closing the last window. + """ + current_window = self.client.window_handle + + for window in self.windows(exclude=[current_window]): + self.client.window_handle = window + if len(self.client.window_handles) > 1: + self.client.close() + + self.client.window_handle = current_window + + def switch_to_top_level_browsing_context(self): + """If the current browsing context selected by WebDriver is a + `` or an `