diff --git a/.travis.yml b/.travis.yml index 131c729c4f1..5d91a7cd0e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -45,8 +45,13 @@ env: - NPROC=2 - TEST_ARGS=--no-pep8 - NOSE_ARGS="--processes=$NPROC --process-timeout=300" + - PYTEST_ARGS="-ra --timeout=300 --durations=25 --cov-report= --cov=lib" # -n $NPROC - PYTHON_ARGS= - DELETE_FONT_CACHE= + - USE_PYTEST=false + #- PYTHONHASHSEED=0 # Workaround for pytest-xdist flaky colletion order + # # https://github.com/pytest-dev/pytest/issues/920 + # # https://github.com/pytest-dev/pytest/issues/1075 matrix: include: @@ -60,6 +65,8 @@ matrix: env: TEST_ARGS=--pep8 - python: 3.5 env: BUILD_DOCS=true + - python: 3.5 + env: USE_PYTEST=true PANDAS=pandas DELETE_FONT_CACHE=1 TEST_ARGS= - python: "nightly" env: PRE=--pre - os: osx @@ -107,10 +114,14 @@ install: # Install dependencies from pypi pip install $PRE python-dateutil $NUMPY pyparsing!=2.1.6 $PANDAS pep8 cycler coveralls coverage pip install $PRE pillow sphinx!=1.3.0 $MOCK numpydoc ipython colorspacious + # Install nose from a build which has partial # support for python36 and suport for coverage output suppressing pip install git+https://github.com/jenshnielsen/nose.git@matplotlibnose + # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124 + pip install $PRE pytest 'pytest-cov>=2.3.1' pytest-timeout pytest-xdist pytest-faulthandler + # We manually install humor sans using the package from Ubuntu 14.10. Unfortunatly humor sans is not # availible in the Ubuntu version used by Travis but we can manually install the deb from a later # version since is it basically just a .ttf file @@ -147,16 +158,21 @@ script: - | echo Testing import of tkagg backend MPLBACKEND="tkagg" python -c 'import matplotlib.pyplot as plt; print(plt.get_backend())' - echo The following args are passed to nose $NOSE_ARGS if [[ $BUILD_DOCS == false ]]; then if [[ $DELETE_FONT_CACHE == 1 ]]; then rm -rf ~/.cache/matplotlib fi export MPL_REPO_DIR=$PWD # needed for pep8-conformance test of the examples - if [[ $TRAVIS_OS_NAME == 'osx' ]]; then - python tests.py $NOSE_ARGS $TEST_ARGS + if [[ $USE_PYTEST == false ]]; then + echo The following args are passed to nose $NOSE_ARGS + if [[ $TRAVIS_OS_NAME == 'osx' ]]; then + python tests.py $NOSE_ARGS $TEST_ARGS + else + gdb -return-child-result -batch -ex r -ex bt --args python $PYTHON_ARGS tests.py $NOSE_ARGS $TEST_ARGS + fi else - gdb -return-child-result -batch -ex r -ex bt --args python $PYTHON_ARGS tests.py $NOSE_ARGS $TEST_ARGS + echo The following args are passed to pytest $PYTEST_ARGS + py.test $PYTEST_ARGS $TEST_ARGS fi else cd doc @@ -171,6 +187,9 @@ script: pip install $PRE requests==2.9.2 linkchecker linkchecker build/html/index.html fi + # Currently disabled because of differece in behaviour + # between `pytest-cov` and `nose-coverage` + #- if [[ $USE_PYTEST == true ]]; then coveralls; fi - rm -rf $HOME/.cache/matplotlib/tex.cache - rm -rf $HOME/.cache/matplotlib/test_cache diff --git a/appveyor.yml b/appveyor.yml index 58d343641fb..e8d0d6c426f 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,6 +14,12 @@ environment: CMD_IN_ENV: "cmd /E:ON /V:ON /C obvci_appveyor_python_build_env.cmd" # Workaround for https://github.com/conda/conda-build/issues/636 PYTHONIOENCODING: "UTF-8" + TEST_ARGS: --no-pep8 + PYTEST_ARGS: -ra --timeout=300 --durations=25 #--cov-report= --cov=lib #-n %NUMBER_OF_PROCESSORS% + USE_PYTEST: no + #PYTHONHASHSEED: 0 # Workaround for pytest-xdist flaky colletion order + # # https://github.com/pytest-dev/pytest/issues/920 + # # https://github.com/pytest-dev/pytest/issues/1075 matrix: # for testing purpose: numpy 1.8 on py2.7, for the rest use 1.10/latest @@ -38,6 +44,13 @@ environment: PYTHON_VERSION: "3.5" TEST_ALL: "no" CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64" + - TARGET_ARCH: "x64" + CONDA_PY: "35" + CONDA_NPY: "110" + PYTHON_VERSION: "3.5" + TEST_ALL: "no" + CONDA_INSTALL_LOCN: "C:\\Miniconda35-x64" + USE_PYTEST: yes - TARGET_ARCH: "x86" CONDA_PY: "27" CONDA_NPY: "18" @@ -58,7 +71,7 @@ platform: build: false init: - - cmd: "ECHO %PYTHON_VERSION% %CONDA_INSTALL_LOCN%" + - cmd: "ECHO %PYTHON_VERSION% PYTEST=%USE_PYTEST% %CONDA_INSTALL_LOCN%" install: - cmd: set PATH=%CONDA_INSTALL_LOCN%;%CONDA_INSTALL_LOCN%\scripts;%PATH%; @@ -82,10 +95,15 @@ install: # same things as the requirements in ci/conda_recipe/meta.yaml # if conda-forge gets a new pyqt, it might be nice to install it as well to have more backends # https://github.com/conda-forge/conda-forge.github.io/issues/157#issuecomment-223536381 - - cmd: conda create -q -n test-environment python=%PYTHON_VERSION% pip setuptools numpy python-dateutil freetype=2.6 msinttypes "tk=8.5" pyparsing pytz tornado "libpng>=1.6.21,<1.7" "zlib=1.2" "cycler>=0.10" nose mock + - conda create -q -n test-environment python=%PYTHON_VERSION% + pip setuptools numpy python-dateutil freetype=2.6 msinttypes "tk=8.5" + pyparsing pytz tornado "libpng>=1.6.21,<1.7" "zlib=1.2" "cycler>=0.10" + nose mock sphinx - activate test-environment - cmd: echo %PYTHON_VERSION% %TARGET_ARCH% - cmd: IF %PYTHON_VERSION% == 2.7 conda install -q functools32 + # pytest-cov>=2.3.1 due to https://github.com/pytest-dev/pytest-cov/issues/124 + - if x%USE_PYTEST% == xyes conda install -q pytest "pytest-cov>=2.3.1" pytest-timeout #pytest-xdist # Let the install prefer the static builds of the libs - set LIBRARY_LIB=%CONDA_PREFIX%\Library\lib @@ -124,7 +142,9 @@ test_script: # Test import of tkagg backend - python -c "import matplotlib as m; m.use('tkagg'); import matplotlib.pyplot as plt; print(plt.get_backend())" # tests - - python tests.py + - if x%USE_PYTEST% == xyes echo The following args are passed to pytest %PYTEST_ARGS% + - if x%USE_PYTEST% == xyes py.test %PYTEST_ARGS% %TEST_ARGS% + - if x%USE_PYTEST% == xno python tests.py %TEST_ARGS% # Generate a html for visual tests - python visual_tests.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000000..e8eb4569b0f --- /dev/null +++ b/conftest.py @@ -0,0 +1,112 @@ +from __future__ import (absolute_import, division, print_function, + unicode_literals) + +import inspect +import os +import pytest +import unittest + +import matplotlib +matplotlib.use('agg') + +from matplotlib import default_test_modules +from matplotlib.testing.decorators import ImageComparisonTest + + +IGNORED_TESTS = { + 'matplotlib': [ + 'test_usetex', + ], +} + + +def blacklist_check(path): + """Check if test is blacklisted and should be ignored""" + head, tests_dir = os.path.split(path.dirname) + if tests_dir != 'tests': + return True + head, top_module = os.path.split(head) + return path.purebasename in IGNORED_TESTS.get(top_module, []) + + +def whitelist_check(path): + """Check if test is not whitelisted and should be ignored""" + left = path.dirname + last_left = None + module_path = path.purebasename + while len(left) and left != last_left: + last_left = left + left, tail = os.path.split(left) + module_path = '.'.join([tail, module_path]) + if module_path in default_test_modules: + return False + return True + + +COLLECT_FILTERS = { + 'none': lambda _: False, + 'blacklist': blacklist_check, + 'whitelist': whitelist_check, +} + + +def is_nose_class(cls): + """Check if supplied class looks like Nose testcase""" + return any(name in ['setUp', 'tearDown'] + for name, _ in inspect.getmembers(cls)) + + +def pytest_addoption(parser): + group = parser.getgroup("matplotlib", "matplotlib custom options") + + group.addoption('--collect-filter', action='store', + choices=COLLECT_FILTERS, default='blacklist', + help='filter tests during collection phase') + + group.addoption('--no-pep8', action='store_true', + help='skip PEP8 compliance tests') + + +def pytest_configure(config): + matplotlib._called_from_pytest = True + matplotlib._init_tests() + + if config.getoption('--no-pep8'): + default_test_modules.remove('matplotlib.tests.test_coding_standards') + IGNORED_TESTS['matplotlib'] += 'test_coding_standards' + + +def pytest_unconfigure(config): + matplotlib._called_from_pytest = False + + +def pytest_ignore_collect(path, config): + if path.ext == '.py': + collect_filter = config.getoption('--collect-filter') + return COLLECT_FILTERS[collect_filter](path) + + +def pytest_pycollect_makeitem(collector, name, obj): + if inspect.isclass(obj): + if issubclass(obj, ImageComparisonTest): + # Workaround `image_compare` decorator as it returns class + # instead of function and this confuses pytest because it crawls + # original names and sees 'test_*', but not 'Test*' in that case + return pytest.Class(name, parent=collector) + + if is_nose_class(obj) and not issubclass(obj, unittest.TestCase): + # Workaround unittest-like setup/teardown names in pure classes + setup = getattr(obj, 'setUp', None) + if setup is not None: + obj.setup_method = lambda self, _: obj.setUp(self) + tearDown = getattr(obj, 'tearDown', None) + if tearDown is not None: + obj.teardown_method = lambda self, _: obj.tearDown(self) + setUpClass = getattr(obj, 'setUpClass', None) + if setUpClass is not None: + obj.setup_class = obj.setUpClass + tearDownClass = getattr(obj, 'tearDownClass', None) + if tearDownClass is not None: + obj.teardown_class = obj.tearDownClass + + return pytest.Class(name, parent=collector) diff --git a/lib/matplotlib/testing/__init__.py b/lib/matplotlib/testing/__init__.py index f47171652d1..0b549b36d08 100644 --- a/lib/matplotlib/testing/__init__.py +++ b/lib/matplotlib/testing/__init__.py @@ -5,6 +5,7 @@ import warnings from contextlib import contextmanager +import matplotlib from matplotlib.cbook import is_string_like, iterable from matplotlib import rcParams, rcdefaults, use @@ -14,16 +15,31 @@ def _is_list_like(obj): return not is_string_like(obj) and iterable(obj) +def is_called_from_pytest(): + """Returns whether the call was done from pytest""" + return getattr(matplotlib, '_called_from_pytest', False) + + def xfail(msg=""): """Explicitly fail an currently-executing test with the given message.""" - from .nose import knownfail - knownfail(msg) + __tracebackhide__ = True + if is_called_from_pytest(): + import pytest + pytest.xfail(msg) + else: + from .nose import knownfail + knownfail(msg) def skip(msg=""): """Skip an executing test with the given message.""" - from nose import SkipTest - raise SkipTest(msg) + __tracebackhide__ = True + if is_called_from_pytest(): + import pytest + pytest.skip(msg) + else: + from nose import SkipTest + raise SkipTest(msg) # stolen from pytest diff --git a/lib/matplotlib/testing/decorators.py b/lib/matplotlib/testing/decorators.py index f26c29e5711..22f6fc32950 100644 --- a/lib/matplotlib/testing/decorators.py +++ b/lib/matplotlib/testing/decorators.py @@ -27,7 +27,7 @@ from matplotlib import rcParams from matplotlib.testing.compare import comparable_formats, compare_images, \ make_test_filename -from . import copy_metadata, skip, xfail +from . import copy_metadata, is_called_from_pytest, skip, xfail from .exceptions import ImageComparisonFailure @@ -37,8 +37,12 @@ def skipif(condition, *args, **kwargs): Optionally specify a reason for better reporting. """ - from .nose.decorators import skipif - return skipif(condition, *args, **kwargs) + if is_called_from_pytest(): + import pytest + return pytest.mark.skipif(condition, *args, **kwargs) + else: + from .nose.decorators import skipif + return skipif(condition, *args, **kwargs) def knownfailureif(fail_condition, msg=None, known_exception_class=None): @@ -53,8 +57,14 @@ def knownfailureif(fail_condition, msg=None, known_exception_class=None): if the exception is an instance of this class. (Default = None) """ - from .nose.decorators import knownfailureif - return knownfailureif(fail_condition, msg, known_exception_class) + if is_called_from_pytest(): + import pytest + strict = fail_condition and fail_condition != 'indeterminate' + return pytest.mark.xfail(condition=fail_condition, reason=msg, + raises=known_exception_class, strict=strict) + else: + from .nose.decorators import knownfailureif + return knownfailureif(fail_condition, msg, known_exception_class) def _do_cleanup(original_units_registry, original_settings): @@ -198,7 +208,7 @@ def remove_text(figure): def test(self): baseline_dir, result_dir = _image_directories(self._func) if self._style != 'classic': - xfail('temporarily disabled until 2.0 tag') + skip('temporarily disabled until 2.0 tag') for fignum, baseline in zip(plt.get_fignums(), self._baseline_images): for extension in self._extensions: will_fail = not extension in comparable_formats() @@ -228,7 +238,7 @@ def test(self): @knownfailureif( will_fail, fail_msg, known_exception_class=ImageComparisonFailure) - def do_test(): + def do_test(fignum, actual_fname, expected_fname): figure = plt.figure(fignum) if self._remove_text: @@ -255,7 +265,7 @@ def do_test(): (self._freetype_version, ft2font.__freetype_version__)) raise - yield (do_test,) + yield do_test, fignum, actual_fname, expected_fname def image_comparison(baseline_images=None, extensions=None, tol=0, diff --git a/lib/matplotlib/tests/test_axes.py b/lib/matplotlib/tests/test_axes.py index c56dc621eb1..91fe5a10f81 100644 --- a/lib/matplotlib/tests/test_axes.py +++ b/lib/matplotlib/tests/test_axes.py @@ -22,7 +22,8 @@ import warnings import matplotlib -from matplotlib.testing.decorators import image_comparison, cleanup, skipif +from matplotlib.testing.decorators import image_comparison, cleanup +from matplotlib.testing import skip import matplotlib.pyplot as plt import matplotlib.markers as mmarkers import matplotlib.patches as mpatches @@ -86,11 +87,11 @@ def test_formatter_ticker(): ax.autoscale_view() -@skipif(LooseVersion(np.__version__) >= LooseVersion('1.11.0'), - reason="Fall out from a fixed numpy bug") @image_comparison(baseline_images=["formatter_large_small"]) def test_formatter_large_small(): # github issue #617, pull #619 + if LooseVersion(np.__version__) >= LooseVersion('1.11.0'): + skip("Fall out from a fixed numpy bug") fig, ax = plt.subplots(1) x = [0.500000001, 0.500000002] y = [1e64, 1.1e64] diff --git a/lib/matplotlib/tests/test_basic.py b/lib/matplotlib/tests/test_basic.py index a5f3a33ba78..d1d6cc9b66e 100644 --- a/lib/matplotlib/tests/test_basic.py +++ b/lib/matplotlib/tests/test_basic.py @@ -2,11 +2,11 @@ unicode_literals) import six +import sys from nose.tools import assert_equal from ..testing.decorators import knownfailureif, skipif -from pylab import * SKIPIF_CONDITION = [] @@ -60,6 +60,8 @@ def test(self): def test_override_builtins(): + import pylab + ok_to_override = { '__name__', '__doc__', @@ -79,9 +81,9 @@ def test_override_builtins(): builtins = sys.modules['__builtin__'] overridden = False - for key in globals().keys(): + for key in dir(pylab): if key in dir(builtins): - if (globals()[key] != getattr(builtins, key) and + if (getattr(pylab, key) != getattr(builtins, key) and key not in ok_to_override): print("'%s' was overridden in globals()." % key) overridden = True diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000000..44da53ce982 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +norecursedirs = .git build ci dist extern release tools unit venv