ENH: Test full output and coverage (#230)

* ENH: Test full output and coverage

* FIX: Add new files to setup.py

* ENH: dvipng

* FIX: Cleaner test

* ENH: Update

* BUG: Remove StringIO

* STY: Better naming and explanations

Author: larsoner

.coveragerc

@@ -0,0 +1,6 @@
+[run]
+branch = True
+source = numpydoc
+include = */numpydoc/*
+omit =
+    */setup.py

.gitignore

@@ -13,3 +13,6 @@ doc/_build
 build
 dist
 doc/_build
+numpydoc/tests/tinybuild/_build
+numpydoc/tests/tinybuild/generated
+MANIFEST

.travis.yml

@@ -13,16 +13,20 @@ cache:
   directories:
     - $HOME/.cache/pip
 before_install:
-  - sudo apt-get install texlive texlive-latex-extra latexmk
+  - sudo apt-get install texlive texlive-latex-extra latexmk dvipng
   - pip install --upgrade pip setuptools  # Upgrade pip and setuptools to get ones with `wheel` support
-  - pip install pytest pytest-cov numpy matplotlib sphinx${SPHINX_SPEC}
+  - pip install pytest pytest-cov numpy matplotlib sphinx${SPHINX_SPEC} codecov
 script:
   - |
     python setup.py sdist
     cd dist
     pip install numpydoc* -v
-  - pytest --pyargs numpydoc
+  - pytest -v --pyargs numpydoc
   - |
     cd ../doc
     make SPHINXOPTS=$SPHINXOPTS html
     make SPHINXOPTS=$SPHINXOPTS latexpdf
+after_script:
+  - |
+    cd ../dist
+    codecov

codecov.yml

@@ -0,0 +1,14 @@
+coverage:
+  precision: 2
+  round: down
+  range: "70...100"
+  status:
+    project:
+      default:
+        target: auto
+        threshold: 0.01
+    patch: false
+    changes: false
+comment:
+  layout: "header, diff, sunburst, uncovered"
+  behavior: default

numpydoc/tests/test_full.py

@@ -0,0 +1,67 @@
+import os.path as op
+import shutil
+
+import pytest
+from sphinx.application import Sphinx
+from sphinx.util.docutils import docutils_namespace
+
+
+# Test framework adapted from sphinx-gallery (BSD 3-clause)
+@pytest.fixture(scope='module')
+def sphinx_app(tmpdir_factory):
+    temp_dir = (tmpdir_factory.getbasetemp() / 'root').strpath
+    src_dir = op.join(op.dirname(__file__), 'tinybuild')
+
+    def ignore(src, names):
+        return ('_build', 'generated')
+
+    shutil.copytree(src_dir, temp_dir, ignore=ignore)
+    # For testing iteration, you can get similar behavior just doing `make`
+    # inside the tinybuild directory
+    src_dir = temp_dir
+    conf_dir = temp_dir
+    out_dir = op.join(temp_dir, '_build', 'html')
+    toctrees_dir = op.join(temp_dir, '_build', 'toctrees')
+    # Avoid warnings about re-registration, see:
+    # https://github.com/sphinx-doc/sphinx/issues/5038
+    with docutils_namespace():
+        app = Sphinx(src_dir, conf_dir, out_dir, toctrees_dir,
+                     buildername='html')
+        # need to build within the context manager
+        # for automodule and backrefs to work
+        app.build(False, [])
+    return app
+
+
+def test_MyClass(sphinx_app):
+    """Test that class documentation is reasonable."""
+    src_dir, out_dir = sphinx_app.srcdir, sphinx_app.outdir
+    class_rst = op.join(src_dir, 'generated',
+                        'numpydoc_test_module.MyClass.rst')
+    with open(class_rst, 'r') as fid:
+        rst = fid.read()
+    assert r'numpydoc\_test\_module' in rst  # properly escaped
+    class_html = op.join(out_dir, 'generated',
+                         'numpydoc_test_module.MyClass.html')
+    with open(class_html, 'r') as fid:
+        html = fid.read()
+    # escaped * chars should no longer be preceded by \'s,
+    # if we see a \* in the output we know it's incorrect:
+    assert r'\*' in html  # XXX should be "not in", bug!
+    # "self" should not be in the parameter list for the class:
+    assert 'self,' in html   # XXX should be "not in", bug!
+    # check xref was embedded properly (dict should link using xref):
+    assert 'stdtypes.html#dict' in html
+
+
+def test_my_function(sphinx_app):
+    """Test that a timings page is created."""
+    out_dir = sphinx_app.outdir
+    function_html = op.join(out_dir, 'generated',
+                            'numpydoc_test_module.my_function.html')
+    with open(function_html, 'r') as fid:
+        html = fid.read()
+    assert r'\*args' not in html
+    assert '*args' in html
+    # check xref (iterable should link using xref):
+    assert 'glossary.html#term-iterable' in html

numpydoc/tests/tinybuild/Makefile

@@ -0,0 +1,11 @@
+all: clean html show
+
+clean:
+	rm -rf _build/*
+	rm -rf generated/
+
+html:
+	sphinx-build -b html -d _build/doctrees . _build/html
+
+show:
+	@python -c "import webbrowser; webbrowser.open_new_tab('file://$(PWD)/_build/html/index.html')"

numpydoc/tests/tinybuild/conf.py

@@ -0,0 +1,22 @@
+import os
+import sys
+path = os.path.dirname(__file__)
+if path not in sys.path:
+    sys.path.insert(0, path)
+import numpydoc_test_module  # noqa
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.intersphinx',
+    'numpydoc',
+]
+project = 'numpydoc_test_module'
+autosummary_generate = True
+source_suffix = '.rst'
+master_doc = 'index'
+exclude_patterns = ['_build']
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3', None),
+}
+nitpicky = True
+highlight_language = 'python3'
+numpydoc_xref_param_type = True

numpydoc/tests/tinybuild/index.rst

@@ -0,0 +1,4 @@
+numpydoc_test_module
+====================
+
+.. automodule:: numpydoc_test_module

numpydoc/tests/tinybuild/numpydoc_test_module.py

@@ -0,0 +1,51 @@
+"""Numpydoc test module.
+
+.. currentmodule:: numpydoc_test_module
+
+.. autosummary::
+   :toctree: generated/
+
+   MyClass
+   my_function
+"""
+
+__all__ = ['MyClass', 'my_function']
+
+
+class MyClass(object):
+    """A class.
+
+    Parameters
+    ----------
+    *args : iterable
+        Arguments.
+    **kwargs : dict
+        Keyword arguments.
+    """
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+
+def my_function(*args, **kwargs):
+    """Return None.
+
+    See [1]_.
+
+    Parameters
+    ----------
+    *args : iterable
+        Arguments.
+    **kwargs : dict
+        Keyword arguments.
+
+    Returns
+    -------
+    out : None
+        The output.
+
+    References
+    ----------
+    .. [1] https://numpydoc.readthedocs.io
+    """
+    return None

setup.cfg

@@ -3,5 +3,5 @@ addopts =
     --showlocals --doctest-modules -ra --cov-report= --cov=numpydoc
     --junit-xml=junit-results.xml --ignore=doc/conf.py
 filterwarnings =
+    ignore:'U' mode is deprecated:DeprecationWarning
     ignore:Using or importing the ABCs.*:DeprecationWarning
-

setup.py

@@ -48,7 +48,12 @@ def read(fname):
     url="https://numpydoc.readthedocs.io",
     license="BSD",
     install_requires=["sphinx >= 1.6.5", 'Jinja2>=2.3'],
-    package_data={'numpydoc': ['tests/test_*.py', 'templates/*.rst']},
-    test_suite = 'nose.collector',
+    package_data={'numpydoc': [
+        'tests/test_*.py',
+        'tests/tinybuild/Makefile',
+        'tests/tinybuild/index.rst',
+        'tests/tinybuild/*.py',
+        'templates/*.rst',
+        ]},
     cmdclass={"sdist": sdist},
 )