|
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +# or more contributor license agreements. See the NOTICE file |
| 3 | +# distributed with this work for additional information |
| 4 | +# regarding copyright ownership. The ASF licenses this file |
| 5 | +# to you under the Apache License, Version 2.0 (the |
| 6 | +# "License"); you may not use this file except in compliance |
| 7 | +# with the License. You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, |
| 12 | +# software distributed under the License is distributed on an |
| 13 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +# KIND, either express or implied. See the License for the |
| 15 | +# specific language governing permissions and limitations |
| 16 | +# under the License. |
| 17 | +"""conftest.py contains configuration for pytest. |
| 18 | +
|
| 19 | +Configuration file for tests in tests/ and scripts/ folders. |
| 20 | +
|
| 21 | +Note that fixtures of higher-scoped fixtures (such as ``session``) are |
| 22 | +instantiated before lower-scoped fixtures (such as ``function``). |
| 23 | +
|
| 24 | +""" |
| 25 | + |
| 26 | +import logging |
| 27 | +import os |
| 28 | +import random |
| 29 | + |
| 30 | +import numpy as np |
| 31 | +import mxnet as mx |
| 32 | +import gluonnlp |
| 33 | +import pytest |
| 34 | + |
| 35 | + |
| 36 | +def pytest_sessionfinish(session, exitstatus): |
| 37 | + if exitstatus == 5: # Don't fail if no tests were run |
| 38 | + session.exitstatus = 0 |
| 39 | + |
| 40 | + |
| 41 | +# * Random seed setup |
| 42 | +def pytest_configure(): |
| 43 | + """Pytest configuration hook to help reproduce test segfaults |
| 44 | +
|
| 45 | + Sets and outputs rng seeds. |
| 46 | +
|
| 47 | + The segfault-debug procedure on a module called test_module.py is: |
| 48 | +
|
| 49 | + 1. run "pytest --verbose test_module.py". A seg-faulting output might be: |
| 50 | +
|
| 51 | + [INFO] np, mx and python random seeds = 4018804151 |
| 52 | + test_module.test1 ... ok |
| 53 | + test_module.test2 ... Illegal instruction (core dumped) |
| 54 | +
|
| 55 | + 2. Copy the module-starting seed into the next command, then run: |
| 56 | +
|
| 57 | + MXNET_MODULE_SEED=4018804151 pytest --log-level=DEBUG --verbose test_module.py |
| 58 | +
|
| 59 | + Output might be: |
| 60 | +
|
| 61 | + [WARNING] **** module-level seed is set: all tests running deterministically **** |
| 62 | + [INFO] np, mx and python random seeds = 4018804151 |
| 63 | + test_module.test1 ... [DEBUG] np and mx random seeds = 3935862516 |
| 64 | + ok |
| 65 | + test_module.test2 ... [DEBUG] np and mx random seeds = 1435005594 |
| 66 | + Illegal instruction (core dumped) |
| 67 | +
|
| 68 | + 3. Copy the segfaulting-test seed into the command: |
| 69 | + MXNET_TEST_SEED=1435005594 pytest --log-level=DEBUG --verbose test_module.py:test2 |
| 70 | + Output might be: |
| 71 | +
|
| 72 | + [INFO] np, mx and python random seeds = 2481884723 |
| 73 | + test_module.test2 ... [DEBUG] np and mx random seeds = 1435005594 |
| 74 | + Illegal instruction (core dumped) |
| 75 | +
|
| 76 | + 3. Finally reproduce the segfault directly under gdb (might need additional os packages) |
| 77 | + by editing the bottom of test_module.py to be |
| 78 | +
|
| 79 | + if __name__ == '__main__': |
| 80 | + logging.getLogger().setLevel(logging.DEBUG) |
| 81 | + test2() |
| 82 | +
|
| 83 | + MXNET_TEST_SEED=1435005594 gdb -ex r --args python test_module.py |
| 84 | +
|
| 85 | + 4. When finished debugging the segfault, remember to unset any exported MXNET_ seed |
| 86 | + variables in the environment to return to non-deterministic testing (a good thing). |
| 87 | + """ |
| 88 | + |
| 89 | + module_seed_str = os.getenv('MXNET_MODULE_SEED') |
| 90 | + if module_seed_str is None: |
| 91 | + seed = np.random.randint(0, np.iinfo(np.int32).max) |
| 92 | + else: |
| 93 | + seed = int(module_seed_str) |
| 94 | + logging.warning('*** module-level seed is set: ' |
| 95 | + 'all tests running deterministically ***') |
| 96 | + print('Setting module np/mx/python random seeds, ' |
| 97 | + 'use MXNET_MODULE_SEED={} to reproduce.'.format(seed)) |
| 98 | + |
| 99 | + np.random.seed(seed) |
| 100 | + mx.npx.random.seed(seed) |
| 101 | + random.seed(seed) |
| 102 | + |
| 103 | + # The MXNET_TEST_SEED environment variable will override MXNET_MODULE_SEED for tests with |
| 104 | + # the 'with_seed()' decoration. Inform the user of this once here at the module level. |
| 105 | + if os.getenv('MXNET_TEST_SEED') is not None: |
| 106 | + logging.warning('*** test-level seed set: all "@with_seed()" ' |
| 107 | + 'tests run deterministically ***') |
| 108 | + |
| 109 | + |
| 110 | +@pytest.hookimpl(tryfirst=True, hookwrapper=True) |
| 111 | +def pytest_runtest_makereport(item, call): |
| 112 | + """Make test outcome available to fixture. |
| 113 | +
|
| 114 | + https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures |
| 115 | + """ |
| 116 | + # execute all other hooks to obtain the report object |
| 117 | + outcome = yield |
| 118 | + rep = outcome.get_result() |
| 119 | + |
| 120 | + # set a report attribute for each phase of a call, which can |
| 121 | + # be "setup", "call", "teardown" |
| 122 | + setattr(item, "rep_" + rep.when, rep) |
| 123 | + |
| 124 | + |
| 125 | +@pytest.fixture(scope='function', autouse=True) |
| 126 | +def function_scope_seed(request): |
| 127 | + """A function scope fixture that manages rng seeds. |
| 128 | +
|
| 129 | + This fixture automatically initializes the python, numpy and mxnet random |
| 130 | + number generators randomly on every test run. |
| 131 | +
|
| 132 | + def test_ok_with_random_data(): |
| 133 | + ... |
| 134 | +
|
| 135 | + To fix the seed used for a test case mark the test function with the |
| 136 | + desired seed: |
| 137 | +
|
| 138 | + @pytest.mark.seed(1) |
| 139 | + def test_not_ok_with_random_data(): |
| 140 | + '''This testcase actually works.''' |
| 141 | + assert 17 == random.randint(0, 100) |
| 142 | +
|
| 143 | + When a test fails, the fixture outputs the seed used. The user can then set |
| 144 | + the environment variable MXNET_TEST_SEED to the value reported, then rerun |
| 145 | + the test with: |
| 146 | +
|
| 147 | + pytest --verbose -s <test_module_name.py> -k <failing_test> |
| 148 | +
|
| 149 | + To run a test repeatedly, install pytest-repeat and add the --count argument: |
| 150 | +
|
| 151 | + pip install pytest-repeat |
| 152 | + pytest --verbose -s <test_module_name.py> -k <failing_test> --count 1000 |
| 153 | +
|
| 154 | + """ |
| 155 | + |
| 156 | + seed = request.node.get_closest_marker('seed') |
| 157 | + env_seed_str = os.getenv('MXNET_TEST_SEED') |
| 158 | + |
| 159 | + if seed is not None: |
| 160 | + seed = seed.args[0] |
| 161 | + assert isinstance(seed, int) |
| 162 | + elif env_seed_str is not None: |
| 163 | + seed = int(env_seed_str) |
| 164 | + else: |
| 165 | + seed = np.random.randint(0, np.iinfo(np.int32).max) |
| 166 | + |
| 167 | + post_test_state = np.random.get_state() |
| 168 | + np.random.seed(seed) |
| 169 | + mx.random.seed(seed) |
| 170 | + random.seed(seed) |
| 171 | + |
| 172 | + seed_message = ('np/mx/python random seeds are set to ' |
| 173 | + '{}, use MXNET_TEST_SEED={} to reproduce.') |
| 174 | + seed_message = seed_message.format(seed, seed) |
| 175 | + |
| 176 | + # Always log seed on DEBUG log level. This makes sure we can find out the |
| 177 | + # value of the seed even if the test case causes a segfault and subsequent |
| 178 | + # teardown code is not run. |
| 179 | + logging.debug(seed_message) |
| 180 | + |
| 181 | + yield # run the test |
| 182 | + |
| 183 | + if request.node.rep_setup.failed: |
| 184 | + logging.info("Setting up a test failed: {}", request.node.nodeid) |
| 185 | + elif request.node.rep_call.outcome == 'failed': |
| 186 | + # Either request.node.rep_setup.failed or request.node.rep_setup.passed |
| 187 | + # should be True |
| 188 | + assert request.node.rep_setup.passed |
| 189 | + # On failure also log seed on INFO log level |
| 190 | + logging.info(seed_message) |
| 191 | + |
| 192 | + np.random.set_state(post_test_state) |
| 193 | + |
| 194 | + |
| 195 | +# * Shared test fixtures |
| 196 | +@pytest.fixture(params=[True, False]) |
| 197 | +def hybridize(request): |
| 198 | + return request.param |
| 199 | + |
| 200 | + |
| 201 | +@pytest.fixture(autouse=True) |
| 202 | +def doctest(doctest_namespace): |
| 203 | + doctest_namespace['np'] = np |
| 204 | + doctest_namespace['gluonnlp'] = gluonnlp |
| 205 | + doctest_namespace['mx'] = mx |
| 206 | + doctest_namespace['gluon'] = mx.gluon |
| 207 | + import doctest |
| 208 | + doctest.ELLIPSIS_MARKER = '-etc-' |
0 commit comments