diff --git a/synthtool/gcp/templates/python_library/noxfile.py.j2 b/synthtool/gcp/templates/python_library/noxfile.py.j2 deleted file mode 100644 index e2a9a1791..000000000 --- a/synthtool/gcp/templates/python_library/noxfile.py.j2 +++ /dev/null @@ -1,545 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Generated by synthtool. DO NOT EDIT! - -from __future__ import absolute_import - -import os -import pathlib -import re -import shutil -from typing import Dict, List -import warnings - -import nox - -FLAKE8_VERSION = "flake8==6.1.0" -BLACK_VERSION = "black[jupyter]==23.7.0" -ISORT_VERSION = "isort==5.11.0" -LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] - -DEFAULT_PYTHON_VERSION = "{{ default_python_version }}" - -UNIT_TEST_PYTHON_VERSIONS: List[str] = [{% for v in unit_test_python_versions %}"{{v}}"{% if not loop.last %}, {% endif %}{% endfor %}] -UNIT_TEST_STANDARD_DEPENDENCIES = [ - "mock", - "asyncmock", - "pytest", - "pytest-cov", - "pytest-asyncio", -] -{%- if unit_test_external_dependencies %} -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [{% for v in unit_test_external_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if unit_test_local_dependencies %} -UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [{% for v in unit_test_local_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if unit_test_dependencies %} -UNIT_TEST_DEPENDENCIES: List[str] = [{% for v in unit_test_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if unit_test_extras %} -UNIT_TEST_EXTRAS: List[str] = [{% for v in unit_test_extras %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_EXTRAS: List[str] = [] -{%- endif %} - -{%- if unit_test_extras_by_python %} -UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {{ '{' }}{% if unit_test_extras_by_python %}{% for python_version, extras in unit_test_extras_by_python.items() %} - "{{python_version}}": [{% for v in extras %} - "{{v}}",{% endfor %} - ],{% endfor %}{% endif %} -} -{%- else %} -UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} -{%- endif %} -{% if system_test_python_versions %} -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = [{% for v in system_test_python_versions %}"{{v}}"{% if not loop.last %}, {% endif %}{% endfor %}] -{%- else %} -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = [] -{%- endif %} -SYSTEM_TEST_STANDARD_DEPENDENCIES: List[str] = [ - "mock", - "pytest", - "google-cloud-testutils", -] -{%- if system_test_external_dependencies %} -SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [{% for v in system_test_external_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if system_test_local_dependencies %} -SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [{% for v in system_test_local_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if system_test_dependencies %} -SYSTEM_TEST_DEPENDENCIES: List[str] = [{% for v in system_test_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if system_test_extras %} -SYSTEM_TEST_EXTRAS: List[str] = [{% for v in system_test_extras %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_EXTRAS: List[str] = [] -{%- endif %} -{%- if system_test_extras_by_python %} -SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {{ '{' }}{% if system_test_extras_by_python %}{% for python_version, extras in system_test_extras_by_python.items() %} - "{{python_version}}": [{% for v in extras %} - "{{v}}",{% endfor %} - ],{% endfor %}{% endif %} -} -{%- else %} -SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} -{%- endif %} - -CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() - -nox.options.sessions = [ - "unit", - "system", - "cover", - "lint", - "lint_setup_py", - "blacken", - "docs", - "docfx", - "format", -] - -# Error if a python version is missing -nox.options.error_on_missing_interpreters = True - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def lint(session): - """Run linters. - - Returns a failure if the linters find linting errors or sufficiently - serious code quality issues. - """ - session.install(FLAKE8_VERSION, BLACK_VERSION) - session.run( - "black", - "--check", - *LINT_PATHS, - ) - session.run("flake8", "google", "tests") - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def blacken(session): - """Run black. Format code to uniform standard.""" - session.install(BLACK_VERSION) - session.run( - "black", - *LINT_PATHS, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def format(session): - """ - Run isort to sort imports. Then run black - to format code to uniform standard. - """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections - session.run( - "isort", - "--fss", - *LINT_PATHS, - ) - session.run( - "black", - *LINT_PATHS, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def lint_setup_py(session): - """Verify that setup.py is valid (including RST check).""" - session.install("docutils", "pygments") - session.run("python", "setup.py", "check", "--restructuredtext", "--strict") - - -def install_unittest_dependencies(session, *constraints): - standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES - session.install(*standard_deps, *constraints) - - if UNIT_TEST_EXTERNAL_DEPENDENCIES: - warnings.warn( - "'unit_test_external_dependencies' is deprecated. Instead, please " - "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", - DeprecationWarning, - ) - session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) - - if UNIT_TEST_LOCAL_DEPENDENCIES: - session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) - - if UNIT_TEST_EXTRAS_BY_PYTHON: - extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) - elif UNIT_TEST_EXTRAS: - extras = UNIT_TEST_EXTRAS - else: - extras = [] - - if extras: - session.install("-e", f".[{','.join(extras)}]", *constraints) - else: - session.install("-e", ".", *constraints) - - -@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) -@nox.parametrize( - "protobuf_implementation", - [ "python", "upb", "cpp" ], -) -def unit(session, protobuf_implementation): - # Install all test dependencies, then install this package in-place. - - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): - session.skip("cpp implementation is not supported in python 3.11+") - - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - install_unittest_dependencies(session, "-c", constraints_path) - - # TODO(https://github.com/googleapis/synthtool/issues/1976): - # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. - # The 'cpp' implementation requires Protobuf<4. - if protobuf_implementation == "cpp": - session.install("protobuf<4") - - # Run py.test against the unit tests. - session.run( - "py.test", - "--quiet", - f"--junitxml=unit_{session.python}_sponge_log.xml", - "--cov=google", - "--cov=tests/unit", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - os.path.join("tests", "unit"), - *session.posargs, - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) - - -def install_systemtest_dependencies(session, *constraints): - - # Use pre-release gRPC for system tests. - # Exclude version 1.52.0rc1 which has a known issue. - # See https://github.com/grpc/grpc/issues/32163 - session.install("--pre", "grpcio!=1.52.0rc1") - - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: - session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_LOCAL_DEPENDENCIES: - session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_DEPENDENCIES: - session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_EXTRAS_BY_PYTHON: - extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) - elif SYSTEM_TEST_EXTRAS: - extras = SYSTEM_TEST_EXTRAS - else: - extras = [] - - if extras: - session.install("-e", f".[{','.join(extras)}]", *constraints) - else: - session.install("-e", ".", *constraints) - - -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def system(session): - """Run the system test suite.""" - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - system_test_path = os.path.join("tests", "system.py") - system_test_folder_path = os.path.join("tests", "system") - - # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. - if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": - session.skip("RUN_SYSTEM_TESTS is set to false, skipping") - # Install pyopenssl for mTLS testing. - if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": - session.install("pyopenssl") - - system_test_exists = os.path.exists(system_test_path) - system_test_folder_exists = os.path.exists(system_test_folder_path) - # Sanity check: only run tests if found. - if not system_test_exists and not system_test_folder_exists: - session.skip("System tests were not found") - - install_systemtest_dependencies(session, "-c", constraints_path) - - # Run py.test against the system tests. - if system_test_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_path, - *session.posargs, - ) - if system_test_folder_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_folder_path, - *session.posargs, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session): - """Run the final coverage report. - - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under={{ cov_level if cov_level != None else '100' }}") - - session.run("coverage", "erase") - - -@nox.session(python="3.10") -def docs(session): - """Build the docs for this library.""" - - session.install("-e", ".") - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "sphinx==4.5.0", - "alabaster", - "recommonmark", - ) - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python="3.10") -def docfx(session): - """Build the docfx yaml files for this library.""" - - session.install("-e", ".") - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "gcp-sphinx-docfx-yaml", - "alabaster", - "recommonmark", - ) - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-T", # show full traceback on exception - "-N", # no colors - "-D", - ( - "extensions=sphinx.ext.autodoc," - "sphinx.ext.autosummary," - "docfx_yaml.extension," - "sphinx.ext.intersphinx," - "sphinx.ext.coverage," - "sphinx.ext.napoleon," - "sphinx.ext.todo," - "sphinx.ext.viewcode," - "recommonmark" - ), - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python="{{ unit_test_python_versions | last }}") -@nox.parametrize( - "protobuf_implementation", - [ "python", "upb", "cpp" ], -) -def prerelease_deps(session, protobuf_implementation): - """Run all tests with prerelease versions of dependencies installed.""" - - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): - session.skip("cpp implementation is not supported in python 3.11+") - - # Install all dependencies - session.install("-e", ".[all, tests, tracing]") - unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES - session.install(*unit_deps_all) - system_deps_all = ( - SYSTEM_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_EXTERNAL_DEPENDENCIES - ) - session.install(*system_deps_all) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - constraints_deps = [ - match.group(1) - for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - session.install(*constraints_deps) - - prerel_deps = [ - "protobuf", - # dependency of grpc - "six", - "grpc-google-iam-v1", - "googleapis-common-protos", - "grpcio", - "grpcio-status", - "google-api-core", - "google-auth", - "proto-plus", - "google-cloud-testutils", - # dependencies of google-cloud-testutils" - "click", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = [ - "requests", - ] - session.install(*other_deps) - - # Print out prerelease package versions - session.run( - "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" - ) - session.run("python", "-c", "import grpc; print(grpc.__version__)") - session.run("python", "-c", "import google.auth; print(google.auth.__version__)") - - session.run( - "py.test", - "tests/unit", - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) - - system_test_path = os.path.join("tests", "system.py") - system_test_folder_path = os.path.join("tests", "system") - - # Only run system tests if found. - if os.path.exists(system_test_path): - session.run( - "py.test", - "--verbose", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_path, - *session.posargs, - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) - if os.path.exists(system_test_folder_path): - session.run( - "py.test", - "--verbose", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_folder_path, - *session.posargs, - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) diff --git a/synthtool/gcp/templates/python_mono_repo_library/noxfile.py.j2 b/synthtool/gcp/templates/python_mono_repo_library/noxfile.py.j2 deleted file mode 100644 index 76eecf4b1..000000000 --- a/synthtool/gcp/templates/python_mono_repo_library/noxfile.py.j2 +++ /dev/null @@ -1,621 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Copyright 2024 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Generated by synthtool. DO NOT EDIT! - -from __future__ import absolute_import -import os -import pathlib -import re -import shutil -from typing import Dict, List -import warnings - -import nox - -BLACK_VERSION = "black[jupyter]==23.7.0" -ISORT_VERSION = "isort==5.11.0" -{% if 'google' in metadata['repo']['distribution_name'] %} -LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] -{% else %} -LINT_PATHS = ["docs", "{{ metadata['repo']['distribution_name'] }}", "tests", "noxfile.py", "setup.py"] -{% endif %} - -DEFAULT_PYTHON_VERSION = "{{ default_python_version }}" - -UNIT_TEST_PYTHON_VERSIONS: List[str] = [{% for v in unit_test_python_versions %}"{{v}}"{% if not loop.last %}, {% endif %}{% endfor %}] -UNIT_TEST_STANDARD_DEPENDENCIES = [ - "mock", - "asyncmock", - "pytest", - "pytest-cov", - "pytest-asyncio", -] -{%- if unit_test_external_dependencies %} -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [{% for v in unit_test_external_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if unit_test_local_dependencies %} -UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [{% for v in unit_test_local_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if unit_test_dependencies %} -UNIT_TEST_DEPENDENCIES: List[str] = [{% for v in unit_test_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if unit_test_extras %} -UNIT_TEST_EXTRAS: List[str] = [{% for v in unit_test_extras %} - "{{v}}",{% endfor %} -] -{%- else %} -UNIT_TEST_EXTRAS: List[str] = [] -{%- endif %} - -{%- if unit_test_extras_by_python %} -UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {{ '{' }}{% if unit_test_extras_by_python %}{% for python_version, extras in unit_test_extras_by_python.items() %} - "{{python_version}}": [{% for v in extras %} - "{{v}}",{% endfor %} - ],{% endfor %}{% endif %} -} -{%- else %} -UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} -{%- endif %} -{% if system_test_python_versions %} -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = [{% for v in system_test_python_versions %}"{{v}}"{% if not loop.last %}, {% endif %}{% endfor %}] -{%- else %} -SYSTEM_TEST_PYTHON_VERSIONS: List[str] = [] -{%- endif %} -SYSTEM_TEST_STANDARD_DEPENDENCIES = [ - "mock", - "pytest", - "google-cloud-testutils", -] -{%- if system_test_external_dependencies %} -SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [{% for v in system_test_external_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_EXTERNAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if system_test_local_dependencies %} -SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [{% for v in system_test_local_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if system_test_dependencies %} -SYSTEM_TEST_DEPENDENCIES: List[str] = [{% for v in system_test_dependencies %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_DEPENDENCIES: List[str] = [] -{%- endif %} -{%- if system_test_extras %} -SYSTEM_TEST_EXTRAS: List[str] = [{% for v in system_test_extras %} - "{{v}}",{% endfor %} -] -{%- else %} -SYSTEM_TEST_EXTRAS: List[str] = [] -{%- endif %} -{%- if system_test_extras_by_python %} -SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {{ '{' }}{% if system_test_extras_by_python %}{% for python_version, extras in system_test_extras_by_python.items() %} - "{{python_version}}": [{% for v in extras %} - "{{v}}",{% endfor %} - ],{% endfor %}{% endif %} -} -{%- else %} -SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = {} -{%- endif %} - -CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() - -# 'docfx' is excluded since it only needs to run in 'docs-presubmit' -nox.options.sessions = [ - "unit", - "system", - "cover", - "lint", - "lint_setup_py", - "blacken", - "docs", -] - -# Error if a python version is missing -nox.options.error_on_missing_interpreters = True - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def lint(session): - """Run linters. - - Returns a failure if the linters find linting errors or sufficiently - serious code quality issues. - """ - session.install("flake8", BLACK_VERSION) - session.run( - "black", - "--check", - *LINT_PATHS, - ) - {% if 'google' in metadata['repo']['distribution_name'] %} - session.run("flake8", "google", "tests") - {% else %} - session.run("flake8", "{{ metadata['repo']['distribution_name'] }}", "tests") - {% endif %} - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def blacken(session): - """Run black. Format code to uniform standard.""" - session.install(BLACK_VERSION) - session.run( - "black", - *LINT_PATHS, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def format(session): - """ - Run isort to sort imports. Then run black - to format code to uniform standard. - """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections - session.run( - "isort", - "--fss", - *LINT_PATHS, - ) - session.run( - "black", - *LINT_PATHS, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def lint_setup_py(session): - """Verify that setup.py is valid (including RST check).""" - session.install("docutils", "pygments") - session.run("python", "setup.py", "check", "--restructuredtext", "--strict") - - -def install_unittest_dependencies(session, *constraints): - standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES - session.install(*standard_deps, *constraints) - - if UNIT_TEST_EXTERNAL_DEPENDENCIES: - warnings.warn( - "'unit_test_external_dependencies' is deprecated. Instead, please " - "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", - DeprecationWarning, - ) - session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) - - if UNIT_TEST_LOCAL_DEPENDENCIES: - session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) - - if UNIT_TEST_EXTRAS_BY_PYTHON: - extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) - elif UNIT_TEST_EXTRAS: - extras = UNIT_TEST_EXTRAS - else: - extras = [] - - if extras: - session.install("-e", f".[{','.join(extras)}]", *constraints) - else: - session.install("-e", ".", *constraints) - - -@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) -@nox.parametrize( - "protobuf_implementation", - [ "python", "upb", "cpp" ], -) -def unit(session, protobuf_implementation): - # Install all test dependencies, then install this package in-place. - - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): - session.skip("cpp implementation is not supported in python 3.11+") - - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - install_unittest_dependencies(session, "-c", constraints_path) - - # TODO(https://github.com/googleapis/synthtool/issues/1976): - # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. - # The 'cpp' implementation requires Protobuf<4. - if protobuf_implementation == "cpp": - session.install("protobuf<4") - - # Run py.test against the unit tests. - session.run( - "py.test", - "--quiet", - f"--junitxml=unit_{session.python}_sponge_log.xml", - {% if metadata['repo']['distribution_name'].startswith('google') %} - "--cov=google", - {% else %} - "--cov={{ metadata['repo']['distribution_name'] }}", - {% endif %} - "--cov=tests/unit", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - os.path.join("tests", "unit"), - *session.posargs, - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) - - -def install_systemtest_dependencies(session, *constraints): - - # Use pre-release gRPC for system tests. - # Exclude version 1.52.0rc1 which has a known issue. - # See https://github.com/grpc/grpc/issues/32163 - session.install("--pre", "grpcio!=1.52.0rc1") - - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: - session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_LOCAL_DEPENDENCIES: - session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_DEPENDENCIES: - session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_EXTRAS_BY_PYTHON: - extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) - elif SYSTEM_TEST_EXTRAS: - extras = SYSTEM_TEST_EXTRAS - else: - extras = [] - - if extras: - session.install("-e", f".[{','.join(extras)}]", *constraints) - else: - session.install("-e", ".", *constraints) - - -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def system(session): - """Run the system test suite.""" - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - system_test_path = os.path.join("tests", "system.py") - system_test_folder_path = os.path.join("tests", "system") - - # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. - if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": - session.skip("RUN_SYSTEM_TESTS is set to false, skipping") - # Install pyopenssl for mTLS testing. - if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": - session.install("pyopenssl") - - system_test_exists = os.path.exists(system_test_path) - system_test_folder_exists = os.path.exists(system_test_folder_path) - # Sanity check: only run tests if found. - if not system_test_exists and not system_test_folder_exists: - session.skip("System tests were not found") - - install_systemtest_dependencies(session, "-c", constraints_path) - - # Run py.test against the system tests. - if system_test_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_path, - *session.posargs, - ) - if system_test_folder_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_folder_path, - *session.posargs, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def cover(session): - """Run the final coverage report. - - This outputs the coverage report aggregating coverage from the unit - test runs (not system test runs), and then erases coverage data. - """ - session.install("coverage", "pytest-cov") - session.run("coverage", "report", "--show-missing", "--fail-under={{ cov_level if cov_level != None else '100' }}") - - session.run("coverage", "erase") - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def docs(session): - """Build the docs for this library.""" - - session.install("-e", ".") - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "sphinx==4.5.0", - "alabaster", - "recommonmark", - ) - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def docfx(session): - """Build the docfx yaml files for this library.""" - - session.install("-e", ".") - session.install( - # We need to pin to specific versions of the `sphinxcontrib-*` packages - # which still support sphinx 4.x. - # See https://github.com/googleapis/sphinx-docfx-yaml/issues/344 - # and https://github.com/googleapis/sphinx-docfx-yaml/issues/345. - "sphinxcontrib-applehelp==1.0.4", - "sphinxcontrib-devhelp==1.0.2", - "sphinxcontrib-htmlhelp==2.0.1", - "sphinxcontrib-qthelp==1.0.3", - "sphinxcontrib-serializinghtml==1.1.5", - "gcp-sphinx-docfx-yaml", - "alabaster", - "recommonmark", - ) - - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) - session.run( - "sphinx-build", - "-T", # show full traceback on exception - "-N", # no colors - "-D", - ( - "extensions=sphinx.ext.autodoc," - "sphinx.ext.autosummary," - "docfx_yaml.extension," - "sphinx.ext.intersphinx," - "sphinx.ext.coverage," - "sphinx.ext.napoleon," - "sphinx.ext.todo," - "sphinx.ext.viewcode," - "recommonmark" - ), - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), - ) - - -@nox.session(python="{{ unit_test_python_versions | last }}") -@nox.parametrize( - "protobuf_implementation", - ["python", "upb", "cpp"], -) -def prerelease_deps(session, protobuf_implementation): - """ - Run all tests with pre-release versions of dependencies installed - rather than the standard non pre-release versions. - Pre-release versions can be installed using - `pip install --pre `. - """ - - if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12", "3.13"): - session.skip("cpp implementation is not supported in python 3.11+") - - # Install all dependencies - session.install("-e", ".") - - # Install dependencies for the unit test environment - unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES - session.install(*unit_deps_all) - - # Install dependencies for the system test environment - system_deps_all = ( - SYSTEM_TEST_STANDARD_DEPENDENCIES - + SYSTEM_TEST_EXTERNAL_DEPENDENCIES - + SYSTEM_TEST_EXTRAS - ) - session.install(*system_deps_all) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - constraints_deps = [ - match.group(1) - for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - # Install dependencies specified in `testing/constraints-X.txt`. - session.install(*constraints_deps) - - # Note: If a dependency is added to the `prerel_deps` list, - # the `core_dependencies_from_source` list in the `core_deps_from_source` - # nox session should also be updated. - prerel_deps = [ - "googleapis-common-protos", - "google-api-core", - "google-auth", - "grpc-google-iam-v1", - "grpcio", - "grpcio-status", - "protobuf", - "proto-plus", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--ignore-installed", dep) - # TODO(https://github.com/grpc/grpc/issues/38965): Add `grpcio-status`` - # to the dictionary below once this bug is fixed. - # TODO(https://github.com/googleapis/google-cloud-python/issues/13643): Add - # `googleapis-common-protos` and `grpc-google-iam-v1` to the dictionary below - # once this bug is fixed. - package_namespaces = { - "google-api-core": "google.api_core", - "google-auth": "google.auth", - "grpcio": "grpc", - "protobuf": "google.protobuf", - "proto-plus": "proto", - } - - version_namespace = package_namespaces.get(dep) - - print(f"Installed {dep}") - if version_namespace: - session.run( - "python", - "-c", - f"import {version_namespace}; print({version_namespace}.__version__)", - ) - - session.run( - "py.test", - "tests/unit", - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) - - -@nox.session(python="{{ unit_test_python_versions | last }}") -@nox.parametrize( - "protobuf_implementation", - ["python", "upb"], -) -def core_deps_from_source(session, protobuf_implementation): - """Run all tests with core dependencies installed from source - rather than pulling the dependencies from PyPI. - """ - - # Install all dependencies - session.install("-e", ".") - - # Install dependencies for the unit test environment - unit_deps_all = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_EXTERNAL_DEPENDENCIES - session.install(*unit_deps_all) - - # Install dependencies for the system test environment - system_deps_all = ( - SYSTEM_TEST_STANDARD_DEPENDENCIES - + SYSTEM_TEST_EXTERNAL_DEPENDENCIES - + SYSTEM_TEST_EXTRAS - ) - session.install(*system_deps_all) - - # Because we test minimum dependency versions on the minimum Python - # version, the first version we test with in the unit tests sessions has a - # constraints file containing all dependencies and extras. - with open( - CURRENT_DIRECTORY - / "testing" - / f"constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", - encoding="utf-8", - ) as constraints_file: - constraints_text = constraints_file.read() - - # Ignore leading whitespace and comment lines. - constraints_deps = [ - match.group(1) - for match in re.finditer( - r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE - ) - ] - - # Install dependencies specified in `testing/constraints-X.txt`. - session.install(*constraints_deps) - - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2358): `grpcio` and - # `grpcio-status` should be added to the list below so that they are installed from source, - # rather than PyPI. - # TODO(https://github.com/googleapis/gapic-generator-python/issues/2357): `protobuf` should be - # added to the list below so that it is installed from source, rather than PyPI - # Note: If a dependency is added to the `core_dependencies_from_source` list, - # the `prerel_deps` list in the `prerelease_deps` nox session should also be updated. - core_dependencies_from_source = [ - f"{CURRENT_DIRECTORY}/../googleapis-common-protos", - "google-api-core @ git+https://github.com/googleapis/python-api-core.git", - "google-auth @ git+https://github.com/googleapis/google-auth-library-python.git", - f"{CURRENT_DIRECTORY}/../grpc-google-iam-v1", - "proto-plus @ git+https://github.com/googleapis/proto-plus-python.git", - ] - - for dep in core_dependencies_from_source: - session.install(dep, "--no-deps", "--ignore-installed") - print(f"Installed {dep}") - - session.run( - "py.test", - "tests/unit", - env={ - "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, - }, - ) diff --git a/tests/test_python_library.py b/tests/test_python_library.py index c68a6eb3c..b68e87fa3 100644 --- a/tests/test_python_library.py +++ b/tests/test_python_library.py @@ -28,127 +28,6 @@ PYTHON_LIBRARY = Path(__file__).parent.parent / "synthtool/gcp/templates/python_library" -@pytest.mark.parametrize( - ["template_kwargs", "expected_text"], - [ - ({}, ["import nox", 'session.install("-e", ".", *constraints)']), - ( - {"unit_test_local_dependencies": ["../testutils", "../unitutils"]}, - [ - """\ -UNIT_TEST_LOCAL_DEPENDENCIES: List[str] = [ - "../testutils", - "../unitutils", -]""", - ], - ), - ( - {"system_test_local_dependencies": ["../testutils", "../sysutils"]}, - [ - """\ -SYSTEM_TEST_LOCAL_DEPENDENCIES: List[str] = [ - "../testutils", - "../sysutils", -]""", - ], - ), - ( - {"unit_test_extras": ["abc", "def"]}, - [ - """\ -UNIT_TEST_EXTRAS: List[str] = [ - "abc", - "def", -]""", - ], - ), - ( - {"system_test_extras": ["abc", "def"]}, - """\ -SYSTEM_TEST_EXTRAS: List[str] = [ - "abc", - "def", -]""", - ), - ( - {"unit_test_extras_by_python": {"3.8": ["abc", "def"]}}, - [ - """\ -UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { - "3.8": [ - "abc", - "def", - ], -}""", - ], - ), - ( - {"system_test_extras_by_python": {"3.8": ["abc", "def"]}}, - [ - """\ -SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { - "3.8": [ - "abc", - "def", - ], -}""", - ], - ), - ( - { - "unit_test_extras": ["tuv", "wxyz"], - "unit_test_extras_by_python": {"3.8": ["abc", "def"]}, - }, - [ - """\ -UNIT_TEST_EXTRAS: List[str] = [ - "tuv", - "wxyz", -]""", - """\ -UNIT_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { - "3.8": [ - "abc", - "def", - ], -}""", - ], - ), - ( - { - "system_test_extras": ["tuv", "wxyz"], - "system_test_extras_by_python": {"3.8": ["abc", "def"]}, - }, - [ - """\ -SYSTEM_TEST_EXTRAS: List[str] = [ - "tuv", - "wxyz", -]""", - """\ -SYSTEM_TEST_EXTRAS_BY_PYTHON: Dict[str, List[str]] = { - "3.8": [ - "abc", - "def", - ], -}""", - ], - ), - ], -) -def test_library_noxfile(template_kwargs, expected_text): - t = templates.Templates(PYTHON_LIBRARY) - result = t.render( - "noxfile.py.j2", - **template_kwargs, - ).read_text() - # Validate Python syntax. - result_code = compile(result, "noxfile.py", "exec") - assert result_code is not None - for expected in expected_text: - assert expected in result - - def test_library_codeowners(): t = templates.Templates(PYTHON_LIBRARY / ".github") result = t.render(