pyproject.toml

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[project]
name = "nemo-evaluator"
version = "0.3.0"
description = "NeMo Evaluator — benchmark environments, pluggable solvers, interceptor proxy, and decision-grade scoring for LLMs"
readme = "README.md"
requires-python = ">=3.12,<3.14"
license = "Apache-2.0"
authors = [
    { name = "NVIDIA" },
    { email = "contact@nvidia.com" },
]
maintainers = [
    { name = "NVIDIA" },
    { email = "contact@nvidia.com" },
]
keywords = [
    "deep learning",
    "evaluations",
    "benchmarks",
    "machine learning",
    "gpu",
    "NLP",
    "LLM",
    "pytorch",
    "NVIDIA",
]
classifiers = [
    "Development Status :: 4 - Beta",
    "Intended Audience :: Science/Research",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: Apache Software License",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Software Development :: Testing",
]
dependencies = [
    "pydantic>=2.0",
    "fastapi>=0.100",
    "uvicorn[standard]>=0.20",
    "aiohttp>=3.9",
    "pyyaml>=6.0",
    "click>=8.0",
    "numpy>=1.24",
    "datasets>=2.14",
    "jinja2>=3.0",
    "starlette>=0.27",
    "boto3>=1.28",
]

[project.optional-dependencies]
scoring = ["sympy>=1.12"]
stats = ["scipy>=1.10"]
ray = ["ray[default]>=2.9"]
skills = ["nemo-skills"]
lm-eval = ["lm_eval>=0.4"]
harbor = ["harbor>=0.3.0,<0.4.0"]
inspect = ["inspect_ai"]
export = ["wandb", "mlflow"]
harnesses = ["nemo-evaluator[lm-eval]"]
all = ["nemo-evaluator[scoring,stats,ray,lm-eval,harbor]"]
docs = [
    "sphinx",
    "sphinx-autobuild",
    "sphinx-copybutton",
    "nvidia-sphinx-theme",
    "sphinxcontrib-mermaid",
    "sphinx-design",
    "myst-parser",
]
dev = [
    "pytest>=7.0",
    "pytest-asyncio>=0.21",
    "pytest-xdist>=3.0",
    "pytest-cov>=4.0",
    "ruff>=0.1",
    "pre-commit>=3.0",
    "nemo-evaluator[scoring,stats,lm-eval,harbor]",
]

[project.urls]
Homepage = "https://docs.nvidia.com/nemo/evaluator/"
Repository = "https://github.com/NVIDIA-NeMo/Evaluator"
Documentation = "https://docs.nvidia.com/nemo/evaluator/"
Changelog = "https://github.com/NVIDIA-NeMo/Evaluator/blob/main/CHANGELOG.md"
Issues = "https://github.com/NVIDIA-NeMo/Evaluator/issues"

[project.scripts]
nel = "nemo_evaluator.cli.main:cli"

[tool.hatch.build.targets.wheel]
packages = ["src/nemo_evaluator"]

[tool.hatch.build.targets.wheel.force-include]
"harbor_datasets/registry_overrides" = "nemo_evaluator/_registry_overrides"

[tool.pytest.ini_options]
testpaths = ["tests"]
asyncio_mode = "auto"
markers = [
    "network: tests that download from HF or hit external services (deselect via: -m 'not network')",
    "integration: integration tests that wire real components together",
    "slow: tests that take > 30s (deselect via: -m 'not slow')",
    "e2e: end-to-end smoke tests requiring infrastructure",
]
filterwarnings = [
    "ignore::DeprecationWarning",
]

[tool.coverage.run]
source = ["nemo_evaluator"]

[tool.coverage.report]
fail_under = 54
show_missing = true
skip_empty = true
exclude_lines = [
    "pragma: no cover",
    "if TYPE_CHECKING:",
    "if __name__ ==",
]

[tool.uv]
managed = true
# Force resolver-wide overrides to break dependency conflicts that come from
# transitive pins we can't easily change upstream. These are honored by `uv`
# (uv pip / uv sync) but ignored by plain pip — Dockerfiles using plain pip
# rely on pip's looser resolver instead.
override-dependencies = [
    # leptonai (pulled in by nemo_run) hard-pins httpx[http2]==0.27.2, which
    # conflicts with litellm==1.83.14's httpx==0.28.1. Relax the pin so the
    # resolver can pick a single version that satisfies litellm.
    "httpx[http2]>=0.28.1",
]
conflicts = [
    [{ extra = "skills" }, { extra = "harbor" }],
    [{ extra = "skills" }, { extra = "all" }],
    [{ extra = "skills" }, { extra = "dev" }],
]

[tool.uv.sources]
# Resolve nemo-skills from git for local/dev installs (uv sync, uv pip).
# Declaring the URL here instead of as a PEP 508 direct reference in
# project.optional-dependencies keeps it out of the published wheel METADATA,
# which PyPI/twine reject ("Can't have direct dependency ...").
nemo-skills = { git = "https://github.com/NVIDIA-NeMo/Skills.git" }

[tool.ruff]
line-length = 120
target-version = "py312"