evaling/pyproject.toml at main · amro/evaling · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
[project]
name = "evaling"
version = "0.2.8"
description = "Compare prompt variants and models from the command line"
readme = "README.md"
license = { file = "LICENSE" }
authors = [{ name = "Amro Mousa" }]
requires-python = ">=3.10"
dependencies = [
    "click>=8.1",
    "httpx>=0.28.1",
    "jinja2>=3.1.6",
    "jsonschema>=4.26.0",
    "pydantic>=2.13.4",
    "pyyaml>=6.0.3",
    "rich>=15.0.0",
]
keywords = ["llm", "eval", "evaluation", "prompt", "testing"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Topic :: Software Development :: Testing",
]

[project.urls]
Homepage = "https://github.com/amro/evaling"
Repository = "https://github.com/amro/evaling"
Documentation = "https://github.com/amro/evaling/tree/main/docs"
Changelog = "https://github.com/amro/evaling/blob/main/CHANGELOG.md"
Issues = "https://github.com/amro/evaling/issues"

[project.scripts]
evaling = "evaling.cli:main"

[project.optional-dependencies]
mcp = [
    # Capped below 3: the server reads the SDK's tool-manager internals to
    # refuse arguments a tool never declared. See _reject_unknown_arguments.
    "mcp>=2.0,<3",
]

[dependency-groups]
dev = [
    "mcp>=2.0,<3",  # same range as the extra above, so tests run what ships
    "pytest>=8",
    "pytest-cov>=5",
    "ruff>=0.8",
    # tomllib is 3.11+, and a test reads this file. Declared rather than left
    # to whichever dependency happens to pull it in on 3.10.
    "tomli>=2; python_version < '3.11'",
]

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["src/evaling"]

[tool.hatch.build.targets.wheel.force-include]
"src/evaling/py.typed" = "evaling/py.typed"

[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-q"
filterwarnings = ["error::pytest.PytestUnraisableExceptionWarning"]
markers = [
    "slow: end-to-end or scale tests (deselect with -m 'not slow')",
    "perf: timing and memory guards (CI runs these alone, without coverage)",
]

[tool.coverage.run]
source = ["evaling"]

[tool.ruff]
line-length = 100
src = ["src", "tests"]

[tool.ruff.lint]
# Preview is on solely for PLW1514; nothing else from preview is selected.
preview = true
explicit-preview-rules = true
select = [
    "E",       # pycodestyle errors
    "W",       # pycodestyle warnings
    "F",       # pyflakes
    "I",       # isort
    "UP",      # pyupgrade
    "B",       # bugbear
    "SIM",     # simplify
    "PLW1514", # text I/O must name its encoding (cp1252 defaults break Windows)
]

# Mutation testing: change the source, see whether a test notices. Scoped to
# the modules that encode invariants, because that is where a test which
# passes with its subject deleted does real damage. `uv run mutmut run`, then
# `uv run mutmut results`. See CONTRIBUTING.md.
[tool.mutmut]
source_paths = ["src/evaling/"]
only_mutate = [
    "*/evaling/privacy.py",
    "*/evaling/secrets.py",
    "*/evaling/reqlog.py",
    "*/evaling/scoring.py",
    "*/evaling/textfile.py",
    "*/evaling/providers/pricing.py",
]
# Every test except the ones that change directory: mutmut resolves its own
# relative source paths inside each test, so a chdir breaks the whole run.
# An exclusion list rather than an inclusion list, because the latter went
# stale the first time a test file was added — 17 survivors turned out to be
# a function whose tests simply were not selected.
# mutmut runs from a copy of the project, so anything the tests read has to
# be copied alongside the source: test_docs reads docs/, the top-level markdown
# and the plugin's markdown, test_plugin reads the plugin and marketplace
# manifests, test_e2e runs the worked examples. Omitting one does not skip a
# test — it fails the unmutated suite, which aborts the whole run before any
# mutant is tried.
also_copy = [
    "docs",
    "examples",
    "plugin",
    ".claude-plugin",
    ".github",
    "tools",
    "README.md",
    "CONTRIBUTING.md",
    "RELEASING.md",
    "CHANGELOG.md",
    "REQUIREMENTS.md",
    "LICENSE",
]
pytest_add_cli_args_test_selection = [
    "tests/",
    "--ignore=tests/test_cli_init.py",
    "--ignore=tests/test_cli_inspect.py",
    "--ignore=tests/test_cli_run.py",
    "--ignore=tests/test_limits_and_cli_extras.py",
    "--ignore=tests/test_mcp_server.py",
    # Runs a subprocess with a different cwd, which trips the same resolution.
    "--deselect=tests/test_provider_command.py::TestWorkingDirectory",
]
pytest_add_cli_args = ["-p", "no:cacheprovider", "-m", "not perf"]