Skip to content

Commit 6c367ae

Browse files
committed
fix: report incomplete JSONL audits and protect child handlers
1 parent 20b6623 commit 6c367ae

6 files changed

Lines changed: 292 additions & 11 deletions

File tree

‎src/logprivacy/integrations/logging_filter.py‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,29 @@ def _iter_effective_handlers(logger: logging.Logger) -> Iterator[logging.Handler
349349
current = current.parent
350350

351351

352+
def _iter_namespace_handlers(logger: logging.Logger) -> Iterator[logging.Handler]:
353+
"""Yield effective and descendant-owned handlers for one logger namespace."""
354+
seen: set[int] = set()
355+
356+
for handler in _iter_effective_handlers(logger):
357+
seen.add(id(handler))
358+
yield handler
359+
360+
root_logger = logging.getLogger()
361+
prefix = None if logger is root_logger else f"{logger.name}."
362+
for candidate in tuple(logging.Logger.manager.loggerDict.values()):
363+
if not isinstance(candidate, logging.Logger) or candidate is logger:
364+
continue
365+
if prefix is not None and not candidate.name.startswith(prefix):
366+
continue
367+
368+
for handler in candidate.handlers:
369+
handler_id = id(handler)
370+
if handler_id not in seen:
371+
seen.add(handler_id)
372+
yield handler
373+
374+
352375
def install_handler_filters(
353376
logger: logging.Logger,
354377
*,
@@ -359,7 +382,7 @@ def install_handler_filters(
359382
root_logger = logging.getLogger()
360383
logger_prefix = None if logger is root_logger else logger.name
361384

362-
for handler in _iter_effective_handlers(logger):
385+
for handler in _iter_namespace_handlers(logger):
363386
existing = next(
364387
(
365388
candidate

‎src/logprivacy/internal/regex_timeout.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
from collections.abc import Callable
1717
from re import Pattern
1818
from types import FrameType
19-
from typing import Any, BinaryIO, cast
19+
from typing import IO, Any, cast
2020

2121
from logprivacy.exceptions import InputLimitExceededError
2222

@@ -106,8 +106,8 @@ def __init__(self) -> None:
106106
assert stdout_pipe is not None
107107

108108
self._process: subprocess.Popen[bytes] = process
109-
self._stdin: BinaryIO = stdin_pipe
110-
self._stdout: BinaryIO = stdout_pipe
109+
self._stdin: IO[bytes] = stdin_pipe
110+
self._stdout: IO[bytes] = stdout_pipe
111111
self._responses: queue.Queue[bytes | None] = queue.Queue()
112112
self._request_lock = threading.Lock()
113113
self._ready = threading.Event()
@@ -298,7 +298,7 @@ def _shutdown_worker() -> None:
298298
worker.close()
299299

300300

301-
def _read_exact(stream: BinaryIO, size: int) -> bytes | None:
301+
def _read_exact(stream: IO[bytes], size: int) -> bytes | None:
302302
chunks: list[bytes] = []
303303
remaining = size
304304
while remaining:

‎src/logprivacy/jsonl/streaming.py‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -249,14 +249,15 @@ def scan_jsonl(
249249
adapters: AdapterRegistry | None = None,
250250
on_error: _OnError = "raise",
251251
) -> Iterator[JSONLScanRecord]:
252-
"""Yield scan findings from a JSON Lines source, line by line.
252+
"""Yield scan findings and incomplete audits from a JSON Lines source.
253253
254254
Each non-empty line is parsed as JSON, then audited for sensitive values.
255-
Yields ``JSONLScanRecord`` for lines that contain findings.
256-
Lines with no findings are not yielded.
255+
A ``JSONLScanRecord`` is yielded when the line contains findings or when the
256+
audit could not inspect the complete value. Complete lines with no findings
257+
are omitted.
257258
258259
``"raise"`` raises ``JSONLProcessingError`` for invalid JSON or UTF-8.
259-
Because scan records only represent findings, ``"skip"`` and
260+
Because scan records represent parsed audits, ``"skip"`` and
260261
``"placeholder"`` both omit invalid lines instead of yielding a synthetic
261262
finding. The original line content is never stored or included in errors.
262263
"""
@@ -286,8 +287,13 @@ def scan_jsonl(
286287
continue
287288

288289
report = cleaner.audit(parsed)
289-
if report.findings:
290-
yield JSONLScanRecord(line_number=line_number, findings=report.findings)
290+
if report.findings or not report.complete:
291+
yield JSONLScanRecord(
292+
line_number=line_number,
293+
findings=report.findings,
294+
complete=report.complete,
295+
limitations=report.limitations,
296+
)
291297

292298

293299
def _validate_on_error(on_error: str) -> None:

‎src/logprivacy/result/types.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,3 +277,5 @@ class JSONLScanRecord:
277277

278278
line_number: int
279279
findings: tuple[Finding, ...]
280+
complete: bool = True
281+
limitations: tuple[str, ...] = ()
Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
from __future__ import annotations
2+
3+
import io
4+
import logging
5+
6+
from logprivacy import LogPrivacyFilter, get_safe_logger
7+
8+
9+
def _restore_logger(
10+
logger: logging.Logger,
11+
*,
12+
handlers: list[logging.Handler],
13+
filters: list[logging.Filter],
14+
level: int,
15+
propagate: bool,
16+
) -> None:
17+
logger.handlers[:] = handlers
18+
logger.filters[:] = filters
19+
logger.setLevel(level)
20+
logger.propagate = propagate
21+
22+
23+
def test_get_safe_logger_protects_existing_child_handler_without_propagation() -> None:
24+
parent = logging.getLogger("child-handler-secure")
25+
child = logging.getLogger("child-handler-secure.worker")
26+
parent_state = {
27+
"handlers": list(parent.handlers),
28+
"filters": list(parent.filters),
29+
"level": parent.level,
30+
"propagate": parent.propagate,
31+
}
32+
child_state = {
33+
"handlers": list(child.handlers),
34+
"filters": list(child.filters),
35+
"level": child.level,
36+
"propagate": child.propagate,
37+
}
38+
stream = io.StringIO()
39+
handler = logging.StreamHandler(stream)
40+
41+
try:
42+
parent.handlers[:] = []
43+
parent.propagate = False
44+
child.handlers[:] = [handler]
45+
child.filters[:] = []
46+
child.setLevel(logging.DEBUG)
47+
child.propagate = False
48+
49+
get_safe_logger("child-handler-secure", level=logging.DEBUG)
50+
child.error("email=%s password=%s", "john@example.com", "secret123")
51+
finally:
52+
_restore_logger(parent, **parent_state)
53+
_restore_logger(child, **child_state)
54+
55+
rendered = stream.getvalue()
56+
assert "john@example.com" not in rendered
57+
assert "secret123" not in rendered
58+
assert "email=[EMAIL] password=[SECRET]" in rendered
59+
60+
61+
def test_get_safe_logger_deduplicates_filters_on_existing_child_handler() -> None:
62+
parent = logging.getLogger("child-handler-deduplicated")
63+
child = logging.getLogger("child-handler-deduplicated.worker")
64+
parent_state = {
65+
"handlers": list(parent.handlers),
66+
"filters": list(parent.filters),
67+
"level": parent.level,
68+
"propagate": parent.propagate,
69+
}
70+
child_state = {
71+
"handlers": list(child.handlers),
72+
"filters": list(child.filters),
73+
"level": child.level,
74+
"propagate": child.propagate,
75+
}
76+
handler = logging.StreamHandler(io.StringIO())
77+
78+
try:
79+
parent.handlers[:] = []
80+
parent.propagate = False
81+
child.handlers[:] = [handler]
82+
child.propagate = False
83+
84+
get_safe_logger("child-handler-deduplicated")
85+
get_safe_logger("child-handler-deduplicated")
86+
87+
scoped = [
88+
candidate
89+
for candidate in handler.filters
90+
if isinstance(candidate, LogPrivacyFilter)
91+
and candidate.logger_prefix == "child-handler-deduplicated"
92+
]
93+
assert len(scoped) == 1
94+
finally:
95+
_restore_logger(parent, **parent_state)
96+
_restore_logger(child, **child_state)
97+
98+
99+
def test_get_safe_logger_does_not_install_filters_on_unrelated_child_handlers() -> None:
100+
parent = logging.getLogger("child-handler-scope")
101+
unrelated = logging.getLogger("other-child-handler.worker")
102+
parent_state = {
103+
"handlers": list(parent.handlers),
104+
"filters": list(parent.filters),
105+
"level": parent.level,
106+
"propagate": parent.propagate,
107+
}
108+
unrelated_state = {
109+
"handlers": list(unrelated.handlers),
110+
"filters": list(unrelated.filters),
111+
"level": unrelated.level,
112+
"propagate": unrelated.propagate,
113+
}
114+
handler = logging.StreamHandler(io.StringIO())
115+
116+
try:
117+
parent.handlers[:] = []
118+
parent.propagate = False
119+
unrelated.handlers[:] = [handler]
120+
unrelated.propagate = False
121+
122+
get_safe_logger("child-handler-scope")
123+
124+
scoped = [
125+
candidate for candidate in handler.filters if isinstance(candidate, LogPrivacyFilter)
126+
]
127+
assert scoped == []
128+
finally:
129+
_restore_logger(parent, **parent_state)
130+
_restore_logger(unrelated, **unrelated_state)
131+
132+
133+
def test_get_safe_logger_does_not_match_similar_namespace_prefix() -> None:
134+
parent = logging.getLogger("app")
135+
unrelated = logging.getLogger("application.worker")
136+
parent_state = {
137+
"handlers": list(parent.handlers),
138+
"filters": list(parent.filters),
139+
"level": parent.level,
140+
"propagate": parent.propagate,
141+
}
142+
unrelated_state = {
143+
"handlers": list(unrelated.handlers),
144+
"filters": list(unrelated.filters),
145+
"level": unrelated.level,
146+
"propagate": unrelated.propagate,
147+
}
148+
handler = logging.StreamHandler(io.StringIO())
149+
150+
try:
151+
parent.handlers[:] = []
152+
parent.propagate = False
153+
unrelated.handlers[:] = [handler]
154+
unrelated.propagate = False
155+
156+
get_safe_logger("app")
157+
158+
assert not any(isinstance(candidate, LogPrivacyFilter) for candidate in handler.filters)
159+
finally:
160+
_restore_logger(parent, **parent_state)
161+
_restore_logger(unrelated, **unrelated_state)
162+
163+
164+
def test_shared_handler_receives_one_scoped_filter() -> None:
165+
parent = logging.getLogger("shared-handler")
166+
child = logging.getLogger("shared-handler.worker")
167+
parent_state = {
168+
"handlers": list(parent.handlers),
169+
"filters": list(parent.filters),
170+
"level": parent.level,
171+
"propagate": parent.propagate,
172+
}
173+
child_state = {
174+
"handlers": list(child.handlers),
175+
"filters": list(child.filters),
176+
"level": child.level,
177+
"propagate": child.propagate,
178+
}
179+
handler = logging.StreamHandler(io.StringIO())
180+
181+
try:
182+
parent.handlers[:] = [handler]
183+
parent.propagate = False
184+
child.handlers[:] = [handler]
185+
child.propagate = False
186+
187+
get_safe_logger("shared-handler")
188+
189+
scoped = [
190+
candidate
191+
for candidate in handler.filters
192+
if isinstance(candidate, LogPrivacyFilter)
193+
and candidate.logger_prefix == "shared-handler"
194+
]
195+
assert len(scoped) == 1
196+
finally:
197+
_restore_logger(parent, **parent_state)
198+
_restore_logger(child, **child_state)
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
from __future__ import annotations
2+
3+
from dataclasses import replace
4+
from pathlib import Path
5+
6+
from logprivacy.jsonl import scan_jsonl
7+
from logprivacy.policy import CleanerPolicy
8+
from logprivacy.result import JSONLScanRecord
9+
10+
11+
def test_scan_jsonl_yields_incomplete_audit_without_findings(tmp_path: Path) -> None:
12+
source = tmp_path / "input.jsonl"
13+
source.write_text('{"outer":{"status":"ok"}}\n', encoding="utf-8")
14+
policy = replace(CleanerPolicy.default(), max_depth=0)
15+
16+
records = list(scan_jsonl(source, policy=policy))
17+
18+
assert len(records) == 1
19+
assert records[0].line_number == 1
20+
assert records[0].findings == ()
21+
assert records[0].complete is False
22+
assert records[0].limitations == ("max_depth",)
23+
24+
25+
def test_scan_jsonl_preserves_findings_when_audit_is_incomplete(tmp_path: Path) -> None:
26+
source = tmp_path / "input.jsonl"
27+
source.write_text(
28+
'{"email":"user@example.com","nested":{"status":"ok"}}\n',
29+
encoding="utf-8",
30+
)
31+
policy = replace(CleanerPolicy.default(), max_items=1)
32+
33+
records = list(scan_jsonl(source, policy=policy))
34+
35+
assert len(records) == 1
36+
assert records[0].complete is False
37+
assert records[0].limitations == ("max_items",)
38+
assert {finding.category for finding in records[0].findings} == {"email"}
39+
40+
41+
def test_scan_jsonl_still_omits_complete_lines_without_findings(tmp_path: Path) -> None:
42+
source = tmp_path / "input.jsonl"
43+
source.write_text('{"status":"ok"}\n', encoding="utf-8")
44+
45+
assert list(scan_jsonl(source)) == []
46+
47+
48+
def test_jsonl_scan_record_defaults_preserve_existing_construction() -> None:
49+
record = JSONLScanRecord(line_number=3, findings=())
50+
51+
assert record.complete is True
52+
assert record.limitations == ()

0 commit comments

Comments
 (0)