Add doc-ci:skip directive and richer run reporting

Authors can now exempt a snippet from execution without polluting the
copyable code: 'doc-ci:skip' in the fence info string, or an HTML comment
'<!-- doc-ci:skip -->' on the line above the fence. Such snippets classify
as directive-skip and are never run. The run reporter now shows captured
error output for failures (the point of the tool) and a clear OK/FAIL
verdict. README's own usage example is marked skip, so the README has no
runnable snippets and CI now dogfoods 'doc-ci run' too. 58 tests passing.

Author: ecogetaway

.github/workflows/ci.yml

@@ -22,3 +22,5 @@ jobs:
         run: pytest
       - name: Dogfood — inventory our own README
         run: doc-ci scan README.md
+      - name: Dogfood — run our own README examples
+        run: doc-ci run README.md

README.md

@@ -37,14 +37,27 @@ README.md:22-26  [bash]  3 line(s)  needs-network
 To actually execute the runnable examples (in a sandbox) and get a pass/fail
 report suitable for CI:
 
-```bash
+```bash doc-ci:skip
 doc-ci run README.md
 doc-ci run docs/ --timeout 20 --json
 ```
 
 `run` exits non-zero if any snippet fails, so it works as a CI gate. Add
 `--json` to either command for machine-readable output.
 
+### Skipping a snippet
+
+An example that shouldn't be executed — it needs a real service, is purely
+illustrative, or is intentionally broken — can be marked so `doc-ci run`
+leaves it alone, without adding noise to the code readers copy. Either:
+
+- put `doc-ci:skip` in the fence info string, right after the language
+  (e.g. an opening fence of `` ```python doc-ci:skip ``), or
+- put `<!-- doc-ci:skip -->` on the line immediately above the fence.
+
+The marker never appears in the copyable code, and the snippet is reported as
+`directive-skip` instead of being run.
+
 ## Safety design (the rule that governs this project)
 
 Documentation snippets are untrusted input. When execution lands, it will be:

src/doc_ci/classifier.py

@@ -24,6 +24,7 @@
 PLACEHOLDER = "placeholder"
 UNSAFE = "unsafe"
 NEEDS_NETWORK = "needs-network"
+DIRECTIVE_SKIP = "directive-skip"
 
 #: Languages the (future) sandbox knows how to run, normalized.
 SUPPORTED_LANGS = {
@@ -105,6 +106,10 @@ def _first_match(code: str, patterns: list[re.Pattern]) -> str | None:
 
 def classify(snippet: Snippet) -> Classification:
     """Classify *snippet* without executing anything."""
+    # An explicit author directive wins over everything else.
+    if snippet.skip:
+        return Classification(DIRECTIVE_SKIP, "doc-ci:skip directive")
+
     lang = SUPPORTED_LANGS.get(snippet.lang)
     if lang is None:
         shown = snippet.lang or "none"

src/doc_ci/cli.py

@@ -78,19 +78,32 @@ def cmd_run(args: argparse.Namespace) -> int:
     if args.json:
         print(json.dumps([r.to_dict() for r in results], indent=2))
     else:
-        for r in results:
-            lang = r.snippet.lang or "no-lang"
-            loc = f"{r.snippet.path}:{r.snippet.start_line}-{r.snippet.end_line}"
-            print(f"{loc}  [{lang}]  {r.status.upper()}  ({r.reason})")
-        counts = summarize(results)
-        summary = ", ".join(f"{k}: {v}" for k, v in sorted(counts.items()))
-        print(f"\n{len(results)} snippet(s)" + (f" — {summary}" if summary else ""))
+        _print_run_report(results)
 
     # CI gate: fail on any failed/errored snippet, or on read errors.
-    failed = summarize(results).get("failed", 0) + summarize(results).get("error", 0)
+    counts = summarize(results)
+    failed = counts.get("failed", 0) + counts.get("error", 0)
     return 1 if (failed or errors) else 0
 
 
+def _print_run_report(results: list) -> None:
+    for r in results:
+        lang = r.snippet.lang or "no-lang"
+        loc = f"{r.snippet.path}:{r.snippet.start_line}-{r.snippet.end_line}"
+        print(f"{r.status.upper():7} {loc}  [{lang}]  {r.reason}")
+        # For real failures, show why: the captured error output, indented.
+        if r.status in ("failed", "error") and r.outcome is not None:
+            detail = (r.outcome.stderr or r.outcome.stdout or "").strip()
+            if detail:
+                for line in detail.splitlines()[-8:]:
+                    print(f"        | {line}")
+
+    counts = summarize(results)
+    summary = ", ".join(f"{k}: {v}" for k, v in sorted(counts.items()))
+    verdict = "FAIL" if (counts.get("failed", 0) or counts.get("error", 0)) else "OK"
+    print(f"\n[{verdict}] {len(results)} snippet(s)" + (f" — {summary}" if summary else ""))
+
+
 def build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
         prog="doc-ci",

src/doc_ci/extractor.py

@@ -5,12 +5,24 @@
 whose first word is treated as the language, closed by a fence of the same
 character at least as long as the opener. Unclosed fences run to end of file,
 matching how most renderers display them.
+
+Authors can mark a snippet to be skipped (never executed by `doc-ci run`)
+without polluting the copyable code, in either of two ways:
+
+    ```python doc-ci:skip            (annotation in the fence info string)
+
+    <!-- doc-ci:skip -->             (HTML comment on the line above the fence)
+    ```python
 """
 
 from __future__ import annotations
 
+import re
 from dataclasses import asdict, dataclass
 
+# Matches the skip directive, tolerating a space after the colon, case-insensitive.
+_SKIP_DIRECTIVE = re.compile(r"doc-ci:\s*skip\b", re.IGNORECASE)
+
 
 @dataclass
 class Snippet:
@@ -21,11 +33,17 @@ class Snippet:
     code: str
     start_line: int  # 1-based line number of the opening fence
     end_line: int  # 1-based line number of the closing fence (or last line if unclosed)
+    info: str = ""  # full info string after the opening fence (lang + any annotations)
+    skip: bool = False  # author asked to skip execution via a doc-ci:skip directive
 
     def to_dict(self) -> dict:
         return asdict(self)
 
 
+def _has_skip_directive(text: str) -> bool:
+    return bool(_SKIP_DIRECTIVE.search(text))
+
+
 def _fence_open(line: str) -> tuple[str, int, str] | None:
     """Return (fence_char, fence_length, info_string) if *line* opens a fence."""
     stripped = line.lstrip(" ")
@@ -62,6 +80,8 @@ def extract_snippets(text: str, path: str = "<string>") -> list[Snippet]:
     fence_char = ""
     fence_len = 0
     lang = ""
+    info = ""
+    skip = False
     buf: list[str] = []
     start_line = 0
     last_line = 0
@@ -73,6 +93,9 @@ def extract_snippets(text: str, path: str = "<string>") -> list[Snippet]:
             if opened is not None:
                 fence_char, fence_len, info = opened
                 lang = info.split()[0].lower() if info else ""
+                # Skip directive: in the info string, or on the preceding line.
+                prev = lines[lineno - 2] if lineno >= 2 else ""
+                skip = _has_skip_directive(info) or _has_skip_directive(prev)
                 in_fence = True
                 buf = []
                 start_line = lineno
@@ -85,6 +108,8 @@ def extract_snippets(text: str, path: str = "<string>") -> list[Snippet]:
                         code="\n".join(buf) + ("\n" if buf else ""),
                         start_line=start_line,
                         end_line=lineno,
+                        info=info,
+                        skip=skip,
                     )
                 )
                 in_fence = False
@@ -99,6 +124,8 @@ def extract_snippets(text: str, path: str = "<string>") -> list[Snippet]:
                 code="\n".join(buf) + ("\n" if buf else ""),
                 start_line=start_line,
                 end_line=last_line,
+                info=info,
+                skip=skip,
             )
         )
 

tests/test_classifier.py

@@ -1,6 +1,7 @@
 """Tests for the snippet classifier. Nothing here executes snippet content."""
 
 from doc_ci.classifier import (
+    DIRECTIVE_SKIP,
     NEEDS_NETWORK,
     PLACEHOLDER,
     RUNNABLE,
@@ -15,6 +16,20 @@ def snip(code: str, lang: str = "bash") -> Snippet:
     return Snippet(path="x.md", lang=lang, code=code, start_line=1, end_line=2)
 
 
+def test_skip_directive_wins_over_everything():
+    # Even an otherwise-runnable snippet is directive-skip when marked.
+    s = Snippet(path="x.md", lang="bash", code="echo hi\n",
+                start_line=1, end_line=2, skip=True)
+    assert classify(s).category == DIRECTIVE_SKIP
+
+
+def test_skip_directive_wins_over_unsafe():
+    # The directive short-circuits before any other check.
+    s = Snippet(path="x.md", lang="bash", code="rm -rf /\n",
+                start_line=1, end_line=2, skip=True)
+    assert classify(s).category == DIRECTIVE_SKIP
+
+
 def test_plain_echo_is_runnable():
     assert classify(snip("echo hello\n")).category == RUNNABLE
 

tests/test_extractor.py

@@ -95,3 +95,44 @@ def test_crlf_tolerance():
     snips = extract_snippets(text)
     assert len(snips) == 1
     assert "echo hi" in snips[0].code
+
+
+def test_info_string_captured():
+    text = "```python title=example.py\nx = 1\n```\n"
+    snips = extract_snippets(text)
+    assert snips[0].info == "python title=example.py"
+    assert snips[0].lang == "python"
+
+
+def test_no_skip_by_default():
+    text = "```bash\necho hi\n```\n"
+    assert extract_snippets(text)[0].skip is False
+
+
+def test_skip_directive_in_info_string():
+    text = "```bash doc-ci:skip\necho hi\n```\n"
+    snips = extract_snippets(text)
+    assert snips[0].skip is True
+    assert snips[0].lang == "bash"  # lang still parsed as first token
+
+
+def test_skip_directive_in_preceding_html_comment():
+    text = "<!-- doc-ci:skip -->\n```bash\necho hi\n```\n"
+    snips = extract_snippets(text)
+    assert snips[0].skip is True
+
+
+def test_skip_directive_tolerates_space_after_colon():
+    text = "```bash doc-ci: skip\necho hi\n```\n"
+    assert extract_snippets(text)[0].skip is True
+
+
+def test_skip_directive_case_insensitive():
+    text = "```bash DOC-CI:SKIP\necho hi\n```\n"
+    assert extract_snippets(text)[0].skip is True
+
+
+def test_directive_two_lines_above_does_not_apply():
+    # Only the immediately preceding line counts, to stay predictable.
+    text = "<!-- doc-ci:skip -->\n\n```bash\necho hi\n```\n"
+    assert extract_snippets(text)[0].skip is False

tests/test_runner.py

@@ -92,6 +92,16 @@ def test_unsupported_language_is_skipped():
     assert box.calls == []
 
 
+def test_skip_directive_snippet_is_not_executed():
+    box = FakeSandbox()
+    marked = Snippet(path="x.md", lang="bash", code="echo hi\n",
+                     start_line=1, end_line=2, skip=True)
+    results = run_snippets([marked], box)
+    assert results[0].status == SKIPPED
+    assert "directive-skip" in results[0].reason
+    assert box.calls == []  # author said skip; never ran
+
+
 def test_runnable_snippet_skipped_when_no_sandbox():
     box = FakeSandbox(available=False)
     results = run_snippets([snip("echo hi\n")], box)