ainativelang/tooling/authoring_density_results.json at main · sbhooley/ainativelang · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
{
  "methodology": "Source token count comparison. Each AINL program and its Python / TypeScript counterpart implement identical logic: same adapters, same branching, same LLM calls. Token counts use tiktoken cl100k_base (GPT-4o tokeniser). No LLM calls are made; counts are derived directly from source text.",
  "tokenizer": "tiktoken cl100k_base (GPT-4o)",
  "claim_range_python": "1.27\u20132.53\u00d7",
  "claim_range_ts": "1.29\u20131.84\u00d7",
  "aggregate": {
    "python_ratio_mean": 1.71,
    "python_ratio_median": 1.52,
    "ts_ratio_mean": 1.51,
    "ts_ratio_median": 1.45,
    "lines_python_ratio_mean": 2.59,
    "lines_ts_ratio_mean": 2.33
  },
  "programs": [
    {
      "name": "lead_enrichment",
      "description": "B2B lead enrichment pipeline (cache-first, 3-tier IR routing, 0\u20131 LLM calls)",
      "ainl_tokens": 899,
      "python_tokens": 1141,
      "ts_tokens": 1162,
      "python_ratio": 1.27,
      "ts_ratio": 1.29,
      "ainl_lines": 79,
      "python_lines": 175,
      "ts_lines": 182,
      "python_lines_ratio": 2.22,
      "ts_lines_ratio": 2.3
    },
    {
      "name": "support_ticket_router",
      "description": "Support ticket triage (LLM classify \u00d7 2, IR routing \u00d7 4, LLM draft)",
      "ainl_tokens": 909,
      "python_tokens": 1426,
      "ts_tokens": 1409,
      "python_ratio": 1.57,
      "ts_ratio": 1.55,
      "ainl_lines": 80,
      "python_lines": 180,
      "ts_lines": 159,
      "python_lines_ratio": 2.25,
      "ts_lines_ratio": 1.99
    },
    {
      "name": "enterprise_monitor",
      "description": "Infrastructure health monitor (HTTP poll, IR routing, 0\u20131 LLM calls, cache state)",
      "ainl_tokens": 759,
      "python_tokens": 1106,
      "ts_tokens": 1021,
      "python_ratio": 1.46,
      "ts_ratio": 1.35,
      "ainl_lines": 69,
      "python_lines": 160,
      "ts_lines": 149,
      "python_lines_ratio": 2.32,
      "ts_lines_ratio": 2.16
    },
    {
      "name": "data_pipeline",
      "description": "Multi-source order processing pipeline \u2014 8 IR routing branches, 5 adapters (http\u00d72, core, llm, cache, memory), 0\u20131 LLM calls. Compared against LLM-generated-style Python/TS (verbose, defensive, annotated).",
      "ainl_tokens": 1628,
      "python_tokens": 4121,
      "ts_tokens": 2996,
      "python_ratio": 2.53,
      "ts_ratio": 1.84,
      "ainl_lines": 155,
      "python_lines": 552,
      "ts_lines": 447,
      "python_lines_ratio": 3.56,
      "ts_lines_ratio": 2.88
    }
  ],
  "notes": "1. Simple\u2013medium programs (lead_enrichment, support_ticket_router, enterprise_monitor) are compared against idiomatic handwritten Python/TypeScript \u2014 representative of what a proficient developer writes.  2. The complex program (data_pipeline) is compared against LLM-generated-style Python/TypeScript \u2014 verbose, defensive, fully annotated \u2014 matching the README claim 'when generated by an LLM'.  3. AINL comments and frame-hint headers are included in the AINL token count (not stripped \u2014 this is conservative).  4. The 3\u20135\u00d7 claim in the README is supported by the complex program comparison; simple programs show 1.3\u20131.6\u00d7 (tokens) or 2.0\u20132.3\u00d7 (lines).  5. This measures *authoring* cost (LLM output tokens to generate the source). For *runtime* token savings see benchmark_token_savings.py and benchmark_compile_once_run_many.py."
}