PERF: Speed up the BBCode conversion hot paths

AGENTS.md

@@ -207,8 +207,10 @@ expect(token).to match_tag_end("b")
   add it to the `mutant.yml` ignore list (with the required comment
   from the skill) — don't reach behind the curtain.
 - No stubbing or mocking the SUT (the class currently being mutated).
-- `MarkdownEscaper` is a hot path. Benchmark (`bundle exec ruby --yjit /tmp/bench_escaper.rb`)
-  before/after any change to `lib/markbridge/renderers/discourse/markdown_escaper.rb`.
+- `MarkdownEscaper` is a hot path. Benchmark before/after any change to
+  `lib/markbridge/renderers/discourse/markdown_escaper.rb` with the
+  isolated escape reports (`bundle exec ruby --yjit bench/bench.rb
+  --single escape_plain` and `--single escape_mixed`).
   Tests over refactors when behavior is equivalent.
 - When writing a `mutant.yml` `ignore` entry per the skill's
   "Unkillable" flow, the inline comment must name the specific
@@ -221,16 +223,30 @@ Tests-only changes can be one commit.**
 ## Performance Notes
 
 **Scanner** (performance-critical):
-- Use index-based access: `@input[@pos]`, not `@input[@pos..@pos]`
+- Byte offsets everywhere: `byteindex`/`byteslice`/`getbyte`, never
+  character indices (`str[pos]` is O(pos) on multibyte input — CRuby has
+  no character-index cache) and never per-char probe strings
+- Every offset must sit on a character boundary (held structurally:
+  jumps land on ASCII matches, advances step over ASCII bytes)
+- Classify bytes with integer range checks, not regexes; `byteindex`
+  returns 0 for a match at the start — nil-check, don't truthiness-check
 - Bounded backtracking: save position, restore on failure
-- Minimize allocations: reuse strings
-- Regex only for character classes
+- Measure with `bench/corpus_bench.rb` (isolating variants) and
+  `bench/alloc_profile.rb` (per-line allocation ranking) before and
+  after changes; always compare the ascii and multi corpora
+
+**Parsing/rendering setup**:
+- `HandlerRegistry.shared_default` / `TagLibrary.shared_default` are
+  built once per process (deep-frozen); the no-customization paths use
+  them. Never mutate them — build a fresh `.default` (or `dup`) for
+  customization
 
 **AST**:
 - Text nodes auto-merge (reduces tree size)
+- `AST::Text` shares frozen input strings (copy-on-write; `merge` dups
+  before its first append) — pass frozen strings on hot paths
 
 **Renderer**:
-- RenderContext uses hash-based cache (O(1) parent lookups)
 - Single-pass tree traversal
 - In-memory rendering (stream large documents yourself)
 

bench/alloc_profile.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+# Allocation profiler for the conversion pipelines.
+#
+#   bundle exec ruby bench/alloc_profile.rb [variant] [ascii|multi]
+#
+# Two instruments:
+#
+# 1. Headline: GC.stat(:total_allocated_objects) delta per post, averaged
+#    over the whole corpus. Use this number to compare branches.
+# 2. Detail: ObjectSpace.trace_object_allocations with GC disabled,
+#    grouped by file:line and by class, ranked. This names the exact
+#    allocation site — profile before optimizing, the ranking is usually
+#    not what intuition predicts.
+#
+# Note: wall-clock benchmarks understate allocation wins — fewer
+# allocations also mean fewer GC cycles under real workloads.
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+require "markbridge/all"
+require "objspace"
+require_relative "corpus"
+
+# ASCII/multibyte corpus pair per source format.
+CORPORA = { bbcode: -> { [Corpus.ascii, Corpus.multibyte] } }.freeze
+
+# Each variant names the corpus pair it runs on and a builder that
+# receives the corpus and returns the per-post work lambda.
+VARIANTS = {
+  "fresh" => [:bbcode, ->(corpus) { ->(i) { Markbridge.bbcode_to_markdown(corpus[i]) } }],
+  "shared" => [
+    :bbcode,
+    lambda do |corpus|
+      handlers = Markbridge::Parsers::BBCode::HandlerRegistry.default
+      renderer = Markbridge::Renderers::Discourse::Renderer.new
+      ->(i) { Markbridge.bbcode_to_markdown(corpus[i], handlers:, renderer:) }
+    end,
+  ],
+  "parse_only" => [
+    :bbcode,
+    lambda do |corpus|
+      parser = Markbridge::Parsers::BBCode::Parser.new
+      ->(i) { parser.parse(corpus[i]) }
+    end,
+  ],
+  "render_only" => [
+    :bbcode,
+    lambda do |corpus|
+      parser = Markbridge::Parsers::BBCode::Parser.new
+      asts = corpus.map { |post| parser.parse(post) }
+      renderer = Markbridge::Renderers::Discourse::Renderer.new
+      ->(i) { renderer.postprocessor.call(renderer.render(asts[i])) }
+    end,
+  ],
+}.freeze
+
+variant = ARGV[0] || "fresh"
+which = ARGV[1] || "ascii"
+
+corpus_key, builder = VARIANTS.fetch(variant) { raise ArgumentError, "unknown variant #{variant}" }
+ascii, multibyte = CORPORA.fetch(corpus_key).call
+corpus = which == "multi" ? multibyte : ascii
+work = builder.call(corpus)
+n = corpus.size
+
+# Warm up (shared-default memoization, autoloads)
+3.times { |k| work.call(k) }
+
+GC.start
+before = GC.stat(:total_allocated_objects)
+n.times { |i| work.call(i) }
+after = GC.stat(:total_allocated_objects)
+puts "#{variant}/#{which}: #{(after - before) / n} objects/post (#{n} posts)"
+
+GC.start
+GC.disable
+ObjectSpace.trace_object_allocations { 20.times { |i| work.call(i) } }
+
+site_counts = Hash.new(0)
+class_counts = Hash.new(0)
+ObjectSpace.each_object do |obj|
+  file = ObjectSpace.allocation_sourcefile(obj)
+  next unless file&.include?("markbridge")
+
+  line = ObjectSpace.allocation_sourceline(obj)
+  site_counts["#{file.sub(%r{.*/markbridge/}, "")}:#{line}"] += 1
+  class_counts[obj.class.to_s] += 1
+end
+GC.enable
+
+puts "\nTop allocation sites (20 posts):"
+site_counts.sort_by { |_, c| -c }.first(40).each { |site, c| puts format("  %6d  %s", c, site) }
+puts "\nTop classes:"
+class_counts.sort_by { |_, c| -c }.first(15).each { |k, c| puts format("  %6d  %s", c, k) }

bench/bench.rb

@@ -8,7 +8,7 @@
 #   bundle exec ruby --yjit bench/bench.rb --isolated # one process per report
 #
 # Suite mode runs all reports in one process. It's faster to run
-# but YJIT compiles 12 hot paths concurrently, so they compete for
+# but YJIT compiles all report paths concurrently, so they compete for
 # the JIT budget and inline-decision heuristics — the resulting
 # numbers tend to under-report per-path throughput, especially on
 # the inline-escape path.
@@ -79,29 +79,6 @@
       "Text with *asterisks* and _underscores_ and `backticks` and [brackets] and |pipes|" \
         "\n\n# Not a heading\n---\nnot a rule",
   },
-  "mixed" => {
-    type: :convert,
-    input: [
-      "[b]bold[/b]",
-      "[quote]x[/quote]",
-      "[list]\n[*]a\n[*]b\n[/list]",
-      "[table][tr][td]1[/td][td]2[/td][/tr][/table]",
-      "[code]x[/code]",
-    ].join("\n\n"),
-  },
-  "large_doc" => {
-    type: :convert,
-    input:
-      (
-        [
-          "[b]bold[/b]",
-          "[quote]x[/quote]",
-          "[list]\n[*]a\n[*]b\n[/list]",
-          "[table][tr][td]1[/td][td]2[/td][/tr][/table]",
-          "[code]x[/code]",
-        ].join("\n\n") + "\n\n"
-      ) * 20,
-  },
   "escape_plain" => {
     type: :escape,
     input: "This is plain text with no special chars. " * 100,

bench/corpus.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+# Deterministic synthetic corpus of forum-post-shaped BBCode documents
+# for bench/corpus_bench.rb and bench/alloc_profile.rb.
+#
+# Two variants with the same structure: ASCII prose and multibyte prose.
+# The multibyte variant exists because character-index string operations
+# in CRuby degrade superlinearly on multibyte input — a corpus without it
+# hides that entire failure mode.
+module Corpus
+  WORDS_ASCII = %w[
+    the
+    quick
+    brown
+    fox
+    jumps
+    over
+    lazy
+    dog
+    while
+    some
+    other
+    words
+    fill
+    sentences
+    about
+    servers
+    databases
+    migration
+    threads
+    posts
+    users
+    forum
+    upgrade
+    version
+    plugin
+    theme
+    category
+    topic
+    reply
+    moderator
+    admin
+  ].freeze
+
+  WORDS_MULTI = %w[
+    das
+    schöne
+    Mädchen
+    läuft
+    über
+    die
+    Straße
+    größer
+    kleiner
+    Übung
+    café
+    naïve
+    résumé
+    Zürich
+    München
+    Österreich
+    straße
+    äöü
+    éàè
+    日本語
+    テスト
+    漢字
+    こんにちは
+    世界
+    привет
+    мир
+    тест
+  ].freeze
+
+  module_function
+
+  def sentence(rng, words, count)
+    Array.new(count) { words[rng.rand(words.size)] }.join(" ")
+  end
+
+  def paragraph(rng, words)
+    "#{Array.new(2 + rng.rand(3)) { sentence(rng, words, 8 + rng.rand(10)) }.join(". ")}."
+  end
+
+  # A post: several paragraphs, sprinkled with BBCode constructs.
+  def post(rng, words)
+    parts = []
+    parts << paragraph(rng, words)
+    parts << "[b]#{sentence(rng, words, 4)}[/b] and [i]#{sentence(rng, words, 3)}[/i]"
+    parts << paragraph(rng, words)
+    parts << construct(rng, words)
+    parts << paragraph(rng, words)
+    parts.join("\n\n")
+  end
+
+  def construct(rng, words)
+    case rng.rand(6)
+    when 0
+      "[quote=\"alice\"]\n#{paragraph(rng, words)}\n[/quote]"
+    when 1
+      items = Array.new(3 + rng.rand(3)) { "[*]#{sentence(rng, words, 5)}" }
+      "[list]\n#{items.join("\n")}\n[/list]"
+    when 2
+      "[code]\ndef hello\n  puts 'x = 1'\nend\n[/code]"
+    when 3
+      "See [url=https://example.com/#{rng.rand(1000)}]#{sentence(rng, words, 3)}[/url] for details."
+    when 4
+      "[table]\n[tr][th]A[/th][th]B[/th][/tr]\n" \
+        "[tr][td]#{words[rng.rand(words.size)]}[/td][td]2[/td][/tr]\n[/table]"
+    when 5
+      "Text with *asterisks*, _underscores_ and -- dashes. #{paragraph(rng, words)}"
+    end
+  end
+
+  def build(words, count: 200, seed: 42)
+    rng = Random.new(seed)
+    Array.new(count) { post(rng, words) }
+  end
+
+  def ascii
+    @ascii ||= build(WORDS_ASCII)
+  end
+
+  def multibyte
+    @multibyte ||= build(WORDS_MULTI)
+  end
+end