From ce09645b2880930c3dc2f1ff9493b037dfa71bb0 Mon Sep 17 00:00:00 2001
From: Andy Shinn <andys@andyshinn.as>
Date: Wed, 2 Jul 2025 21:29:46 -0500
Subject: [PATCH 1/2] handle base64 encoded data atom in the tokenizer

---
 src/kicad/tokenizer.ts       |  3 ++-
 test/kicad/tokenizer.test.ts | 22 ++++++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/src/kicad/tokenizer.ts b/src/kicad/tokenizer.ts
index d6846ba1..f6d94f8d 100644
--- a/src/kicad/tokenizer.ts
+++ b/src/kicad/tokenizer.ts
@@ -64,6 +64,7 @@ function is_atom(c: string) {
             "=",
             "~",
             "$",
+            "|",
         ].includes(c)
     );
 }
@@ -109,7 +110,7 @@ export function* tokenize(input: string) {
                 state = State.number;
                 start_idx = i;
                 continue;
-            } else if (is_alpha(c) || ["*", "&", "$", "/", "%"].includes(c)) {
+            } else if (is_alpha(c) || ["*", "&", "$", "/", "%", "|"].includes(c)) {
                 state = State.atom;
                 start_idx = i;
                 continue;
diff --git a/test/kicad/tokenizer.test.ts b/test/kicad/tokenizer.test.ts
index e3cac66e..0f3ab942 100644
--- a/test/kicad/tokenizer.test.ts
+++ b/test/kicad/tokenizer.test.ts
@@ -123,6 +123,28 @@ suite("kicad.tokenizer.tokenize(): s-expression tokenizer", function () {
             CLOSE_TOKEN,
         ]);
     });
+
+    test("with embedded data containing pipes", function () {
+        const tokens = tokenizer.tokenize(
+            "(data |KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI)",
+        );
+        assert_tokens(tokens, [
+            OPEN_TOKEN,
+            [ATOM, "data"],
+            [ATOM, "|KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI"],
+            CLOSE_TOKEN,
+        ]);
+    });
+
+    test("with pipe character in middle of atom", function () {
+        const tokens = tokenizer.tokenize("(test |middle|end)");
+        assert_tokens(tokens, [
+            OPEN_TOKEN,
+            [ATOM, "test"],
+            [ATOM, "|middle|end"],
+            CLOSE_TOKEN,
+        ]);
+    });
 });
 
 suite("kicad.tokenizer.listify()", function () {

From 3cec0ad3b8335af1ed29744e5942dfef8b3ab47d Mon Sep 17 00:00:00 2001
From: Andy Shinn <andys@andyshinn.as>
Date: Wed, 2 Jul 2025 21:47:29 -0500
Subject: [PATCH 2/2] ok prettier, i see you...

---
 src/kicad/tokenizer.ts       | 5 ++++-
 test/kicad/tokenizer.test.ts | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/kicad/tokenizer.ts b/src/kicad/tokenizer.ts
index f6d94f8d..7b4498b6 100644
--- a/src/kicad/tokenizer.ts
+++ b/src/kicad/tokenizer.ts
@@ -110,7 +110,10 @@ export function* tokenize(input: string) {
                 state = State.number;
                 start_idx = i;
                 continue;
-            } else if (is_alpha(c) || ["*", "&", "$", "/", "%", "|"].includes(c)) {
+            } else if (
+                is_alpha(c) ||
+                ["*", "&", "$", "/", "%", "|"].includes(c)
+            ) {
                 state = State.atom;
                 start_idx = i;
                 continue;
diff --git a/test/kicad/tokenizer.test.ts b/test/kicad/tokenizer.test.ts
index 0f3ab942..88ffc9a7 100644
--- a/test/kicad/tokenizer.test.ts
+++ b/test/kicad/tokenizer.test.ts
@@ -131,7 +131,10 @@ suite("kicad.tokenizer.tokenize(): s-expression tokenizer", function () {
         assert_tokens(tokens, [
             OPEN_TOKEN,
             [ATOM, "data"],
-            [ATOM, "|KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI"],
+            [
+                ATOM,
+                "|KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI",
+            ],
             CLOSE_TOKEN,
         ]);
     });