Handle base64 encoded data atom in the tokenizer

src/kicad/tokenizer.ts

@@ -64,6 +64,7 @@ function is_atom(c: string) {
             "=",
             "~",
             "$",
+            "|",
         ].includes(c)
     );
 }
@@ -109,7 +110,10 @@ export function* tokenize(input: string) {
                 state = State.number;
                 start_idx = i;
                 continue;
-            } else if (is_alpha(c) || ["*", "&", "$", "/", "%"].includes(c)) {
+            } else if (
+                is_alpha(c) ||
+                ["*", "&", "$", "/", "%", "|"].includes(c)
+            ) {
                 state = State.atom;
                 start_idx = i;
                 continue;

test/kicad/tokenizer.test.ts

@@ -123,6 +123,31 @@ suite("kicad.tokenizer.tokenize(): s-expression tokenizer", function () {
             CLOSE_TOKEN,
         ]);
     });
+
+    test("with embedded data containing pipes", function () {
+        const tokens = tokenizer.tokenize(
+            "(data |KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI)",
+        );
+        assert_tokens(tokens, [
+            OPEN_TOKEN,
+            [ATOM, "data"],
+            [
+                ATOM,
+                "|KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI",
+            ],
+            CLOSE_TOKEN,
+        ]);
+    });
+
+    test("with pipe character in middle of atom", function () {
+        const tokens = tokenizer.tokenize("(test |middle|end)");
+        assert_tokens(tokens, [
+            OPEN_TOKEN,
+            [ATOM, "test"],
+            [ATOM, "|middle|end"],
+            CLOSE_TOKEN,
+        ]);
+    });
 });
 
 suite("kicad.tokenizer.listify()", function () {