diff --git a/src/kicad/tokenizer.ts b/src/kicad/tokenizer.ts index d6846ba1..7b4498b6 100644 --- a/src/kicad/tokenizer.ts +++ b/src/kicad/tokenizer.ts @@ -64,6 +64,7 @@ function is_atom(c: string) { "=", "~", "$", + "|", ].includes(c) ); } @@ -109,7 +110,10 @@ export function* tokenize(input: string) { state = State.number; start_idx = i; continue; - } else if (is_alpha(c) || ["*", "&", "$", "/", "%"].includes(c)) { + } else if ( + is_alpha(c) || + ["*", "&", "$", "/", "%", "|"].includes(c) + ) { state = State.atom; start_idx = i; continue; diff --git a/test/kicad/tokenizer.test.ts b/test/kicad/tokenizer.test.ts index e3cac66e..88ffc9a7 100644 --- a/test/kicad/tokenizer.test.ts +++ b/test/kicad/tokenizer.test.ts @@ -123,6 +123,31 @@ suite("kicad.tokenizer.tokenize(): s-expression tokenizer", function () { CLOSE_TOKEN, ]); }); + + test("with embedded data containing pipes", function () { + const tokens = tokenizer.tokenize( + "(data |KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI)", + ); + assert_tokens(tokens, [ + OPEN_TOKEN, + [ATOM, "data"], + [ + ATOM, + "|KLUv/aCvzgcAAAAQiVBORw0KGgoAAAANSUhEUgAABiAAAANoCAYAAABJLCIrAAAABHNCSVQICAgI", + ], + CLOSE_TOKEN, + ]); + }); + + test("with pipe character in middle of atom", function () { + const tokens = tokenizer.tokenize("(test |middle|end)"); + assert_tokens(tokens, [ + OPEN_TOKEN, + [ATOM, "test"], + [ATOM, "|middle|end"], + CLOSE_TOKEN, + ]); + }); }); suite("kicad.tokenizer.listify()", function () {