Split change key

jackzhxng · jackzhxng · commit 7898f9fb85dc · 2025-03-27T10:06:05.000-07:00
diff --git a/src/pre_tokenizer.cpp b/src/pre_tokenizer.cpp
@@ -78,7 +78,7 @@ PreTokenizerConfig& PreTokenizerConfig::parse_json(const json& json_config) {
   type = json_config.at("type");
   if (type == "Split") {
     try {
-      pattern = json_config.at("pattern");
+      pattern = json_config.at("pattern").at("Regex");
     } catch (json::out_of_range&) {
     }
   } else if (type == "Digits") {

Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ PreTokenizerConfig& PreTokenizerConfig::parse_json(const json& json_config) {`
`78`	`78`	`type = json_config.at("type");`
`79`	`79`	`if (type == "Split") {`
`80`	`80`	`try {`
`81`		`- pattern = json_config.at("pattern");`
	`81`	`+ pattern = json_config.at("pattern").at("Regex");`
`82`	`82`	`} catch (json::out_of_range&) {`
`83`	`83`	`}`
`84`	`84`	`} else if (type == "Digits") {`