Remove old tokenizer/ directory in ExecuTorch (#9728)

Summary:
X-link: pytorch-labs/tokenizers#39

Pull Request resolved: #9728

See what happens

Differential Revision: D72007597

Author: larryliu0820

.ci/scripts/test_phi_3_mini.sh

@@ -56,7 +56,7 @@ cmake_build_phi_3_mini() {
 prepare_tokenizer() {
   echo "Downloading and converting tokenizer.model"
   wget -O tokenizer.model "https://huggingface.co/microsoft/Phi-3-mini-128k-instruct/resolve/main/tokenizer.model?download=true"
-  $PYTHON_EXECUTABLE -m executorch.extension.llm.tokenizer.tokenizer -t tokenizer.model -o tokenizer.bin
+  $PYTHON_EXECUTABLE -m pytorch_tokenizers.tools.llama2c.convert -t tokenizer.model -o tokenizer.bin
 }
 
 # Export phi-3-mini model to pte

CMakeLists.txt

@@ -751,7 +751,7 @@ if(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR)
 endif()
 
 if(EXECUTORCH_BUILD_EXTENSION_LLM)
-  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizer)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/llm/tokenizers)
 endif()
 
 if(EXECUTORCH_BUILD_EXTENSION_MODULE)

examples/models/llama/TARGETS

@@ -202,7 +202,7 @@ runtime.python_library(
         ":export_library",
         "//executorch/examples/models/llama/tokenizer:tiktoken_py",
         "//executorch/extension/llm/export:export_lib",
-        "//executorch/extension/llm/tokenizer:tokenizer_py_lib",
+        "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
         "//executorch/extension/pybindings:portable_lib",
     ],
 )

examples/models/llama/eval_llama_lib.py

@@ -15,13 +15,13 @@
 from executorch.examples.models.llama.export_llama_lib import (
     get_quantizer_and_quant_params,
 )
-from executorch.examples.models.llama.tokenizer.tiktoken import Tokenizer as Tiktoken
+from pytorch_tokenizers.tiktoken import TiktokenTokenizer as Tiktoken
 
 from executorch.extension.llm.export.builder import LLMEdgeManager
-from executorch.extension.llm.tokenizer.tokenizer import (
-    Tokenizer as SentencePieceTokenizer,
+from pytorch_tokenizers.llama2c import (
+    Llama2cTokenizer as SentencePieceTokenizer,
 )
-from executorch.extension.llm.tokenizer.utils import get_tokenizer
+from pytorch_tokenizers import get_tokenizer
 from lm_eval.evaluator import simple_evaluate
 from torch.nn import CrossEntropyLoss
 from tqdm import tqdm

examples/models/llama/evaluate/eager_eval.py

@@ -8,10 +8,10 @@
 from typing import Optional, Union
 
 import torch
-from executorch.examples.models.llama.tokenizer.tiktoken import Tokenizer as Tiktoken
-from executorch.extension.llm.tokenizer.tokenizer import (
-    Tokenizer as SentencePieceTokenizer,
+from pytorch_tokenizers.llama2c import (
+    Llama2cTokenizer as SentencePieceTokenizer,
 )
+from pytorch_tokenizers.tiktoken import TiktokenTokenizer as Tiktoken
 
 from lm_eval.models.huggingface import HFLM as eval_wrapper
 

examples/models/llama/runner/generation.py

@@ -10,7 +10,7 @@
 
 import torch
 
-from executorch.extension.llm.tokenizer.utils import get_tokenizer
+from pytorch_tokenizers import get_tokenizer
 
 
 def sample_top_p(probs, p):

examples/models/llama/tokenizer/targets.bzl

@@ -16,7 +16,6 @@ def define_common_targets():
         ],
         exported_deps = [
             "//pytorch/tokenizers:tiktoken",
-            "//executorch/extension/llm/tokenizer:tiktoken", # TODO: remove
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",

examples/models/llama/tokenizer/test/test_tiktoken.cpp

@@ -10,7 +10,7 @@
 
 #include <vector>
 
-#include <executorch/extension/llm/tokenizer/tiktoken.h>
+#include <pytorch/tokenizers/tiktoken.h>
 
 #include <gtest/gtest.h>
 
@@ -21,9 +21,9 @@
 using namespace ::testing;
 
 using ::example::Version;
-using ::executorch::extension::llm::Tokenizer;
-using ::executorch::runtime::Error;
-using ::executorch::runtime::Result;
+using ::tokenizers::Tokenizer;
+using ::tokenizers::Error;
+using ::tokenizers::Result;
 
 static std::string get_resource_path(const std::string& name) {
 #ifdef EXECUTORCH_FB_BUCK
@@ -36,7 +36,7 @@ static std::string get_resource_path(const std::string& name) {
 class MultimodalTiktokenV5ExtensionTest : public Test {
  public:
   void SetUp() override {
-    tokenizer_ = std::make_unique<executorch::extension::llm::Tiktoken>(
+    tokenizer_ = std::make_unique<tokenizers::Tiktoken>(
         example::get_multimodal_special_tokens(), 0, 1);
     modelPath_ = get_resource_path("test_tiktoken_tokenizer.model");
   }

examples/models/llava/export_llava.py

@@ -46,8 +46,8 @@
 )
 
 from executorch.extension.llm.export.builder import DType, LLMEdgeManager
-from executorch.extension.llm.tokenizer.tokenizer import Tokenizer
 from executorch.util.activation_memory_profiler import generate_memory_trace
+from pytorch_tokenizers.llama2c import Llama2cTokenizer as Tokenizer
 from torch.export import Dim
 from torch.nn.attention import SDPBackend
 

examples/qualcomm/oss_scripts/llama/CMakeLists.txt

@@ -5,7 +5,7 @@
 # LICENSE file in the root directory of this source tree.
 
 # model sharding with custom op
-set(CUSTOM_OP_SRCS_FILE 
+set(CUSTOM_OP_SRCS_FILE
   "${EXECUTORCH_SOURCE_DIR}/extension/llm/custom_ops/op_fallback.cpp"
 )
 add_library(custom_ops ${CUSTOM_OP_SRCS_FILE})
@@ -35,7 +35,7 @@ list(
 list(
   APPEND
   _llama_runner__srcs
-  ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizer/tiktoken.cpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/../../../../extension/llm/tokenizers/src/tiktoken.cpp
   ${CMAKE_CURRENT_SOURCE_DIR}/../../../models/llama/tokenizer/llama_tiktoken.cpp
 )
 

extension/llm/export/TARGETS

@@ -40,6 +40,6 @@ runtime.python_library(
         "//executorch/exir:lib",
         "//executorch/exir/backend:backend_details",
         "//executorch/extension/export_util:export_util",
-        "//executorch/extension/llm/tokenizer:tokenizer_py_lib",
+        "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
     ],
 )

extension/llm/export/builder.py

@@ -35,7 +35,7 @@
 from executorch.extension.export_util.utils import export_to_edge, save_pte_program
 
 from executorch.extension.llm.export.export_passes import RemoveRedundantTransposes
-from executorch.extension.llm.tokenizer.utils import get_tokenizer
+from pytorch_tokenizers import get_tokenizer
 from torch.ao.quantization.quantize_pt2e import convert_pt2e, prepare_pt2e
 from torch.ao.quantization.quantizer import Quantizer
 from torch.ao.quantization.quantizer.composable_quantizer import ComposableQuantizer