Skip to content

Commit 6ff9598

Browse files
authored
Remove old tokenizer/ directory in ExecuTorch
Differential Revision: D72007597 Pull Request resolved: #39
1 parent 567473d commit 6ff9598

File tree

6 files changed

+5
-87
lines changed

6 files changed

+5
-87
lines changed

pytorch_tokenizers/tools/__init__.py

Whitespace-only changes.

pytorch_tokenizers/tools/llama2c/convert.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
import argparse
1313

14-
from pytorch_tokenizers import Llama2cTokenizer
14+
from pytorch_tokenizers.llama2c import Llama2cTokenizer
1515

1616

1717
if __name__ == "__main__":

pytorch_tokenizers/tools/llama2c/targets.bzl

+3
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ def define_common_targets():
2424
external_deps = [
2525
"sentencepiece-py",
2626
],
27+
deps = [
28+
"//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
29+
],
2730
)
2831

2932
runtime.python_binary(

setup.py

+1-2
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,5 @@
1414
version="0.1.0",
1515
long_description=long_description,
1616
long_description_content_type="text/markdown",
17-
packages=find_packages(where="pytorch_tokenizers"),
18-
package_dir={"": "pytorch_tokenizers"},
17+
packages=find_packages(),
1918
)

tools/llama2c/convert.py

-52
This file was deleted.

tools/llama2c/targets.bzl

-32
This file was deleted.

0 commit comments

Comments
 (0)