Pin the version of tokenizers on macOS Python3.6 to compile it

Author: hankcs

hanlp/version.py

@@ -2,7 +2,7 @@
 # Author: hankcs
 # Date: 2019-12-28 19:26
 
-__version__ = '2.1.0-beta.53'
+__version__ = '2.1.0-beta.54'
 """HanLP version"""
 
 

setup.py

@@ -13,11 +13,16 @@
     exec(fp.read(), version)
 
 FASTTEXT = 'fasttext-wheel==0.9.2'
-if sys.version_info >= (3, 10):
+sys_version_info = sys.version_info
+if sys_version_info >= (3, 10):
     TENSORFLOW = ['tensorflow>=2.8.0']
 else:
     TENSORFLOW = ['tensorflow==2.6.0', 'keras==2.6.0', 'protobuf<3.19']
 
+TOKENIZERS = []
+if (sys_version_info.major, sys_version_info.minor) == (3, 6) and sys.platform == 'darwin':
+    TOKENIZERS = ['tokenizers==0.10.3']
+
 extras_require = {
     'amr': [
         'penman==1.2.1',
@@ -66,6 +71,7 @@
         'hanlp-common>=0.0.19',
         'hanlp-trie>=0.0.4',
         'hanlp-downloader',
+        *TOKENIZERS,
     ],
     extras_require=extras_require,
     python_requires='>=3.6',