fix decode error (#618)

* * fix decode error

* * start the coverage earlier

Author: HYLcool

data_juicer/utils/constant.py

@@ -172,14 +172,15 @@ def get_access_log(cls, dj_cfg=None, dataset=None):
                 elif 'jsonl' in dj_cfg.dataset_path:
                     tmp_f_name = dj_cfg.dataset_path. \
                         replace('.jsonl', '.tmp.jsonl')
-                    with open(dj_cfg.dataset_path, 'r') as orig_file:
+                    with open(dj_cfg.dataset_path, 'r',
+                              encoding='utf-8') as orig_file:
                         first_line = orig_file.readline()
 
                 assert tmp_f_name is not None and first_line is not None, \
                     'error when loading the first line, when ' \
                     f'dj_cfg.dataset_path={dj_cfg.dataset_path}'
 
-                with open(tmp_f_name, 'w') as tmp_file:
+                with open(tmp_f_name, 'w', encoding='utf-8') as tmp_file:
                     tmp_file.write(first_line)
 
                 tmp_dj_cfg.dataset_path = tmp_f_name

tests/run.py

@@ -12,6 +12,10 @@
 import unittest
 import coverage
 
+# start the coverage immediately
+cov = coverage.Coverage(include='data_juicer/**')
+cov.start()
+
 from loguru import logger
 
 from data_juicer.utils.unittest_utils import set_clear_model_flag, get_partial_test_cases
@@ -91,9 +95,7 @@ def gather_test_cases(test_dir, pattern, tag, mode='partial'):
 
 
 def main():
-    cov = coverage.Coverage(include='data_juicer/**')
-    cov.start()
-
+    global cov
     runner = unittest.TextTestRunner()
     test_suite = gather_test_cases(os.path.abspath(args.test_dir),
                                    args.pattern, args.tag, args.mode)