Sync codebase

Author: hauntsaninja

src/lib.rs

@@ -455,7 +455,10 @@ impl CoreBPE {
         let decoder: HashMap<usize, Vec<u8>> =
             encoder.iter().map(|(k, v)| (*v, k.clone())).collect();
 
-        assert!(encoder.len() == decoder.len());
+        assert!(
+            encoder.len() == decoder.len(),
+            "Encoder and decoder must be of equal length; maybe you had duplicate token indices in your encoder?"
+        );
 
         let special_tokens_decoder: HashMap<usize, Vec<u8>> = special_tokens_encoder
             .iter()

tiktoken/load.py

@@ -21,7 +21,9 @@ def read_file(blobpath: str) -> bytes:
         with blobfile.BlobFile(blobpath, "rb") as f:
             return f.read()
     # avoiding blobfile for public files helps avoid auth issues, like MFA prompts
-    return requests.get(blobpath).content
+    resp = requests.get(blobpath)
+    resp.raise_for_status()
+    return resp.content
 
 
 def read_file_cached(blobpath: str) -> bytes: