Update src/datasets/utils/file_utils.py

Co-authored-by: Quentin Lhoest <[email protected]>
huggingface · Mar 10, 2023 · 7d491e1 · 7d491e1
1 parent 01ab083
commit 7d491e1
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/src/datasets/utils/file_utils.py b/src/datasets/utils/file_utils.py
@@ -501,7 +501,8 @@ def get_from_cache(
         elif scheme not in ("http", "https"):
             response = fsspec_head(url)
             # use the hash of the response as a pseudo ETag to detect changes
-            etag = json.dumps(response, sort_keys=True) if use_etag else None
+            # s3fs uses "ETag", gcsfs uses "etag"
+            etag = (response.get("ETag", None) or response.get("etag", None)) if use_etag else None
             connected = True
         try:
             response = http_head(