Modify is_remote_filesystem to return True for FUSE-mounted paths

src/datasets/builder.py

@@ -54,6 +54,7 @@
 from .download.streaming_download_manager import StreamingDownloadManager, xopen
 from .features import Features
 from .filesystems import (
+    is_fuse_mounted_path,
     is_remote_filesystem,
     rename,
 )
@@ -757,6 +758,8 @@ def download_and_prepare(
         # output_dir can be a remote bucket on GCS or S3 (when using BeamBasedBuilder for distributed data processing)
         fs_token_paths = fsspec.get_fs_token_paths(output_dir, storage_options=storage_options)
         self._fs: fsspec.AbstractFileSystem = fs_token_paths[0]
+        if is_fuse_mounted_path(output_dir):
+            self._fs = create_fuse_file_system(self._fs)
         is_local = not is_remote_filesystem(self._fs)
         self._output_dir = fs_token_paths[2][0] if is_local else self._fs.unstrip_protocol(fs_token_paths[2][0])
 

src/datasets/filesystems/__init__.py

@@ -1,12 +1,15 @@
 import importlib
+import os
 import shutil
 import threading
 from typing import List
 
 import fsspec
 import fsspec.asyn
+import psutil
 
 from . import compression
+from .fusefilesystem import FuseFileSystem
 from .hffilesystem import HfFileSystem
 
 
@@ -41,6 +44,25 @@ def extract_path_from_uri(dataset_path: str) -> str:
     return dataset_path
 
 
+def is_fuse_mounted_path(filepath):
+    """
+    Determine whether a given file path resides on a FUSE-mounted filesystem.
+
+    Args:
+        filepath (`str`):
+            Path to a file.
+    """
+    filepath = os.path.abspath(filepath)
+
+    partitions = psutil.disk_partitions(all=True)
+    fuse_partitions = [p.mountpoint for p in partitions if "fuse" in p.fstype]
+
+    for fuse_path in fuse_partitions:
+        if filepath.startswith(fuse_path):
+            return True
+    return False
+
+
 def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool:
     """
     Validates if filesystem has remote protocol.
@@ -49,10 +71,11 @@ def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool:
         fs (`fsspec.spec.AbstractFileSystem`):
             An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or [`datasets.filesystems.S3FileSystem`].
     """
-    if fs is not None and fs.protocol != "file":
-        return True
-    else:
+    if fs is None:
         return False
+    if isinstance(fs, FuseFileSystem) or fs.protocol != "file":
+        return True
+    return False
 
 
 def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str):

src/datasets/filesystems/fusefilesystem.py

@@ -0,0 +1,16 @@
+from fsspec.implementations.local import LocalFileSystem
+
+
+class FuseFileSystem(LocalFileSystem):
+    """
+    `datasets.filesystems.FuseFileSystem` is a subclass of `fsspec`'s [`LocalFileSystem`](https://filesystem-spec.readthedocs.io/en/latest/api.html#fsspec.implementations.local.LocalFileSystem).
+
+    Users can use this class to access FUSE-mounted files. The `datasets` library treats this class as a remote file
+    system to make file moving and renaming more efficient.
+    """
+
+
+def create_fuse_file_system(fs: LocalFileSystem):
+    """Creates a FuseFileSystem by copying over attributes from an `LocalFileSystem`."""
+    fuse_fs = FuseFileSystem()
+    fuse_fs.__dict__.update(fs.__dict__)