Merge branch 'fsspec:master' into expose-connection-errors

Author: btbest

.github/workflows/main.yaml

@@ -32,7 +32,7 @@ jobs:
       - name: Setup conda
         uses: conda-incubator/setup-miniconda@v3
         with:
-          environment-file: ci/environment-py38.yml
+          environment-file: ci/environment-linux.yml
           python-version: ${{ matrix.PY }}
 
       - name: Run Tests
@@ -43,7 +43,7 @@ jobs:
 
   win:
     name: pytest-win
-    runs-on: windows-2019
+    runs-on: windows-2022
 
     env:
       CIRUN: true
@@ -75,22 +75,6 @@ jobs:
           python-version: "3.11"
       - uses: pre-commit/action@main
 
-#  typecheck:
-#    runs-on: ubuntu-latest
-#    steps:
-#      - name: Checkout
-#        uses: actions/checkout@v4
-#
-#      - name: Setup conda
-#        uses: conda-incubator/setup-miniconda@v3
-#        with:
-#          environment-file: ci/environment-typecheck.yml
-#
-#      - name: mypy
-#        shell: bash -l {0}
-#        run: |
-#          mypy fsspec
-#
   downstream:
     name: downstream
     runs-on: ubuntu-24.04

.pre-commit-config.yaml

@@ -13,11 +13,10 @@ repos:
       - id: check-json
       - id: check-yaml
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    # Ruff version.
-    rev: v0.9.2
+    rev: v0.12.2
     hooks:
       # Run the linter.
-      - id: ruff
+      - id: ruff-check
         args: [ --fix, "--show-fixes"]
       - id: ruff-format
         types_or: [python]

README.md

@@ -96,3 +96,11 @@ filesystem_spec repository to setup pre-commit hooks. ``black`` will now be run
 before you commit, reformatting any changed files. You can format without
 committing via ``pre-commit run`` or skip these checks with ``git commit
 --no-verify``.
+
+## Support
+
+Work on this repository is supported in part by:
+
+"Anaconda, Inc. - Advancing AI through open source."
+
+<a href="https://anaconda.com/"><img src="https://camo.githubusercontent.com/b8555ef2222598ed37ce38ac86955febbd25de7619931bb7dd3c58432181d3b6/68747470733a2f2f626565776172652e6f72672f636f6d6d756e6974792f6d656d626572732f616e61636f6e64612f616e61636f6e64612d6c617267652e706e67" alt="anaconda logo" width="40%"/></a>

ci/environment-py38.yml renamed to ci/environment-linux.yml

@@ -7,17 +7,17 @@ User Functions
 --------------
 
 .. autosummary::
-   fsspec.available_compressions
-   fsspec.available_protocols
-   fsspec.filesystem
-   fsspec.fuse.run
-   fsspec.generic.rsync
-   fsspec.get_filesystem_class
-   fsspec.get_mapper
-   fsspec.gui.FileSelector
-   fsspec.open
-   fsspec.open_files
-   fsspec.open_local
+   available_compressions
+   available_protocols
+   filesystem
+   fuse.run
+   generic.rsync
+   get_filesystem_class
+   get_mapper
+   gui.FileSelector
+   open
+   open_files
+   open_local
 
 .. autofunction:: fsspec.available_compressions
 .. autofunction:: fsspec.available_protocols
@@ -36,24 +36,24 @@ Base Classes
 ------------
 
 .. autosummary::
-   fsspec.archive.AbstractArchiveFileSystem
-   fsspec.asyn.AsyncFileSystem
-   fsspec.callbacks.Callback
-   fsspec.callbacks.DotPrinterCallback
-   fsspec.callbacks.NoOpCallback
-   fsspec.callbacks.TqdmCallback
-   fsspec.core.BaseCache
-   fsspec.core.OpenFile
-   fsspec.core.OpenFiles
-   fsspec.core.get_fs_token_paths
-   fsspec.core.url_to_fs
-   fsspec.dircache.DirCache
-   fsspec.FSMap
-   fsspec.generic.GenericFileSystem
-   fsspec.registry.register_implementation
-   fsspec.spec.AbstractBufferedFile
-   fsspec.spec.AbstractFileSystem
-   fsspec.spec.Transaction
+   archive.AbstractArchiveFileSystem
+   asyn.AsyncFileSystem
+   callbacks.Callback
+   callbacks.DotPrinterCallback
+   callbacks.NoOpCallback
+   callbacks.TqdmCallback
+   core.BaseCache
+   core.OpenFile
+   core.OpenFiles
+   core.get_fs_token_paths
+   core.url_to_fs
+   dircache.DirCache
+   FSMap
+   generic.GenericFileSystem
+   registry.register_implementation
+   spec.AbstractBufferedFile
+   spec.AbstractFileSystem
+   spec.Transaction
 
 .. autoclass:: fsspec.archive.AbstractArchiveFileSystem
    :members:
@@ -107,30 +107,31 @@ Built-in Implementations
 ------------------------
 
 .. autosummary::
-   fsspec.implementations.arrow.ArrowFSWrapper
-   fsspec.implementations.arrow.HadoopFileSystem
-   fsspec.implementations.cached.CachingFileSystem
-   fsspec.implementations.cached.SimpleCacheFileSystem
-   fsspec.implementations.cached.WholeFileCacheFileSystem
-   fsspec.implementations.dask.DaskWorkerFileSystem
-   fsspec.implementations.data.DataFileSystem
-   fsspec.implementations.dbfs.DatabricksFileSystem
-   fsspec.implementations.dirfs.DirFileSystem
-   fsspec.implementations.ftp.FTPFileSystem
-   fsspec.implementations.git.GitFileSystem
-   fsspec.implementations.github.GithubFileSystem
-   fsspec.implementations.http.HTTPFileSystem
-   fsspec.implementations.jupyter.JupyterFileSystem
-   fsspec.implementations.libarchive.LibArchiveFileSystem
-   fsspec.implementations.local.LocalFileSystem
-   fsspec.implementations.memory.MemoryFileSystem
-   fsspec.implementations.reference.ReferenceFileSystem
-   fsspec.implementations.reference.LazyReferenceMapper
-   fsspec.implementations.sftp.SFTPFileSystem
-   fsspec.implementations.smb.SMBFileSystem
-   fsspec.implementations.tar.TarFileSystem
-   fsspec.implementations.webhdfs.WebHDFS
-   fsspec.implementations.zip.ZipFileSystem
+   implementations.arrow.ArrowFSWrapper
+   implementations.arrow.HadoopFileSystem
+   implementations.cached.CachingFileSystem
+   implementations.cached.SimpleCacheFileSystem
+   implementations.cached.WholeFileCacheFileSystem
+   implementations.dask.DaskWorkerFileSystem
+   implementations.data.DataFileSystem
+   implementations.dbfs.DatabricksFileSystem
+   implementations.dirfs.DirFileSystem
+   implementations.ftp.FTPFileSystem
+   implementations.gist.GistFileSystem
+   implementations.git.GitFileSystem
+   implementations.github.GithubFileSystem
+   implementations.http.HTTPFileSystem
+   implementations.jupyter.JupyterFileSystem
+   implementations.libarchive.LibArchiveFileSystem
+   implementations.local.LocalFileSystem
+   implementations.memory.MemoryFileSystem
+   implementations.reference.ReferenceFileSystem
+   implementations.reference.LazyReferenceMapper
+   implementations.sftp.SFTPFileSystem
+   implementations.smb.SMBFileSystem
+   implementations.tar.TarFileSystem
+   implementations.webhdfs.WebHDFS
+   implementations.zip.ZipFileSystem
 
 .. autoclass:: fsspec.implementations.arrow.ArrowFSWrapper
    :members: __init__
@@ -162,6 +163,9 @@ Built-in Implementations
 .. autoclass:: fsspec.implementations.ftp.FTPFileSystem
    :members: __init__
 
+.. autoclass:: fsspec.implementations.gist.GistFileSystem
+   :members: __init__
+
 .. autoclass:: fsspec.implementations.git.GitFileSystem
    :members: __init__
 
@@ -231,6 +235,7 @@ documentation carefully before using any particular package.
 - `irods`_ for access to iRODS servers, with protocol "irods://"
 - `lakefs`_ for lakeFS data lakes, with protocol "lakefs://"
 - `morefs`_ for `OverlayFileSystem`, `DictFileSystem`, and others
+- `obstore`_: zero-dependency access to Amazon S3, Google Cloud Storage, and Azure Blob Storage using the underlying Rust `object_store`_ library, with protocols "s3://", "gs://", and "abfs://".
 - `ocifs`_ for access to Oracle Cloud Object Storage, with protocol "oci://"
 - `ocilake`_ for OCI Data Lake storage
 - `ossfs`_ for Alibaba Cloud (Aliyun) Object Storage System (OSS)
@@ -246,6 +251,7 @@ documentation carefully before using any particular package.
 - `wandbfsspec`_ to access Weights & Biases (experimental)
 - `webdav4`_ for WebDAV, with protocol "webdav://" or "dav://"
 - `xrootd`_ for xrootd, with protocol "root://"
+- `msgraphfs`_ for Microsoft storage (ie Sharepoint) using the drive API through Microsoft Graph, with protocol "msgd://"
 
 .. _abfs: https://github.com/dask/adlfs
 .. _adl: https://github.com/dask/adlfs
@@ -266,6 +272,8 @@ documentation carefully before using any particular package.
 .. _irods: https://github.com/xwcl/irods_fsspec
 .. _lakefs: https://github.com/aai-institute/lakefs-spec
 .. _morefs: https://github.com/iterative/morefs
+.. _object_store: https://docs.rs/object_store/latest/object_store/
+.. _obstore: https://developmentseed.org/obstore/latest/
 .. _ocifs: https://ocifs.readthedocs.io/en/latest/
 .. _ocilake: https://github.com/oracle/ocifs
 .. _ossfs: https://github.com/fsspec/ossfs
@@ -280,19 +288,20 @@ documentation carefully before using any particular package.
 .. _wandbfsspec: https://github.com/alvarobartt/wandbfsspec
 .. _webdav4: https://github.com/skshetry/webdav4
 .. _xrootd: https://github.com/CoffeaTeam/fsspec-xrootd
+.. _msgraphfs: https://github.com/acsone/msgraphfs
 
 .. _readbuffering:
 
 Read Buffering
 --------------
 
 .. autosummary::
-   fsspec.caching.BlockCache
-   fsspec.caching.BytesCache
-   fsspec.caching.MMapCache
-   fsspec.caching.ReadAheadCache
-   fsspec.caching.FirstChunkCache
-   fsspec.caching.BackgroundBlockCache
+   caching.BlockCache
+   caching.BytesCache
+   caching.MMapCache
+   caching.ReadAheadCache
+   caching.FirstChunkCache
+   caching.BackgroundBlockCache
 
 .. autoclass:: fsspec.caching.BlockCache
    :members:
@@ -317,7 +326,7 @@ Utilities
 
 .. autosummary::
 
-   fsspec.utils.read_block
+   utils.read_block
 
 .. autofunction:: fsspec.utils.read_block
 

ci/environment-typecheck.yml

@@ -1,6 +1,64 @@
 Changelog
 =========
 
+2025.9.0
+--------
+
+Enhancements
+
+- include Last_modified info from HTTP headers to info (#1909)
+- add optional semaphore to async-wrapper (#1908)
+
+Fixes
+
+- ensure cachingFSs show correct protocol (#1897)
+- fix simplecache cat_ranges (#1892)
+
+Other
+
+- Style (#1894)
+
+
+2025.7.0
+--------
+
+Enhancements
+
+- only download HTML mime for http listing (#1889)
+- add tos:// to registry (#1878)
+
+Fixes
+
+- use st_birthtime in localFS, if available (#1883)
+- allow cat_* in simplecache (#1881)
+- remove deprecated asyncio use (#1862)
+- create event loop if it doesn't exist (#1857)
+
+Other
+
+- remove references to py38 (#1888)
+- ruff updates (#1887, 1864)
+- github rate limits in CI (#1879, 1877)
+- acknowledge Anaconda support (#1876)
+- add obstore to known implementations (#1875)
+- add Microsoft storage to known implementations (#1853)
+- use builtins zstd for py3.14 (#1874)
+- gdrivefs -> gdrive_fsspec (#1858)
+- windows version in CI (#1855)
+- error message typo (#1854)
+
+
+2025.5.1
+--------
+
+Enhancements
+
+- file system for GitHub gists (#1791)
+
+Other
+
+- doc fixes (#1847, 1848)
+
 2025.5.0
 --------