upgrade CacheControl to use the more memory-efficient SeparateBodyFileCache

Closes: #598

Author: mr-c

mypy-stubs/cachecontrol/cache.pyi

@@ -1,5 +1,12 @@
+from datetime import datetime
+from typing import IO
+
 class BaseCache:
     def get(self, key: str) -> bytes | None: ...
-    def set(self, key: str, value: bytes, expires: int | None = None) -> None: ...
+    def set(self, key: str, value: bytes, expires: int | datetime | None = None) -> None: ...
     def delete(self, key: str) -> None: ...
     def close(self) -> None: ...
+
+class SeparateBodyBaseCache(BaseCache):
+    def set_body(self, key: str, body: bytes) -> None: ...
+    def get_body(self, key: str) -> IO[bytes] | None: ...

mypy-stubs/cachecontrol/caches/__init__.pyi

@@ -1,11 +1,3 @@
-# Stubs for cachecontrol.caches (Python 2)
-#
-# NOTE: This dynamically typed stub was automatically generated by stubgen.
+from cachecontrol.caches.file_cache import SeparateBodyFileCache as SeparateBodyFileCache
 
-from typing import Any
-
-from .file_cache import FileCache as FileCache
-
-# from .redis_cache import RedisCache as RedisCache
-
-notice = ...  # type: Any
+__all__ = ["SeparateBodyFileCache"]

mypy-stubs/cachecontrol/caches/file_cache.pyi

@@ -1,31 +1,34 @@
-from os import PathLike
-from typing import ContextManager
-
-from ..cache import BaseCache as BaseCache
-from ..controller import CacheController as CacheController
+from cachecontrol.cache import SeparateBodyBaseCache
+from datetime import datetime
+from filelock import BaseFileLock
+from pathlib import Path
+from typing import IO, ContextManager
 
 class _LockClass:
     path: str
 
 _lock_class = ContextManager[_LockClass]
 
-class FileCache(BaseCache):
+class _FileCacheMixin:
     directory: str
     forever: bool
     filemode: int
     dirmode: int
     lock_class: _lock_class | None = None
     def __init__(
         self,
-        directory: str | PathLike[str],
-        forever: bool = ...,
-        filemode: int = ...,
-        dirmode: int = ...,
-        use_dir_lock: bool | None = ...,
-        lock_class: _lock_class | None = ...,
+        directory: str | Path,
+        forever: bool = False,
+        filemode: int = 384,
+        dirmode: int = 448,
+        lock_class: type[BaseFileLock] | None = None,
     ) -> None: ...
     @staticmethod
     def encode(x: str) -> str: ...
-    def get(self, key: str) -> None | bytes: ...
-    def set(self, key: str, value: bytes, expires: int | None = None) -> None: ...
+    def get(self, key: str) -> bytes | None: ...
+    def set(self, key: str, value: bytes, expires: int | datetime | None = None) -> None: ...
+
+class SeparateBodyFileCache(_FileCacheMixin, SeparateBodyBaseCache):
+    def get_body(self, key: str) -> IO[bytes] | None: ...
+    def set_body(self, key: str, body: bytes) -> None: ...
     def delete(self, key: str) -> None: ...

pyproject.toml

@@ -8,7 +8,8 @@ requires = [
     "types-dataclasses",
     "importlib_resources>=1.4;python_version<'3.9'",
     "ruamel.yaml>=0.17.6, < 0.19",
-    "types-setuptools"
+    "types-setuptools",
+    "CacheControl[filecache] >= 0.13.1, < 0.15"
 ]
 build-backend = "setuptools.build_meta"
 

schema_salad/metaschema.py

@@ -139,13 +139,13 @@ def __init__(
             self.fetcher = copyfrom.fetcher
         else:
             import requests
-            from cachecontrol.caches import FileCache
+            from cachecontrol.caches import SeparateBodyFileCache
             from cachecontrol.wrapper import CacheControl
 
             root = pathlib.Path(os.environ.get("HOME", tempfile.gettempdir()))
             session = CacheControl(
                 requests.Session(),
-                cache=FileCache(root / ".cache" / "salad"),
+                cache=SeparateBodyFileCache(root / ".cache" / "salad"),
             )
             self.fetcher: Fetcher = DefaultFetcher({}, session)
 

schema_salad/python_codegen_support.py

@@ -136,13 +136,13 @@ def __init__(
             self.fetcher = copyfrom.fetcher
         else:
             import requests
-            from cachecontrol.caches import FileCache
+            from cachecontrol.caches import SeparateBodyFileCache
             from cachecontrol.wrapper import CacheControl
 
             root = pathlib.Path(os.environ.get("HOME", tempfile.gettempdir()))
             session = CacheControl(
                 requests.Session(),
-                cache=FileCache(root / ".cache" / "salad"),
+                cache=SeparateBodyFileCache(root / ".cache" / "salad"),
             )
             self.fetcher: Fetcher = DefaultFetcher({}, session)
 

schema_salad/ref_resolver.py

@@ -22,7 +22,7 @@
 )
 
 import requests
-from cachecontrol.caches import FileCache
+from cachecontrol.caches import SeparateBodyFileCache
 from cachecontrol.wrapper import CacheControl
 from rdflib.exceptions import ParserError
 from rdflib.graph import Graph
@@ -179,10 +179,12 @@ def __init__(
                 root = pathlib.Path(os.environ.get("HOME", tempfile.gettempdir()))
                 self.session = CacheControl(
                     requests.Session(),
-                    cache=FileCache(root / ".cache" / "salad"),
+                    cache=SeparateBodyFileCache(root / ".cache" / "salad"),
                 )
             elif isinstance(doc_cache, str):
-                self.session = CacheControl(requests.Session(), cache=FileCache(doc_cache))
+                self.session = CacheControl(
+                    requests.Session(), cache=SeparateBodyFileCache(doc_cache)
+                )
         else:
             self.session = session
 

setup.py

@@ -88,7 +88,7 @@
     "ruamel.yaml >= 0.17.6, < 0.19",
     "rdflib >= 4.2.2, < 8.0.0",
     "mistune>=3,<3.1",
-    "CacheControl[filecache] >= 0.11.7, < 0.15",
+    "CacheControl[filecache] >= 0.13.1, < 0.15",
     "mypy_extensions",
     "importlib_resources>=1.4;python_version<'3.9'",
 ]