Bring changes from v7.11.4 to main

.env

@@ -43,6 +43,9 @@ ASSET_SERVER_URL=http://host.docker.internal/web_asset_store.xml
 # Make sure to set the `ASSET_SERVER_KEY` to a unique value
 ASSET_SERVER_KEY=your_asset_server_access_key
 
+# Information to connect to a Redis database
+# Specify will use this database as a process broker and storage for temporary
+# values
 REDIS_HOST=redis
 REDIS_PORT=6379
 REDIS_DB_INDEX=0

.github/workflows/test.yml

@@ -73,6 +73,11 @@ jobs:
         options:
           --health-cmd="mariadb-admin ping" --health-interval=5s
           --health-timeout=2s --health-retries=3
+      redis:
+        image: redis:latest
+        ports:
+          - 6379
+
 
     steps:
       - uses: actions/checkout@v4
@@ -128,6 +133,8 @@ jobs:
           echo "MIGRATOR_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
           echo "APP_USER_NAME = 'MasterUser'" >> specifyweb/settings/local_specify_settings.py
           echo "APP_USER_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
+          echo "REDIS_HOST = '127.0.0.1'" >> specifyweb/settings/local_specify_settings.py
+          echo "REDIS_PORT = ${{ job.services.redis.ports[6379] }}" >> specifyweb/settings/local_specify_settings.py
 
       - name: Need these files to be present
         run:

specifyweb/backend/businessrules/rules/attachment_rules.py

@@ -30,8 +30,8 @@ def attachment_jointable_save(sender, obj):
 
     attachee = get_attachee(obj)
     obj.attachment.tableid = attachee.specify_model.tableId
-    scopetype, scope = Scoping(attachee)()
-    obj.attachment.scopetype, obj.attachment.scopeid = scopetype, scope.id
+    scopetype, scope = Scoping.from_instance(attachee)
+    obj.attachment.scopetype, obj.attachment.scopeid = scopetype.value, scope.id
     obj.attachment.save()
 
 

specifyweb/backend/redis_cache/connect.py

@@ -0,0 +1,116 @@
+from redis import Redis
+from django.conf import settings
+
+class RedisConnection:
+    def __init__(self,
+                 host=getattr(settings, "REDIS_HOST", None),
+                 port=getattr(settings, "REDIS_PORT", None),
+                 db_index=getattr(settings, "REDIS_DB_INDEX", 0),
+                 decode_responses=True):
+        if None in (host, port, db_index):
+            raise ValueError(
+                "Redis is not correctly configured", host, port, db_index)
+        self.host = host
+        self.port = port
+        self.db_index = db_index
+        self.decode_responses = decode_responses
+        self.connection = Redis(
+            host=self.host,
+            port=self.port,
+            db=self.db_index,
+            decode_responses=self.decode_responses
+        )
+
+    def delete(self, key: str):
+        return self.connection.delete(key)
+
+
+class RedisDataType:
+    def __init__(self, established: RedisConnection) -> None:
+        self._established = established
+
+    @property
+    def connection(self):
+        return self._established.connection
+
+    def delete(self, key: str):
+        return self._established.delete(key)
+
+class RedisList(RedisDataType):
+    """
+    See https://redis.io/docs/latest/develop/data-types/lists/
+    """
+
+    def left_push(self, key: str, value) -> int:
+        return self.connection.lpush(key, value)
+
+    def right_push(self, key: str, value) -> int:
+        return self.connection.rpush(key, value)
+
+    def right_pop(self, key: str) -> str | bytes | None:
+        return self.connection.rpop(key)
+
+    def left_pop(self, key: str) -> str | bytes | None:
+        return self.connection.lpop(key)
+
+    def length(self, key: str) -> int:
+        return self.connection.llen(key)
+
+    def range(self, key: str, start_index: int, end_index: int) -> list[str] | list[bytes]:
+        return self.connection.lrange(key, start_index, end_index)
+
+    def trim(self, key: str, start_index: int, end_index: int) -> list[str] | list[bytes]:
+        return self.connection.ltrim(key, start_index, end_index)
+
+    def blocking_left_pop(self, key: str, timeout: int) -> str | bytes | None:
+        response = self.connection.blpop(key, timeout=timeout)
+        if response is None:
+            return None
+        _filled_list_key, item = response
+        return item
+
+class RedisSet(RedisDataType):
+    """
+    See https://redis.io/docs/latest/develop/data-types/sets/
+    """
+    def add(self, key: str, *values: str) -> int:
+        return self.connection.sadd(key, *values)
+
+    def is_member(self, key: str, value: str) -> bool:
+        is_member = int(self.connection.sismember(key, value))
+        return is_member == 1
+
+    def remove(self, key: str, value: str):
+        return self.connection.srem(key, value)
+
+    def size(self, key: str) -> int:
+        return self.connection.scard(key)
+
+    def members(self, key: str) -> set[str]:
+        return self.connection.smembers(key)
+
+    def union(self, *keys: str) -> set[str]:
+        return self.connection.sunion(*keys)
+
+    def intersection(self, *keys: str) -> set[str]:
+        return self.connection.sinter(*keys)
+
+    def difference(self, *keys: str) -> set[str]:
+        return self.connection.sdiff(*keys)
+
+class RedisString(RedisDataType):
+    """
+    See https://redis.io/docs/latest/develop/data-types/strings/
+    """
+
+    def set(self, key, value, time_to_live=None, override_existing=True):
+        flags = {
+            "ex": time_to_live,
+            "nx": not override_existing
+        }
+        self.connection.set(key, value, **flags)
+
+    def get(self, key, delete_key=False) -> str | bytes | None:
+        if delete_key:
+            return self.connection.getdel(key)
+        return self.connection.get(key)

specifyweb/backend/redis_cache/rqueue.py

@@ -0,0 +1,61 @@
+import json
+
+from typing import Callable, Generator, Iterable, cast
+
+from specifyweb.backend.redis_cache.connect import RedisConnection, RedisList, RedisSet
+
+type Serialized = str | bytes | bytearray
+
+type Serializer[T] = Callable[[T], str]
+type Deserializer[T] = Callable[[Serialized], T]
+
+def default_serializer(obj) -> str:
+    return str(obj)
+
+def default_deserializer(serialized: Serialized):
+    return serialized
+
+class RedisQueue[T]:
+    def __init__(self, connection: RedisConnection, key: str,
+                 serializer: Serializer[T] | None = None,
+                 deserializer: Deserializer[T] | None = None):
+        self.connection = RedisList(connection)
+        self.key = key
+        self.serializer = serializer or cast(Serializer[T], default_serializer)
+        self.deserializer = deserializer or cast(Deserializer[T], default_deserializer)
+
+    def key_name(self, *name_parts: str | None):
+        key_name = "_".join([self.key, *(part for part in name_parts if part is not None)])
+        return key_name
+
+    def push(self, *objs: T, sub_key: str | None = None) -> int:
+        key_name = self.key_name(sub_key)
+        return self.connection.right_push(key_name, *self._serialize_objs(*objs))
+
+    def pop(self, sub_key: str | None = None) -> T | None:
+        key_name = self.key_name(sub_key)
+        popped = self.connection.left_pop(key_name)
+        if popped is None:
+            return None
+        return self.deserializer(popped)
+
+    def wait_and_pop(self, timeout: int = 0, sub_key: str | None = None) -> T:
+        key_name = self.key_name(sub_key)
+        popped = self.connection.blocking_left_pop(key_name, timeout)
+        if popped is None:
+            raise TimeoutError("No items in queue after timeout")
+        return self.deserializer(popped)
+
+    def peek(self, sub_key: str | None = None) -> T | None:
+        key_name = self.key_name(sub_key)
+        top_value = self._deserialize_objs(*self.connection.range(key_name, 0, 0))
+        if len(top_value) == 0:
+            return None
+        return top_value[0]
+
+    def _serialize_objs(self, *objs: T) -> Generator[str, None, None]:
+        return (self.serializer(obj) for obj in objs)
+
+    def _deserialize_objs(self, *serialized: Serialized):
+        return tuple(self.deserializer(obj) for obj in serialized)
+

specifyweb/backend/stored_queries/execution.py

@@ -51,7 +51,6 @@ class QuerySort:
     def by_id(sort_id: QUREYFIELD_SORT_T):
         return QuerySort.SORT_TYPES[sort_id]
 
-
 def DefaultQueryFormatterProps():
     return ObjectFormatterProps(
         format_agent_type=False,

specifyweb/backend/workbench/upload/parse.py

@@ -44,7 +44,12 @@ class ParseSucess(NamedTuple):
 ParseResult = ParseSucess | ParseFailure
 
 
-def parse_field(table_name: str, field_name: str, raw_value: str, formatter: ScopedFormatter | None = None) -> ParseResult:
+def parse_field(
+        table_name: str,
+        field_name: str,
+        raw_value: str,
+        formatter: ScopedFormatter | None = None
+        ) -> ParseResult:
     table = datamodel.get_table_strict(table_name)
     field = table.get_field_strict(field_name)
 
@@ -170,7 +175,11 @@ def parse_date(table: Table, field_name: str, dateformat: str, value: str) -> Pa
     return ParseFailure('badDateFormat', {'value': value, 'format': dateformat})
 
 
-def parse_formatted(uiformatter: ScopedFormatter, table: Table, field: Field | Relationship, value: str) -> ParseResult:
+def parse_formatted(
+        uiformatter: ScopedFormatter,
+        table: Table,
+        field: Field | Relationship,
+        value: str) -> ParseResult:
     try:
         canonicalized = uiformatter(table, value)
     except FormatMismatch as e:

specifyweb/backend/workbench/upload/scoping.py

@@ -1,11 +1,12 @@
 
 
 from functools import reduce
-from typing import Any, cast
+from typing import Any, Callable, cast
 from collections.abc import Callable
 
 from specifyweb.specify.datamodel import datamodel, Table, is_tree_table
 from specifyweb.specify.utils.func import CustomRepr
+from specifyweb.specify.utils.autonumbering import AutonumberingLockDispatcher
 from specifyweb.specify.models_utils.load_datamodel import DoesNotExistError
 from specifyweb.specify import models
 from specifyweb.backend.trees.utils import get_default_treedef
@@ -113,7 +114,8 @@ def extend_columnoptions(
         fieldname: str, 
         row: Row | None = None,
         toOne: dict[str, Uploadable] | None = None,
-        context: ScopeContext | None = None
+        context: ScopeContext | None = None,
+        lock_dispatcher: Callable[[], AutonumberingLockDispatcher] | None = None
     ) -> ExtendedColumnOptions:
 
     context = context or ScopeContext()
@@ -125,7 +127,7 @@ def extend_columnoptions(
 
     ui_formatter = get_or_defer_formatter(collection, tablename, fieldname, row, toOne, context)
     scoped_formatter = (
-        None if ui_formatter is None else ui_formatter.apply_scope(collection)
+        None if ui_formatter is None else ui_formatter.apply_scope(collection, lock_dispatcher)
     )
 
     if tablename.lower() == "collectionobjecttype" and fieldname.lower() == "name":
@@ -256,7 +258,11 @@ def get_or_defer_formatter(
 
 
 def apply_scoping_to_uploadtable(
-    ut: UploadTable, collection, context: ScopeContext | None = None, row=None
+    ut: UploadTable,
+    collection,
+    context: ScopeContext | None = None,
+    row=None,
+    lock_dispatcher: Callable[[], AutonumberingLockDispatcher] | None = None
 ) -> ScopedUploadTable:
     # IMPORTANT:
     # before this comment, collection is untrusted and unreliable
@@ -276,7 +282,7 @@ def apply_scoping_to_uploadtable(
 
     apply_scoping = lambda key, value: get_deferred_scoping(
         key, table.django_name, value, row, ut, context
-    ).apply_scoping(collection, context, row)
+    ).apply_scoping(collection, context, row, lock_dispatcher=lock_dispatcher)
 
     to_ones = {
         key: adjuster(apply_scoping(key, value), key)
@@ -299,7 +305,14 @@ def _backref(key):
     scoped_table = ScopedUploadTable(
         name=ut.name,
         wbcols={
-            f: extend_columnoptions(colopts, collection, table.name, f, row, ut.toOne, context)
+            f: extend_columnoptions(colopts,
+                                    collection,
+                                    table.name,
+                                    f,
+                                    row,
+                                    ut.toOne,
+                                    context,
+                                    lock_dispatcher=lock_dispatcher)
             for f, colopts in ut.wbcols.items()
         },
         static=ut.static,
@@ -347,7 +360,10 @@ def set_order_number(
     return tmr._replace(strong_ignore=[*tmr.strong_ignore, *to_ignore])
 
 
-def apply_scoping_to_treerecord(tr: TreeRecord, collection, context: ScopeContext | None = None) -> ScopedTreeRecord:
+def apply_scoping_to_treerecord(tr: TreeRecord,
+                                collection,
+                                context: ScopeContext | None = None, 
+                                lock_dispatcher: Callable[[], AutonumberingLockDispatcher] | None = None) -> ScopedTreeRecord:
     table = datamodel.get_table_strict(tr.name)
 
     treedef = get_default_treedef(table, collection)
@@ -376,7 +392,11 @@ def apply_scoping_to_treerecord(tr: TreeRecord, collection, context: ScopeContex
             else r._replace(treedef_id=treedef.id) if r.treedef_id is None  # Adjust treeid for parsed JSON plans
             else r
         ): {
-            f: extend_columnoptions(colopts, collection, table.name, f)
+            f: extend_columnoptions(colopts,
+                collection,
+                table.name,
+                f,
+                lock_dispatcher=lock_dispatcher)
             for f, colopts in cols.items()
         }
         for r, cols in tr.ranks.items()