diff --git a/changelog.md b/changelog.md index 4a3e9c162..d3e413c5c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,15 @@ # Changelog + +## [3.5.2] - not yet released + +All following changes are relevant only for the case when file server is +S3 bucket/CDN: + +- poll nodes of type document for the status of preview thumbnail (only for S3 + CDN setups) +- adds `/api/documents/thumbnail-img-status/` endpoint + + ## [3.5.1] - 2025-05-11 ### Changes diff --git a/papermerge/core/alembic/README b/papermerge/core/alembic/README index c2f91d81e..de9ecff8e 100644 --- a/papermerge/core/alembic/README +++ b/papermerge/core/alembic/README @@ -26,3 +26,9 @@ Navigate back and forth: $ alembic downgrade -1 $ alembic upgrade +1 ``` + +To view migrations in chronological order: + +``` +alembic history +``` diff --git a/papermerge/core/alembic/versions/2518bf648ffd_add_group_ownership_related_columns.py b/papermerge/core/alembic/versions/2518bf648ffd_add_group_ownership_related_columns.py index 5d6ece33d..0d028408e 100644 --- a/papermerge/core/alembic/versions/2518bf648ffd_add_group_ownership_related_columns.py +++ b/papermerge/core/alembic/versions/2518bf648ffd_add_group_ownership_related_columns.py @@ -17,7 +17,6 @@ down_revision: Union[str, None] = "1a5a9bffcad4" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None -conn = op.get_bind() def upgrade() -> None: @@ -47,87 +46,84 @@ def upgrade() -> None: with op.batch_alter_table("document_types") as batch_op: batch_op.alter_column("user_id", existing_type=sa.UUID(), nullable=True) - if conn.dialect.name != "sqlite": - op.drop_constraint( - "unique document type per user", "document_types", type_="unique" - ) - op.create_unique_constraint( - "unique document type per user", - "document_types", - ["name", "user_id"], - ) - op.create_unique_constraint( - "unique document type per group", - "document_types", - ["name", "group_id"], - ) - op.create_foreign_key( - "document_types_group_id_fkey", - "document_types", - "groups", - ["group_id"], - ["id"], - ) - op.create_check_constraint( - constraint_name="check__user_id_not_null__or__group_id_not_null", - table_name="document_types", - condition="user_id IS NOT NULL OR group_id IS NOT NULL", - ) + op.drop_constraint( + "unique document type per user", "document_types", type_="unique" + ) + op.create_unique_constraint( + "unique document type per user", + "document_types", + ["name", "user_id"], + ) + op.create_unique_constraint( + "unique document type per group", + "document_types", + ["name", "group_id"], + ) + op.create_foreign_key( + "document_types_group_id_fkey", + "document_types", + "groups", + ["group_id"], + ["id"], + ) + op.create_check_constraint( + constraint_name="check__user_id_not_null__or__group_id_not_null", + table_name="document_types", + condition="user_id IS NOT NULL OR group_id IS NOT NULL", + ) #### groups op.add_column("groups", sa.Column("home_folder_id", sa.Uuid(), nullable=True)) op.add_column("groups", sa.Column("inbox_folder_id", sa.Uuid(), nullable=True)) - if conn.dialect.name != "sqlite": - op.create_foreign_key( - "groups_inbox_folder_id_fkey", - "groups", - "folders", - ["inbox_folder_id"], - ["node_id"], - ondelete="CASCADE", - deferrable=True, - ) - op.create_foreign_key( - "groups_home_folder_id_fkey", - "groups", - "folders", - ["home_folder_id"], - ["node_id"], - ondelete="CASCADE", - deferrable=True, - ) + op.create_foreign_key( + "groups_inbox_folder_id_fkey", + "groups", + "folders", + ["inbox_folder_id"], + ["node_id"], + ondelete="CASCADE", + deferrable=True, + ) + op.create_foreign_key( + "groups_home_folder_id_fkey", + "groups", + "folders", + ["home_folder_id"], + ["node_id"], + ondelete="CASCADE", + deferrable=True, + ) ### nodes op.add_column("nodes", sa.Column("group_id", sa.Uuid(), nullable=True)) with op.batch_alter_table("nodes") as batch_op: batch_op.alter_column("user_id", existing_type=sa.UUID(), nullable=True) - if conn.dialect.name != "sqlite": - op.drop_constraint("unique title per parent per user", "nodes", type_="unique") - op.create_unique_constraint( - "unique title per parent per user", - "nodes", - ["parent_id", "title", "user_id"], - ) - op.create_unique_constraint( - "unique title per parent per group", - "nodes", - ["parent_id", "title", "group_id"], - ) - op.create_foreign_key( - "nodes_group_id_fkey", - "nodes", - "groups", - ["group_id"], - ["id"], - ondelete="CASCADE", - use_alter=True, - ) - op.create_check_constraint( - constraint_name="check__user_id_not_null__or__group_id_not_null", - table_name="nodes", - condition="user_id IS NOT NULL OR group_id IS NOT NULL", - ) + op.drop_constraint("unique title per parent per user", "nodes", type_="unique") + op.create_unique_constraint( + "unique title per parent per user", + "nodes", + ["parent_id", "title", "user_id"], + ) + op.create_unique_constraint( + "unique title per parent per group", + "nodes", + ["parent_id", "title", "group_id"], + ) + op.create_foreign_key( + "nodes_group_id_fkey", + "nodes", + "groups", + ["group_id"], + ["id"], + ondelete="CASCADE", + use_alter=True, + ) + op.create_check_constraint( + constraint_name="check__user_id_not_null__or__group_id_not_null", + table_name="nodes", + condition="user_id IS NOT NULL OR group_id IS NOT NULL", + ) # tags op.add_column("tags", sa.Column("group_id", sa.Uuid(), nullable=True)) @@ -135,20 +131,17 @@ def upgrade() -> None: with op.batch_alter_table("tags") as batch_op: batch_op.alter_column("user_id", existing_type=sa.UUID(), nullable=True) - if conn.dialect.name != "sqlite": - op.drop_constraint("unique tag name per user", "tags", type_="unique") - op.create_unique_constraint( - "unique tag name per user", "tags", ["name", "user_id"] - ) - op.create_unique_constraint( - "unique tag name per group", "tags", ["name", "group_id"] - ) - op.create_foreign_key(None, "tags", "groups", ["group_id"], ["id"]) - op.create_check_constraint( - constraint_name="check__user_id_not_null__or__group_id_not_null", - table_name="tags", - condition="user_id IS NOT NULL OR group_id IS NOT NULL", - ) + op.drop_constraint("unique tag name per user", "tags", type_="unique") + op.create_unique_constraint("unique tag name per user", "tags", ["name", "user_id"]) + op.create_unique_constraint( + "unique tag name per group", "tags", ["name", "group_id"] + ) + op.create_foreign_key(None, "tags", "groups", ["group_id"], ["id"]) + op.create_check_constraint( + constraint_name="check__user_id_not_null__or__group_id_not_null", + table_name="tags", + condition="user_id IS NOT NULL OR group_id IS NOT NULL", + ) # ### end Alembic commands ### @@ -156,11 +149,8 @@ def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### # tags - if conn.dialect.name != "sqlite": - op.drop_constraint("unique tag name per user/group", "tags", type_="unique") - op.create_unique_constraint( - "unique tag name per user", "tags", ["name", "user_id"] - ) + op.drop_constraint("unique tag name per user/group", "tags", type_="unique") + op.create_unique_constraint("unique tag name per user", "tags", ["name", "user_id"]) with op.batch_alter_table("tags") as batch_op: batch_op.alter_column("user_id", existing_type=sa.UUID(), nullable=False) @@ -168,16 +158,15 @@ def downgrade() -> None: op.drop_column("tags", "group_id") # nodes - if conn.dialect.name != "sqlite": - op.drop_constraint("nodes_group_id_fkey", "nodes", type_="foreignkey") - op.drop_constraint( - "unique title per parent per user/group", "nodes", type_="unique" - ) - op.create_unique_constraint( - "unique title per parent per user", - "nodes", - ["parent_id", "title", "user_id"], - ) + op.drop_constraint("nodes_group_id_fkey", "nodes", type_="foreignkey") + op.drop_constraint( + "unique title per parent per user/group", "nodes", type_="unique" + ) + op.create_unique_constraint( + "unique title per parent per user", + "nodes", + ["parent_id", "title", "user_id"], + ) with op.batch_alter_table("nodes") as batch_op: batch_op.alter_column( @@ -187,24 +176,22 @@ def downgrade() -> None: op.drop_column("nodes", "group_id") # groups - if conn.dialect.name != "sqlite": - op.drop_constraint("groups_home_folder_id_fkey", "groups", type_="foreignkey") - op.drop_constraint("groups_inbox_folder_id_fkey", "groups", type_="foreignkey") + op.drop_constraint("groups_home_folder_id_fkey", "groups", type_="foreignkey") + op.drop_constraint("groups_inbox_folder_id_fkey", "groups", type_="foreignkey") op.drop_column("groups", "inbox_folder_id") op.drop_column("groups", "home_folder_id") # document_types - if conn.dialect.name != "sqlite": - op.drop_constraint( - "document_types_group_id_fkey", "document_types", type_="foreignkey" - ) - op.drop_constraint( - "unique document type per user/group", "document_types", type_="unique" - ) - op.create_unique_constraint( - "unique document type per user", "document_types", ["name", "user_id"] - ) + op.drop_constraint( + "document_types_group_id_fkey", "document_types", type_="foreignkey" + ) + op.drop_constraint( + "unique document type per user/group", "document_types", type_="unique" + ) + op.create_unique_constraint( + "unique document type per user", "document_types", ["name", "user_id"] + ) with op.batch_alter_table("document_types") as batch_op: batch_op.alter_column("user_id", existing_type=sa.UUID(), nullable=False) @@ -212,10 +199,9 @@ def downgrade() -> None: op.drop_column("document_types", "group_id") # custom_fields - if conn.dialect.name != "sqlite": - op.drop_constraint( - "custom_fields_group_id_fkey", "custom_fields", type_="foreignkey" - ) + op.drop_constraint( + "custom_fields_group_id_fkey", "custom_fields", type_="foreignkey" + ) with op.batch_alter_table("custom_fields") as batch_op: batch_op.alter_column("user_id", existing_type=sa.UUID(), nullable=False) diff --git a/papermerge/core/alembic/versions/a03014b93c1e_add_documents_preview_status_field.py b/papermerge/core/alembic/versions/a03014b93c1e_add_documents_preview_status_field.py new file mode 100644 index 000000000..09ee94081 --- /dev/null +++ b/papermerge/core/alembic/versions/a03014b93c1e_add_documents_preview_status_field.py @@ -0,0 +1,29 @@ +"""add documents.preview_status and documents.preview_error fields + +Revision ID: a03014b93c1e +Revises: 2118951c4d90 +Create Date: 2025-05-12 07:25:19.171857 + +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision: str = "a03014b93c1e" +down_revision: Union[str, None] = "2118951c4d90" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("documents", sa.Column("preview_status", sa.String(), nullable=True)) + op.add_column("documents", sa.Column("preview_error", sa.String(), nullable=True)) + + +def downgrade() -> None: + op.drop_column("documents", "preview_error") + op.drop_column("documents", "preview_status") diff --git a/papermerge/core/cache/__init__.py b/papermerge/core/cache/__init__.py new file mode 100644 index 000000000..9154a50d9 --- /dev/null +++ b/papermerge/core/cache/__init__.py @@ -0,0 +1,13 @@ +from papermerge.core import config +from .empty import Client as EmptyClient +from .redis_client import Client as RedisClient + +settings = config.get_settings() + +redis_url = settings.papermerge__redis__url +cache_enabled = settings.papermerge__main__cache_enabled + +if redis_url and cache_enabled: + client = RedisClient(redis_url) +else: + client = EmptyClient() diff --git a/papermerge/core/cache/empty.py b/papermerge/core/cache/empty.py new file mode 100644 index 000000000..50901c7f9 --- /dev/null +++ b/papermerge/core/cache/empty.py @@ -0,0 +1,10 @@ +class Client: + + def get(self, key): + return None + + def set(self, key, value, ex: int = 60): ... + + +def get_client(): + return Client() diff --git a/papermerge/core/cache/redis_client.py b/papermerge/core/cache/redis_client.py new file mode 100644 index 000000000..0d0cd33b5 --- /dev/null +++ b/papermerge/core/cache/redis_client.py @@ -0,0 +1,21 @@ +import redis + + +class Client: + def __init__(self, url): + self.url = url + self.client = redis.from_url(url) + + def get(self, key): + if self.client.exists(key): + return self.client.get(key).decode("utf-8") + + return None + + def set(self, key, value, ex: int = 60): + """ex is number of SECONDS until key expires""" + self.client.set(key, value, ex) + + +def get_client(url): + return Client(url) diff --git a/papermerge/core/cloudfront.py b/papermerge/core/cloudfront.py index 659dd4c32..5d2a78b02 100644 --- a/papermerge/core/cloudfront.py +++ b/papermerge/core/cloudfront.py @@ -7,8 +7,24 @@ from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import padding from papermerge.core.config import settings +from papermerge.core.cache import client as cache + + +PEM_PRIVATE_KEY_STRING = "pem-private-key-string" +PEM_PRIVATE_KEY_TTL = 600 + def rsa_signer(message): + private_key_string = cache.get(PEM_PRIVATE_KEY_STRING) + + if PEM_PRIVATE_KEY_STRING is not None: + private_key = serialization.load_pem_private_key( + private_key_string, + password=None, + backend=default_backend() + ) + return private_key.sign(message, padding.PKCS1v15(), hashes.SHA1()) + _kpath = settings.papermerge__main__cf_sign_url_private_key if _kpath is None: raise ValueError( @@ -22,11 +38,18 @@ def rsa_signer(message): ) with open(key_path, 'rb') as key_file: + private_key_string = key_file.read() private_key = serialization.load_pem_private_key( - key_file.read(), + private_key_string, password=None, backend=default_backend() ) + cache.set( + PEM_PRIVATE_KEY_STRING, + private_key_string, + ex=PEM_PRIVATE_KEY_TTL + ) + return private_key.sign(message, padding.PKCS1v15(), hashes.SHA1()) diff --git a/papermerge/core/config.py b/papermerge/core/config.py index 737e6cbe0..5e0514569 100644 --- a/papermerge/core/config.py +++ b/papermerge/core/config.py @@ -7,6 +7,9 @@ class FileServer(str, Enum): LOCAL = 'local' S3 = 's3' + # Don't use "s3-local-test" in production! + # It is meant only for local testing + S3_LOCAL_TEST = 's3-local-test' # used only for testing class Settings(BaseSettings): @@ -19,6 +22,7 @@ class Settings(BaseSettings): papermerge__main__cf_sign_url_key_id: str | None = None papermerge__main__cf_domain: str | None = None papermerge__main__timezone: str = 'Europe/Berlin' + papermerge__main__cache_enabled: bool = False papermerge__database__url: str = "sqlite:////db/db.sqlite3" papermerge__redis__url: str | None = None papermerge__ocr__default_lang_code: str = 'deu' diff --git a/papermerge/core/constants.py b/papermerge/core/constants.py index 9d9b8fbaa..d720a8107 100644 --- a/papermerge/core/constants.py +++ b/papermerge/core/constants.py @@ -1,3 +1,5 @@ +from enum import Enum + INBOX_TITLE = "inbox" HOME_TITLE = "home" CTYPE_FOLDER = "folder" @@ -22,7 +24,11 @@ # bulk remove of docs thumbnails S3_WORKER_REMOVE_DOCS_THUMBNAIL = "s3_worker_remove_docs_thumbnail" S3_WORKER_REMOVE_PAGE_THUMBNAIL = "s3_worker_remove_page_thumbnail" -S3_WORKER_GENERATE_PREVIEW = "s3_worker_generate_preview" +# generate document thumbnail preview i.e. one single image +# as preview for the whole document +S3_WORKER_GENERATE_DOC_THUMBNAIL = "s3_worker_generate_doc_thumbnail" +# generate preview image(s) for one or multiple document pages +S3_WORKER_GENERATE_PAGE_IMAGE = "s3_worker_generate_page_image" WORKER_OCR_DOCUMENT = "worker_ocr_document" # path_tmpl_worker: move one document (based on path template) PATH_TMPL_MOVE_DOCUMENT = "path_tmpl_move_document" @@ -38,3 +44,21 @@ class ContentType: IMAGE_JPEG = "image/jpeg" IMAGE_PNG = "image/png" IMAGE_TIFF = "image/tiff" + + +class ImagePreviewStatus(str, Enum): + """Image preview status + + 1. If database field `preview_status` is NULL -> + image preview was not considered yet i.e. client + have not asked for it yet. + 2. "pending" - image preview was scheduled, as client has asked + for it, but has not started yet + 3. "ready - image preview complete: + a. preview image was generated + b. preview image was uploaded to S3 + 4. "failed" image preview failed + """ + READY = "ready" + PENDING = "pending" + FAILED = "failed" diff --git a/papermerge/core/dbapi.py b/papermerge/core/dbapi.py index 144f98799..9a6ce9ca1 100644 --- a/papermerge/core/dbapi.py +++ b/papermerge/core/dbapi.py @@ -23,7 +23,8 @@ update_doc_type, update_doc_cfv, get_doc_cfv, - get_doc_ver_pages + get_doc_ver_pages, + get_docs_thumbnail_img_status ) from .features.nodes.db.api import get_nodes, get_folder from .features.roles.db.api import ( @@ -72,6 +73,7 @@ "update_doc_cfv", "get_docs_count_by_type", "upload", + "get_docs_thumbnail_img_status", "create_document_type", "get_document_types", "get_document_type", diff --git a/papermerge/core/features/document/db/api.py b/papermerge/core/features/document/db/api.py index fd10c1359..3102bedf8 100644 --- a/papermerge/core/features/document/db/api.py +++ b/papermerge/core/features/document/db/api.py @@ -14,7 +14,7 @@ from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import joinedload - +from papermerge.core.features.document import s3 from papermerge.core.db.engine import Session from papermerge.core.utils.misc import copy_file from papermerge.core import schema, orm, constants, tasks @@ -858,3 +858,62 @@ def get_last_ver_pages( ) return db_session.execute(stmt).scalars().all() + + +def get_docs_thumbnail_img_status( + db_session: Session, + doc_ids: list[uuid.UUID] +) -> Tuple[list[schema.DocumentPreviewImageStatus], list[uuid.UUID]]: + """Gets image preview statuses for given docIDs + + Response is a tuple. First item of the tuple is list of + statuses and second item of the tuple is the list of document + IDs for which image preview field has value NULL i.e. was not considered yet + or maybe was reseted. + """ + + fserver = settings.papermerge__main__file_server + + stmt = select( + orm.Document.id.label("doc_id"), + orm.Document.preview_status + ).select_from(orm.Document).where( + orm.Document.id.in_(doc_ids) + ) + + doc_ids_not_yet_considered_for_preview = [] + items = [] + if fserver in (config.FileServer.S3.value, config.FileServer.S3_LOCAL_TEST): + for row in db_session.execute(stmt): + url = None + + if row.preview_status == constants.ImagePreviewStatus.READY: + # image URL is returned if only and only if image + # preview is ready (generated and uploaded to S3) + if fserver == config.FileServer.S3: + # Real world CDN setup + url = s3.doc_thumbnail_signed_url(row.doc_id) + elif config.FileServer.S3_LOCAL_TEST: + # Testing setup for CDN + url = f"/api/thumbnails/{row.doc_id}" + + if row.preview_status is None: + doc_ids_not_yet_considered_for_preview.append(row.doc_id) + + item = schema.DocumentPreviewImageStatus( + doc_id=row.doc_id, + status=row.preview_status, + preview_image_url=url + ) + items.append(item) + else: + # Non-CDN setup + for row in db_session.execute(stmt): + item = schema.DocumentPreviewImageStatus( + doc_id=row.doc_id, + status=constants.ImagePreviewStatus.READY, + preview_image_url=f"/api/thumbnails/{row.doc_id}" + ) + items.append(item) + + return items, doc_ids_not_yet_considered_for_preview diff --git a/papermerge/core/features/document/db/orm.py b/papermerge/core/features/document/db/orm.py index 2a1ff7533..9c31aed44 100644 --- a/papermerge/core/features/document/db/orm.py +++ b/papermerge/core/features/document/db/orm.py @@ -24,6 +24,17 @@ class Document(Node): ocr: Mapped[bool] = mapped_column(default=False) ocr_status: Mapped[str] = mapped_column(default=OCRStatusEnum.unknown) + # `preview_status` + # NULL = no preview available -> thumbnail_url will be empty + # Ready = preview available -> thumbnail_url will point to preview image + # Failed = preview generation failed -> thumbnail_url is empty + # in which case `preview_error` will contain error why preview + # generation failed + preview_status: Mapped[str] = mapped_column(nullable=True) + # `preview_error` + # only for troubleshooting purposes. Relevant only in case + # `preview_status` = Failed + preview_error: Mapped[str] = mapped_column(nullable=True) document_type: Mapped[DocumentType] = relationship( # noqa: F821 primaryjoin="DocumentType.id == Document.document_type_id" ) diff --git a/papermerge/core/features/document/router.py b/papermerge/core/features/document/router.py index f41f456dd..8bc7a1f28 100644 --- a/papermerge/core/features/document/router.py +++ b/papermerge/core/features/document/router.py @@ -3,7 +3,7 @@ import uuid from typing import Annotated -from fastapi import APIRouter, HTTPException, Security, UploadFile, status +from fastapi import APIRouter, HTTPException, Security, UploadFile, status, Query from sqlalchemy.exc import NoResultFound from papermerge.core import exceptions as exc @@ -186,15 +186,6 @@ def upload_file( content_type=file.headers.get("content-type"), ) - if config.papermerge__main__file_server == FileServer.S3: - # generate preview using `s3_worker` - # it will, as well, upload previews to s3 storage - send_task( - const.S3_WORKER_GENERATE_PREVIEW, - kwargs={"doc_id": str(doc.id)}, - route_name="s3preview", - ) - if error: raise HTTPException(status_code=400, detail=error.model_dump()) @@ -314,3 +305,57 @@ def get_documents_by_type( num_pages=int(total_count / page_size) + 1, items=items, ) + + +@router.get( + "/thumbnail-img-status/", + responses={ + status.HTTP_403_FORBIDDEN: { + "description": f"No `{scopes.NODE_VIEW}` permission on one of the documents", + "content": OPEN_API_GENERIC_JSON_DETAIL, + } + }, +) +@utils.docstring_parameter(scope=scopes.NODE_VIEW) +def get_document_doc_thumbnail_status( + user: Annotated[schema.User, Security(get_current_user, scopes=[scopes.NODE_VIEW])], + doc_ids: list[uuid.UUID] = Query(), +) -> list[schema.DocumentPreviewImageStatus]: + """ + Get documents thumbnail image preview status + + Receives as input a list of document IDs (i.e. node IDs). + + In case of CDN setup, for each document with NULL value in `preview_status` + field - one `S3worker` task will be scheduled for generating respective + document thumbnail. + + Required scope: `{scope}` + """ + + doc_ids_not_yet_considered = [] + with db.Session() as db_session: + for doc_id in doc_ids: + if not dbapi_common.has_node_perm( + db_session, + node_id=doc_id, + codename=scopes.NODE_VIEW, + user_id=user.id, + ): + raise exc.HTTP403Forbidden() + + response, doc_ids_not_yet_considered = dbapi.get_docs_thumbnail_img_status( + db_session, doc_ids=doc_ids + ) + + fserver = config.papermerge__main__file_server + if fserver in (FileServer.S3.value, FileServer.S3_LOCAL_TEST.value): + if len(doc_ids_not_yet_considered) > 0: + for doc_id in doc_ids_not_yet_considered: + send_task( + const.S3_WORKER_GENERATE_DOC_THUMBNAIL, + kwargs={"doc_id": str(doc_id)}, + route_name="s3preview", + ) + + return response diff --git a/papermerge/core/features/document/s3.py b/papermerge/core/features/document/s3.py new file mode 100644 index 000000000..b94f03e8d --- /dev/null +++ b/papermerge/core/features/document/s3.py @@ -0,0 +1,24 @@ +from uuid import UUID + +from papermerge.core import pathlib as plib +from papermerge.core import config + +settings = config.get_settings() + +VALID_FOR_SECONDS = 600 + + +def doc_thumbnail_signed_url(uid: UUID) -> str: + from papermerge.core.cloudfront import sign_url + + resource_path = plib.thumbnail_path(uid) + prefix = settings.papermerge__main__prefix + if prefix: + url = f"https://{settings.papermerge__main__cf_domain}/{prefix}/{resource_path}" + else: + url = f"https://{settings.papermerge__main__cf_domain}/{resource_path}" + + return sign_url( + url, + valid_for=VALID_FOR_SECONDS, + ) diff --git a/papermerge/core/features/document/schema.py b/papermerge/core/features/document/schema.py index b2098f91f..70a94d4fd 100644 --- a/papermerge/core/features/document/schema.py +++ b/papermerge/core/features/document/schema.py @@ -15,6 +15,7 @@ from papermerge.core import pathlib as plib from papermerge.core.types import OCRStatusEnum from papermerge.core import config +from papermerge.core.features.document import s3 settings = config.get_settings() @@ -133,7 +134,8 @@ class Page(BaseModel): @field_validator("svg_url", mode="before") @classmethod def svg_url_value(cls, value, info: ValidationInfo) -> str: - if settings.papermerge__main__file_server == "local": + file_server = settings.papermerge__main__file_server + if file_server in (config.FileServer.LOCAL, config.FileServer.S3_LOCAL_TEST): return f"/api/pages/{info.data['id']}/svg" s3_url = _s3_page_svg_url(info.data["id"]) # UUID of the page here @@ -142,7 +144,8 @@ def svg_url_value(cls, value, info: ValidationInfo) -> str: @field_validator("jpg_url", mode="before") @classmethod def jpg_url_value(cls, value, info: ValidationInfo) -> str: - if settings.papermerge__main__file_server == "local": + file_server = settings.papermerge__main__file_server + if file_server in (config.FileServer.LOCAL, config.FileServer.S3_LOCAL_TEST): return f"/api/pages/{info.data['id']}/jpg" s3_url = _s3_page_thumbnail_url( @@ -173,7 +176,8 @@ class DocumentVersion(BaseModel): @field_validator("download_url", mode="before") def download_url_validator(cls, _, info): - if settings.papermerge__main__file_server == config.FileServer.LOCAL.value: + file_server = settings.papermerge__main__file_server + if file_server in (config.FileServer.LOCAL, config.FileServer.S3_LOCAL_TEST): return f"/api/document-versions/{info.data['id']}/download" return _s3_docver_download_url(info.data["id"], info.data["file_name"]) @@ -189,7 +193,15 @@ def thumbnail_url(value, info): ThumbnailUrl = Annotated[str | None, Field(validate_default=True)] -class Document(BaseModel): +class DocumentNode(BaseModel): + """Document without versions + + The point of this class is to be used when listing folders/documents in + which case info about document versions (and their pages etc) is not + required (generating document version info in context of CDN is very + slow as for each page of each doc ver signed URL must be computed) + """ + id: UUID title: str ctype: Literal["document"] @@ -199,10 +211,10 @@ class Document(BaseModel): parent_id: UUID | None document_type_id: UUID | None = None breadcrumb: list[tuple[UUID, str]] = [] - versions: list[DocumentVersion] | None = [] ocr: bool = True # will this document be OCRed? ocr_status: OCRStatusEnum = OCRStatusEnum.unknown thumbnail_url: ThumbnailUrl = None + preview_status: str | None = None user_id: UUID | None = None group_id: UUID | None = None owner_name: str | None = None @@ -211,16 +223,40 @@ class Document(BaseModel): @field_validator("thumbnail_url", mode="before") def thumbnail_url_validator(cls, value, info): - if settings.papermerge__main__file_server == config.FileServer.LOCAL.value: + file_server = settings.papermerge__main__file_server + if file_server == config.FileServer.LOCAL: return f"/api/thumbnails/{info.data['id']}" - # if it is not local, then it is s3 + cloudfront - return _s3_doc_thumbnail_url(info.data["id"]) + # if it is not local, then it is s3 + CDN/cloudfront + if ( + "preview_status" in info.data + and info.data["preview_status"] == const.ImagePreviewStatus.READY + ): + if file_server == config.FileServer.S3: + # give client back signed URL only in case preview image + # was successfully uploaded to S3 backend. + # `preview_status` is set to ready/failed by s3 worker + # after preview image upload to s3 succeeds/fails + return s3.doc_thumbnail_signed_url(info.data["id"]) + else: + return f"/api/thumbnails/{info.data['id']}" + + return None # Config model_config = ConfigDict(from_attributes=True) +class Document(DocumentNode): + versions: list[DocumentVersion] | None = [] + + +class DocumentPreviewImageStatus(BaseModel): + doc_id: UUID + status: str | None + preview_image_url: str | None = None + + class NewDocument(BaseModel): # UUID may be present to allow custom IDs # See https://github.com/papermerge/papermerge-core/issues/325 @@ -256,22 +292,6 @@ class Thumbnail(BaseModel): size: int -def _s3_doc_thumbnail_url(uid: UUID) -> str: - from papermerge.core.cloudfront import sign_url - - resource_path = plib.thumbnail_path(uid) - prefix = settings.papermerge__main__prefix - if prefix: - url = f"https://{settings.papermerge__main__cf_domain}/{prefix}/{resource_path}" - else: - url = f"https://{settings.papermerge__main__cf_domain}/{resource_path}" - - return sign_url( - url, - valid_for=600, # valid for 600 seconds - ) - - def _s3_page_thumbnail_url(uid: UUID, size: int) -> str: from papermerge.core.cloudfront import sign_url diff --git a/papermerge/core/features/nodes/db/api.py b/papermerge/core/features/nodes/db/api.py index 3dd955a55..5c1b94f55 100644 --- a/papermerge/core/features/nodes/db/api.py +++ b/papermerge/core/features/nodes/db/api.py @@ -137,9 +137,9 @@ def get_paginated_nodes( if node.ctype == "folder": items.append(schema.Folder.model_validate(node)) else: - items.append(schema.Document.model_validate(node)) + items.append(schema.DocumentNode.model_validate(node)) - return PaginatedResponse[Union[schema.Document, schema.Folder]]( + return PaginatedResponse[Union[schema.DocumentNode, schema.Folder]]( page_size=page_size, page_number=page_number, num_pages=num_pages, diff --git a/papermerge/core/features/nodes/router.py b/papermerge/core/features/nodes/router.py index 4aceb8546..91a03b456 100644 --- a/papermerge/core/features/nodes/router.py +++ b/papermerge/core/features/nodes/router.py @@ -42,7 +42,7 @@ def get_node( parent_id: UUID, user: Annotated[schema.User, Security(get_current_user, scopes=[scopes.NODE_VIEW])], params: CommonQueryParams = Depends(), -) -> PaginatedResponse[Union[schema.Document, schema.Folder]]: +) -> PaginatedResponse[Union[schema.DocumentNode, schema.Folder]]: """Returns list of *paginated* direct descendants of `parent_id` node Required scope: `{scope}` diff --git a/papermerge/core/schema.py b/papermerge/core/schema.py index aa2cbed8a..6fe194512 100644 --- a/papermerge/core/schema.py +++ b/papermerge/core/schema.py @@ -10,6 +10,7 @@ ) from .features.document.schema import ( Document, + DocumentNode, NewDocument, DocumentVersion, Page, @@ -23,7 +24,8 @@ MovePagesIn, MovePagesOut, ExtractStrategy, - MoveStrategy + MoveStrategy, + DocumentPreviewImageStatus ) from .features.users.schema import User, CreateUser, UserDetails, UpdateUser, ChangeUserPassword, UserHomes, UserInboxes, UserHome, UserInbox from .features.custom_fields.schema import CustomField, UpdateCustomField, CustomFieldType, CustomFieldValue @@ -43,8 +45,10 @@ 'UpdateNode', 'MoveNode', 'Document', + 'DocumentNode', 'NewDocument', 'DocumentVersion', + 'DocumentPreviewImageStatus', 'Page', 'MovePage', 'User', diff --git a/ui2/src/actions/thumbnailActions.ts b/ui2/src/actions/thumbnailActions.ts new file mode 100644 index 000000000..01f54ef23 --- /dev/null +++ b/ui2/src/actions/thumbnailActions.ts @@ -0,0 +1,43 @@ +import {AppThunk} from "@/app/types" // adjust based on your setup +import { + documentThumbnailErrorUpdated, + documentThumbnailUpdated +} from "@/features/nodes/nodesSlice" +import { + documentThumbnailErrorUpdated as sharedNodesdocumentThumbnailErrorUpdated, + documentThumbnailUpdated as sharedNodesdocumentThumbnailUpdated +} from "@/features/shared_nodes/sharedNodesSlice" + +export const updateAllThumbnails = + (document_id: string, thumbnail_url: string | null): AppThunk => + dispatch => { + dispatch( + documentThumbnailUpdated({ + document_id: document_id, + thumbnail_url: thumbnail_url + }) + ) + dispatch( + sharedNodesdocumentThumbnailUpdated({ + document_id: document_id, + thumbnail_url: thumbnail_url + }) + ) + } + +export const updateErrorAllThumbnails = + (docId: string, error: string): AppThunk => + dispatch => { + dispatch( + documentThumbnailErrorUpdated({ + document_id: docId, + error: error + }) + ) + dispatch( + sharedNodesdocumentThumbnailErrorUpdated({ + document_id: docId, + error: error + }) + ) + } diff --git a/ui2/src/app/types.ts b/ui2/src/app/types.ts index 1032afba5..38a759098 100644 --- a/ui2/src/app/types.ts +++ b/ui2/src/app/types.ts @@ -3,3 +3,7 @@ import {store} from "@/app/store" export type AppStore = typeof store export type RootState = ReturnType export type AppDispatch = AppStore["dispatch"] +export type AppThunk = ( + dispatch: AppDispatch, + getState: () => RootState +) => ReturnType diff --git a/ui2/src/components/NodeThumbnail/Thumbnail.tsx b/ui2/src/components/NodeThumbnail/Thumbnail.tsx new file mode 100644 index 000000000..6af8cd3de --- /dev/null +++ b/ui2/src/components/NodeThumbnail/Thumbnail.tsx @@ -0,0 +1,25 @@ +import useDocumentThumbnail from "@/hooks/DocumentThumbnail" +import {Loader} from "@mantine/core" +import ThumbnailPlaceholder from "./ThumbnailPlaceholder" + +interface Args { + nodeID: string +} + +export default function Thumbnail({nodeID}: Args) { + const {data, isLoading, isError, error} = useDocumentThumbnail({nodeID}) + + if (isLoading) { + return + } + + if (isError) { + return + } + + if (data) { + return + } + + return +} diff --git a/ui2/src/components/NodeThumbnail/ThumbnailPlaceholder.tsx b/ui2/src/components/NodeThumbnail/ThumbnailPlaceholder.tsx new file mode 100644 index 000000000..73ae7437f --- /dev/null +++ b/ui2/src/components/NodeThumbnail/ThumbnailPlaceholder.tsx @@ -0,0 +1,28 @@ +import {Tooltip} from "@mantine/core" + +interface Args { + error: string | null +} + +export default function ThumbnailPlaceholder({error}: Args) { + return ( + + + + + + + + + + ) +} diff --git a/ui2/src/features/nodes/apiSlice.ts b/ui2/src/features/nodes/apiSlice.ts index 359ab3229..56e9abc1c 100644 --- a/ui2/src/features/nodes/apiSlice.ts +++ b/ui2/src/features/nodes/apiSlice.ts @@ -11,13 +11,7 @@ import type { SortMenuColumn, SortMenuDirection } from "@/types" -import { - getBaseURL, - getDefaultHeaders, - getRemoteUserID, - getWSURL, - imageEncode -} from "@/utils" +import {getRemoteUserID, getWSURL} from "@/utils" import { documentMovedNotifReceived, documentsMovedNotifReceived @@ -57,7 +51,6 @@ export type PaginatedArgs = { sortColumn: SortMenuColumn } -import {RootState} from "@/app/types" import {PAGINATION_DEFAULT_ITEMS_PER_PAGES} from "@/cconstants" export const apiSliceWithNodes = apiSlice.injectEndpoints({ @@ -160,55 +153,6 @@ export const apiSliceWithNodes = apiSlice.injectEndpoints({ ws.close() } }), - getDocumentThumbnail: builder.query({ - //@ts-ignore - queryFn: async (node_id, queryApi) => { - const state = queryApi.getState() as RootState - - if (!node_id) { - console.error("Node ID is empty or null") - return "Node ID is empty or null" - } - - const node = - state.nodes.entities[node_id] || state.sharedNodes.entities[node_id] - - if (!node) { - console.error( - `Node with ID=${node_id} not found in state.nodes.entities` - ) - return `Node ID = ${node_id} not found` - } - - const thumbnails_url = node.thumbnail_url - const headers = getDefaultHeaders() - let url - - if (thumbnails_url && !thumbnails_url.startsWith("/api/")) { - // cloud URL e.g. aws cloudfront URL - url = thumbnails_url - } else { - // use backend server URL (which may differ from frontend's URL) - url = `${getBaseURL(true)}${thumbnails_url}` - } - - if (!thumbnails_url || !url) { - console.error( - `Thumbnail URL for Node ID=${node_id} is undefined or null` - ) - return "node does not have thumbnail" - } - - try { - const response = await fetch(url, {headers: headers}) - const resp2 = await response.arrayBuffer() - const encodedData = imageEncode(resp2, "image/jpeg") - return {data: encodedData} - } catch (err) { - return {err} - } - } - }), addNewFolder: builder.mutation({ query: folder => ({ url: "/nodes/", @@ -291,6 +235,5 @@ export const { useUpdateNodeTagsMutation, useGetNodeTagsQuery, useDeleteNodesMutation, - useGetDocumentThumbnailQuery, useMoveNodesMutation } = apiSliceWithNodes diff --git a/ui2/src/features/nodes/components/Commander/NodesCommander/Node/Document/Document.tsx b/ui2/src/features/nodes/components/Commander/NodesCommander/Node/Document/Document.tsx index 0cd2b08d3..0040f7f94 100644 --- a/ui2/src/features/nodes/components/Commander/NodesCommander/Node/Document/Document.tsx +++ b/ui2/src/features/nodes/components/Commander/NodesCommander/Node/Document/Document.tsx @@ -11,12 +11,12 @@ import { selectSelectedNodeIds } from "@/features/ui/uiSlice" +import Thumbnail from "@/components/NodeThumbnail/Thumbnail" import Tags from "@/features/nodes/components/Commander/NodesCommander/Node/Tags" import type {NodeType, PanelMode} from "@/types" import classes from "./Document.module.scss" import PanelContext from "@/contexts/PanelContext" -import {useGetDocumentThumbnailQuery} from "@/features/nodes/apiSlice" type Args = { node: NodeType @@ -38,7 +38,7 @@ export default function Document({ selectSelectedNodeIds(s, mode) ) as Array const currentFolderID = useAppSelector(s => selectCurrentNodeID(s, mode)) - const {data} = useGetDocumentThumbnailQuery(node.id) + const dispatch = useAppDispatch() const tagNames = node.tags.map(t => t.name) @@ -76,7 +76,7 @@ export default function Document({ onClick(node)}> {node.is_shared && } - +
{node.title}
 diff --git a/ui2/src/features/nodes/components/Commander/NodesCommander/NodesCommander.tsx b/ui2/src/features/nodes/components/Commander/NodesCommander/NodesCommander.tsx index 2122e67e1..08cf8e392 100644 --- a/ui2/src/features/nodes/components/Commander/NodesCommander/NodesCommander.tsx +++ b/ui2/src/features/nodes/components/Commander/NodesCommander/NodesCommander.tsx @@ -45,7 +45,7 @@ import { selectDraggedPages, selectLastPageSize } from "@/features/ui/uiSlice" -import type {ExtractPagesResponse, NType, NodeType, PanelMode} from "@/types" +import type {ExtractPagesResponse, NType, PanelMode} from "@/types" import classes from "./Commander.module.scss" import {useTranslation} from "react-i18next" @@ -54,7 +54,7 @@ import {DropFilesModal} from "./DropFiles" import DropNodesModal from "./DropNodesDialog" import ExtractPagesModal from "./ExtractPagesModal" import FolderNodeActions from "./FolderNodeActions" -import Node from "./Node" +import NodesList from "./NodesList" export default function Commander() { const {t} = useTranslation() @@ -259,22 +259,19 @@ export default function Commander() { root.render(image) } - const nodes = data.items.map((n: NodeType) => ( - - )) - let commanderContent: JSX.Element - if (nodes.length > 0) { + if (data.items.length > 0) { commanderContent = ( <> - {nodes} + + + void + onNodeDrag: () => void + onNodeDragStart: (nodeID: string, event: React.DragEvent) => void +} + +export default function NodesList({ + items, + onClick, + onNodeDrag, + onNodeDragStart +}: Args) { + const dispatch = useAppDispatch() + const documentIds = useMemo(() => items.map(n => n.id), [items]) + const {previews} = usePreviewPolling(documentIds, { + pollIntervalMs: 3000, + maxRetries: 10 + }) + + useEffect(() => { + Object.entries(previews).forEach(([docId, preview]) => { + dispatch(updateAllThumbnails(docId, preview.url)) + }) + }, [previews]) + + return items.map((n: NodeType) => ( + + )) +} diff --git a/ui2/src/features/nodes/nodesSlice.ts b/ui2/src/features/nodes/nodesSlice.ts index cda5e4b7c..3e36fa199 100644 --- a/ui2/src/features/nodes/nodesSlice.ts +++ b/ui2/src/features/nodes/nodesSlice.ts @@ -20,10 +20,40 @@ import {apiSliceWithNodes} from "./apiSlice" const nodeAdapter = createEntityAdapter() const initialState = nodeAdapter.getInitialState() +interface DocumentThumbnailUpdated { + document_id: string + thumbnail_url: string | null +} + +interface DocumentThumbnailErrorUpdated { + document_id: string + error: string +} + const nodesSlice = createSlice({ name: "nodes", initialState, reducers: { + documentThumbnailUpdated: ( + state, + action: PayloadAction + ) => { + const payload = action.payload + const node = state.entities[payload.document_id] + if (node && payload.thumbnail_url) { + node.thumbnail_url = payload.thumbnail_url + } + }, + documentThumbnailErrorUpdated: ( + state, + action: PayloadAction + ) => { + const payload = action.payload + const node = state.entities[payload.document_id] + if (node && payload.error) { + node.thumbnail_preview_error = payload.error + } + }, documentsMovedNotifReceived: ( _state, action: PayloadAction @@ -72,8 +102,12 @@ const nodesSlice = createSlice({ } }) -export const {documentsMovedNotifReceived, documentMovedNotifReceived} = - nodesSlice.actions +export const { + documentsMovedNotifReceived, + documentMovedNotifReceived, + documentThumbnailUpdated, + documentThumbnailErrorUpdated +} = nodesSlice.actions export default nodesSlice.reducer @@ -92,6 +126,38 @@ export const selectNodesByIds = createSelector( } ) +export const selectDocumentThumbnailURL = ( + state: RootState, + nodeID: string +): null | string => { + const node = + state.nodes.entities[nodeID] || state.sharedNodes.entities[nodeID] + + if (!node) { + return null + } + + if (!node.thumbnail_url) { + return null + } + + return node.thumbnail_url +} + +export const selectDocumentThumbnailError = ( + state: RootState, + nodeID: string +): null | string => { + const node = + state.nodes.entities[nodeID] || state.sharedNodes.entities[nodeID] + + if (!node) { + return null + } + + return node.thumbnail_preview_error +} + export const moveNodesListeners = (startAppListening: AppStartListening) => { startAppListening({ matcher: apiSliceWithNodes.endpoints.moveNodes.matchFulfilled, diff --git a/ui2/src/features/shared_nodes/components/SharedCommander/Node/Document/Document.tsx b/ui2/src/features/shared_nodes/components/SharedCommander/Node/Document/Document.tsx index 87ba455c3..a75c585be 100644 --- a/ui2/src/features/shared_nodes/components/SharedCommander/Node/Document/Document.tsx +++ b/ui2/src/features/shared_nodes/components/SharedCommander/Node/Document/Document.tsx @@ -12,8 +12,8 @@ import Tags from "@/features/nodes/components/Commander/NodesCommander/Node/Tags import type {NodeType, PanelMode} from "@/types" import classes from "./Document.module.scss" +import Thumbnail from "@/components/NodeThumbnail/Thumbnail" import PanelContext from "@/contexts/PanelContext" -import {useGetDocumentThumbnailQuery} from "@/features/nodes/apiSlice" type Args = { node: NodeType @@ -26,7 +26,6 @@ export default function Document({node, onClick, cssClassNames}: Args) { const selectedIds = useAppSelector(s => selectSelectedNodeIds(s, mode) ) as Array - const {data} = useGetDocumentThumbnailQuery(node.id) const dispatch = useAppDispatch() const tagNames = node.tags.map(t => t.name) @@ -45,7 +44,7 @@ export default function Document({node, onClick, cssClassNames}: Args) { > onClick(node)}> - +
{node.title}
 diff --git a/ui2/src/features/shared_nodes/sharedNodesSlice.ts b/ui2/src/features/shared_nodes/sharedNodesSlice.ts index 6c4973e43..8b7e032a7 100644 --- a/ui2/src/features/shared_nodes/sharedNodesSlice.ts +++ b/ui2/src/features/shared_nodes/sharedNodesSlice.ts @@ -14,6 +14,16 @@ import { } from "@reduxjs/toolkit" import {apiSliceWithSharedNodes} from "./apiSlice" +interface DocumentThumbnailUpdated { + document_id: string + thumbnail_url: string | null +} + +interface DocumentThumbnailErrorUpdated { + document_id: string + error: string +} + const sharedNodesAdapter = createEntityAdapter() const initialState = sharedNodesAdapter.getInitialState() @@ -21,6 +31,26 @@ const sharedNodesSlice = createSlice({ name: "sharedNode", initialState, reducers: { + documentThumbnailUpdated: ( + state, + action: PayloadAction + ) => { + const payload = action.payload + const node = state.entities[payload.document_id] + if (node && payload.thumbnail_url) { + node.thumbnail_url = payload.thumbnail_url + } + }, + documentThumbnailErrorUpdated: ( + state, + action: PayloadAction + ) => { + const payload = action.payload + const node = state.entities[payload.document_id] + if (node && payload.error) { + node.thumbnail_preview_error = payload.error + } + }, documentsMovedNotifReceived: ( _state, action: PayloadAction @@ -55,8 +85,12 @@ const sharedNodesSlice = createSlice({ } }) -export const {documentsMovedNotifReceived, documentMovedNotifReceived} = - sharedNodesSlice.actions +export const { + documentsMovedNotifReceived, + documentMovedNotifReceived, + documentThumbnailUpdated, + documentThumbnailErrorUpdated +} = sharedNodesSlice.actions export default sharedNodesSlice.reducer diff --git a/ui2/src/hooks/DocumentThumbnail.ts b/ui2/src/hooks/DocumentThumbnail.ts new file mode 100644 index 000000000..c93840277 --- /dev/null +++ b/ui2/src/hooks/DocumentThumbnail.ts @@ -0,0 +1,108 @@ +import {useAppSelector} from "@/app/hooks" +import { + selectDocumentThumbnailError, + selectDocumentThumbnailURL +} from "@/features/nodes/nodesSlice" +import {getBaseURL, getDefaultHeaders, imageEncode} from "@/utils" +import {useEffect, useState} from "react" + +interface Args { + nodeID: string +} + +type ThumbnailState = { + data: string | null + isLoading: boolean + isError: boolean + error: string | null +} + +export default function useDocumentThumbnail({nodeID}: Args) { + const [state, setState] = useState({ + data: null, + isLoading: true, + isError: false, + error: null + }) + const thumbnail_url = useAppSelector(s => + selectDocumentThumbnailURL(s, nodeID) + ) + const thumbnail_error = useAppSelector(s => + selectDocumentThumbnailError(s, nodeID) + ) + const headers = getDefaultHeaders() + let url: string + + useEffect(() => { + if (thumbnail_error) { + setState({ + data: null, + isLoading: false, + isError: true, + error: thumbnail_error + }) + return + } + }, [thumbnail_error]) + + useEffect(() => { + let isMounted = true + if (!thumbnail_url) { + if (isMounted) { + setState({data: null, isLoading: true, isError: false, error: null}) + } + return + } + + if (thumbnail_url && !thumbnail_url.startsWith("/api/")) { + // cloud URL e.g. aws cloudfront URL + url = thumbnail_url + } else { + // use backend server URL (which may differ from frontend's URL) + url = `${getBaseURL(true)}${thumbnail_url}` + } + + const fetchThumbnail = async () => { + try { + const response = await fetch(url, {headers: headers}) + const resp2 = await response.arrayBuffer() + const encodedData = imageEncode(resp2, "image/jpeg") + + if (isMounted) { + setState({ + data: encodedData, + isLoading: false, + isError: false, + error: null + }) + } + } catch (err) { + if (isMounted) { + setState({ + data: null, + isLoading: false, + isError: true, + error: (err as Error).message + }) + } + } + } + + fetchThumbnail() + + return () => { + isMounted = false + } + }, [thumbnail_url]) + + if (thumbnail_error) { + return { + data: null, + isLoading: false, + isError: true, + error: thumbnail_error + } + } + + return state +} diff --git a/ui2/src/hooks/PrevewPolling.ts b/ui2/src/hooks/PrevewPolling.ts new file mode 100644 index 000000000..077b80e76 --- /dev/null +++ b/ui2/src/hooks/PrevewPolling.ts @@ -0,0 +1,138 @@ +import {getBaseURL, getDefaultHeaders} from "@/utils" +import {useEffect, useRef, useState} from "react" + +interface DocumentPreview { + status: string + url: string | null +} + +interface PreviewStatusResponseItem { + doc_id: string + status: string + preview_image_url: string | null +} + +interface UsePreviewPollingOptions { + pollIntervalMs?: number + maxRetries?: number +} + +interface UsePreviewPollingResult { + previews: Record + allReady: boolean + isLoading: boolean + error: Error | null +} + +function toDocIdsQueryParams(docIds: string[]): string { + const params = new URLSearchParams() + docIds.forEach(id => params.append("doc_ids", id)) + return params.toString() +} + +const usePreviewPolling = ( + documentIds: string[], + {pollIntervalMs = 3000, maxRetries = 20}: UsePreviewPollingOptions = {} +): UsePreviewPollingResult => { + const [previews, setPreviews] = useState>({}) + const [allReady, setAllReady] = useState(false) + const [isLoading, setIsLoading] = useState(true) + const [error, setError] = useState(null) + + const retryCount = useRef(0) + const intervalRef = useRef(null) + const headers = getDefaultHeaders() + + useEffect(() => { + if (retryCount.current > maxRetries) { + return + } + + if (!documentIds || documentIds.length === 0) { + return + } + + const pollPreviewStatuses = async () => { + try { + const queryString = toDocIdsQueryParams(documentIds) + const res = await fetch( + `${getBaseURL()}/api/documents/thumbnail-img-status/?${queryString}`, + {headers: headers} + ) + if (!res.ok) throw new Error(`HTTP ${res.status}`) + + const data: PreviewStatusResponseItem[] = await res.json() + setError(null) + setIsLoading(false) + + setPreviews(prev => { + const updated: Record = {...prev} + + let complete = true + + data.forEach(({doc_id, status, preview_image_url}) => { + const newPreview: DocumentPreview = { + status, + url: preview_image_url || null + } + + updated[doc_id] = newPreview + + if (status !== "ready") { + complete = false + } + }) + + setAllReady(complete) + + if (complete && intervalRef.current !== null) { + clearInterval(intervalRef.current) + } + + return updated + }) + } catch (err: unknown) { + const errorObj = err instanceof Error ? err : new Error("Unknown error") + setError(errorObj) + retryCount.current += 1 + if (retryCount.current >= maxRetries && intervalRef.current !== null) { + clearInterval(intervalRef.current) + } + } + } + + pollPreviewStatuses() // First run + intervalRef.current = window.setInterval( + pollPreviewStatuses, + pollIntervalMs + ) + + return () => { + if (intervalRef.current !== null) { + clearInterval(intervalRef.current) + } + } + }, [documentIds, pollIntervalMs, maxRetries]) + + if (retryCount.current > maxRetries) { + const newError = new Error( + `Failed to get thumbnails after ${maxRetries} retries` + ) + + return { + previews, + allReady, + isLoading: false, + error: newError + } + } + + return { + previews, + allReady, + isLoading, + error + } +} + +export default usePreviewPolling diff --git a/ui2/src/types.ts b/ui2/src/types.ts index 2a98d1945..dacc9220a 100644 --- a/ui2/src/types.ts +++ b/ui2/src/types.ts @@ -142,6 +142,7 @@ export type NodeType = NType & { ocr_status: OcrStatusEnum ocr: boolean thumbnail_url: string | null + thumbnail_preview_error: string | null breadcrumb: Array<[string, string]> document_type_id?: string is_shared: boolean diff --git a/ui2/src/utils.ts b/ui2/src/utils.ts index 46ec43edf..d46bf6c8b 100644 --- a/ui2/src/utils.ts +++ b/ui2/src/utils.ts @@ -122,7 +122,6 @@ export function getDefaultHeaders(): Record { "Remote-Email": remote_email || "", "Remote-Name": remote_name || "" } - console.log(headers) } if (token) {