improve send_callback function with callback url validation, restricting redirection and limiting response size

.env.example

@@ -80,7 +80,7 @@ CELERY_ENABLE_UTC=true
 CELERY_TIMEZONE=Asia/Kolkata
 
 
-# Callback Timeouts (in seconds)
+# Callback Timeouts and size limit(in seconds and MB respectively)
 CALLBACK_CONNECT_TIMEOUT = 3
 CALLBACK_READ_TIMEOUT = 10
 

backend/app/api/routes/collections.py

@@ -26,7 +26,7 @@
     DeletionRequest,
     CollectionPublic,
 )
-from app.utils import APIResponse, load_description
+from app.utils import APIResponse, load_description, validate_callback_url
 from app.services.collections import (
     create_collection as create_service,
     delete_collection as delete_service,
@@ -81,6 +81,9 @@ def create_collection(
     current_user: CurrentUserOrgProject,
     request: CreationRequest,
 ):
+    if request.callback_url:
+        validate_callback_url(str(request.callback_url))
+
     collection_job_crud = CollectionJobCrud(session, current_user.project_id)
     collection_job = collection_job_crud.create(
         CollectionJobCreate(
@@ -130,6 +133,9 @@ def delete_collection(
     collection_id: UUID = FastPath(description="Collection to delete"),
     request: CallbackRequest | None = Body(default=None),
 ):
+    if request and request.callback_url:
+        validate_callback_url(str(request.callback_url))
+
     _ = CollectionCrud(session, current_user.project_id).read_one(collection_id)
 
     deletion_request = DeletionRequest(

backend/app/api/routes/documents.py

@@ -10,6 +10,7 @@
     Query,
     UploadFile,
 )
+from pydantic import HttpUrl
 from fastapi import Path as FastPath
 
 from app.api.deps import CurrentUserOrgProject, SessionDep
@@ -32,7 +33,12 @@
     build_document_schema,
     build_document_schemas,
 )
-from app.utils import APIResponse, get_openai_client, load_description
+from app.utils import (
+    APIResponse,
+    get_openai_client,
+    load_description,
+    validate_callback_url,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -111,6 +117,9 @@ async def upload_doc(
     callback_url: str
     | None = Form(None, description="URL to call to report doc transformation status"),
 ):
+    if callback_url:
+        validate_callback_url(callback_url)
+
     source_format, actual_transformer = pre_transform_validation(
         src_filename=src.filename,
         target_format=target_format,

backend/app/api/routes/llm.py

@@ -5,7 +5,7 @@
 from app.api.deps import AuthContextDep, SessionDep
 from app.models import LLMCallRequest, LLMCallResponse, Message
 from app.services.llm.jobs import start_job
-from app.utils import APIResponse, load_description
+from app.utils import APIResponse, validate_callback_url, load_description
 
 
 logger = logging.getLogger(__name__)
@@ -45,6 +45,9 @@ def llm_call(
     project_id = _current_user.project.id
     organization_id = _current_user.organization.id
 
+    if request.callback_url:
+        validate_callback_url(str(request.callback_url))
+
     start_job(
         db=_session,
         request=request,

backend/app/core/config.py

@@ -118,7 +118,7 @@ def AWS_S3_BUCKET(self) -> str:
     CELERY_ENABLE_UTC: bool = True
     CELERY_TIMEZONE: str = "UTC"
 
-    # callback timeouts
+    # callback timeouts and limits
     CALLBACK_CONNECT_TIMEOUT: int = 3
     CALLBACK_READ_TIMEOUT: int = 10
 

backend/app/core/util.py

@@ -2,8 +2,6 @@
 from datetime import datetime, timezone
 
 from fastapi import HTTPException
-from requests import Session, RequestException
-from pydantic import BaseModel, HttpUrl
 
 from openai import OpenAI
 
@@ -24,19 +22,6 @@ def raise_from_unknown(error: Exception, status_code=500):
     raise HTTPException(status_code=status_code, detail=str(error))
 
 
-def post_callback(url: HttpUrl, payload: BaseModel):
-    errno = 0
-    with Session() as session:
-        response = session.post(str(url), json=payload.model_dump())
-        try:
-            response.raise_for_status()
-        except RequestException as err:
-            logger.warning(f"Callback failure: {err}")
-            errno += 1
-
-    return not errno
-
-
 def configure_openai(credentials: dict) -> tuple[OpenAI, bool]:
     """
     Configure OpenAI client with the provided credentials.