Add backend support for custom behaviors + validation endpoint (#2505)

Backend support for #2151 

Adds support for specifying custom behaviors via a list of strings.

When workflows are added or modified, minimal backend validation is done
to ensure that all custom behavior URLs are valid URLs (after removing
the git prefix and custom query arguments).

A separate `POST /crawlconfigs/validate/custom-behavior` endpoint is
also added, which can be used to validate a custom behavior URL. It
performs the same syntax check as above and then:
- For URL directly to behavior file, ensures URL resolves and returns a
2xx/3xx status code
- For Git repositories, uses `git ls-remote` to ensure they exist (and
that branch exists if specified)

---------

Co-authored-by: Ilya Kreymer <[email protected]>

Author: tw4l

backend/Dockerfile

@@ -1,5 +1,10 @@
 FROM docker.io/library/python:3.12-slim
 
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends git \
+	&& apt-get purge -y --auto-remove \
+	&& rm -rf /var/lib/apt/lists/*
+
 WORKDIR /app
 
 ADD requirements.txt /app

backend/btrixcloud/crawlconfigs.py

@@ -4,7 +4,7 @@
 
 # pylint: disable=too-many-lines
 
-from typing import List, Union, Optional, TYPE_CHECKING, cast
+from typing import List, Union, Optional, TYPE_CHECKING, cast, Dict, Tuple
 
 import asyncio
 import json
@@ -15,8 +15,9 @@
 from uuid import UUID, uuid4
 import urllib.parse
 
-import pymongo
+import aiohttp
 from fastapi import APIRouter, Depends, HTTPException, Query
+import pymongo
 
 from .pagination import DEFAULT_PAGE_SIZE, paginated_format
 from .models import (
@@ -43,8 +44,9 @@
     CrawlConfigDeletedResponse,
     CrawlerProxy,
     CrawlerProxies,
+    ValidateCustomBehavior,
 )
-from .utils import dt_now, slug_from_name, validate_regexes
+from .utils import dt_now, slug_from_name, validate_regexes, is_url
 
 if TYPE_CHECKING:
     from .orgs import OrgOps
@@ -231,6 +233,10 @@ async def add_crawl_config(
                 exclude = [exclude]
             validate_regexes(exclude)
 
+        if config_in.config.customBehaviors:
+            for url in config_in.config.customBehaviors:
+                self._validate_custom_behavior_url_syntax(url)
+
         now = dt_now()
         crawlconfig = CrawlConfig(
             id=uuid4(),
@@ -291,6 +297,23 @@ async def add_crawl_config(
             execMinutesQuotaReached=exec_mins_quota_reached,
         )
 
+    def _validate_custom_behavior_url_syntax(self, url: str) -> Tuple[bool, List[str]]:
+        """Validate custom behaviors are valid URLs after removing custom git syntax"""
+        git_prefix = "git+"
+        is_git_repo = False
+
+        if url.startswith(git_prefix):
+            is_git_repo = True
+            url = url[len(git_prefix) :]
+
+        parts = url.split("?")
+        url = parts[0]
+
+        if not is_url(url):
+            raise HTTPException(status_code=400, detail="invalid_custom_behavior")
+
+        return is_git_repo, parts
+
     def ensure_quota_page_limit(self, crawlconfig: CrawlConfig, org: Organization):
         """ensure page limit is set to no greater than quota page limit, if any"""
         if org.quotas.maxPagesPerCrawl and org.quotas.maxPagesPerCrawl > 0:
@@ -356,6 +379,10 @@ async def update_crawl_config(
                 exclude = [exclude]
             validate_regexes(exclude)
 
+        if update.config and update.config.customBehaviors:
+            for url in update.config.customBehaviors:
+                self._validate_custom_behavior_url_syntax(url)
+
         # indicates if any k8s crawl config settings changed
         changed = False
         changed = changed or (
@@ -1070,6 +1097,75 @@ async def re_add_all_scheduled_cron_jobs(self):
                     flush=True,
                 )
 
+    async def _validate_behavior_git_repo(self, repo_url: str, branch: str = ""):
+        """Validate git repository and branch, if specified, exist and are reachable"""
+        cmd = f"git ls-remote {repo_url} HEAD"
+        proc = await asyncio.create_subprocess_shell(cmd)
+        if await proc.wait() > 0:
+            raise HTTPException(
+                status_code=404,
+                detail="custom_behavior_not_found",
+            )
+
+        if branch:
+            await asyncio.sleep(0.5)
+            git_remote_cmd = (
+                f"git ls-remote --exit-code --heads {repo_url} refs/heads/{branch}"
+            )
+            proc = await asyncio.create_subprocess_shell(git_remote_cmd)
+            if await proc.wait() > 0:
+                raise HTTPException(
+                    status_code=404,
+                    detail="custom_behavior_branch_not_found",
+                )
+
+    async def _validate_behavior_url(self, url: str):
+        """Validate behavior file exists at url"""
+        try:
+            timeout = aiohttp.ClientTimeout(total=10)
+            async with aiohttp.ClientSession(timeout=timeout) as session:
+                async with session.get(url) as resp:
+                    if resp.status >= 400:
+                        raise HTTPException(
+                            status_code=404,
+                            detail="custom_behavior_not_found",
+                        )
+        # pylint: disable=raise-missing-from
+        except aiohttp.ClientError:
+            raise HTTPException(
+                status_code=404,
+                detail="custom_behavior_not_found",
+            )
+
+    async def validate_custom_behavior(self, url: str) -> Dict[str, bool]:
+        """Validate custom behavior URL
+
+        Implemented:
+        - Ensure URL is valid (after removing custom git prefix and syntax)
+        - Ensure URL returns status code < 400
+        - Ensure git repository can be reached by git ls-remote and that branch
+        exists, if provided
+        """
+        git_branch = ""
+
+        is_git_repo, url_parts = self._validate_custom_behavior_url_syntax(url)
+        url = url_parts[0]
+
+        if is_git_repo and len(url_parts) > 1:
+            query_str = url_parts[1]
+            try:
+                git_branch = urllib.parse.parse_qs(query_str)["branch"][0]
+            # pylint: disable=broad-exception-caught
+            except (KeyError, IndexError):
+                pass
+
+        if is_git_repo:
+            await self._validate_behavior_git_repo(url, branch=git_branch)
+        else:
+            await self._validate_behavior_url(url)
+
+        return {"success": True}
+
 
 # ============================================================================
 # pylint: disable=too-many-locals
@@ -1316,6 +1412,14 @@ async def re_add_all_scheduled_cron_jobs(
         asyncio.create_task(ops.re_add_all_scheduled_cron_jobs())
         return {"success": True}
 
+    @router.post("/validate/custom-behavior", response_model=SuccessResponse)
+    async def validate_custom_behavior(
+        behavior: ValidateCustomBehavior,
+        # pylint: disable=unused-argument
+        org: Organization = Depends(org_crawl_dep),
+    ):
+        return await ops.validate_custom_behavior(behavior.customBehavior)
+
     org_ops.router.include_router(router)
 
     return ops

backend/btrixcloud/models.py

@@ -333,6 +333,7 @@ class RawCrawlConfig(BaseModel):
 
     logging: Optional[str] = None
     behaviors: Optional[str] = "autoscroll,autoplay,autofetch,siteSpecific"
+    customBehaviors: List[str] = []
 
     userAgent: Optional[str] = None
 
@@ -541,6 +542,8 @@ class CrawlConfigDefaults(BaseModel):
 
     exclude: Optional[List[str]] = None
 
+    customBehaviors: List[str] = []
+
 
 # ============================================================================
 class CrawlConfigAddedResponse(BaseModel):
@@ -592,6 +595,13 @@ class CrawlConfigDeletedResponse(BaseModel):
     status: str
 
 
+# ============================================================================
+class ValidateCustomBehavior(BaseModel):
+    """Input model for validating custom behavior URL/Git reference"""
+
+    customBehavior: str
+
+
 # ============================================================================
 
 ### CRAWLER VERSIONS ###

backend/btrixcloud/utils.py

@@ -11,6 +11,7 @@
 
 from datetime import datetime, timezone
 from typing import Optional, Dict, Union, List, Any
+from urllib.parse import urlparse
 from uuid import UUID
 
 from fastapi import HTTPException
@@ -100,6 +101,15 @@ def is_falsy_bool(stri: Optional[str]) -> bool:
     return False
 
 
+def is_url(url: str) -> bool:
+    """Check if string is a valid URL"""
+    try:
+        result = urlparse(url)
+        return all([result.scheme, result.netloc])
+    except ValueError:
+        return False
+
+
 def str_list_to_bools(str_list: List[str], allow_none=True) -> List[Union[bool, None]]:
     """Return version of input string list cast to bool or None, ignoring other values"""
     output: List[Union[bool, None]] = []

backend/test/conftest.py

@@ -326,7 +326,7 @@ def crawler_config_id_only(_crawler_create_config_only):
     return _crawler_config_id
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def sample_crawl_data():
     return {
         "runNow": False,