Add API handlers and API Docker image

.github/workflows/publish_remote_api_image.yml

@@ -0,0 +1,56 @@
+name: Build and Publish Remote-Vector-Index-Builder API Image to Docker
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'remote_vector_index_builder/app/**'
+      - '.github/workflows/publish_remote_api_image.yml'
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build-and-publish-api-image:
+    name: Build and Publish Remote-Vector-Index-Builder API Image
+    if: github.repository == 'opensearch-project/remote-vector-index-builder'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Build Docker Image
+        run : |
+          docker build  -f ./remote_vector_index_builder/app/Dockerfile . -t opensearchstaging/remote-vector-index-builder:api-1.0.0
+          docker tag opensearchstaging/remote-vector-index-builder:api-1.0.0 opensearchstaging/remote-vector-index-builder:api-latest
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ secrets.REMOTE_VECTOR_DOCKER_ROLE }}
+          aws-region: us-east-1
+
+      - name: Retrieve Values
+        id: retrieve-values
+        run: |
+          DOCKERHUB_PASSWORD=`aws secretsmanager get-secret-value --secret-id jenkins-staging-dockerhub-credential --query SecretString --output text`
+          echo "::add-mask::$DOCKERHUB_PASSWORD"
+          echo "dockerhub-password=$DOCKERHUB_PASSWORD" >> $GITHUB_OUTPUT
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.REMOTE_VECTOR_DOCKER_USERNAME }}
+          password: ${{ steps.retrieve-values.outputs.dockerhub-password }}
+
+      - name: Push Docker Image
+        run : |
+          docker push opensearchstaging/remote-vector-index-builder:api-1.0.0
+          docker push opensearchstaging/remote-vector-index-builder:api-latest
+      - name: Runner Cleanups
+        if: always()
+        run: |
+          docker logout

.github/workflows/run-tests.yml

@@ -38,6 +38,7 @@ jobs:
           run: |
             python -m pip install --upgrade pip
             python -m pip install -r remote_vector_index_builder/core/requirements.txt
+            python -m pip install -r remote_vector_index_builder/app/requirements.txt
             python -m pip install -r test_remote_vector_index_builder/requirements.txt
 
         - name: Run Linting - flake8

remote_vector_index_builder/app/Dockerfile

@@ -0,0 +1,20 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+FROM opensearchstaging/remote-vector-index-builder:core-latest
+
+WORKDIR /remote_vector_index_builder
+
+COPY ./remote_vector_index_builder/app/requirements.txt /remote_vector_index_builder/app/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /remote_vector_index_builder/app/requirements.txt
+
+COPY ./remote_vector_index_builder/app /remote_vector_index_builder/app
+
+ENV PYTHONPATH='${PYTHONPATH}:/tmp/faiss/build/faiss/python:/remote_vector_index_builder'
+RUN ["python", "app/test_imports.py"]
+CMD ["fastapi", "run", "app/main.py", "--port", "80"]

remote_vector_index_builder/app/__init__.py

@@ -0,0 +1,6 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.

remote_vector_index_builder/app/base/__init__.py

@@ -0,0 +1,6 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.

remote_vector_index_builder/app/base/config.py

@@ -0,0 +1,35 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+from pydantic_settings import BaseSettings
+from app.storage.types import RequestStoreType
+from typing import Optional
+
+
+class Settings(BaseSettings):
+    """
+    Settings class for the application. Pulls the settings
+    from the Docker container environment variables
+    """
+
+    # Request Store settings
+    request_store_type: RequestStoreType = RequestStoreType.MEMORY
+
+    # In memory settings
+    request_store_max_size: int = 1000000
+    request_store_ttl_seconds: Optional[int] = 600
+
+    # Resource Manager settings, in bytes
+    gpu_memory_limit: float = 24.0 * 10**9
+    cpu_memory_limit: float = 32.0 * 10**9
+
+    # Workflow Executor settings
+    max_workers: int = 5
+
+    # Service settings
+    service_name: str = "remote-vector-index-builder-api"
+    log_level: str = "INFO"

remote_vector_index_builder/app/base/exceptions.py

@@ -0,0 +1,22 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+class ApiError(Exception):
+    """Base exception for api errors"""
+
+    pass
+
+
+class HashCollisionError(ApiError):
+    """Raised when there's a hash collision in the Request Store"""
+
+    pass
+
+
+class CapacityError(ApiError):
+    """Raised when the worker does not have enough capacity to fulfill the request"""
+
+    pass

remote_vector_index_builder/app/base/resources.py

@@ -0,0 +1,107 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+
+import threading
+
+
+class ResourceManager:
+    """
+    A thread-safe resource manager that tracks and manages GPU and CPU memory allocations.
+
+    This class provides mechanisms to safely allocate and release memory resources
+    in a multi-threaded environment, ensuring that memory usage doesn't exceed
+    the specified limits.
+
+    Attributes:
+        _total_gpu_memory (float): Total available GPU memory in bytes
+        _total_cpu_memory (float): Total available CPU memory in bytes
+        _available_gpu_memory (float): Currently available GPU memory in bytes
+        _available_cpu_memory (float): Currently available CPU memory in bytes
+        _lock (threading.Lock): Thread lock for synchronization
+    """
+
+    def __init__(self, total_gpu_memory: float, total_cpu_memory: float):
+        """
+        Initialize the ResourceManager with specified GPU and CPU memory limits.
+
+        Args:
+            total_gpu_memory (float): Total GPU memory available for allocation, in bytes
+            total_cpu_memory (float): Total CPU memory available for allocation, in bytes
+        """
+        self._total_gpu_memory = total_gpu_memory
+        self._total_cpu_memory = total_cpu_memory
+        self._available_gpu_memory = total_gpu_memory
+        self._available_cpu_memory = total_cpu_memory
+        self._lock = threading.Lock()
+
+    # TODO: separate this function into CPU and GPU specific allocation checks
+    def can_allocate(self, gpu_memory: float, cpu_memory: float) -> bool:
+        """
+        Check if the requested amount of GPU and CPU memory can be allocated.
+
+        Args:
+            gpu_memory (float): Amount of GPU memory requested, in bytes
+            cpu_memory (float): Amount of CPU memory requested, in bytes
+
+        Returns:
+            bool: True if the requested memory can be allocated, False otherwise
+        """
+        with self._lock:
+            return (
+                self._available_gpu_memory >= gpu_memory
+                and self._available_cpu_memory >= cpu_memory
+            )
+
+    def allocate(self, gpu_memory: float, cpu_memory: float) -> bool:
+        """
+        Attempt to allocate the specified amount of GPU and CPU memory.
+
+        Args:
+            gpu_memory (float): Amount of GPU memory to allocate, in bytes
+            cpu_memory (float): Amount of CPU memory to allocate, in bytes
+
+        Returns:
+            bool: True if allocation was successful, False if insufficient resources
+        """
+        if not self.can_allocate(gpu_memory, cpu_memory):
+            return False
+        with self._lock:
+            self._available_gpu_memory -= gpu_memory
+            self._available_cpu_memory -= cpu_memory
+            return True
+
+    def release(self, gpu_memory: float, cpu_memory: float) -> None:
+        """
+        Release previously allocated GPU and CPU memory back to the pool.
+
+        Args:
+            gpu_memory (float): Amount of GPU memory to release, in bytes
+            cpu_memory (float): Amount of CPU memory to release, in bytes
+        """
+        with self._lock:
+            self._available_gpu_memory += gpu_memory
+            self._available_cpu_memory += cpu_memory
+
+    def get_available_gpu_memory(self) -> float:
+        """
+        Get the current amount of available GPU memory.
+
+        Returns:
+            float: Amount of available GPU memory in bytes
+        """
+        with self._lock:
+            return self._available_gpu_memory
+
+    def get_available_cpu_memory(self) -> float:
+        """
+        Get the current amount of available GPU memory.
+
+        Returns:
+            float: Amount of available GPU memory in bytes
+        """
+        with self._lock:
+            return self._available_cpu_memory

remote_vector_index_builder/app/executors/__init__.py

@@ -0,0 +1,6 @@
+# Copyright OpenSearch Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.