Adds Github action to trigger tests upon raising PR

.github/workflows/tests.yml

@@ -0,0 +1,151 @@
+name: Run Tests
+
+on:
+  push:
+    branches: [main,beta]
+  pull_request:
+    branches: [main,beta]
+    types: [opened, synchronize, reopened, ready_for_review]
+
+permissions:
+  contents: read
+  checks: write
+  pull-requests: write
+
+jobs:
+  test-main:
+    name: Test Main SDK (Python 3.9)
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Backup pyproject.toml
+        run: |
+          cp pyproject.toml pyproject.toml.bak
+
+      - name: Remove additional dependencies
+        run: |
+          sed -i.bak '/^additional_dev = \[$/,/]$/d' pyproject.toml
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Set up Python 3.9
+        run: uv python install 3.9
+
+      - name: Install dependencies (dev only)
+        run: |
+          uv sync --python 3.9
+
+      - name: Run main tests (excluding CrewAI)
+        env:
+          MAXIM_API_KEY: ${{ secrets.MAXIM_API_KEY }}
+          MAXIM_BASE_URL: ${{ secrets.MAXIM_BASE_URL }}
+          MAXIM_DATASET_ID: ${{ secrets.MAXIM_DATASET_ID }}
+          MAXIM_WORKSPACE_ID: ${{ secrets.MAXIM_WORKSPACE_ID }}
+          MAXIM_LOG_REPO_ID: ${{ secrets.MAXIM_LOG_REPO_ID }}
+          MAXIM_PROMPT_CHAIN_VERSION_ID: ${{ secrets.MAXIM_PROMPT_CHAIN_VERSION_ID }}
+          MAXIM_ASSISTANT_PROMPT_VERSION_ID: ${{ secrets.MAXIM_ASSISTANT_PROMPT_VERSION_ID }}
+          MAXIM_ASSISTANT_PROMPT_CHAIN_VERSION_ID: ${{ secrets.MAXIM_ASSISTANT_PROMPT_CHAIN_VERSION_ID }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+          AZURE_OPENAI_KEY: ${{ secrets.AZURE_OPENAI_KEY }}
+          PORTKEY_API_KEY: ${{ secrets.PORTKEY_API_KEY }}
+          PORTKEY_VIRTUAL_KEY: ${{ secrets.PORTKEY_VIRTUAL_KEY }}
+          LLAMAINDEX_API_KEY: ${{ secrets.LLAMAINDEX_API_KEY }}
+          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
+          TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}
+          FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
+          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          MAXIM_PROMPT_1_ID: ${{ secrets.MAXIM_PROMPT_1_ID }}
+          MAXIM_PROMPT_2_ID: ${{ secrets.MAXIM_PROMPT_2_ID }}
+          MAXIM_PROMPT_3_ID: ${{ secrets.MAXIM_PROMPT_3_ID }}
+          MAXIM_PROMPT_1_VERSION_1_ID: ${{ secrets.MAXIM_PROMPT_1_VERSION_1_ID }}
+          MAXIM_PROMPT_1_VERSION_3_ID: ${{ secrets.MAXIM_PROMPT_1_VERSION_3_ID }}
+          MAXIM_PROMPT_1_VERSION_4_ID: ${{ secrets.MAXIM_PROMPT_1_VERSION_4_ID }}
+          MAXIM_PROMPT_1_VERSION_5_ID: ${{ secrets.MAXIM_PROMPT_1_VERSION_5_ID }}
+          MAXIM_FOLDER_1_ID: ${{ secrets.MAXIM_FOLDER_1_ID }}
+          MAXIM_FOLDER_2_ID: ${{ secrets.MAXIM_FOLDER_2_ID }}
+          PROMPT_VERSION_ID: ${{ secrets.PROMPT_VERSION_ID }}
+          FOLDER_ID: ${{ secrets.FOLDER_ID }}
+          MAXIM_PROMPT_3_VERSION_2_ID: ${{ secrets.MAXIM_PROMPT_3_VERSION_2_ID }}
+          MAXIM_TEST_RUN_PROMPT_VERSION_ID: ${{ secrets.MAXIM_TEST_RUN_PROMPT_VERSION_ID }}
+          MAXIM_TEST_RUN_ASSISTANT_PROMPT_VERSION_ID: ${{ secrets.MAXIM_TEST_RUN_ASSISTANT_PROMPT_VERSION_ID }}
+          MAXIM_TEST_RUN_ASSISTANT_PROMPT_CHAIN_VERSION_ID: ${{ secrets.MAXIM_TEST_RUN_ASSISTANT_PROMPT_CHAIN_VERSION_ID }}
+        run: |
+          uv run pytest maxim/tests/test_prompts.py maxim/tests/test_openai.py maxim/tests/test_llamaindex.py maxim/tests/test_together.py maxim/tests/test_fireworks.py maxim/tests/test_groq.py maxim/tests/test_add_dataset_entries_comprehensive.py maxim/tests/test_maxim_core_simple.py --junitxml=junit/main-test-results.xml
+
+      - name: Upload main test results
+        uses: actions/upload-artifact@v4
+        if: success() || failure()    # run this step even if previous step failed
+        with:
+          name: main-test-results
+          path: junit/main-test-results.xml
+
+      - name: Publish main test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: success() || failure()    # run this step even if previous step failed
+        with:
+          junit_files: "junit/main-test-results.xml"
+          check_name: "Main Test Results"
+
+      - name: Restore pyproject.toml
+        run: |
+          mv pyproject.toml.bak pyproject.toml
+
+  additional-tests:
+    name: Test Additional Integrations (Python 3.10)
+    runs-on: ubuntu-latest
+    if: github.event.pull_request.draft == false
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          version: "latest"
+
+      - name: Backup .python-version
+        run: |
+          cp .python-version .python-version.bak
+
+      - name: Update .python-version for CrewAI
+        run: |
+          echo "3.10.0" > .python-version
+
+      - name: Set up Python 3.10
+        run: uv python install 3.10
+
+      - name: Install dependencies (CrewAI only)
+        run: |
+          uv sync --python 3.10
+
+      - name: Run additional integration tests
+        run: |
+          uv run pytest maxim/tests/test_crewai.py --junitxml=junit/additional-test-results.xml
+
+      - name: Upload additional test results
+        uses: actions/upload-artifact@v4
+        if: success() || failure()    # run this step even if previous step failed
+        with:
+          name: additional-test-results
+          path: junit/additional-test-results.xml
+
+      - name: Publish additional test results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: success() || failure()    # run this step even if previous step failed
+        with:
+          junit_files: "junit/additional-test-results.xml"
+          check_name: "Additional Test Results"
+
+      - name: Restore .python-version
+        if: always()    # run this step even if previous steps failed
+        run: |
+          mv .python-version.bak .python-version

maxim/filter_objects.py

@@ -75,10 +75,16 @@ def parse_incoming_query(incoming_query: str) -> List[RuleType]:
                     value = bool(value)
                     parsed = True
                 if not parsed:
-                    if type(value) == bool:
+                    try:
+                        value = int(value)
+                        parsed = True
+                    except ValueError:
+                        pass
+                if not parsed:
+                    if isinstance(value, bool):
                         value = bool(value)
                         parsed = True
-                    elif type(value) is int:
+                    elif isinstance(value, int):
                         value = int(value)
                         parsed = True
                 if not parsed:

maxim/logger/portkey/portkey.py

@@ -134,6 +134,7 @@ def create(self, *args, **kwargs):
                     .get("message", {})
                     .get("content", "")
                 )
+            if trace is not None:
                 trace.end()
         except Exception as e:
             scribe().warning(
@@ -201,6 +202,7 @@ async def create(self, *args, **kwargs):
                     .get("message", {})
                     .get("content", "")
                 )
+            if trace is not None:
                 trace.end()
         except Exception as e:
             scribe().warning(

maxim/tests/test_add_dataset_entries_comprehensive.py

@@ -7,7 +7,11 @@
 from maxim import Maxim, Config
 from maxim.models.dataset import DatasetEntry, Variable, DatasetEntryWithRowNo, FileVariablePayload, VariableFileAttachment
 from maxim.logger.components.attachment import FileAttachment, FileDataAttachment, UrlAttachment
+from dotenv import load_dotenv
+load_dotenv()
 
+MAXIM_BASE_URL = os.getenv("MAXIM_BASE_URL")
+MAXIM_API_KEY = os.getenv("MAXIM_API_KEY")
 
 class TestAddDatasetEntriesComprehensive(unittest.TestCase):
     """Comprehensive test suite for the updated add_dataset_entries method."""
@@ -17,13 +21,9 @@ def setUp(self) -> None:
         if hasattr(Maxim, "_instance"):
             delattr(Maxim, "_instance")
 
-        # Set up test environment variables
-        os.environ["MAXIM_API_KEY"] = "test-api-key"
-        os.environ["MAXIM_BASE_URL"] = "https://app.getmaxim.ai"
-
         config = Config(
-            api_key="test-api-key",
-            base_url="https://app.getmaxim.ai",
+            api_key=MAXIM_API_KEY,
+            base_url=MAXIM_BASE_URL,
             debug=True,
             raise_exceptions=True
         )
@@ -34,10 +34,6 @@ def tearDown(self) -> None:
         """Clean up after tests."""
         if hasattr(Maxim, "_instance"):
             delattr(Maxim, "_instance")
-        if "MAXIM_API_KEY" in os.environ:
-            del os.environ["MAXIM_API_KEY"]
-        if "MAXIM_BASE_URL" in os.environ:
-            del os.environ["MAXIM_BASE_URL"]
 
     def _create_mock_response(self, content: bytes, headers: dict = None) -> Mock:
         """Create a mock HTTP response."""
@@ -175,7 +171,7 @@ def test_add_dataset_entries_with_file_attachments(self) -> None:
                 name="test.txt",
                 mime_type="text/plain"
             )
-            
+
             entry = DatasetEntry(entry={
                 "input": Variable(type="text", payload="Input with file"),
                 "files": Variable(type="file", payload=[file_attachment])
@@ -186,7 +182,7 @@ def test_add_dataset_entries_with_file_attachments(self) -> None:
 
             # Verify file upload process was triggered
             self.assertEqual(mock_client.request.call_count, 4)  # total_rows + entries + upload_url + patch
-            
+
         finally:
             # Clean up temp file
             os.unlink(temp_file_path)
@@ -201,7 +197,7 @@ def test_add_dataset_entries_with_url_attachment(self) -> None:
             b'{"data": {"url": "https://signed-url.com", "key": "datasets/test-dataset-id/entry123/test-file-key"}}'
         )
         mock_patch_response = self._create_mock_response(b'{"success": true}')
-        
+
         self._setup_mock_network_calls([
             mock_total_rows_response,
             mock_add_entries_response,
@@ -215,7 +211,7 @@ def test_add_dataset_entries_with_url_attachment(self) -> None:
             name="image.jpg",
             mime_type="image/jpeg"
         )
-        
+
         entry = DatasetEntry(entry={
             "input": Variable(type="text", payload="Input with URL"),
             "images": Variable(type="file", payload=[url_attachment])
@@ -240,7 +236,7 @@ def test_add_dataset_entries_with_file_data_attachment(self) -> None:
             b'{"data": {"url": "https://signed-url.com", "key": "datasets/test-dataset-id/entry123/test-file-key"}}'
         )
         mock_patch_response = self._create_mock_response(b'{"success": true}')
-        
+
         mock_client = self._setup_mock_network_calls([
             mock_total_rows_response,
             mock_add_entries_response,
@@ -254,7 +250,7 @@ def test_add_dataset_entries_with_file_data_attachment(self) -> None:
             name="data.bin",
             mime_type="application/octet-stream"
         )
-        
+
         entry = DatasetEntry(entry={
             "input": Variable(type="text", payload="Input with file data"),
             "data": Variable(type="file", payload=[file_data_attachment])
@@ -272,7 +268,7 @@ def test_add_dataset_entries_mixed_input_types(self) -> None:
         mock_add_entries_response = self._create_mock_response(
             b'{"data": {"ids": ["entry1", "entry2"], "cells": []}}'
         )
-        
+
         mock_client = self._setup_mock_network_calls([
             mock_total_rows_response,
             mock_add_entries_response
@@ -283,7 +279,7 @@ def test_add_dataset_entries_mixed_input_types(self) -> None:
             "input": Variable(type="text", payload="Object input"),
             "output": Variable(type="json", payload={"result": "object"}),
         })
-        
+
         dict_entry = {
             "input": "Dict input",
             "output": {"result": "dict"}

maxim/tests/test_add_dataset_entries_integration.py

@@ -118,7 +118,7 @@ def test_add_dataset_entries_with_dictionary_input(self) -> None:
         entries = [
             {
                 "Input": "How do I calculate the area of a circle?",
-                "Expected Output": {
+                "Test": {
                     "formula": "A = π × r²",
                     "explanation": "The area of a circle is π times the radius squared",
                     "example": "For radius 5: A = π × 25 = 78.54 square units",
@@ -129,7 +129,7 @@ def test_add_dataset_entries_with_dictionary_input(self) -> None:
             },
             {
                 "Input": "What is 15 + 27?",
-                "Expected Output": {
+                "Test": {
                     "answer": 42,
                     "method": "simple addition",
                     "step_by_step": ["15 + 27", "= 42"],
@@ -195,7 +195,7 @@ def test_add_dataset_entries_with_dataset_entry_objects(self) -> None:
         # Create DatasetEntry objects with correct column structure
         entry1 = DatasetEntry(entry={
             "Input": Variable(type="text", payload="What is the capital of France?"),
-            "Expected Output": Variable(type="json", payload={
+            "Test": Variable(type="json", payload={
                 "answer": "Paris",
                 "confidence": 0.95,
                 "reasoning": "Paris is the capital and largest city of France",
@@ -206,7 +206,7 @@ def test_add_dataset_entries_with_dataset_entry_objects(self) -> None:
 
         entry2 = DatasetEntry(entry={
             "Input": Variable(type="text", payload="Explain the process of photosynthesis"),
-            "Expected Output": Variable(type="json", payload={
+            "Test": Variable(type="json", payload={
                 "answer": "Photosynthesis is the process by which plants convert sunlight into energy",
                 "confidence": 0.98,
                 "key_components": ["chlorophyll", "sunlight", "carbon dioxide", "water"],
@@ -265,7 +265,7 @@ def test_add_dataset_entries_with_file_attachments(self) -> None:
 
             entry_with_file = DatasetEntry(entry={
                 "Input": Variable(type="text", payload="Integration test with file attachment"),
-                "Expected Output": Variable(type="json", payload={
+                "Test": Variable(type="json", payload={
                     "result": "Successfully processed file attachment",
                     "file_type": "text",
                     "confidence": 0.95
@@ -283,7 +283,7 @@ def test_add_dataset_entries_with_file_attachments(self) -> None:
 
             entry_with_data = DatasetEntry(entry={
                 "Input": Variable(type="text", payload="Integration test with binary data"),
-                "Expected Output": Variable(type="json", payload={
+                "Test": Variable(type="json", payload={
                     "result": "Successfully processed binary data",
                     "file_size_bytes": len(binary_data),
                     "confidence": 0.97
@@ -349,7 +349,7 @@ def test_mixed_entry_types_integration(self) -> None:
         # Create mixed entry types with correct columns
         dict_entry = {
             "Input": "What are the benefits of renewable energy?",
-            "Expected Output": {
+            "Test": {
                 "benefits": ["environmental protection", "sustainability", "cost savings"],
                 "types": ["solar", "wind", "hydro", "geothermal"],
                 "impact": "positive long-term effects on climate",
@@ -360,7 +360,7 @@ def test_mixed_entry_types_integration(self) -> None:
 
         dataset_entry = DatasetEntry(entry={
             "Input": Variable(type="text", payload="Explain machine learning in simple terms"),
-            "Expected Output": Variable(type="json", payload={
+            "Test": Variable(type="json", payload={
                 "definition": "Machine learning is AI that learns from data to make predictions",
                 "key_concepts": ["algorithms", "training", "predictions", "patterns"],
                 "applications": ["image recognition", "natural language processing", "recommendation systems"],

maxim/tests/test_agno.py

@@ -1,11 +1,12 @@
 import unittest
 import os
-from uuid import uuid4
+from dotenv import load_dotenv
 
 from maxim.logger import Logger
 from maxim.logger.agno import instrument_agno
 from maxim.tests.mock_writer import inject_mock_writer
 
+load_dotenv()
 
 class TestAgnoIntegration(unittest.TestCase):
     def test_agno_generation_logging(self):
@@ -19,8 +20,8 @@ def test_agno_generation_logging(self):
         # Setup Maxim logger with mock writer
         logger = Logger(
             {"id": os.getenv("MAXIM_LOG_REPO_ID", "test-repo")},
-            api_key="test-api-key",
-            base_url="https://app.getmaxim.ai",
+            api_key=os.getenv("MAXIM_API_KEY"),
+            base_url=os.getenv("MAXIM_BASE_URL"),
         )
         mock_writer = inject_mock_writer(logger)
         instrument_agno(logger)