Skip to content

Feature/tests and docs #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Aparnap2 wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Feature/tests and docs #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
68cea36
feat: Add OpsMate agent tests and Makefile for deployment testing
Aparnap2 Jan 24, 2026
ff9a41f
chore: add .env.test for Docker-based testing
Aparnap2 Jan 24, 2026
d53d410
feat: add E2E API tests and LLM evaluation tests
Aparnap2 Jan 24, 2026
a1df97a
feat: Add ExecOps intelligent agentic system with OpenRouter
Aparnap2 Jan 24, 2026
eb85d0c
feat: implement multi-agent orchestration system with intelligent agents
Aparnap2 Jan 24, 2026
8dce7a5
feat: add production intelligence infrastructure
Aparnap2 Jan 24, 2026
c1ef022
docs: organize documentation and clean up codebase
Aparnap2 Jan 26, 2026
fce68f4
docs: add 100% testing checklist with all categories
Aparnap2 Jan 27, 2026
4a2ba29
tests: add comprehensive unit tests for coverage
Aparnap2 Jan 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ai-service/.deepeval/.deepeval-cache.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"test_cases_lookup_map": {"{\"actual_output\": \"PR #123 implements a feature for issue LIN-456.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is PR #123 implementing?\", \"retrieval_context\": [\"Developer has 5 recent commits\", \"Issue LIN-456 is in IN_PROGRESS state\", \"PR #123 implements feature for LIN-456\"]}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.7, "success": false, "score": 0.5, "reason": "The score is 0.50 because the actual output does not provide a direct answer to the question 'What is PR #123 implementing?'. The statement 'The statement does not directly address the input question about what PR #123 is implementing.' indicates that the output lacks relevant information.", "strictMode": false, "evaluationModel": "qwen2.5-coder:3b (Ollama)", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"PR #123 implements a feature.\",\n \"It addresses issue LIN-456.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement does not directly address the input question about what PR #123 is implementing.\"\n }\n]"}, "metric_configuration": {"threshold": 0.7, "evaluation_model": "qwen2.5-coder:3b (Ollama)", "strict_mode": false, "include_reason": true}}]}}}
1 change: 1 addition & 0 deletions ai-service/.deepeval/.latest_test_run.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"testRunData": {"testCases": [{"name": "test_neo4j_context_relevance", "input": "What is PR #123 implementing?", "actualOutput": "PR #123 implements a feature for issue LIN-456.", "retrievalContext": ["PR #123 implements feature for LIN-456", "Issue LIN-456 is in IN_PROGRESS state", "Developer has 5 recent commits"], "success": false, "metricsData": [{"name": "Answer Relevancy", "threshold": 0.7, "success": false, "score": 0.5, "reason": "The score is 0.50 because the actual output does not provide a direct answer to the question 'What is PR #123 implementing?'. The statement 'The statement does not directly address the input question about what PR #123 is implementing.' indicates that the output lacks relevant information.", "strictMode": false, "evaluationModel": "qwen2.5-coder:3b (Ollama)", "evaluationCost": 0.0, "verboseLogs": "Statements:\n[\n \"PR #123 implements a feature.\",\n \"It addresses issue LIN-456.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement does not directly address the input question about what PR #123 is implementing.\"\n }\n]"}], "runDuration": 37.60427986899958, "evaluationCost": 0.0, "order": 0}], "conversationalTestCases": [], "metricsScores": [{"metric": "Answer Relevancy", "scores": [0.5], "passes": 0, "fails": 1, "errors": 0}], "prompts": [], "testPassed": 0, "testFailed": 1, "runDuration": 37.62747733799915, "evaluationCost": 0.0}}
1 change: 1 addition & 0 deletions ai-service/.deepeval/.temp_test_run_data.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"testCases": [], "conversationalTestCases": [], "metricsScores": [], "runDuration": 0.0}
28 changes: 28 additions & 0 deletions ai-service/.env.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Test environment with actual Docker containers
GITHUB_TOKEN=ghp_test
GITHUB_REPO_OWNER=test-owner
GITHUB_REPO_NAME=test-repo
SLACK_WEBHOOK_URL=https://hooks.slack.com/test

# Neo4j (neo4j:echoteam123)
NEO4J_URI=bolt://localhost:7687
NEO4J_USER=neo4j
NEO4J_PASSWORD=echoteam123

# Redis (echoteam-redis on port 6380)
REDIS_URL=redis://localhost:6380

# PostgreSQL (postgres-pgvector on port 5432)
DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres

# AWS (mocked in tests)
AWS_ACCESS_KEY_ID=test
AWS_SECRET_ACCESS_KEY=test
AWS_REGION=us-east-1

# LLM
OLLAMA_BASE_URL=http://localhost:11434
USE_LLM_COMPLIANCE=false

# Checkpointer
USE_POSTGRES_CHECKPOINTER=false
245 changes: 245 additions & 0 deletions ai-service/Makefile.opsmate
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,245 @@
# OpsMate Deployment Testing Rig
#
# Sequential container testing without docker-compose (per RAM constraints)
# Usage: make -f Makefile.opsmate <target>
#
# Prerequisites:
# - Docker installed
# - At least 4GB RAM available

.PHONY: help network redis neo4j app test tests cleanup stop-all

# Network name for container communication
NETWORK_NAME := opsmate-net

# Container names
REDIS_CONTAINER := opsmate-redis
NEO4J_CONTAINER := opsmate-neo4j
APP_CONTAINER := opsmate-app

# Ports
REDIS_PORT := 6379
NEO4J_BOLT_PORT := 7687
NEO4J_HTTP_PORT := 7474
APP_PORT := 8000

# Environment
COMPOSE_FILE ?= docker-compose.yml
DOCKERFILE ?= Dockerfile

# =============================================================================
# Help
# =============================================================================

help:
@echo "OpsMate Testing Rig - Sequential Container Testing"
@echo ""
@echo "Usage: make -f Makefile.opsmate <target>"
@echo ""
@echo "Targets:"
@echo " network - Create Docker network"
@echo " redis - Start Redis container"
@echo " neo4j - Start Neo4j container"
@echo " app-build - Build the app container"
@echo " app - Start the app container"
@echo " test - Run all tests"
@echo " tests - Run agent-specific tests"
@echo " cleanup - Remove test containers and network"
@echo " stop-all - Stop all running containers"
@echo " health - Check all service health"
@echo ""
@echo "One-liner (full stack):"
@echo " make network redis neo4j app-build app test"

# =============================================================================
# Network Setup
# =============================================================================

network:
@echo "Creating Docker network: $(NETWORK_NAME)"
@docker network create $(NETWORK_NAME) 2>/dev/null || echo "Network already exists"
@echo "Network created successfully"

# =============================================================================
# Redis (Task Queue)
# =============================================================================

redis:
@echo "Starting Redis container..."
@docker run -d \
--name $(REDIS_CONTAINER) \
--net $(NETWORK_NAME) \
-p $(REDIS_PORT9 \
):637redis:alpine \
Comment on lines +72 to +73
Copy link
Contributor

@gemini-code-assist gemini-code-assist bot Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

There appears to be a typo in the port mapping. It seems like the intention was to map port 6379 to 6379 for the Redis container, but there's an extra 9 and redis:alpine is in the wrong place. This will cause the Redis container to fail to start correctly.

-p $(REDIS_PORT9 \
		):637redis:alpine

2>/dev/null || echo "Redis may already be running"
Comment on lines +67 to +74
Copy link

@coderabbitai coderabbitai bot Jan 27, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

🧩 Analysis chain

🏁 Script executed:

find . -name "Makefile.opsmate" -type f

Repository: Aparnap2/ExecOps

Length of output: 89

🏁 Script executed:

cat -n ./ai-service/Makefile.opsmate | sed -n '60,80p'

Repository: Aparnap2/ExecOps

Length of output: 935

🏁 Script executed:

grep -n "REDIS_PORT\|REDIS_CONTAINER\|NETWORK_NAME" ./ai-service/Makefile.opsmate | head -20

Repository: Aparnap2/ExecOps

Length of output: 1200

Fix malformed Redis docker run command

The port mapping and image reference are corrupted (lines 72–73). The $(REDIS_PORT9 variable reference is invalid, and the image name is incorrectly concatenated with part of a port number. This will break make redis.

🐛 Proposed fix 
 	`@docker` run -d \
 		--name $(REDIS_CONTAINER) \
 		--net $(NETWORK_NAME) \
-		-p $(REDIS_PORT9 \
-		):637redis:alpine \
+		-p $(REDIS_PORT):6379 \
+		redis:alpine \
 		2>/dev/null || echo "Redis may already be running"
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
redis:
@echo "Starting Redis container..."
@docker run -d \
--name $(REDIS_CONTAINER) \
--net $(NETWORK_NAME) \
-p $(REDIS_PORT9 \
):637redis:alpine \
2>/dev/null || echo "Redis may already be running"
redis:
`@echo` "Starting Redis container..."
`@docker` run -d \
--name $(REDIS_CONTAINER) \
--net $(NETWORK_NAME) \
-p $(REDIS_PORT):6379 \
redis:alpine \
2>/dev/null || echo "Redis may already be running"
🤖 Prompt for AI Agents 
In `@ai-service/Makefile.opsmate` around lines 67 - 74, The redis Makefile target
has a malformed docker run command: the port variable is mistyped as
$(REDIS_PORT9 and the image/tokenization is corrupted ("-p $(REDIS_PORT9
\n\t):637redis:alpine"). Fix the redis target (function/target name: redis) to
use the correct port variable $(REDIS_PORT), ensure the -p mapping is "-p
$(REDIS_PORT):6379", and restore the image reference to "redis:alpine"; keep the
existing references to $(REDIS_CONTAINER) and $(NETWORK_NAME) and preserve the
2>/dev/null || echo fallback behavior.

@echo "Redis started on port $(REDIS_PORT)"
@sleep 2
@echo "Testing Redis connection..."
@docker run --rm --net $(NETWORK_NAME) redis:alpine redis-cli -h $(REDIS_CONTAINER) ping
@echo "Redis is healthy"

redis-stop:
@docker stop $(REDIS_CONTAINER) 2>/dev/null || true
@docker rm $(REDIS_CONTAINER) 2>/dev/null || true

# =============================================================================
# Neo4j (Graph Brain)
# =============================================================================

neo4j:
@echo "Starting Neo4j container..."
@docker run -d \
--name $(NEO4J_CONTAINER) \
--net $(NETWORK_NAME) \
-p $(NEO4J_HTTP_PORT):7474 \
-p $(NEO4J_BOLT_PORT):7687 \
-e NEO4J_AUTH=neo4j/testpassword \
-e NEO4J_PLUGINS='["apoc", "graph-data-science"]' \
neo4j:community \
2>/dev/null || echo "Neo4j may already be running"
@echo "Neo4j started (HTTP: $(NEO4J_HTTP_PORT), Bolt: $(NEO4J_BOLT_PORT))"
@sleep 5
@echo "Testing Neo4j connection..."
@curl -s -u neo4j:testpassword \
-H "Content-Type: application/json" \
-X POST http://localhost:$(NEO4J_HTTP_PORT)/db/neo4j/tx/commit \
-d '{"statements":[{"statement":"RETURN 1 as test"}]}' | grep -q "test" && echo "Neo4j is healthy" || echo "Neo4j may still be starting"

neo4j-stop:
@docker stop $(NEO4J_CONTAINER) 2>/dev/null || true
@docker rm $(NEO4J_CONTAINER) 2>/dev/null || true

# =============================================================================
# App Build
# =============================================================================

app-build:
@echo "Building OpsMate app container..."
@docker build -t $(APP_CONTAINER):latest -f $(DOCKERFILE) . --no-cache
@echo "App container built successfully"

app-build-no-cache:
@echo "Building OpsMate app container (no cache)..."
@docker build -t $(APP_CONTAINER):latest -f $(DOCKERFILE) .
@echo "App container built successfully"

# =============================================================================
# App
# =============================================================================

app: app-build
@echo "Starting OpsMate app container..."
@docker run -d \
--name $(APP_CONTAINER) \
--net $(NETWORK_NAME) \
-p $(APP_PORT):8000 \
-e REDIS_URL=redis://$(REDIS_CONTAINER):6379 \
-e NEO4J_URI=bolt://$(NEO4J_CONTAINER):7687 \
-e NEO4J_USER=neo4j \
-e NEO4J_PASSWORD=testpassword \
-e DATABASE_URL=postgresql://postgres:postgres@postgres:5432/postgres \
-e USE_POSTGRES_CHECKPOINTER=false \
$(APP_CONTAINER):latest \
2>/dev/null || echo "App may already be running"
@echo "App started on port $(APP_PORT)"
@sleep 3
@echo "Testing app health..."
@curl -s http://localhost:$(APP_PORT)/health || echo "App may still be starting"

app-stop:
@docker stop $(APP_CONTAINER) 2>/dev/null || true
@docker rm $(APP_CONTAINER) 2>/dev/null || true

# =============================================================================
# Tests
# =============================================================================

test: tests
@echo ""
@echo "=========================================="
@echo "All tests completed!"
@echo "=========================================="

tests:
@echo "Running OpsMate agent tests..."
@echo ""
@echo "1. Running Hunter (Zombie Hunter) tests..."
@.venv/bin/pytest tests/test_hunter.py -v --tb=short || echo "Hunter tests completed with status: $$?"
@echo ""
@echo "2. Running Watchman (Night Watchman) tests..."
@.venv/bin/pytest tests/test_watchman.py -v --tb=short || echo "Watchman tests completed with status: $$?"
@echo ""
@echo "3. Running Guard (Access Guard) tests..."
@.venv/bin/pytest tests/test_guard.py -v --tb=short || echo "Guard tests completed with status: $$?"
@echo ""
@echo "4. Running full test suite..."
@.venv/bin/pytest tests/ -v --tb=short -x || echo "Tests completed"

test-hunter:
@echo "Running Hunter tests..."
@.venv/bin/pytest tests/test_hunter.py -v --tb=short

test-watchman:
@echo "Running Watchman tests..."
@.venv/bin/pytest tests/test_watchman.py -v --tb=short

test-guard:
@echo "Running Guard tests..."
@.venv/bin/pytest tests/test_guard.py -v --tb=short

test-e2e:
@echo "Running E2E API tests..."
@echo "Triggering Zombie Scan..."
@curl -s -X POST http://localhost:$(APP_PORT)/agents/hunter/scan \
-H "Authorization: Bearer test_token" | head -c 500
@echo ""
@echo "Triggering Access Audit..."
@curl -s -X POST http://localhost:$(APP_PORT)/agents/guard/audit | head -c 500
@echo ""

# =============================================================================
# Health Checks
# =============================================================================

health:
@echo "Checking service health..."
@echo ""
@echo "Redis: $$(docker run --rm --net $(NETWORK_NAME) redis:alpine redis-cli -h $(REDIS_CONTAINER) ping 2>/dev/null || echo 'NOT RUNNING')"
@echo "Neo4j: $$(curl -s -o /dev/null -w '%{http_code}' -u neo4j:testpassword http://localhost:$(NEO4J_HTTP_PORT)/ 2>/dev/null || echo 'NOT RUNNING')"
@echo "App: $$(curl -s -o /dev/null -w '%{http_code}' http://localhost:$(APP_PORT)/health 2>/dev/null || echo 'NOT RUNNING')"

# =============================================================================
# Cleanup
# =============================================================================

cleanup: stop-all
@echo "Removing Docker network..."
@docker network rm $(NETWORK_NAME) 2>/dev/null || echo "Network already removed"
@echo "Cleanup complete"

stop-all:
@echo "Stopping all containers..."
@docker stop $(REDIS_CONTAINER) $(NEO4J_CONTAINER) $(APP_CONTAINER) 2>/dev/null || true
@docker rm $(REDIS_CONTAINER) $(NEO4J_CONTAINER) $(APP_CONTAINER) 2>/dev/null || true
@echo "All containers stopped"

# =============================================================================
# Quick Start (all-in-one)
# =============================================================================

quick-start: network redis neo4j app-build app
@echo ""
@echo "=========================================="
@echo "OpsMate is running!"
@echo "=========================================="
@echo "Redis: localhost:$(REDIS_PORT)"
@echo "Neo4j: localhost:$(NEO4J_HTTP_PORT) (neo4j/testpassword)"
@echo "App: localhost:$(APP_PORT)"
@echo ""
@echo "Run 'make -f Makefile.opsmate test' to verify"

quick-test: network redis neo4j app-build app tests
@echo ""
@echo "=========================================="
@echo "Quick test complete!"
@echo "=========================================="
Loading