added envs to conductor, reference by scripts

Author: Mitchell Shiell

Makefile

@@ -1,16 +1,16 @@
 # Define all phony targets (targets that don't create files)
-.PHONY: phaseOne phaseTwo down clean cleanVolumes mockData
+.PHONY: dev-phase-one dev-stage clean-data reset-volumes load-sample-data
 
 # Start Phase One development environment
-phaseOne:
+phase-one:
 	PROFILE=phaseOne docker compose -f ./docker-compose.phaseOne.yml --profile phaseOne up --attach conductor 
 
-# Start Phase Two development environment
-stageDev:
+# Start Stage development environment
+stage-dev:
 	PROFILE=stageDev docker compose -f ./docker-compose.phaseOne.yml --profile stageDev up --attach conductor
 
 # Gracefully shutdown all containers while preserving volumes
-down:
+shutdown:
 	@{ \
 		printf "\033[1;36mConductor:\033[0m Checking for containers...\n"; \
 		if docker compose -f ./docker-compose.phaseOne.yml ps -a -q 2>/dev/null | grep -q .; then \
@@ -25,7 +25,7 @@ down:
 	}
 
 # Shutdown all containers and remove all volumes (WARNING: Deletes all data)
-downVolumes:
+reset:
 	@{ \
 		printf "\033[1;33mWarning:\033[0m This will remove all containers AND their volumes. Data will be lost.\n"; \
 		read -p "Are you sure you want to continue? [y/N] " confirm; \
@@ -46,11 +46,11 @@ downVolumes:
 	}
 
 # Load sample data into Elasticsearch
-mockData:
-	PROFILE=mockData docker compose -f ./docker-compose.phaseOne.yml --profile mockData up --attach conductor
+load-data:
+	PROFILE=data docker compose -f ./docker-compose.phaseOne.yml --profile data up --attach conductor
 
 # Remove all documents from Elasticsearch (preserves index structure)
-clean:
+clean-data:
 	@echo "\033[1;33mWarning:\033[0m This will delete ALL data from the Elasticsearch index."
 	@echo "This action cannot be undone."
 	@/bin/bash -c 'read -p "Are you sure you want to continue? [y/N] " confirm; \

README.md

@@ -50,7 +50,7 @@ previous phase's foundation while introducing new capabilities.
 1. **Clone the repo branch**
 
 ```
-git clone -b prelude https://github.com/overture-stack/conductor.git
+git clone -b preludeV2.1 https://github.com/overture-stack/conductor.git
 ```
 
 2. **Build the Stage image using the dockerfile** For phaseOne run:

apps/readme.md

@@ -1,6 +1,6 @@
 # Apps folder
 
-This folder contains the following
+This folder containers a
 
 ## CSV-Processor
 

docker-compose.phaseOne.yml

@@ -1,6 +1,6 @@
 services:
   conductor:
-    profiles: ['phaseOne', 'phaseTwo', 'phaseThree', 'platform', 'stageDev', 'mockData', 'clean']
+    profiles: ['phaseOne', 'phaseTwo', 'phaseThree', 'platform', 'stageDev', 'data', 'clean']
     image: node:18-alpine
     container_name: conductor
     ports:
@@ -15,8 +15,22 @@ services:
       - ./scripts:/scripts
       - ./volumes/health:/health
     environment:
-      - PROFILE=${PROFILE:-platform}
-
+      PROFILE: ${PROFILE:-platform}
+      ES_URL: http://elasticsearch:9200
+      ES_USER: elastic
+      ES_PASS: myelasticpassword
+      ARRANGER_FILE_URL: http://arranger-file:5050/graphql
+      ARRANGER_TABULAR_URL: http://arranger-tabular:5051/graphql
+      FILE_INDEX_NAME: file-index
+      FILE_ES_TEMPLATE_FILE: /usr/share/elasticsearch/config/file_data_index_template.json
+      FILE_ES_TEMPLATE_NAME: file_template
+      FILE_ES_ALIAS_NAME: file_centric
+      TABULAR_INDEX_NAME: tabular-index
+      TABULAR_ES_TEMPLATE_FILE: /usr/share/elasticsearch/config/tabular_data_index_template.json
+      TABULAR_ES_TEMPLATE_NAME: tabular_template
+      TABULAR_ES_ALIAS_NAME: tabular_centric
+      STAGE_URL: http://stage:3000
+      TABULAR_DATA_FILE: /data/tabularData.csv
     command: >
       sh -c '
         apk add --no-cache --quiet curl >/dev/null 2>&1
@@ -48,7 +62,7 @@ services:
               chmod +x scripts/deployments/stageDev.sh
               scripts/deployments/stageDev.sh
               ;;
-            mockData)
+            data)
               echo "Running mock data submission..."
               chmod +x scripts/services/phaseOne/submitMockData.sh
               scripts/services/phaseOne/submitMockData.sh
@@ -90,6 +104,7 @@ services:
       discovery.type: single-node
       cluster.name: workflow.elasticsearch
       ES_JAVA_OPTS: -Xms512m -Xmx2048m
+      ES_USER: elastic
       ELASTIC_PASSWORD: myelasticpassword
       xpack.security.enabled: 'true'
     logging:

scripts/deployments/phaseThree.sh

@@ -0,0 +1,52 @@
+# Scripts
+
+## Deployments
+
+Deployment scripts are an ordered collection of service scripts made to
+systematically deploy platform infrastructure.
+
+## Services
+
+Service scripts are modular and designed to perform specific tasks (like health
+checks, data setup, and service validation) that can be easily referenced and
+executed by deployment scripts. The plug-and-play setup allows for flexible
+deployment configurations where scripts can be added, removed, or reordered as
+needed.
+
+To reduce coginitive overload we have organized setup scripts by the phase they
+are introduced in. All configurable variables are located at the top of the
+script and are configurable from the enviorment variables of the conductor
+service within the docker compose file.
+
+### PhaseOne.sh
+
+Configurations:
+
+- `DEBUG`: Boolean flag to enable/disable debug logging
+- `SCRIPT_DIR`: Base directory path for service scripts
+
+Key Functions:
+
+- `debug()`: Handles conditional debug output when DEBUG=true
+- `rs()`: (Run Script) Utility function that:
+  - Verifies script existence
+  - Sets execute permissions
+  - Executes scripts with explicit shell command
+  - Includes debug logging of permissions
+
+Deployment Sequence:
+
+1. Removes any existing health check files (These files, when present, flag
+   downstream services to start)
+2. Initializes Elasticsearch
+3. Sets up File Data in Elasticsearch
+4. Sets up Tabular Data in Elasticsearch
+5. Updates Conductor health status (by creating a `healthcheck` file)
+6. Performs Stage verification
+7. Validates Arranger setup
+
+The script includes progress tracking with colored output and completes by
+displaying the portal's access URL (default: http://localhost:3000). Each step
+includes error handling and debug logging capabilities when enabled.
+
+## Services

scripts/deployments/phaseTwo.sh

@@ -1,9 +1,7 @@
 #!/bin/sh
 
-# Define configurations
-RETRY_DELAY=20
-ARRANGER_FILE_URL="http://arranger-file:5050/graphql"
-ARRANGER_TABULAR_URL="http://arranger-tabular:5051/graphql"
+# Define configurations with environment variables and defaults
+RETRY_DELAY="${RETRY_DELAY:-20}"
 
 # Check Arrangers
 echo -e "Checking Arranger services"

scripts/readme.md

@@ -1,9 +1,9 @@
 #!/bin/sh
 
 # Configuration
-ES_URL="http://elasticsearch:9200"
-INDEX_NAME="tabular-index"
-ES_AUTH="elastic:myelasticpassword"
+ES_URL="${ES_URL:-http://elasticsearch:9200}"
+INDEX_NAME="${INDEX_NAME:-tabular-index}"
+ES_AUTH="${ES_USER}:${ES_PASS}" 
 
 echo -e "\033[1;36mElasticsearch:\033[0m Clearing data from index $INDEX_NAME"
 

scripts/services/phaseOne/arrangerCheck.sh

@@ -1,8 +1,7 @@
 # Basic health check configuration
 MAX_RETRIES=10
 RETRY_COUNT=0
-ES_URL="http://elasticsearch:9200"
-ES_AUTH="elastic:myelasticpassword"  # Matches your ELASTIC_PASSWORD env var
+ES_AUTH="${ES_USER}:${ES_PASS}" 
 
 # Check both authentication and cluster health
 until curl -s -f -u $ES_AUTH "$ES_URL/_cluster/health?wait_for_status=yellow" > /dev/null; do

scripts/services/phaseOne/clearElasticsearchData.sh

@@ -1,32 +1,27 @@
 #!/bin/sh
 
-TEMPLATE_FILE="/usr/share/elasticsearch/config/file_data_index_template.json"
-ES_URL="http://elasticsearch:9200"
-AUTH="-u elastic:myelasticpassword"
-INDEX_NAME="file-index"
-TEMPLATE_NAME="file_template"
-ALIAS_NAME="file_centric"
+ES_AUTH="${ES_USER}:${ES_PASS}"
 
 # Check template file
-[ ! -f "$TEMPLATE_FILE" ] && printf "\033[1;31mError:\033[0m Template file not found at $TEMPLATE_FILE\n" && exit 1
+[ ! -f "$FILE_ES_TEMPLATE_FILE" ] && printf "\033[1;31mError:\033[0m Template file not found at $FILE_ES_TEMPLATE_FILE\n" && exit 1
 
 # Set up template if it doesn't exist
 printf "\033[1;36mConductor:\033[0m Setting up the Elasticsearch file index template\n"
-if ! curl -s $AUTH "$ES_URL/_template/$TEMPLATE_NAME" | grep -q "\"index_patterns\""; then 
-   curl -s $AUTH -X PUT "$ES_URL/_template/$TEMPLATE_NAME" \
-       -H "Content-Type: application/json" -d @"$TEMPLATE_FILE" > /dev/null && \
+if ! curl -s -u "$ES_AUTH" "$ES_URL/_template/$FILE_ES_TEMPLATE_NAME" | grep -q "\"index_patterns\""; then 
+   curl -s -u "$ES_AUTH" -X PUT "$ES_URL/_template/$FILE_ES_TEMPLATE_NAME" \
+       -H "Content-Type: application/json" -d @"$FILE_ES_TEMPLATE_FILE" > /dev/null && \
    printf "\033[1;32mSuccess:\033[0m Elasticsearch file index template created successfully\n"
 else
    printf "\033[1;36mElasticsearch (File):\033[0m File Index template already exists, skipping creation\n"
 fi
 
 # Create index with alias if it doesn't exist
 printf "\033[1;36mConductor:\033[0m Setting up the Elasticsearch file index and alias\n"
-if ! curl -s -f $AUTH -X GET "$ES_URL/$INDEX_NAME" > /dev/null 2>&1; then
+if ! curl -s -f -u "$ES_AUTH" -X GET "$ES_URL/$FILE_INDEX_NAME" > /dev/null 2>&1; then
    printf "\033[1;36mElasticsearch (File):\033[0m Index does not exist, creating file index\n"
-   response=$(curl -s -w "\n%{http_code}" $AUTH -X PUT "$ES_URL/$INDEX_NAME" \
+   response=$(curl -s -w "\n%{http_code}" -u "$ES_AUTH" -X PUT "$ES_URL/$FILE_INDEX_NAME" \
        -H "Content-Type: application/json" \
-       -d "{\"aliases\": {\"$ALIAS_NAME\": {}}}")
+       -d "{\"aliases\": {\"$FILE_ES_ALIAS_NAME\": {}}}")
 
    http_code=$(echo "$response" | tail -n1)
    if [ "$http_code" != "200" ] && [ "$http_code" != "201" ]; then

scripts/services/phaseOne/elasticsearchCheck.sh

@@ -1,41 +1,35 @@
 #!/bin/sh
-
-TEMPLATE_FILE="/usr/share/elasticsearch/config/tabular_data_index_template.json"
-ES_URL="http://elasticsearch:9200"
-AUTH="-u elastic:myelasticpassword"
-INDEX_NAME="tabular-index"
-TEMPLATE_NAME="tabular_template"
-ALIAS_NAME="tabular_centric"
+ES_AUTH="${ES_USER}:${ES_PASS}"
 
 # Check template file
-[ ! -f "$TEMPLATE_FILE" ] && printf "\033[1;31mError:\033[0m Template file not found at $TEMPLATE_FILE\n" && exit 1
+[ ! -f "$TABULAR_ES_TEMPLATE_FILE" ] && printf "\033[1;31mError:\033[0m Template file not found at $TABULAR_ES_TEMPLATE_FILE\n" && exit 1
 
 # Set up template if it doesn't exist
 printf "\033[1;36mConductor:\033[0m Setting up the Elasticsearch tabular index template\n"
-if ! curl -s $AUTH "$ES_URL/_template/$TEMPLATE_NAME" | grep -q "\"index_patterns\""; then 
-   curl -s $AUTH -X PUT "$ES_URL/_template/$TEMPLATE_NAME" \
-       -H "Content-Type: application/json" -d @"$TEMPLATE_FILE" > /dev/null && \
-   printf "\033[1;32mSuccess:\033[0m Elasticsearch tabular index template created successfully\n"
+if ! curl -s -u "$ES_AUTH" "$ES_URL/_template/$TABULAR_ES_TEMPLATE_NAME" | grep -q "\"index_patterns\""; then 
+  curl -s -u "$ES_AUTH" -X PUT "$ES_URL/_template/$TABULAR_ES_TEMPLATE_NAME" \
+      -H "Content-Type: application/json" -d @"$TABULAR_ES_TEMPLATE_FILE" > /dev/null && \
+  printf "\033[1;32mSuccess:\033[0m Elasticsearch tabular index template created successfully\n"
 else
-   printf "\033[1;36mElasticsearch (Tabular):\033[0m Tabular Index template already exists, skipping creation\n"
+  printf "\033[1;36mElasticsearch (Tabular):\033[0m Tabular Index template already exists, skipping creation\n"
 fi
 
 # Create index with alias if it doesn't exist
 printf "\033[1;36mConductor:\033[0m Setting up the Elasticsearch tabular index and alias\n"
-if ! curl -s -f $AUTH -X GET "$ES_URL/$INDEX_NAME" > /dev/null 2>&1; then
-   printf "\033[1;36mElasticsearch (Tabular):\033[0m Index does not exist, creating tabular index\n"
-   response=$(curl -s -w "\n%{http_code}" $AUTH -X PUT "$ES_URL/$INDEX_NAME" \
-       -H "Content-Type: application/json" \
-       -d "{\"aliases\": {\"$ALIAS_NAME\": {}}}")
-   
-   http_code=$(echo "$response" | tail -n1)
-   if [ "$http_code" != "200" ] && [ "$http_code" != "201" ]; then
-       printf "\033[1;31mError:\033[0m Failed to create tabular index. HTTP Code: $http_code\n"
-       exit 1
-   fi
-   printf "\033[1;32mSuccess:\033[0m Tabular index and alias created\n"
+if ! curl -s -f -u "$ES_AUTH" -X GET "$ES_URL/$TABULAR_INDEX_NAME" > /dev/null 2>&1; then
+  printf "\033[1;36mElasticsearch (Tabular):\033[0m Index does not exist, creating tabular index\n"
+  response=$(curl -s -w "\n%{http_code}" -u "$ES_AUTH" -X PUT "$ES_URL/$TABULAR_INDEX_NAME" \
+      -H "Content-Type: application/json" \
+      -d "{\"aliases\": {\"$TABULAR_ES_ALIAS_NAME\": {}}}")
+  
+  http_code=$(echo "$response" | tail -n1)
+  if [ "$http_code" != "200" ] && [ "$http_code" != "201" ]; then
+      printf "\033[1;31mError:\033[0m Failed to create tabular index. HTTP Code: $http_code\n"
+      exit 1
+  fi
+  printf "\033[1;32mSuccess:\033[0m Tabular index and alias created\n"
 else
-   printf "\033[1;36mElasticsearch (Tabular):\033[0m Tabular index already exists\n"
+  printf "\033[1;36mElasticsearch (Tabular):\033[0m Tabular index already exists\n"
 fi
 
 printf "\033[1;32mSuccess:\033[0m Elasticsearch tabular setup complete\n"

scripts/services/phaseOne/elasticsearchSetupFileData.sh

@@ -5,7 +5,6 @@ RETRY_COUNT=0
 MAX_RETRIES=10          
 RETRY_DELAY=5           
 TIMEOUT=10              
-STAGE_URL="http://stage:3000"
 
 printf "\033[1;36mConductor:\033[0m Checking if Stage is reachable\n"
 

scripts/services/phaseOne/elasticsearchSetupTabularData.sh

@@ -3,10 +3,9 @@
 # Configuration
 MAX_RETRIES=10
 RETRY_COUNT=0
-CSV_PROCESSOR_PATH="/csv-processor"
-ES_URL="http://elasticsearch:9200"
-DATA_FILE="/data/tabularData.csv"
-INDEX_NAME="tabular-index"
+CSV_PROCESSOR_PATH="${CSV_PROCESSOR_PATH:-/csv-processor}"
+DATA_FILE="${TABULAR_DATA_FILE}"
+INDEX_NAME="${TABULAR_INDEX_NAME}"
 
 # Check if data file exists
 if [ ! -f "$DATA_FILE" ]; then