Monitoring updates

mxd/performance-tests/experiment_controller.sh

@@ -16,16 +16,17 @@ terraform_dir="$(dirname "$0")/.."
 test_pod_context="kind-mxd"
 test_environment_context="kind-mxd"
 is_debug=true
-
+IS_MONITORING_ENABLED=false
 
 #Override defaults with provided command line arguments
-while getopts "f:t:x:y:d" opt; do
+while getopts "f:t:x:y:d:m" opt; do
     case $opt in
         f) path_to_test_configuration=$OPTARG;;
         t) terraform_dir=$OPTARG;;
         x) test_pod_context=$OPTARG;;
         y) test_environment_context=$OPTARG;;
         d) is_debug=$OPTARG;;
+        m) IS_MONITORING_ENABLED=true;;
         \?) cat help.txt; exit 1;;
     esac
 done
@@ -56,6 +57,41 @@ print_error_log_and_exit() {
   exit 1
 }
 
+#######################################
+# Sets up monitoring components
+# Globals: IS_MONITORING_ENABLED
+# Outputs:
+#   Installs monitoring components if enabled
+#######################################
+setup_monitoring() {
+  if [[ $IS_MONITORING_ENABLED == true ]]; then
+    print_info_log "Monitoring enabled"
+
+    # Add Helm repositories
+    helm repo add jaegertracing https://jaegertracing.github.io/helm-charts
+    helm repo add jetstack https://charts.jetstack.io
+    helm repo update
+
+    # Check if cert-manager is already installed
+    if ! kubectl get namespace cert-manager &>/dev/null; then
+        # Install cert-manager
+        kubectl create namespace cert-manager
+        helm install cert-manager jetstack/cert-manager --namespace cert-manager --version v1.5.3
+        kubectl get pods -n cert-manager
+        kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.5.3/cert-manager.crds.yaml
+    else
+        print_info_log "Cert-manager is already installed"
+    fi
+
+    # Install Jaeger
+    helm install jaeger jaegertracing/jaeger-operator
+    kubectl apply -f jaeger-instance.yaml
+
+    # Install Open Telemetry Collector
+    helm upgrade --install otel-collector-cluster open-telemetry/opentelemetry-collector --values otel-collector-values.yaml
+    print_info_log "To access JaegerUI please use port forwarding: kubectl port-forward <jaeger-pod-name> <local-port>:<jaeger-ui-port>"
+  fi
+}
 
 #######################################
 # Sets up test environment
@@ -84,6 +120,8 @@ setup_test_environment() {
     | print_debug_log \
     || print_error_log_and_exit "Failed to create configmap with name custom-property"
 
+  setup_monitoring
+
   print_info_log "Init terraform"
   terraform -chdir="${terraform_dir}" init \
     >> "${TERRAFORM_LOGFILE}" \

mxd/performance-tests/jaeger-instance.yaml

@@ -0,0 +1,6 @@
+apiVersion: jaegertracing.io/v1
+kind: Jaeger
+metadata:
+  name: my-jaeger
+spec:
+  strategy: allInOne

mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py

@@ -1,85 +1,115 @@
 import json
-import matplotlib.pyplot as plt # to install: pip3 install plotly
-import numpy as np
-import mplcursors  # to install: pip3 install mplcursors
+import plotly.graph_objects as go
 import os
-import random
-
-############ Required Input ############
-root_folder = './Ergebnisse'
-values = ['meanResTime', 'sampleCount']
-processes = ['Get Transfer State', 'Initiate Transfer']
-
-######################################################################################
-# A function for extracting values from a given file
-def extract_values(file_path):
-    with open(file_path, 'r') as file:
-        lines = file.readlines()
-    values = {}
-    for line in lines:
-        if 'OEM_PLANTS' in line:
-            values['OEM_PLANTS'] = int(line.split('=')[1].strip())
-        elif 'OEM_CARS_INITIAL' in line:
-            values['OEM_CARS_INITIAL'] = int(line.split('=')[1].strip())
-        elif 'PARTS_PER_CAR' in line:
-            values['PARTS_PER_CAR'] = int(line.split('=')[1].strip())
-        elif 'CARS_PRODUCED_PER_INTERVALL' in line:
-            values['CARS_PRODUCED_PER_INTERVALL'] = int(line.split('=')[1].strip())
-    return values
-
-# List comprehension to gather all directories under the specified root_folder
-directories = [os.path.join(root_folder, o) for o in os.listdir(root_folder) 
-    if os.path.isdir(os.path.join(root_folder,o))]
-
-# Iterates over each action and value from the lists and creates a new figure for each combination
-for action_to_consider in processes:
-    for value_idx, value in enumerate(values):
-        plt.figure(figsize=(15, 4))
-        plt.title('Aggregation of Performance Test Results: {} for {}'.format(value, action_to_consider))
-
-        stats = []
-        labels = []
-        meta_values_list = []
-
-        # Goes through each directory to find the metadata and statistics files
-        for idx, directory in enumerate(directories):
-            metadata_path = os.path.join(directory, 'metadata.txt')  
-            statistics_path = os.path.join(directory, 'dashboard/statistics.json')  
-
-            # If both files exist in the directory, it calls the function to extract values from the metadata file
-            if os.path.exists(metadata_path) and os.path.exists(statistics_path):
-                meta_values = extract_values(metadata_path)
-
-                # Opens the statistics.json file and loads its content
-                with open(statistics_path, 'r') as f:
-                    stats_data = json.load(f)
-
-                # Appends the statistics for each action and the associated labels for the plot
-                action = stats_data[action_to_consider]
-                stats.append((meta_values['OEM_PLANTS'], action[value]))
-                labels.append('Plants: {}, {}: {:.2f}'.format(meta_values['OEM_PLANTS'], value, round(action[value],2)))
-                meta_values_list.append(meta_values)
-
-        # Plots the statistics data as a line plot and a scatter plot
-        plt.plot([s[0] for s in stats], [s[1] for s in stats])
-        colors = [ (random.random(), random.random(), random.random()) for _ in range(len(stats)) ]
-        scatter = plt.scatter([s[0] for s in stats], [s[1] for s in stats], color=colors)
-
-        # Adds labels to each point on the scatter plot (that x-axis matches OEM_PLANTS values)
-        for i, text in enumerate(labels):
-            plt.text(stats[i][0], stats[i][1], text)
-
-        # Add cursor
-        cursor = mplcursors.cursor(scatter, hover=True)
-        @cursor.connect("add")
-        def on_add(sel):
-            i = sel.target.index
-            sel.annotation.set_text('Plants: {}, Cars: {}, Parts/Car: {}, Cars/Interval: {}'.format(
-                stats[i][0],
-                meta_values_list[i]['OEM_CARS_INITIAL'],
-                meta_values_list[i]['PARTS_PER_CAR'],
-                meta_values_list[i]['CARS_PRODUCED_PER_INTERVALL']))
-
-        plt.xlabel('OEM_PLANTS')
-        plt.ylabel(value)
-        plt.show()
+import sys
+
+def load_metadata(metadata_file):
+    """
+    Load metadata from a text file.
+    """
+    metadata = {}
+    current_category = None
+
+    with open(metadata_file, 'r') as f:
+        for line in f:
+            line = line.strip()
+            print("Processing line:", line)  # Debug print to check each line
+            if line.startswith('#'):
+                current_category = line[1:].strip()
+                metadata[current_category] = {}
+            elif line:
+                key, value = line.split('=')
+                metadata[current_category][key.strip()] = value.strip()
+
+    print("Parsed Metadata Keys:", metadata.keys())  # Debug print parsed metadata keys
+    return metadata
+
+def load_stats(stats_file):
+    """
+    Load statistics from a JSON file.
+    """
+    with open(stats_file, 'r') as f:
+        stats_data = json.load(f)
+
+    return stats_data
+
+def sort_processes(processes, metadata):
+    """
+    Sort processes based on specified criteria.
+    """
+    sorted_processes = sorted(processes, key=lambda x: (
+        int(metadata[x].get('OEM_CARS_INITIAL', 0)),
+        int(metadata[x].get('SUPPLIER_PARTS_INITIAL', 0)),
+        int(metadata[x].get('OEM_PLANTS', 0)),
+        int(metadata[x].get('SUPPLIER_PLANTS', 0)),
+        int(metadata[x].get('SUPPLIER_FLEET_MANAGERS', 0)),
+        int(metadata[x].get('ADDITIONAL_CONTRACT_DEFINITIONS_OEM', 0)),
+        int(metadata[x].get('ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER', 0)),
+    ))
+    print("Sorted Processes:", sorted_processes)
+    for process in sorted_processes:
+        print(f"Metadata for {process}: {metadata[process]}")
+    return sorted_processes
+
+def plot_data(processes, stats_data, metadata, output_file):
+    """
+    Plot median response time data for each process as bar charts and save to HTML.
+    """
+    fig = go.Figure()
+
+    for process in processes:
+        stats = stats_data.get(process, {})
+        median_res_time = stats.get('medianResTime')  # Check if 'medianResTime' exists in stats
+        if median_res_time is not None:
+            process_info = metadata[process]
+            process_label = f"{process}\nCars: {process_info['OEM_CARS_INITIAL']}, Parts: {process_info['SUPPLIER_PARTS_INITIAL']}, OEM Plants: {process_info['OEM_PLANTS']}, Supplier Plants: {process_info['SUPPLIER_PLANTS']}, Fleet Managers: {process_info['SUPPLIER_FLEET_MANAGERS']}"
+            print(f"Adding trace for {process} with medianResTime {median_res_time}")
+            fig.add_trace(go.Bar(x=[process_label], y=[median_res_time], name=process_label))
+
+    fig.update_layout(title='Median Response Time for JMeter Calls',
+                      xaxis_title='Evaluation Scenarios',
+                      yaxis_title='Median Response Time',
+                      barmode='group')  # Change to 'group' for multiple bars per scenario
+
+    print("Final Figure:", fig)  # Print the figure object for debugging
+    fig.write_html(output_file)
+
+def process_folders(root_folder, output_file):
+    """
+    Process folders inside the root folder to get metadata and statistics files.
+    """
+    processes = []
+    stats_data = {}
+    metadata = {}
+
+    for folder in os.listdir(root_folder):
+        folder_path = os.path.join(root_folder, folder)
+        if os.path.isdir(folder_path):
+            metadata_file = os.path.join(folder_path, 'metadata.txt')
+            dashboard_folder = os.path.join(folder_path, 'dashboard')
+            stats_file = os.path.join(dashboard_folder, 'statistics.json')
+
+            if os.path.exists(metadata_file) and os.path.exists(stats_file):
+                metadata[folder] = load_metadata(metadata_file)
+                stats_data.update(load_stats(stats_file))
+                processes.append(folder)
+
+    sorted_processes = sort_processes(processes, metadata)
+    plot_data(sorted_processes, stats_data, metadata, output_file)
+
+def main(root_folder=None, output_file=None):
+    """
+    Main function to aggregate and plot data from folders.
+    """
+    if root_folder is None:
+        root_folder = '.'  # Set current folder if no root_folder provided
+
+    if output_file is None:
+        output_file = 'output.html'  # Set default output file name if not provided
+
+    process_folders(root_folder, output_file)
+
+if __name__ == "__main__":
+    root_folder = sys.argv[1] if len(sys.argv) > 1 else None
+    output_file = sys.argv[2] if len(sys.argv) > 2 else None
+    main(root_folder, output_file)

mxd/performance-tests/otel-collector-values.yaml

@@ -0,0 +1,20 @@
+mode: deployment
+
+replicaCount: 1
+
+presets:
+  # enables the k8sclusterreceiver and adds it to the metrics pipelines
+  clusterMetrics:
+    enabled: true
+  # enables the k8sobjectsreceiver to collect events only and adds it to the logs pipelines
+  kubernetesEvents:
+    enabled: true
+
+config:
+  exporters:
+    otlp:
+      endpoint: "my-jaeger-collector.default:9411" # the endpoint of Jaeger instance
+  service:
+    pipelines:
+      traces:
+        exporters: [otlp]