From deb2410ce1f7f44f0403e30ff3abc9d0bc18268f Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Wed, 10 Jan 2024 11:09:44 +0530 Subject: [PATCH 01/34] Initial changes needed for gardener deployment * Disable Ingress * Update Static IPs for Keycloak and MIW --- mxd/{ingress.tf => ingress.tfignore} | 0 mxd/keycloak.tf | 17 +++++++++-------- mxd/keycloak/miw_test_realm.json | 2 +- mxd/miw.tf | 2 +- .../connector/{ingress.tf => ingress.tfignore} | 0 mxd/modules/connector/outputs.tf | 16 ++++++++-------- mxd/outputs.tf | 14 +++++++------- mxd/performance-tests/Dockerfile | 7 +++++++ mxd/performance-tests/build-docker.sh | 3 +++ mxd/performance-tests/performance-test.yaml | 9 +++++++++ 10 files changed, 45 insertions(+), 25 deletions(-) rename mxd/{ingress.tf => ingress.tfignore} (100%) rename mxd/modules/connector/{ingress.tf => ingress.tfignore} (100%) create mode 100644 mxd/performance-tests/Dockerfile create mode 100644 mxd/performance-tests/build-docker.sh create mode 100644 mxd/performance-tests/performance-test.yaml diff --git a/mxd/ingress.tf b/mxd/ingress.tfignore similarity index 100% rename from mxd/ingress.tf rename to mxd/ingress.tfignore diff --git a/mxd/keycloak.tf b/mxd/keycloak.tf index 3c390ff0..fc575d7a 100644 --- a/mxd/keycloak.tf +++ b/mxd/keycloak.tf @@ -117,12 +117,13 @@ resource "kubernetes_config_map" "keycloak_env" { KEYCLOAK_ADMIN_PASSWORD = "admin" # the KC_HOSTNAME must be known in advance, so that Keycloak's token contain valid `iss` claims - KC_HOSTNAME = local.keycloak-ip - KC_HEALTH_ENABLED = true - MIW_BPN = var.miw-bpn - ALICE_BPN = var.alice-bpn - BOB_BPN = var.bob-bpn - TRUDY_BPN = var.trudy-bpn + KC_HOSTNAME = local.keycloak-ip + KC_HEALTH_ENABLED = true + MIW_BPN = var.miw-bpn + ALICE_BPN = var.alice-bpn + BOB_BPN = var.bob-bpn + TRUDY_BPN = var.trudy-bpn + PROXY_ADDRESS_FORWARDING = true } } @@ -146,6 +147,6 @@ resource "kubernetes_service" "keycloak" { } locals { - keycloak-ip = "10.96.103.80" - keycloak-url = "${local.keycloak-ip}:${var.keycloak-port}" + keycloak-ip = "100.104.103.180" + keycloak-url = "http://${local.keycloak-ip}:${var.keycloak-port}" } \ No newline at end of file diff --git a/mxd/keycloak/miw_test_realm.json b/mxd/keycloak/miw_test_realm.json index 4232c019..0c2f8bf1 100644 --- a/mxd/keycloak/miw_test_realm.json +++ b/mxd/keycloak/miw_test_realm.json @@ -26,7 +26,7 @@ "oauth2DeviceCodeLifespan": 600, "oauth2DevicePollingInterval": 5, "enabled": true, - "sslRequired": "external", + "sslRequired": "none", "registrationAllowed": false, "registrationEmailAsUsername": false, "rememberMe": false, diff --git a/mxd/miw.tf b/mxd/miw.tf index 3a1ad980..561cf85d 100644 --- a/mxd/miw.tf +++ b/mxd/miw.tf @@ -130,7 +130,7 @@ resource "kubernetes_service" "miw" { } locals { - miw-ip = "10.96.81.222" + miw-ip = "100.104.81.222" miw-url = "${local.miw-ip}:${var.miw-api-port}" keycloak-realm = "miw_test" } diff --git a/mxd/modules/connector/ingress.tf b/mxd/modules/connector/ingress.tfignore similarity index 100% rename from mxd/modules/connector/ingress.tf rename to mxd/modules/connector/ingress.tfignore diff --git a/mxd/modules/connector/outputs.tf b/mxd/modules/connector/outputs.tf index 7de93be5..55011c65 100644 --- a/mxd/modules/connector/outputs.tf +++ b/mxd/modules/connector/outputs.tf @@ -28,14 +28,14 @@ output "database-name" { value = var.database-name } -output "urls" { - value = { - management = local.management_url - health = local.health_url - proxy = local.proxy_url - public = local.public_url - } -} +#output "urls" { +# value = { +# management = local.management_url +# health = local.health_url +# proxy = local.proxy_url +# public = local.public_url +# } +#} output "node-ip" { value = kubernetes_service.controlplane-service.spec.0.cluster_ip diff --git a/mxd/outputs.tf b/mxd/outputs.tf index 70407f8d..218a9626 100644 --- a/mxd/outputs.tf +++ b/mxd/outputs.tf @@ -62,13 +62,13 @@ output "miw-database-credentials" { } } -output "bob-urls" { - value = module.bob-connector.urls -} - -output "alice-urls" { - value = module.alice-connector.urls -} +#output "bob-urls" { +# value = module.bob-connector.urls +#} +# +#output "alice-urls" { +# value = module.alice-connector.urls +#} output "bob-node-ip" { value = module.bob-connector.node-ip diff --git a/mxd/performance-tests/Dockerfile b/mxd/performance-tests/Dockerfile new file mode 100644 index 00000000..0295f2fa --- /dev/null +++ b/mxd/performance-tests/Dockerfile @@ -0,0 +1,7 @@ +FROM justb4/jmeter:5.5 + +WORKDIR $JMETER_HOME/mxd-performance-evaluation + +COPY mxd-performance-evaluation . + +ENTRYPOINT ["./run_experiment.sh"] diff --git a/mxd/performance-tests/build-docker.sh b/mxd/performance-tests/build-docker.sh new file mode 100644 index 00000000..90c6c7db --- /dev/null +++ b/mxd/performance-tests/build-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +docker build -t hemantxpatel/mxd-performance-test . diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml new file mode 100644 index 00000000..183578ac --- /dev/null +++ b/mxd/performance-tests/performance-test.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Pod +metadata: + name: mxd-performance-test +spec: + containers: + - name: mxd-performance-test + image: hemantxpatel/mxd-performance-test + restartPolicy: Never From e2986add4dfb653559e43791d91823ed5a31ebe6 Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Thu, 11 Jan 2024 12:49:50 +0530 Subject: [PATCH 02/34] Separate Postgres server for each components --- mxd/keycloak.tf | 14 ++++- mxd/main.tf | 10 ++-- mxd/miw.tf | 18 ++++-- mxd/modules/connector/main.tf | 11 +++- mxd/modules/connector/variables.tf | 4 -- mxd/{postgres.tf => modules/postgres/main.tf} | 57 +++++++------------ mxd/modules/postgres/outputs.tf | 42 ++++++++++++++ mxd/modules/postgres/variables.tf | 34 +++++++++++ mxd/outputs.tf | 16 ++---- mxd/variables.tf | 12 ---- 10 files changed, 139 insertions(+), 79 deletions(-) rename mxd/{postgres.tf => modules/postgres/main.tf} (67%) create mode 100644 mxd/modules/postgres/outputs.tf create mode 100644 mxd/modules/postgres/variables.tf diff --git a/mxd/keycloak.tf b/mxd/keycloak.tf index fc575d7a..60cb9426 100644 --- a/mxd/keycloak.tf +++ b/mxd/keycloak.tf @@ -17,6 +17,14 @@ # SPDX-License-Identifier: Apache-2.0 # +module "keycloak-postgres" { + source = "./modules/postgres" + database-name = "keycloak" + database-username = "keycloak" + database-password = "keycloak" + database-port = var.postgres-port +} + resource "kubernetes_deployment" "keycloak" { metadata { name = "keycloak" @@ -109,9 +117,9 @@ resource "kubernetes_config_map" "keycloak_env" { data = { KC_DB = "postgres" KC_DB_SCHEMA = "public" - KC_DB_PASSWORD = local.kc-pg-pwd - KC_DB_USERNAME = var.keycloak-db-user - KC_DB_URL = "jdbc:postgresql://${local.pg-host}/${var.keycloak-database}" + KC_DB_PASSWORD = module.keycloak-postgres.database-password + KC_DB_USERNAME = module.keycloak-postgres.database-username + KC_DB_URL = "jdbc:postgresql://${module.keycloak-postgres.database-url}/${module.keycloak-postgres.database-name}" KEYCLOAK_MIW_PUBLIC_CLIENT = "miw_public" KEYCLOAK_ADMIN = "admin" KEYCLOAK_ADMIN_PASSWORD = "admin" diff --git a/mxd/main.tf b/mxd/main.tf index c726fd38..e6ff985c 100644 --- a/mxd/main.tf +++ b/mxd/main.tf @@ -52,11 +52,10 @@ module "alice-connector" { source = "./modules/connector" humanReadableName = "alice" participantId = var.alice-bpn - database-host = local.pg-ip database-name = "alice" database-credentials = { - user = "postgres" - password = "postgres" + user = "alice" + password = "alice" } ssi-config = { miw-url = "http://${kubernetes_service.miw.metadata.0.name}:${var.miw-api-port}" @@ -73,11 +72,10 @@ module "bob-connector" { source = "./modules/connector" humanReadableName = "bob" participantId = var.bob-bpn - database-host = local.pg-ip database-name = "bob" database-credentials = { - user = "postgres" - password = "postgres" + user = "bob" + password = "bob" } ssi-config = { miw-url = "http://${kubernetes_service.miw.metadata.0.name}:${var.miw-api-port}" diff --git a/mxd/miw.tf b/mxd/miw.tf index 561cf85d..34d2f4b4 100644 --- a/mxd/miw.tf +++ b/mxd/miw.tf @@ -17,6 +17,14 @@ # SPDX-License-Identifier: Apache-2.0 # +module "miw-postgres" { + source = "./modules/postgres" + database-name = "miw" + database-username = "miw" + database-password = "miw" + database-port = var.postgres-port +} + resource "kubernetes_deployment" "miw" { metadata { name = "miw" @@ -79,12 +87,12 @@ resource "kubernetes_config_map" "miw-config" { name = "miw-config" } data = { - DB_HOST = local.pg-ip - DB_PORT = var.postgres-port + DB_HOST = module.miw-postgres.database-host + DB_PORT = module.miw-postgres.database-port DB_USER = "postgres" - DB_NAME = var.miw-database - DB_USER_NAME = var.miw-db-user - DB_PASSWORD = local.miw-pg-pwd + DB_NAME = module.miw-postgres.database-name + DB_USER_NAME = module.miw-postgres.database-username + DB_PASSWORD = module.miw-postgres.database-password KEYCLOAK_CLIENT_ID = "miw_private_client" ENCRYPTION_KEY = "Woh9waid4Ei5eez0aitieghoow9so4oe" diff --git a/mxd/modules/connector/main.tf b/mxd/modules/connector/main.tf index 8799723e..a84915f0 100644 --- a/mxd/modules/connector/main.tf +++ b/mxd/modules/connector/main.tf @@ -17,6 +17,15 @@ # SPDX-License-Identifier: Apache-2.0 # +module "postgres" { + source = "../postgres" + + database-name = var.database-name + database-username = var.database-credentials.user + database-password = var.database-credentials.password + database-port = var.database-port +} + resource "helm_release" "connector" { name = lower(var.humanReadableName) force_update = true @@ -112,5 +121,5 @@ resource "random_string" "aes_key_raw" { locals { aes_key_b64 = base64encode(random_string.aes_key_raw.result) client_secret = base64encode(random_string.kc_client_secret.result) - jdbcUrl = "jdbc:postgresql://${var.database-host}:${var.database-port}/${var.database-name}" + jdbcUrl = "jdbc:postgresql://${module.postgres.database-url}/${var.database-name}" } \ No newline at end of file diff --git a/mxd/modules/connector/variables.tf b/mxd/modules/connector/variables.tf index 73941f1f..c13e86fc 100644 --- a/mxd/modules/connector/variables.tf +++ b/mxd/modules/connector/variables.tf @@ -33,10 +33,6 @@ variable "participantId" { description = "Participant ID of the connector. In Catena-X, this MUST be the BPN" } -variable "database-host" { - description = "IP address (ClusterIP) or host name of the postgres database host" - -} variable "database-port" { default = 5432 description = "Port where the Postgres database is reachable, defaults to 5432." diff --git a/mxd/postgres.tf b/mxd/modules/postgres/main.tf similarity index 67% rename from mxd/postgres.tf rename to mxd/modules/postgres/main.tf index 36bd1b56..cf17b445 100644 --- a/mxd/postgres.tf +++ b/mxd/modules/postgres/main.tf @@ -19,9 +19,9 @@ resource "kubernetes_deployment" "postgres" { metadata { - name = "postgres" + name = local.app-name labels = { - App = "postgres" + App = local.app-name } } @@ -29,19 +29,19 @@ resource "kubernetes_deployment" "postgres" { replicas = 1 selector { match_labels = { - App = "postgres" + App = local.app-name } } template { metadata { labels = { - App = "postgres" + App = local.app-name } } spec { container { image = local.pg-image - name = "postgres" + name = local.app-name env_from { config_map_ref { @@ -92,7 +92,7 @@ resource "kubernetes_deployment" "postgres" { # ConfigMap that contains SQL statements to initialize the DB, create a "miw" DB, etc. resource "kubernetes_config_map" "postgres-config" { metadata { - name = "pg-initdb-config" + name = "${local.app-name}-initdb-config" } ## Create databases for keycloak and MIW, create users and assign privileges @@ -100,21 +100,11 @@ resource "kubernetes_config_map" "postgres-config" { POSTGRES_USER = "postgres" POSTGRES_PASSWORD = "postgres" "init.sql" = < Date: Mon, 22 Jan 2024 12:41:57 +0530 Subject: [PATCH 03/34] Database connection pooling configuration --- mxd/modules/connector/main.tf | 8 ++++++++ mxd/modules/connector/values.yaml | 4 ++-- mxd/performance-tests/performance-test.yaml | 1 + 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/mxd/modules/connector/main.tf b/mxd/modules/connector/main.tf index a84915f0..32d1e29c 100644 --- a/mxd/modules/connector/main.tf +++ b/mxd/modules/connector/main.tf @@ -56,6 +56,14 @@ resource "helm_release" "connector" { env : { "TX_SSI_ENDPOINT_AUDIENCE" : "http://${kubernetes_service.controlplane-service.metadata.0.name}:8084/api/v1/dsp" "EDC_DSP_CALLBACK_ADDRESS" : "http://${kubernetes_service.controlplane-service.metadata.0.name}:8084/api/v1/dsp" + "EDC_DATASOURCE_ASSET_POOL_CONNECTIONS_MAX-TOTAL" : 32 + "EDC_DATASOURCE_POLICY_POOL_CONNECTIONS_MAX-TOTAL" : 32 + "EDC_DATASOURCE_CONTRACTDEFINITION_POOL_CONNECTIONS_MAX-TOTAL" : 32 + "EDC_DATASOURCE_CONTRACTNEGOTIATION_POOL_CONNECTIONS_MAX-TOTAL" : 32 + "EDC_DATASOURCE_TRANSFERPROCESS_POOL_CONNECTIONS_MAX-TOTAL" : 32 + "EDC_DATASOURCE_EDR_POOL_CONNECTIONS_MAX-TOTAL" : 32 + "EDC_DATASOURCE_BPN_POOL_CONNECTIONS_MAX-TOTAL" : 32 + "JAVA_TOOL_OPTIONS" : "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1044 -Dcom.sun.management.jmxremote.port=3333 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" } ssi : { miw : { diff --git a/mxd/modules/connector/values.yaml b/mxd/modules/connector/values.yaml index 4328f6af..4e0cfeb8 100644 --- a/mxd/modules/connector/values.yaml +++ b/mxd/modules/connector/values.yaml @@ -34,7 +34,7 @@ controlplane: authKey: password image: pullPolicy: Never - tag: "latest" +# tag: "latest" # repository: "edc-controlplane-postgresql-hashicorp-vault" securityContext: # avoids some errors in the log: cannot write temp files of large multipart requests when R/O @@ -52,7 +52,7 @@ dataplane: port: 1044 image: pullPolicy: Never - tag: "latest" +# tag: "latest" # repository: "edc-dataplane-hashicorp-vault" securityContext: # avoids some errors in the log: cannot write temp files of large multipart requests when R/O diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index 183578ac..671ff566 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -6,4 +6,5 @@ spec: containers: - name: mxd-performance-test image: hemantxpatel/mxd-performance-test + args: ["-q", "M"] restartPolicy: Never From ccfbf26d26817c050412d55ba3e2d58b82975014 Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Fri, 9 Feb 2024 10:58:34 +0530 Subject: [PATCH 04/34] MIW Changes --- mxd/miw.tf | 2 +- mxd/modules/connector/main.tf | 15 ++++++++------- mxd/performance-tests/performance-test.yaml | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/mxd/miw.tf b/mxd/miw.tf index 34d2f4b4..2175a687 100644 --- a/mxd/miw.tf +++ b/mxd/miw.tf @@ -49,7 +49,7 @@ resource "kubernetes_deployment" "miw" { spec { container { name = "miw" - image = "tractusx/managed-identity-wallet:main" + image = "hemantxpatel/managed-identity-wallet:main" image_pull_policy = "Always" port { diff --git a/mxd/modules/connector/main.tf b/mxd/modules/connector/main.tf index 32d1e29c..efe80457 100644 --- a/mxd/modules/connector/main.tf +++ b/mxd/modules/connector/main.tf @@ -56,14 +56,15 @@ resource "helm_release" "connector" { env : { "TX_SSI_ENDPOINT_AUDIENCE" : "http://${kubernetes_service.controlplane-service.metadata.0.name}:8084/api/v1/dsp" "EDC_DSP_CALLBACK_ADDRESS" : "http://${kubernetes_service.controlplane-service.metadata.0.name}:8084/api/v1/dsp" - "EDC_DATASOURCE_ASSET_POOL_CONNECTIONS_MAX-TOTAL" : 32 - "EDC_DATASOURCE_POLICY_POOL_CONNECTIONS_MAX-TOTAL" : 32 - "EDC_DATASOURCE_CONTRACTDEFINITION_POOL_CONNECTIONS_MAX-TOTAL" : 32 - "EDC_DATASOURCE_CONTRACTNEGOTIATION_POOL_CONNECTIONS_MAX-TOTAL" : 32 - "EDC_DATASOURCE_TRANSFERPROCESS_POOL_CONNECTIONS_MAX-TOTAL" : 32 - "EDC_DATASOURCE_EDR_POOL_CONNECTIONS_MAX-TOTAL" : 32 - "EDC_DATASOURCE_BPN_POOL_CONNECTIONS_MAX-TOTAL" : 32 "JAVA_TOOL_OPTIONS" : "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1044 -Dcom.sun.management.jmxremote.port=3333 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" + "EDC_HOSTNAME" : "${var.humanReadableName}-tractusx-connector-controlplane" + "EDC_BLOBSTORE_ENDPOINT_TEMPLATE" : local.edc-blobstore-endpoint-template + "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_SOURCETYPES" : "HttpData,AmazonS3,AzureStorage" + "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_DESTINATIONTYPES" : "HttpProxy,AmazonS3,AzureStorage" + "EDC_DATASOURCE_POLICY-MONITOR_NAME" : "policy-monitor" + "EDC_DATASOURCE_POLICY-MONITOR_USER" : var.database-credentials.user + "EDC_DATASOURCE_POLICY-MONITOR_PASSWORD" : var.database-credentials.password + "EDC_DATASOURCE_POLICY-MONITOR_URL" : local.jdbcUrl } ssi : { miw : { diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index 671ff566..8e9b7c95 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -6,5 +6,5 @@ spec: containers: - name: mxd-performance-test image: hemantxpatel/mxd-performance-test - args: ["-q", "M"] + args: ["-q", "S"] restartPolicy: Never From f854fa937f7fa67fcdcaf90fe4d4f0ca547cfddb Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Fri, 9 Feb 2024 11:01:03 +0530 Subject: [PATCH 05/34] Remove unused envs --- mxd/modules/connector/main.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/mxd/modules/connector/main.tf b/mxd/modules/connector/main.tf index efe80457..6404c5e6 100644 --- a/mxd/modules/connector/main.tf +++ b/mxd/modules/connector/main.tf @@ -56,9 +56,7 @@ resource "helm_release" "connector" { env : { "TX_SSI_ENDPOINT_AUDIENCE" : "http://${kubernetes_service.controlplane-service.metadata.0.name}:8084/api/v1/dsp" "EDC_DSP_CALLBACK_ADDRESS" : "http://${kubernetes_service.controlplane-service.metadata.0.name}:8084/api/v1/dsp" - "JAVA_TOOL_OPTIONS" : "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=1044 -Dcom.sun.management.jmxremote.port=3333 -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" "EDC_HOSTNAME" : "${var.humanReadableName}-tractusx-connector-controlplane" - "EDC_BLOBSTORE_ENDPOINT_TEMPLATE" : local.edc-blobstore-endpoint-template "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_SOURCETYPES" : "HttpData,AmazonS3,AzureStorage" "EDC_DATAPLANE_SELECTOR_DEFAULTPLANE_DESTINATIONTYPES" : "HttpProxy,AmazonS3,AzureStorage" "EDC_DATASOURCE_POLICY-MONITOR_NAME" : "policy-monitor" From 138a7162a67e16ae9086904f18ed26741558d3f1 Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Tue, 6 Feb 2024 20:57:39 +0530 Subject: [PATCH 06/34] fix: Generate private / public key pair for transfer proxy --- mxd/modules/connector/main.tf | 17 ++++++++++++++++- mxd/modules/connector/values.yaml | 2 ++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/mxd/modules/connector/main.tf b/mxd/modules/connector/main.tf index 6404c5e6..1e5be2e9 100644 --- a/mxd/modules/connector/main.tf +++ b/mxd/modules/connector/main.tf @@ -46,7 +46,18 @@ resource "helm_release" "connector" { "postStart" : [ "sh", "-c", - "sleep 5 && /bin/vault kv put secret/client-secret content=${local.client_secret} && /bin/vault kv put secret/aes-keys content=${local.aes_key_b64} && /bin/vault kv put secret/${var.ssi-config.oauth-secretalias} content=${var.ssi-config.oauth-clientsecret}" + join(" && ", [ + "sleep 5", + "/bin/vault kv put secret/client-secret content=${local.client_secret}", + "/bin/vault kv put secret/aes-keys content=${local.aes_key_b64}", + "/bin/vault kv put secret/${var.ssi-config.oauth-secretalias} content=${var.ssi-config.oauth-clientsecret}", + "/bin/vault kv put secret/edc.aws.access.key content=${var.minio-config.minio-username}", + "/bin/vault kv put secret/edc.aws.secret.access.key content=${var.minio-config.minio-password}", + "/bin/vault kv put secret/${var.azure-account-name}-key content=${var.azure-account-key}", + "/bin/vault kv put secret/${var.azure-account-name}-sas content='${local.azure-sas-token}'", + "/bin/vault kv put secret/transferProxyTokenSignerPrivateKey content='${tls_private_key.transfer_proxy_privatekey.private_key_pem}'", + "/bin/vault kv put secret/transferProxyTokenSignerPublicKey content='${tls_private_key.transfer_proxy_privatekey.public_key_pem}'", + ]) ] } } @@ -125,6 +136,10 @@ resource "random_string" "aes_key_raw" { length = 16 } +resource "tls_private_key" "transfer_proxy_privatekey" { + algorithm = "ED25519" +} + locals { aes_key_b64 = base64encode(random_string.aes_key_raw.result) client_secret = base64encode(random_string.kc_client_secret.result) diff --git a/mxd/modules/connector/values.yaml b/mxd/modules/connector/values.yaml index 4e0cfeb8..1d397b33 100644 --- a/mxd/modules/connector/values.yaml +++ b/mxd/modules/connector/values.yaml @@ -78,6 +78,8 @@ vault: token: root secretNames: transferProxyTokenEncryptionAesKey: aes-keys + transferProxyTokenSignerPrivateKey: transferProxyTokenSignerPrivateKey + transferProxyTokenSignerPublicKey: transferProxyTokenSignerPublicKey # this must be set through CLI args: --set vault.secrets=$YOUR_VAULT_SECRETS where YOUR_VAULT_SECRETS should # # be a string in the format "key1:secret1;key2:secret2;..." secrets: From 6d773a8502501236e3b8c900d5a11e770097219b Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Fri, 9 Feb 2024 11:17:06 +0530 Subject: [PATCH 07/34] Remove minio / azure config --- mxd/modules/connector/main.tf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mxd/modules/connector/main.tf b/mxd/modules/connector/main.tf index 1e5be2e9..32d96dc0 100644 --- a/mxd/modules/connector/main.tf +++ b/mxd/modules/connector/main.tf @@ -51,10 +51,6 @@ resource "helm_release" "connector" { "/bin/vault kv put secret/client-secret content=${local.client_secret}", "/bin/vault kv put secret/aes-keys content=${local.aes_key_b64}", "/bin/vault kv put secret/${var.ssi-config.oauth-secretalias} content=${var.ssi-config.oauth-clientsecret}", - "/bin/vault kv put secret/edc.aws.access.key content=${var.minio-config.minio-username}", - "/bin/vault kv put secret/edc.aws.secret.access.key content=${var.minio-config.minio-password}", - "/bin/vault kv put secret/${var.azure-account-name}-key content=${var.azure-account-key}", - "/bin/vault kv put secret/${var.azure-account-name}-sas content='${local.azure-sas-token}'", "/bin/vault kv put secret/transferProxyTokenSignerPrivateKey content='${tls_private_key.transfer_proxy_privatekey.private_key_pem}'", "/bin/vault kv put secret/transferProxyTokenSignerPublicKey content='${tls_private_key.transfer_proxy_privatekey.public_key_pem}'", ]) From 4f66b34dbc1dd53aa6b971825bbdfdaae653d6d2 Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Fri, 9 Feb 2024 17:31:10 +0530 Subject: [PATCH 08/34] update latest image tag --- mxd/modules/connector/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mxd/modules/connector/values.yaml b/mxd/modules/connector/values.yaml index 1d397b33..6b54fab7 100644 --- a/mxd/modules/connector/values.yaml +++ b/mxd/modules/connector/values.yaml @@ -34,7 +34,7 @@ controlplane: authKey: password image: pullPolicy: Never -# tag: "latest" + tag: "latest" # repository: "edc-controlplane-postgresql-hashicorp-vault" securityContext: # avoids some errors in the log: cannot write temp files of large multipart requests when R/O @@ -52,7 +52,7 @@ dataplane: port: 1044 image: pullPolicy: Never -# tag: "latest" + tag: "latest" # repository: "edc-dataplane-hashicorp-vault" securityContext: # avoids some errors in the log: cannot write temp files of large multipart requests when R/O From 76607e7e0e378e2404ae0091f2886b17e1385902 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta Date: Fri, 16 Feb 2024 11:54:04 +0200 Subject: [PATCH 09/34] feat: create script orchestrating a sequence of performance tests runs --- .../custom_experiment.properties | 38 +++++ mxd/performance-tests/performance-test.yaml | 16 +- mxd/performance-tests/run-sml.sh | 146 ++++++++++++++++++ 3 files changed, 198 insertions(+), 2 deletions(-) create mode 100644 mxd/performance-tests/custom_experiment.properties create mode 100755 mxd/performance-tests/run-sml.sh diff --git a/mxd/performance-tests/custom_experiment.properties b/mxd/performance-tests/custom_experiment.properties new file mode 100644 index 00000000..a163e6d2 --- /dev/null +++ b/mxd/performance-tests/custom_experiment.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=180 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/1fwngcu1 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_SECOND=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=1 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=1 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=0 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=0 \ No newline at end of file diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index 8e9b7c95..91b00adb 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,6 +5,18 @@ metadata: spec: containers: - name: mxd-performance-test - image: hemantxpatel/mxd-performance-test - args: ["-q", "S"] + image: ciprian2398/mxd-performance-test:0.0.8-SNAPSHOT + envFrom: + - configMapRef: + name: special-config + volumeMounts: + - name: property-volume + mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties + subPath: custom_experiment.properties + + volumes: + - name: property-volume + configMap: + name: custom-property + restartPolicy: Never diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh new file mode 100755 index 00000000..4909517d --- /dev/null +++ b/mxd/performance-tests/run-sml.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# Function to display help information +function display_help { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " -m MODE Set t-shirt size(s) as a coma-separated list (e.g., \"S,M,L,C\")" + echo " C - custom property file provided with -c param will be used" + echo " S-small, M-medium, L-large" + echo " Default S" + echo " -l LOG_MSG Set the log message which indicates the finis of test execution (default: \"Test Completed\")" + echo " -p POD_NAME Set the pod name (default: \"mxd-performance-test\")" + echo " -g OUTPUT Set the generated output file (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar\")" + echo " -t TERRAFORM Set the Terraform directory (default: \"/Users/ciprian/IdeaProjects/tutorial-resources/mxd/\")" + echo " -c CUSTOM Set the custom properties file (default: \"custom_experiment.properties\")" + echo " -o LOG_FILE Set the log file name (default: \"sml_script_[current_datetime].logs\")" + echo " -d DEBUG Enable debug mode (default: true)" + echo " -help Display this help message" + exit 0 +} + +# List of all available t-shirt sizes +#MODE=("S" "M" "L") +MODE=("S") +LOG_MESSAGE="Test Completed" +POD_NAME="mxd-performance-test" +GENERATED_OUTPUT_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar" +TERRAFORM_CHDIR="/Users/ciprian/IdeaProjects/tutorial-resources/mxd/" +CUSTOM_PROPERTIES="custom_experiment.properties" +LOGFILE="sml_script_$(date +%d-%m-%YT%H-%M-%S).logs" +IS_DEBUG=true + +# Parse command-line options +while getopts "m:l:p:g:t:c:o:d:h:" opt; do + case $opt in + m) + IFS=',' read -ra MODE <<< "$OPTARG" + ;; + l) LOG_MESSAGE=$OPTARG;; + p) POD_NAME=$OPTARG;; + g) GENERATED_OUTPUT_FILE=$OPTARG;; + t) TERRAFORM_CHDIR=$OPTARG;; + c) CUSTOM_PROPERTIES=$OPTARG;; + o) LOGFILE=$OPTARG;; + d) IS_DEBUG=$OPTARG;; + h) display_help;; + \?) echo "Invalid option: -$OPTARG" >&2; display_help exit 1;; + esac +done + +# Parse command-line options +while getopts "m:l:p:g:t:c:o:d:" opt; do + case $opt in + m) + IFS=',' read -ra MODE <<< "$OPTARG" + ;; + l) LOG_MESSAGE=$OPTARG;; + p) POD_NAME=$OPTARG;; + g) GENERATED_OUTPUT_FILE=$OPTARG;; + t) TERRAFORM_CHDIR=$OPTARG;; + c) CUSTOM_PROPERTIES=$OPTARG;; + o) LOGFILE=$OPTARG;; + d) IS_DEBUG=$OPTARG;; + \?) echo "Invalid option: -$OPTARG" >&2; exit 1;; + esac +done + +# Prints informational messages +function info { + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[32m INFO \033[0m $@" +} + +# Prints debug messages +function debug { + read IN + if [[ $IS_DEBUG == true ]]; then + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[33m DEBUG \033[0m $IN" + fi +} + +# Prints error messages and exits with error code +function error_exit { + read IN + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[31m ERROR \033[0m $@ original error: $IN" + exit 1 +} + +# Initializes the test +function init { + local size=$1 + info "Testing $size t-shirt size" + kubectl create configmap special-config --from-literal=JMETER_SCRIPT="" --from-literal=T_SHIRT_SIZE="$size" | debug || error_exit "Failed to create special-config configmap" + kubectl create configmap custom-property --from-file=custom_experiment.properties=$CUSTOM_PROPERTIES | debug || error_exit "Failed to create configmap custom-property" + info "Init terraform" + terraform -chdir="$TERRAFORM_CHDIR" init >> "$LOGFILE" || error_exit "Failed to initialize Terraform" + info "Apply terraform" + terraform -chdir="$TERRAFORM_CHDIR" apply -auto-approve >> "$LOGFILE" || error_exit "Failed to apply Terraform" + info "Start the performance-test container" + kubectl apply -f performance-test.yaml | debug || error_exit "Failed to start performance-test container" + info "Waiting for container ready state" + kubectl wait --for=condition=ready "pod/$POD_NAME" | debug || error_exit "Container failed to reach ready state" +} + +# Copies output file when tests are ready +function copyFileWhenTestsReady { + local size=$1 + info "Waiting for the tests to finish ..." + while true; do + logs=$(kubectl logs --tail=5 "$POD_NAME" 2>/dev/null) + if echo "$logs" | grep -q "$LOG_MESSAGE"; then + info "Log message found in the logs." + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "output_${size}.tar" | debug || error_exit "Failed to copy output file" + info "Test Report downloaded with name output_${size}.tar" + break + else + echo "Waiting for log message" | debug + sleep 5 + fi + done +} + +# Cleans up resources +function cleanup { + info "Destroying terraform" + terraform -chdir="$TERRAFORM_CHDIR" destroy -auto-approve >> "$LOGFILE" || error_exit "Failed to destroy Terraform" + info "Destroying pod" + kubectl delete pod "$POD_NAME" | debug || error_exit "Failed to delete pod" + info "Destroying configmap" + kubectl delete configmap special-config | debug || error_exit "Failed to delete configmap special-config" + info "Destroying configmap" + kubectl delete configmap custom-property | debug || error_exit "Failed to delete configmap custom-property" + info "Waiting for the pod to be deleted" + kubectl wait --for=delete "pod/$POD_NAME" + info "Execution finished." + exit 0 +} + +# Execute cleanup when CONTROL+C is invoked +trap "cleanup" INT + +# Main loop +for char in "${MODE[@]}"; do + init "$char" + copyFileWhenTestsReady "$char" + cleanup +done \ No newline at end of file From dddefaa4fb62b981e56cf6bffa3a4d8fe39cab08 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta Date: Mon, 19 Feb 2024 12:16:59 +0200 Subject: [PATCH 10/34] feat: new docker image --- mxd/performance-tests/custom_experiment.properties | 2 +- mxd/performance-tests/performance-test.yaml | 2 +- mxd/performance-tests/run-sml.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mxd/performance-tests/custom_experiment.properties b/mxd/performance-tests/custom_experiment.properties index a163e6d2..22b840e4 100644 --- a/mxd/performance-tests/custom_experiment.properties +++ b/mxd/performance-tests/custom_experiment.properties @@ -35,4 +35,4 @@ SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT CARS_REQUESTED_PER_INTERVALL=1 ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=0 -ADDITIONAL_CONTRACT_DEFINITIONS_OEM=0 \ No newline at end of file +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=0 diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index 91b00adb..e6ccb01f 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: mxd-performance-test - image: ciprian2398/mxd-performance-test:0.0.8-SNAPSHOT + image: ciprian2398/mxd-performance-test:0.0.9-SNAPSHOT envFrom: - configMapRef: name: special-config diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh index 4909517d..ae05ce05 100755 --- a/mxd/performance-tests/run-sml.sh +++ b/mxd/performance-tests/run-sml.sh @@ -109,7 +109,7 @@ function copyFileWhenTestsReady { logs=$(kubectl logs --tail=5 "$POD_NAME" 2>/dev/null) if echo "$logs" | grep -q "$LOG_MESSAGE"; then info "Log message found in the logs." - kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "output_${size}.tar" | debug || error_exit "Failed to copy output file" + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "output_${size}_$(date +%d-%m-%YT%H-%M-%S).tar" | debug || error_exit "Failed to copy output file" info "Test Report downloaded with name output_${size}.tar" break else From 269968ffef60e7ecb0d1c6dec9c51a85166e8d51 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta Date: Mon, 19 Feb 2024 19:05:53 +0200 Subject: [PATCH 11/34] feat: discard t-shirt size --- mxd/performance-tests/Dockerfile | 7 - mxd/performance-tests/build-docker.sh | 3 - mxd/performance-tests/performance-test.yaml | 5 +- mxd/performance-tests/run-sml.sh | 131 ++++++++++-------- .../large_experiment.properties | 38 +++++ .../medium_experiment.properties | 38 +++++ ...edium_experiment_1000_contracts.properties | 38 +++++ ...medium_experiment_100_contracts.properties | 38 +++++ .../medium_experiment_10_contracts.properties | 38 +++++ .../small_experiment.properties | 38 +++++ 10 files changed, 303 insertions(+), 71 deletions(-) delete mode 100644 mxd/performance-tests/Dockerfile delete mode 100644 mxd/performance-tests/build-docker.sh create mode 100644 mxd/performance-tests/test-configurations/large_experiment.properties create mode 100644 mxd/performance-tests/test-configurations/medium_experiment.properties create mode 100644 mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties create mode 100644 mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties create mode 100644 mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties create mode 100644 mxd/performance-tests/test-configurations/small_experiment.properties diff --git a/mxd/performance-tests/Dockerfile b/mxd/performance-tests/Dockerfile deleted file mode 100644 index 0295f2fa..00000000 --- a/mxd/performance-tests/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM justb4/jmeter:5.5 - -WORKDIR $JMETER_HOME/mxd-performance-evaluation - -COPY mxd-performance-evaluation . - -ENTRYPOINT ["./run_experiment.sh"] diff --git a/mxd/performance-tests/build-docker.sh b/mxd/performance-tests/build-docker.sh deleted file mode 100644 index 90c6c7db..00000000 --- a/mxd/performance-tests/build-docker.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env sh - -docker build -t hemantxpatel/mxd-performance-test . diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index e6ccb01f..5480bda0 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,10 +5,7 @@ metadata: spec: containers: - name: mxd-performance-test - image: ciprian2398/mxd-performance-test:0.0.9-SNAPSHOT - envFrom: - - configMapRef: - name: special-config + image: ciprian2398/mxd-performance-test:0.1.0-SNAPSHOT volumeMounts: - name: property-volume mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh index ae05ce05..6cec1e20 100755 --- a/mxd/performance-tests/run-sml.sh +++ b/mxd/performance-tests/run-sml.sh @@ -2,69 +2,45 @@ # Function to display help information function display_help { - echo "Usage: $0 [OPTIONS]" echo "Options:" - echo " -m MODE Set t-shirt size(s) as a coma-separated list (e.g., \"S,M,L,C\")" - echo " C - custom property file provided with -c param will be used" - echo " S-small, M-medium, L-large" - echo " Default S" - echo " -l LOG_MSG Set the log message which indicates the finis of test execution (default: \"Test Completed\")" - echo " -p POD_NAME Set the pod name (default: \"mxd-performance-test\")" - echo " -g OUTPUT Set the generated output file (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar\")" - echo " -t TERRAFORM Set the Terraform directory (default: \"/Users/ciprian/IdeaProjects/tutorial-resources/mxd/\")" - echo " -c CUSTOM Set the custom properties file (default: \"custom_experiment.properties\")" - echo " -o LOG_FILE Set the log file name (default: \"sml_script_[current_datetime].logs\")" - echo " -d DEBUG Enable debug mode (default: true)" - echo " -help Display this help message" + echo " -f Set the the path to file/folder with experiment files (default: \"test-configurations/small_experiment.properties\")" + echo " -l Set the log message which indicates the finish of test execution (default: \"Test Completed\")" + echo " -p Set the pod name (default: \"mxd-performance-test\")" + echo " -g Set the generated output file path (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar\")" + echo " -s Set the generated output_slim file path (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output_slim.tar\")" + echo " -t Set the Terraform directory (default: \"/Users/ciprian/IdeaProjects/tutorial-resources/mxd/\")" + echo " -c Set the destination name name of experiment properties file when mounting on the pod (default: \"custom_experiment.properties\")" + echo " -o Set the terraform log file name (default: \"sml_script_[current_datetime].logs\")" + echo " -d Enable debug mode (default: true)" exit 0 } -# List of all available t-shirt sizes -#MODE=("S" "M" "L") -MODE=("S") +PROVIDED_PATH="test-configurations/small_experiment.properties" LOG_MESSAGE="Test Completed" POD_NAME="mxd-performance-test" GENERATED_OUTPUT_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar" +GENERATED_OUTPUT_SLIM_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output_slim.tar" TERRAFORM_CHDIR="/Users/ciprian/IdeaProjects/tutorial-resources/mxd/" CUSTOM_PROPERTIES="custom_experiment.properties" LOGFILE="sml_script_$(date +%d-%m-%YT%H-%M-%S).logs" IS_DEBUG=true # Parse command-line options -while getopts "m:l:p:g:t:c:o:d:h:" opt; do +while getopts "f:l:p:g:s:t:c:o:d:" opt; do case $opt in - m) - IFS=',' read -ra MODE <<< "$OPTARG" - ;; + f) PROVIDED_PATH=$OPTARG;; l) LOG_MESSAGE=$OPTARG;; p) POD_NAME=$OPTARG;; g) GENERATED_OUTPUT_FILE=$OPTARG;; + s) GENERATED_OUTPUT_SLIM_FILE=$OPTARG;; t) TERRAFORM_CHDIR=$OPTARG;; c) CUSTOM_PROPERTIES=$OPTARG;; o) LOGFILE=$OPTARG;; d) IS_DEBUG=$OPTARG;; - h) display_help;; \?) echo "Invalid option: -$OPTARG" >&2; display_help exit 1;; esac done -# Parse command-line options -while getopts "m:l:p:g:t:c:o:d:" opt; do - case $opt in - m) - IFS=',' read -ra MODE <<< "$OPTARG" - ;; - l) LOG_MESSAGE=$OPTARG;; - p) POD_NAME=$OPTARG;; - g) GENERATED_OUTPUT_FILE=$OPTARG;; - t) TERRAFORM_CHDIR=$OPTARG;; - c) CUSTOM_PROPERTIES=$OPTARG;; - o) LOGFILE=$OPTARG;; - d) IS_DEBUG=$OPTARG;; - \?) echo "Invalid option: -$OPTARG" >&2; exit 1;; - esac -done - # Prints informational messages function info { echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[32m INFO \033[0m $@" @@ -80,17 +56,16 @@ function debug { # Prints error messages and exits with error code function error_exit { - read IN - echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[31m ERROR \033[0m $@ original error: $IN" - exit 1 + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[31m ERROR \033[0m $@" + #cleanup + #exit 1 } # Initializes the test function init { - local size=$1 - info "Testing $size t-shirt size" - kubectl create configmap special-config --from-literal=JMETER_SCRIPT="" --from-literal=T_SHIRT_SIZE="$size" | debug || error_exit "Failed to create special-config configmap" - kubectl create configmap custom-property --from-file=custom_experiment.properties=$CUSTOM_PROPERTIES | debug || error_exit "Failed to create configmap custom-property" + local experiment_file=$1 + info "Adding ${experiment_file} on pod using custom-property configmap" + kubectl create configmap custom-property --from-file="${CUSTOM_PROPERTIES}"="${experiment_file}" | debug || error_exit "Failed to create configmap with name custom-property" info "Init terraform" terraform -chdir="$TERRAFORM_CHDIR" init >> "$LOGFILE" || error_exit "Failed to initialize Terraform" info "Apply terraform" @@ -103,14 +78,16 @@ function init { # Copies output file when tests are ready function copyFileWhenTestsReady { - local size=$1 + local experiment_file=$1 info "Waiting for the tests to finish ..." while true; do logs=$(kubectl logs --tail=5 "$POD_NAME" 2>/dev/null) if echo "$logs" | grep -q "$LOG_MESSAGE"; then info "Log message found in the logs." - kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "output_${size}_$(date +%d-%m-%YT%H-%M-%S).tar" | debug || error_exit "Failed to copy output file" - info "Test Report downloaded with name output_${size}.tar" + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "${experiment_file}.tar" + info "Test Report downloaded with name output_${experiment_file}.tar" + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_SLIM_FILE}" "${experiment_file}_slim.tar" + info "Test Report downloaded with name output_${experiment_file}_slim.tar" break else echo "Waiting for log message" | debug @@ -126,8 +103,6 @@ function cleanup { info "Destroying pod" kubectl delete pod "$POD_NAME" | debug || error_exit "Failed to delete pod" info "Destroying configmap" - kubectl delete configmap special-config | debug || error_exit "Failed to delete configmap special-config" - info "Destroying configmap" kubectl delete configmap custom-property | debug || error_exit "Failed to delete configmap custom-property" info "Waiting for the pod to be deleted" kubectl wait --for=delete "pod/$POD_NAME" @@ -135,12 +110,54 @@ function cleanup { exit 0 } -# Execute cleanup when CONTROL+C is invoked -trap "cleanup" INT +function startForSingleFile { + local experiment_file=$1 + info Start experiment for file $experiment_file -# Main loop -for char in "${MODE[@]}"; do - init "$char" - copyFileWhenTestsReady "$char" + init "$experiment_file" + copyFileWhenTestsReady "$experiment_file" cleanup -done \ No newline at end of file +} + +function startForDirectory { + local folder_path=$1 + # Use an array to store the all properties file in provided folder + local files_array=() + + # Populate the array using a loop + while IFS= read -r -d '' file; do + files_array+=("$file") + done < <(find "$folder_path" -type f -name "*$extension" -print0) + + # Show detected files + for file in "${files_array[@]}"; do + info "Detected following properties file: $file" + done + + # Start test for one file at a time + for file in "${files_array[@]}"; do + startForSingleFile "$file" + done +} + +function main { + if [ -e "$PROVIDED_PATH" ]; then + if [ -f "$PROVIDED_PATH" ]; then + info "$PROVIDED_PATH is a file." + startForSingleFile "$PROVIDED_PATH" + elif [ -d "$PROVIDED_PATH" ]; then + info "$PROVIDED_PATH is a directory." + startForDirectory "$PROVIDED_PATH" + else + info "$PROVIDED_PATH exists but is neither a file nor a directory." + fi + else + info "$PROVIDED_PATH does not exist." + fi +} + +# Execute cleanup when CONTROL+C is invoked +trap cleanup INT + +# Script entry point +main \ No newline at end of file diff --git a/mxd/performance-tests/test-configurations/large_experiment.properties b/mxd/performance-tests/test-configurations/large_experiment.properties new file mode 100644 index 00000000..ec629b0b --- /dev/null +++ b/mxd/performance-tests/test-configurations/large_experiment.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_SECOND=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=10 +OEM_CARS_INITIAL=1000000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=10 +SUPPLIER_PARTS_INITIAL=1000000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=0 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=0 diff --git a/mxd/performance-tests/test-configurations/medium_experiment.properties b/mxd/performance-tests/test-configurations/medium_experiment.properties new file mode 100644 index 00000000..a2b7d564 --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_SECOND=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=0 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=0 diff --git a/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties new file mode 100644 index 00000000..46b63ac1 --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_SECOND=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=1000 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=1000 diff --git a/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties new file mode 100644 index 00000000..b5de0825 --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_SECOND=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=100 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=100 diff --git a/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties new file mode 100644 index 00000000..1d775891 --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_SECOND=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=10 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=10 diff --git a/mxd/performance-tests/test-configurations/small_experiment.properties b/mxd/performance-tests/test-configurations/small_experiment.properties new file mode 100644 index 00000000..22b840e4 --- /dev/null +++ b/mxd/performance-tests/test-configurations/small_experiment.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=180 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/1fwngcu1 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_SECOND=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=1 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=1 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=0 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=0 From 57b4976462ab6f19c268ff9e738d1282a8312409 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta Date: Tue, 20 Feb 2024 18:18:49 +0200 Subject: [PATCH 12/34] fix: cleanup_and_exit and extension --- mxd/performance-tests/run-sml.sh | 8 +++++++- .../medium_experiment_10000_contracts.properties} | 12 ++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) rename mxd/performance-tests/{custom_experiment.properties => test-configurations/medium_experiment_10000_contracts.properties} (80%) diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh index 6cec1e20..b89677ea 100755 --- a/mxd/performance-tests/run-sml.sh +++ b/mxd/performance-tests/run-sml.sh @@ -24,6 +24,7 @@ TERRAFORM_CHDIR="/Users/ciprian/IdeaProjects/tutorial-resources/mxd/" CUSTOM_PROPERTIES="custom_experiment.properties" LOGFILE="sml_script_$(date +%d-%m-%YT%H-%M-%S).logs" IS_DEBUG=true +extension=".properties" # Parse command-line options while getopts "f:l:p:g:s:t:c:o:d:" opt; do @@ -107,6 +108,11 @@ function cleanup { info "Waiting for the pod to be deleted" kubectl wait --for=delete "pod/$POD_NAME" info "Execution finished." +} + +# Cleans up resources and exit +function cleanup_and_exit { + cleanup exit 0 } @@ -157,7 +163,7 @@ function main { } # Execute cleanup when CONTROL+C is invoked -trap cleanup INT +trap cleanup_and_exit INT # Script entry point main \ No newline at end of file diff --git a/mxd/performance-tests/custom_experiment.properties b/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties similarity index 80% rename from mxd/performance-tests/custom_experiment.properties rename to mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties index 22b840e4..11013f29 100644 --- a/mxd/performance-tests/custom_experiment.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties @@ -1,8 +1,8 @@ # General Parameters -EXPERIMENT_DURATION_SEC=180 +EXPERIMENT_DURATION_SEC=1800 EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ -BACKEND_SERVICE=https://requestbin.myworkato.com/1fwngcu1 +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 INTERVALS_PER_SECOND=10 @@ -14,7 +14,7 @@ OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password -OEM_PLANTS=1 +OEM_PLANTS=5 OEM_CARS_INITIAL=10000 PARTS_PER_CAR=100 CARS_PRODUCED_PER_INTERVALL=1 @@ -26,7 +26,7 @@ SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password -SUPPLIER_PLANTS=1 +SUPPLIER_PLANTS=5 SUPPLIER_PARTS_INITIAL=10000 PARTS_PRODUCED_PER_INTERVALL=50 SUPPLIER_FLEET_MANAGERS=1 @@ -34,5 +34,5 @@ SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT CARS_REQUESTED_PER_INTERVALL=1 -ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=0 -ADDITIONAL_CONTRACT_DEFINITIONS_OEM=0 +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=10000 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=10000 From 8a9795b64bb3adae7f368b3a45b806adbc98252d Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Fri, 23 Feb 2024 14:38:30 +0530 Subject: [PATCH 13/34] Update policy request --- mxd/postman/mxd-seed.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mxd/postman/mxd-seed.json b/mxd/postman/mxd-seed.json index b1593d9a..4108c267 100644 --- a/mxd/postman/mxd-seed.json +++ b/mxd/postman/mxd-seed.json @@ -1,10 +1,10 @@ { "info": { - "_postman_id": "f7ae8abb-bac9-4ba6-89e2-e5cfb86c0b23", + "_postman_id": "1be67f78-c3fb-44f7-93cb-9c73d67ce795", "name": "MXD Management API", "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", "_exporter_id": "252261", - "_collection_link": "https://winter-rocket-884715.postman.co/workspace/SAP~ff4cf631-05f4-48ab-a1e0-f716cf2efc25/collection/252261-f7ae8abb-bac9-4ba6-89e2-e5cfb86c0b23?action=share&source=collection_link&creator=252261" + "_collection_link": "https://winter-rocket-884715.postman.co/workspace/SAP~ff4cf631-05f4-48ab-a1e0-f716cf2efc25/collection/252261-1be67f78-c3fb-44f7-93cb-9c73d67ce795?action=share&source=collection_link&creator=252261" }, "item": [ { @@ -110,7 +110,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"@context\": {\n \"odrl\": \"http://www.w3.org/ns/odrl/2/\"\n },\n \"@type\": \"PolicyDefinitionRequestDto\",\n \"@id\": \"1\",\n \"policy\": {\n\t\t\"@type\": \"Policy\",\n\t\t\"odrl:permission\" : [{\n\t\t\t\"odrl:action\" : \"USE\",\n\t\t\t\"odrl:constraint\" : {\n\t\t\t\t\"@type\": \"LogicalConstraint\",\n\t\t\t\t\"odrl:or\" : [{\n\t\t\t\t\t\"@type\" : \"Constraint\",\n\t\t\t\t\t\"odrl:leftOperand\" : \"BusinessPartnerNumber\",\n\t\t\t\t\t\"odrl:operator\" : {\n \"@id\": \"odrl:eq\"\n },\n\t\t\t\t\t\"odrl:rightOperand\" : \"{{POLICY_BPN}}\"\n\t\t\t\t}]\n\t\t\t}\n\t\t}]\n }\n}", + "raw": "{\n \"@context\": {\n \"odrl\": \"http://www.w3.org/ns/odrl/2/\"\n },\n \"@type\": \"PolicyDefinitionRequestDto\",\n \"@id\": \"1\",\n \"policy\": {\n\t\t\"@type\": \"odrl:Set\",\n\t\t\"odrl:permission\" : [{\n\t\t\t\"odrl:action\" : \"USE\",\n\t\t\t\"odrl:constraint\" : {\n\t\t\t\t\"@type\": \"LogicalConstraint\",\n\t\t\t\t\"odrl:or\" : [{\n\t\t\t\t\t\"@type\" : \"Constraint\",\n\t\t\t\t\t\"odrl:leftOperand\" : \"BusinessPartnerNumber\",\n\t\t\t\t\t\"odrl:operator\" : {\n \"@id\": \"odrl:eq\"\n },\n\t\t\t\t\t\"odrl:rightOperand\" : \"{{POLICY_BPN}}\"\n\t\t\t\t}]\n\t\t\t}\n\t\t}]\n }\n}", "options": { "raw": { "language": "json" From c494bf3d00156a90cacdc1b4886fcd853bb934cc Mon Sep 17 00:00:00 2001 From: ciprian-nicuta Date: Tue, 27 Feb 2024 12:22:24 +0200 Subject: [PATCH 14/34] fix: new image --- mxd/performance-tests/performance-test.yaml | 2 +- .../test-configurations/large_experiment.properties | 2 +- .../test-configurations/medium_experiment.properties | 2 +- .../medium_experiment_10000_contracts.properties | 2 +- .../medium_experiment_1000_contracts.properties | 2 +- .../medium_experiment_100_contracts.properties | 2 +- .../medium_experiment_10_contracts.properties | 2 +- .../test-configurations/small_experiment.properties | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index 5480bda0..7d749052 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: mxd-performance-test - image: ciprian2398/mxd-performance-test:0.1.0-SNAPSHOT + image: hemantxpatel/mxd-performance-test:0.1.5-SNAPSHOT volumeMounts: - name: property-volume mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties diff --git a/mxd/performance-tests/test-configurations/large_experiment.properties b/mxd/performance-tests/test-configurations/large_experiment.properties index ec629b0b..b2688917 100644 --- a/mxd/performance-tests/test-configurations/large_experiment.properties +++ b/mxd/performance-tests/test-configurations/large_experiment.properties @@ -5,7 +5,7 @@ EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 -INTERVALS_PER_SECOND=10 +INTERVALS_PER_MINUTE=10 # OEM Parameters diff --git a/mxd/performance-tests/test-configurations/medium_experiment.properties b/mxd/performance-tests/test-configurations/medium_experiment.properties index a2b7d564..b40b5bfb 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment.properties @@ -5,7 +5,7 @@ EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 -INTERVALS_PER_SECOND=10 +INTERVALS_PER_MINUTE=10 # OEM Parameters diff --git a/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties index 11013f29..61f3695c 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties @@ -5,7 +5,7 @@ EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 -INTERVALS_PER_SECOND=10 +INTERVALS_PER_MINUTE=10 # OEM Parameters diff --git a/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties index 46b63ac1..85f96c42 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties @@ -5,7 +5,7 @@ EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 -INTERVALS_PER_SECOND=10 +INTERVALS_PER_MINUTE=10 # OEM Parameters diff --git a/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties index b5de0825..38a36684 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties @@ -5,7 +5,7 @@ EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 -INTERVALS_PER_SECOND=10 +INTERVALS_PER_MINUTE=10 # OEM Parameters diff --git a/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties index 1d775891..9727e921 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties @@ -5,7 +5,7 @@ EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 -INTERVALS_PER_SECOND=10 +INTERVALS_PER_MINUTE=10 # OEM Parameters diff --git a/mxd/performance-tests/test-configurations/small_experiment.properties b/mxd/performance-tests/test-configurations/small_experiment.properties index 22b840e4..67483075 100644 --- a/mxd/performance-tests/test-configurations/small_experiment.properties +++ b/mxd/performance-tests/test-configurations/small_experiment.properties @@ -1,11 +1,11 @@ # General Parameters -EXPERIMENT_DURATION_SEC=180 +EXPERIMENT_DURATION_SEC=1800 EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ BACKEND_SERVICE=https://requestbin.myworkato.com/1fwngcu1 MAX_NEGOTIATION_POLL_COUNT=100 MAX_TRANSFER_POLL_COUNT=100 -INTERVALS_PER_SECOND=10 +INTERVALS_PER_MINUTE=10 # OEM Parameters From b2ceb0256b9c007973ca88a0f6da507d196149a3 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta Date: Tue, 27 Feb 2024 12:42:03 +0200 Subject: [PATCH 15/34] feat: move mxd-performance-evaluation repo to a local folder *https://github.com/alexandrudanciu/mxd-performance-evaluation.git --- .../mxd-performance-evaluation/Dockerfile | 7 + .../Graphics/Initial_Counter.svg | 3 + .../Graphics/Sources/Inital_Counter.drawio | 85 + .../Graphics/Sources/Supplier_OEM.drawio | 244 +++ .../Graphics/Sources/TestSetup.drawio | 48 + .../Graphics/Sources/Test_Deployment.drawio | 169 ++ .../Graphics/Supplier_OEM.svg | 3 + .../Graphics/TestSetup.svg | 3 + .../Graphics/Test_Deployment.svg | 3 + .../mxd-performance-evaluation/README.md | 163 ++ .../build-docker.sh | 5 + .../measurement_interval.jmx | 1735 +++++++++++++++++ .../results_aggregation.py | 85 + .../run_experiment.sh | 47 + .../mxd-performance-evaluation/setup.jmx | 1076 ++++++++++ .../mxd-performance-evaluation/tear_down.jmx | 710 +++++++ 16 files changed, 4386 insertions(+) create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Dockerfile create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Supplier_OEM.drawio create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/TestSetup.drawio create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Test_Deployment.drawio create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Supplier_OEM.svg create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/TestSetup.svg create mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Test_Deployment.svg create mode 100644 mxd/performance-tests/mxd-performance-evaluation/README.md create mode 100755 mxd/performance-tests/mxd-performance-evaluation/build-docker.sh create mode 100755 mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx create mode 100644 mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py create mode 100755 mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh create mode 100755 mxd/performance-tests/mxd-performance-evaluation/setup.jmx create mode 100755 mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx diff --git a/mxd/performance-tests/mxd-performance-evaluation/Dockerfile b/mxd/performance-tests/mxd-performance-evaluation/Dockerfile new file mode 100644 index 00000000..08f58049 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Dockerfile @@ -0,0 +1,7 @@ +FROM justb4/jmeter:5.5 + +WORKDIR $JMETER_HOME/mxd-performance-evaluation + +COPY ["run_experiment.sh", "setup.jmx", "measurement_interval.jmx", "tear_down.jmx", "./"] + +ENTRYPOINT ["./run_experiment.sh"] diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg new file mode 100644 index 00000000..10cb955c --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg @@ -0,0 +1,3 @@ + + +

Sources:
Factory-Icon: Flaticon (2024): fabrik_861164, in: https://www.flaticon.com/de/kostenloses-icon/fabrik_861164 (22.01.24)
Engine-Icon: Flaticon (2024): auto-motor_5557384, in: https://www.flaticon.com/de/kostenloses-icon/auto-motor_5557384 (22.01.24)

Sources:...
SetUp OEM
SetUp OEM
OEM Plants
OEM Plants
/setup.jmx
/setup.jmx
/measurement_interval.jmx
/measurement_in...
Initial
Initial
1
1
.
.
.
.
.
.
1
1
n
n
.
.
.
.
.
.Text is not SVG - cannot display \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio new file mode 100644 index 00000000..6dc0d615 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Supplier_OEM.drawio b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Supplier_OEM.drawio new file mode 100644 index 00000000..c9124b4c --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Supplier_OEM.drawio @@ -0,0 +1,244 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/TestSetup.drawio b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/TestSetup.drawio new file mode 100644 index 00000000..2219f2a4 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/TestSetup.drawio @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Test_Deployment.drawio b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Test_Deployment.drawio new file mode 100644 index 00000000..5a4b88ee --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Test_Deployment.drawio @@ -0,0 +1,169 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Supplier_OEM.svg b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Supplier_OEM.svg new file mode 100644 index 00000000..070506a3 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Supplier_OEM.svg @@ -0,0 +1,3 @@ + + +
OEM Plants
OEM Plants
Supplier Fleet Managers
Supplier Fleet Managers
Parts produced per time interval (PARTS_PRODUCED_PER_INTERVALL)
Parts produced per time interval...
Cars requested per time interval
(CARS_REQUESTED_PER_INTERVALL)
Cars requested per time in...

Sources:
Factory-Icon: Flaticon (2024): fabrik_861164, in: https://www.flaticon.com/de/kostenloses-icon/fabrik_861164 (22.01.24)
Engine-Icon: Flaticon (2024): auto-motor_5557384, in: https://www.flaticon.com/de/kostenloses-icon/auto-motor_5557384 (22.01.24)
Bill-Icon: Flaticon (2024): rechnung_1449943, in: https://www.flaticon.com/de/kostenloses-icon/rechnung_1449943 (22.01.24)
Production-Line-Icon: Flaticon (2024): production-line_9420117, in: https://www.flaticon.com/de/kostenloses-icon/production-line_9420117 (22.01.24)
Office-Icon: Flaticon (2024): buro_1086562, in: https://www.flaticon.com/de/kostenloses-icon/buro_1086562 (22.01.24)

Sources:...
Cars produced per time interval (CARS_PRODUCED_PER_INTERVALL)
Cars produced per time interval (...
 Supplier Plants
 Supplier Plants
Number of OEM plants
(OEM_PLANTS)
Number of OEM plants...
Number of Supplier plants
(SUPPLIER_PLANTS)
Number of Supplier pl...
Number of Fleet Managers
(FLEET_MANAGERS)
Number of Fleet Manag...Text is not SVG - cannot display \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/TestSetup.svg b/mxd/performance-tests/mxd-performance-evaluation/Graphics/TestSetup.svg new file mode 100644 index 00000000..50843e2f --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/TestSetup.svg @@ -0,0 +1,3 @@ + + +
System Under Test
System Under Test
Part Twins
Part Twins
OEM
OEM
Control Plane
Control Plane
Data Plane
Data Plane
Car Twins
Car Twins
Supplier
Supplier
Control Plane
Control Plane
Data Plane
Data Plane
Workload Driver
Workload DriverText is not SVG - cannot display \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Test_Deployment.svg b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Test_Deployment.svg new file mode 100644 index 00000000..c9cfe009 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Test_Deployment.svg @@ -0,0 +1,3 @@ + + +
Kubernetes Cluster
Kubernetes Cluster
Worker Node
Worker Node
OEM Pod
OEM Pod
OEM_EDC.jar
OEM_EDC.jar
OEM Container
OEM Container
JMeter Pod
JMeter Pod
JMeter Container
JMeter Container
Supplier Pod
Supplier Pod
Supplier_EDC.jar
Supplier_EDC.jar
Supplier Container
Supplier Container
JMeterScript
JMeterScript
Database Pod
Database Pod
Database Container
Database Container
OEM Postgres Database
OEM Postgres Database
Database Pod
Database Pod
Database Container
Database Container
Supplier Postgres Database
Supplier Postgres Da...
MIW Pod
MIW Pod
Managed Identity Wallet
Managed Identity Wal...
MIW Container
MIW Container
Database Pod
Database Pod
Database Container
Database Container
MIW Postgres Database
MIW Postgres DatabaseText is not SVG - cannot display \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/README.md b/mxd/performance-tests/mxd-performance-evaluation/README.md new file mode 100644 index 00000000..2b32d62a --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/README.md @@ -0,0 +1,163 @@ +# EDC Performance Tests Introduction + +## Introduction +This document describes the design of the Eclipse Dataspace Connector (EDC) Performance Tests. +The EDC is the central communication component in Catena-X, which as a connector implements a framework agreement for sovereign, cross-organizational data exchange. + +### Goals of the test +The test is intended to measures the performance in scenarios similar to reality of the automotive industry. +Due to the fact that the EDC maps many such scenarios and use cases, it was designed as a parameter-based, customizable test. +The test defines a framework that can be adapted to real business scenarios using defined parameters. +This is an attempt to cover a broad spectrum of use cases and their performance. + +To ensure the credibility of the test, the test is based on defined characteristics: + +- Real-world: The performance workload should have the characteristics of real-world systems. +- Complexity: The test should capture intra- and inter-company business processes from two counter parties. +- Openness and version independence: The test should be able to be implemented using different versions of the EDC and not be limited to a specific use case scenario. + +## Application +The test and its framework are based on the Java-based test tool "Apache JMeter". The documentation and further information on this tool can be found at Link: https://jmeter.apache.org +The test scripts are implemented exclusively as .jmx files, an XML-coded script that uses JMeter as the default file format for testing. + +In the first stage of these performance tests, the EDC is simulated using Minimum Tractus-X Dataspace (MXD). Further information on this can be found under the following link: https://eclipse-tractusx.github.io/docs-kits/kits/tractusx-edc/docs/kit/adoption-view/Adoption%20View/ + +# Test Design + +## Definition of General Terms +| Term | Explanation| +|------------------------------|------------------------------| +| OEM | An OEM or "Original Equipment Manufacturer" manufactures a part or component that is used in another company's product | +| Supplier | An Supplier represents the end of the supply chain in the test scenario and manufactures the end product | + +## Test Setup +The relationship between the test setup and the setup of the Minimum Tractus-X Dataspace is visualized in the following figure. It should be emphasized that both the OEM and the supplier act as data recipients and senders. + +Visualization of the test setup + +### Deployment of the Test +In order to create a test environment that is as realistic as possible, a deployment in a Kubernetes cluster is used for the test, which in turn deploys a worker node. This worker node of the cluster is characterized by the deployment of two EDCs, each for OEM and supplier and the respective connectors. The contents of the two EDCs are stored in a Postgres database. +The actual test and thus the workload for the EDCs is carried out by a JMeter script. The entire context is visualized below. + +Deployment SetUp + +## Functional structure of the test +The experiment is divided into three different phases. Each phase is implemented as own .jmx file. + +### Setup +During the setup phase an initial set of asset, policy and contract data are loaded into the connectors. The purpose of these data sets is introduce a certain level of data aging before conducting measurements. + +### Measurement Interval +This phase represents the actual performance evaluation. The script runs through all standard processes that can be executed by the EDC. The process starts with the creation of assets, policies and contract definitions based on the number of mock-up data from the previous process. This is followed by further EDC processes such as "Create Catalogue" until the data transfer is negotiated and executed, as described in the EDC wiki: +https://eclipse-tractusx.github.io/docs-kits/category/connector-kit/ +In order to test this section as realistically as possible, the process is carried out for both OEMs ("OEM Plants") and suppliers ("Supplier Plants"). The section also contains the "Supplier Fleet Manager" process, a process designed to simulate employees at the supplier who request digital twins of the cars. + +### Tear Down +During the tear-down phase, all previously created assets, policies and contracts are deleted from the connectors. + +## Definition of Properties (experiment.properties) + +### General Parameters + +| Property | Explanation| +|------------------------------|------------------------------| +| EXPERIMENT_DURATION_SEC | Duration of the experiment, during which the threads of the experiment are executed as often as possible. The unit is defined in seconds. | +| EDC_NAMESPACE | URL as an Permanent identifier of the EDC and its version. | +| BACKEND_SERVICE | URL to reference the backend system of the EDCs. | +| MAX_NEGOTIATION_POLL_COUNT | Number of maximum status checks of "FINALIZED" state of the negotiation between the two initiated EDCs Alice and Bob. | +| MAX_TRANSFER_POLL_COUNT | Number of maximum status checks of "FINALIZED" state of the transfer between the two initiated EDCs Alice and Bob. | +| INTERVALS_PER_MINUTE | Number of simulated time intervals per minute. | + +### OEM Parameters + +| Property | Explanation| +|------------------------------|------------------------------| +| OEM_ID | Unique identifier of the OEM (Business Partner Number). | +| OEM_MANAGEMENT_URL | URL referencing the Management API of the OEM's EDC. | +| OEM_PROTOCOL_URL | URL referencing the Dataspace Protocol of the OEM Connector. | +| OEM_API_KEY | Specific, secret key to access the Management API of the OEM's EDC. | +| OEM_PLANTS | Number of OEM plants manufacturing cars. Each plant is modeled as individual thread. | +| OEM_CARS_INITIAL | Number of inital produced cars in the "SetUp" process for creating mockup data in the database for further processes | +| PARTS_PER_CAR | Number of part twins requested by the OEM from the supplier during the production of a car. | +| CARS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | + +### Supplier Parameters + +| Property | Explanation| +|------------------------------|------------------------------| +| SUPPLIER_ID | Unique identifier of the Supplier (Business Partner Number) | +| SUPPLIER_MANAGEMENT_URL | URL referencing the Management API of the Supplier EDC. | +| SUPPLIER_PROTOCOL_URL | URL referencing the Dataspace Protocol of the Supplier Connector. | +| SUPPLIER_API_KEY | Specific, secret key to access the Management API of the Supplier EDC. | +| SUPPLIER_PLANTS | Number of supplier plants manufacturing parts. Each plant is modeled as individual thread. | +| PARTS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | +| SUPPLIER_FLEET_MANAGERS | Number of employees requesting car twins. | +| CARS_REQUESTED_PER_INTERVALL | Number of car twins requested by the supplier per time interval per employee. | +| SUPPLIER_PARTS_INITIAL | Number of inital produced parts in the "SetUp" process for creating mockup data in the database for further processes | + +## Functional structure in dependency to the parameter + +### Visualization +The number of times the processes, which depicts the execution of the production of the cars, the parts required for them and the invoices created for them, are executed in the test setup depends on the three parameters "OEM_PLANTS", "SUPPLIER_PLANTS" and "SUPPLIER FLEET MANAGER". This relationship is illustrated in the following figure: + +Dependency between OEM and Supplier + +The sub-process of executing the above processes is characterized by the number of cars tested. The parameter "OEM_CARS_INITIAL" can be used to define the number of mock-up cars to be created in the setup.jmx. This parameter is then passed to the measurement_interval.jmx and defines the start of the number of cars produced there (starting with counter amount 1). + +Initial Counter + +# User Guide + +## Installation of the required software +The minimum Tractus-X Dataspace is required to carry out the tests in a local environment. +The software and tools required for this are described in the following wiki: +https://github.com/sap-contributions/eclipse-tractusx_tutorial-resources/blob/main/mxd/1_MXD_Introduction.md + +The setup of the dataspace is explained further in the following wiki: +https://github.com/sap-contributions/eclipse-tractusx_tutorial-resources/blob/main/mxd/2_MXD_Setup.md + +After installing the Tractus-X Dataspace, the test software must now be installed. +To do this, the Jmeter tool must first be installed. Download here: +https://jmeter.apache.org/download_jmeter.cgi +Add the JMETER_HOME environment variable and let it reference the /bin directory of JMeter. + +To run the Jmeter tool, a Java Runtime Environment (JRE) is required, which can be installed here: +https://www.java.com/de/download/manual.jsp + +## Running the Evaluation +Once all tools and software have been installed and the Tractus-X Dataspace has been deployed as explained in the wiki, the test can be executed: + +1. [OPTIONAL] Review and update the connectors properties in the three different .properties files. A brief explanation of all properties can be found in the User Guide of this document. +2. Execute the run_experiment.sh script (Bash4 required). To do this, enter the corresponding arguments (two required), on which parameters the execution should be carried out and which of the three phases of the experiment (as described in the test design) should be executed. + +**.properties -files "-q": (First argument)** + +| Naming | Default Values | +|------------------------------|------------------------------| +| S = small_experiment.properties | Supplier/OEM Plants = 1, Initial cars/parts = 10000 | +| M = medium_experiment.properties | Supplier/OEM Plants = 5, Initial cars/parts = 100000 | +| L = medium_experiment.properties | Supplier/OEM Plants = 10, Initial cars/parts = 1000000 | + +**.jmx -files "-t": (Second argument)** +| Letter | Execution of | +|------------------------------|------------------------------| +| S | setup.jmx | +| M | measurement.jmx | +| T | tear_down.jmx | +| SMT | Sequential execution of all three files | + +3. Enjoy the test results! + +## Test results +After executing the shell script, the test results can be viewed at /Output/measurement_interval/index.html. +Of relevance is the sub-item "Statistics" in index.html. This sub-item contains, among other things: + +| Naming | Explanation | +|------------------------------|------------------------------| +| Executions | Number of processes performed | +| Response Time (ms) | Number in milliseconds of how long it takes the EDC to respond | +| Throughput | Ratio of executions to response time | +| Network (KB/sec) | Data throughput of the EDC in KB | + + +All further explanations can be found in the JMeter documentation: https://jmeter.apache.org/usermanual/generating-dashboard.html \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh new file mode 100755 index 00000000..e1943be5 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +docker build -t hemantxpatel/mxd-performance-test:0.1.4-SNAPSHOT . + +docker image push hemantxpatel/mxd-performance-test:0.1.4-SNAPSHOT \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx new file mode 100755 index 00000000..41aaab79 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx @@ -0,0 +1,1735 @@ + + + + + false + false + + + + EXPERIMENT_DURATION_SEC + ${__P(EXPERIMENT_DURATION_SEC, 60)} + = + + + EDC_NAMESPACE + = + ${__P(EDC_NAMESPACE, https://w3id.org/edc/v0.0.1/ns/)} + + + BACKEND_SERVICE + = + ${__P(BACKEND_SERVICE, https://requestbin.myworkato.com/z8zu6nz9)} + + + MAX_NEGOTIATION_POLL_COUNT + ${__P(MAX_NEGOTIATION_POLL_COUNT, 100)} + = + + + MAX_TRANSFER_POLL_COUNT + ${__P(MAX_TRANSFER_POLL_COUNT, 100)} + = + + + LOOP_COUNT + ${__P(LOOP_COUNT, 50)} + = + + + THREAD_COUNT + ${__P(THREAD_COUNT, 1)} + = + + + INTERVALS_PER_MINUTE + ${__P(INTERVALS_PER_MINUTE, 10)} + = + + + OEM_MANAGEMENT_URL + ${__P(OEM_MANAGEMENT_URL, http://localhost/alice/management)} + = + + + OEM_PROTOCOL_URL + = + ${__P(OEM_PROTOCOL_URL, http://alice-controlplane:8084/api/v1/dsp)} + + + OEM_ID + = + ${__P(OEM_ID, BPNL000000000001)} + + + OEM_API_KEY + ${__P(OEM_API_KEY, password)} + = + + + OEM_PLANTS + ${__P(OEM_PLANTS, 2)} + = + + + OEM_CARS_INITIAL + ${__P(OEM_CARS_INITIAL, 1000)} + = + + + PARTS_PER_CAR + ${__P(PARTS_PER_CAR, 100)} + = + + + CARS_PRODUCED_PER_INTERVALL + ${__P(CARS_PRODUCED_PER_INTERVALL, 1)} + = + + + SUPPLIER_MANAGEMENT_URL + ${__P(SUPPLIER_MANAGEMENT_URL, http://localhost/bob/management)} + = + + + SUPPLIER_PROTOCOL_URL + = + ${__P(SUPPLIER_PROTOCOL_URL, http://bob-controlplane:8084/api/v1/dsp)} + + + SUPPLIER_ID + = + ${__P(SUPPLIER_ID, BPNL000000000002)} + + + SUPPLIER_API_KEY + ${__P(SUPPLIER_API_KEY, password)} + = + + + SUPPLIER_PLANTS + ${__P(SUPPLIER_PLANTS, 2)} + = + + + PARTS_PRODUCED_PER_INTERVALL + ${__P(PARTS_PRODUCED_PER _INTERVALL, 50)} + = + + + SUPPLIER_FLEET_MANAGERS + ${__P(SUPPLIER_FLEET_MANAGERS, 1)} + = + + + SUPPLIER_PARTS_PLANTS_COUNT + ${__P(SUPPLIER_PARTS_PLANTS_COUNT)} + = + + + SUPPLIER_FLEET_COUNT + ${__P(SUPPLIER_FLEET_COUNT)} + = + + + CARS_REQUESTED_PER_INTERVALL + ${__P(CARS_REQUESTED_PER_INTERVALL, 1)} + = + + + SUPPLIER_PARTS_INITIAL + ${__P(SUPPLIER_PARTS_INITIAL, 1000)} + = + + + + false + + + + startnextloop + + -1 + false + + + false + true + ${EXPERIMENT_DURATION_SEC} + + true + ${OEM_PLANTS} + + + + false + true + false + + + + 0 + ${INTERVALS_PER_MINUTE} + + + + + + NEGOTIATION_ID + NEGOTIATION_STATE + TRANSFER_ID + TRANSFER_STATE + + + + + + + + + + true + Local Variables to be reset after each loop + + + + ${__intSum(1,${OEM_CARS_INITIAL})} + + 1 + PROVIDE_COUNTER + + false + + + + 1 + + 1 + CONSUME_COUNTER + + false + + + + true + + + + false + { + "@context": {}, + "@id": "${PROVIDE_COUNTER}", + "properties": { + "description": "Product EDC Demo Asset" + }, + "dataAddress": { + "@type": "DataAddress", + "type": "HttpData", + "baseUrl": "https://jsonplaceholder.typicode.com/todos" + } +} + = + + + + ${OEM_MANAGEMENT_URL}/v3/assets + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + true + + + + false + { + "@context": { + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "PolicyDefinitionRequestDto", + "@id": "${PROVIDE_COUNTER}", + "policy": { + "@type": "Policy", + "odrl:permission" : [{ + "odrl:action" : "USE", + "odrl:constraint" : { + "@type": "LogicalConstraint", + "odrl:or" : [{ + "@type" : "Constraint", + "odrl:leftOperand" : "BusinessPartnerNumber", + "odrl:operator" : { + "@id": "odrl:eq" + }, + "odrl:rightOperand" : "${SUPPLIER_ID}" + }] + } + }] + } +} + = + + + + ${OEM_MANAGEMENT_URL}/v2/policydefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + true + + + + false + { + "@context": {}, + "@id": "${PROVIDE_COUNTER}", + "@type": "ContractDefinition", + "accessPolicyId": "${PROVIDE_COUNTER}", + "contractPolicyId": "${PROVIDE_COUNTER}", + "assetsSelector" : { + "@type" : "CriterionDto", + "operandLeft": "${EDC_NAMESPACE}id", + "operator": "=", + "operandRight": "${PROVIDE_COUNTER}" + } +} + = + + + + ${OEM_MANAGEMENT_URL}/v2/contractdefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "CatalogRequest", + "counterPartyAddress": "${SUPPLIER_PROTOCOL_URL}", + "protocol": "dataspace-protocol-http", + "querySpec": { + "offset": 0, + "limit": 10, + "filterExpression": { + "@type": "CriterionDto", + "operandLeft": "https://w3id.org/edc/v0.0.1/ns/id", + "operator": "=", + "operandRight": "${CONSUME_COUNTER}" + } + } +} + = + + + + ${OEM_MANAGEMENT_URL}/v2/catalog/request + POST + true + false + false + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + CONTRACT_OFFER_ID + ${__eval($['dcat:dataset'][?(@.id=="${CONSUME_COUNTER}")]['odrl:hasPolicy']['@id'])} + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "https://w3id.org/edc/v0.0.1/ns/ContractRequest", + "counterPartyAddress": "${SUPPLIER_PROTOCOL_URL}", + "protocol": "dataspace-protocol-http", + "providerId": "${SUPPLIER_ID}", + "offer": { + "offerId": "${CONTRACT_OFFER_ID}", + "assetId": "${CONSUME_COUNTER}", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@type": "odrl:Set", + "@id": "${CONSUME_COUNTER}", + "odrl:permission" : { + "odrl:action": { + "odrl:type": "USE" + }, + "odrl:constraint" : { + "odrl:or" : { + "odrl:leftOperand" : "BusinessPartnerNumber", + "odrl:operator" : { + "@id": "odrl:eq" + }, + "odrl:rightOperand" : "${OEM_ID}" + } + } + }, + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": { + "@id": "${CONSUME_COUNTER}" + } + } + }, + "callbackAddresses": [] +} + = + + + + ${OEM_MANAGEMENT_URL}/v2/contractnegotiations + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + NEGOTIATION_ID + $['@id'] + 1 + + + + + true + false + + + + ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && +${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} + Loop while Negotiation State is FINALIZED + + + + 500 + + + + false + + + + ${OEM_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID}/state + GET + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + NEGOTIATION_STATE + $['state'] + + + + + + + groovy + + + true + var currentState = vars.get("NEGOTIATION_STATE"); + +if(!"FINALIZED".equals( currentState)) { + SampleResult.setSuccessful(false); + SampleResult.setResponseData("Negotiation State is not 'FINALIZED' after ${vars.get('MAX_NEGOTIATION_POLL_COUNT')} attempts. Current State: ${currentState}"); +} + + + + + + false + + + + ${OEM_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID} + GET + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + CONTRACT_AGREEMENT_ID + $['contractAgreementId'] + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "https://w3id.org/edc/v0.0.1/ns/TransferRequest", + "protocol": "dataspace-protocol-http", + "counterPartyAddress": "${SUPPLIER_PROTOCOL_URL}", + "connectorId": "${SUPPLIER_ID}", + "contractId": "${CONTRACT_AGREEMENT_ID}", + "assetId": "${CONSUME_COUNTER}", + "dataDestination": { + "type": "HttpProxy" + }, + "privateProperties": { + "receiverHttpEndpoint": "${BACKEND_SERVICE}" + }, + + "transferType": { + "contentType": "application/octet-stream", + "isFinite": true + } +} + = + + + + ${OEM_MANAGEMENT_URL}/v2/transferprocesses + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + TRANSFER_ID + $['@id'] + 1 + + + + + true + false + + + + ${__jexl3("${TRANSFER_STATE}" != "STARTED" && +${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} + Loop while Transfer State is STARTED + + + + 500 + + + + false + + + + ${OEM_MANAGEMENT_URL}/v2/transferprocesses/${TRANSFER_ID}/state + GET + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + TRANSFER_STATE + $['state'] + + + + + + + groovy + + + true + var currentState = vars.get("TRANSFER_STATE"); + +if(!"STARTED".equals( currentState)) { + SampleResult.setSuccessful(false); + SampleResult.setResponseData("Transfer State is not 'COMPLETED' after ${vars.get('MAX_TRANSFER_POLL_COUNT')} attempts. Current State: ${currentState}"); +} + + + + + + true + + saveConfig + + + true + true + true + + true + true + true + true + true + true + true + true + true + true + false + true + true + false + true + 0 + true + true + true + true + true + true + true + true + true + + + output/measurement_interval_oem_plants_errors.xml + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + SetUp OEM + + + + + startnextloop + + -1 + false + + ${SUPPLIER_PLANTS} + + false + true + ${EXPERIMENT_DURATION_SEC} + + true + + + + false + true + false + + + + 0 + ${INTERVALS_PER_MINUTE} + + + + + + NEGOTIATION_ID + NEGOTIATION_STATE + TRANSFER_ID + TRANSFER_STATE + + + + + + + + + + true + Local Variables to be reset after each loop + + + + ${__intSum(1,${SUPPLIER_PARTS_INITIAL})} + + 1 + ID + + false + + + + true + + + + false + { + "@context": {}, + "@id": "${ID}", + "properties": { + "description": "Product EDC Demo Asset" + }, + "dataAddress": { + "@type": "DataAddress", + "type": "HttpData", + "baseUrl": "https://jsonplaceholder.typicode.com/todos" + } +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v3/assets + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + true + + + + false + { + "@context": { + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "PolicyDefinitionRequestDto", + "@id": "${ID}", + "policy": { + "@type": "Policy", + "odrl:permission" : [{ + "odrl:action" : "USE", + "odrl:constraint" : { + "@type": "LogicalConstraint", + "odrl:or" : [{ + "@type" : "Constraint", + "odrl:leftOperand" : "BusinessPartnerNumber", + "odrl:operator" : { + "@id": "odrl:eq" + }, + "odrl:rightOperand" : "${OEM_ID}" + }] + } + }] + } +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + true + + + + false + { + "@context": {}, + "@id": "${ID}", + "@type": "ContractDefinition", + "accessPolicyId": "${ID}", + "contractPolicyId": "${ID}", + "assetsSelector" : { + "@type" : "CriterionDto", + "operandLeft": "${EDC_NAMESPACE}id", + "operator": "=", + "operandRight": "${ID}" + } +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + true + + saveConfig + + + true + true + true + + true + true + true + true + true + true + true + true + true + true + false + true + true + false + true + 0 + true + true + true + true + true + true + true + true + true + + + output/measurement_interval_supplier_plants_errors.xml + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + SetUp OEM + + + + + startnextloop + + -1 + false + + ${SUPPLIER_FLEET_MANAGERS} + + false + true + ${EXPERIMENT_DURATION_SEC} + + true + + + + false + true + false + + + + 0 + ${INTERVALS_PER_MINUTE} + + + + + 1 + + 1 + CONSUME_COUNT + + false + Consumes the current value from the OEM. + + + + + NEGOTIATION_ID + NEGOTIATION_STATE + TRANSFER_ID + TRANSFER_STATE + + + + + + + + + + true + Local Variables to be reset after each loop + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "CatalogRequest", + "counterPartyAddress": "${OEM_PROTOCOL_URL}", + "protocol": "dataspace-protocol-http", + "querySpec": { + "offset": 0, + "limit": 10, + "filterExpression": { + "@type": "CriterionDto", + "operandLeft": "https://w3id.org/edc/v0.0.1/ns/id", + "operator": "=", + "operandRight": "${CONSUME_COUNT}" + } + } +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/catalog/request + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + CONTRACT_OFFER_ID + ${__eval($['dcat:dataset'][?(@.id=="${CONSUME_COUNT}")]['odrl:hasPolicy']['@id'])} + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/", + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "https://w3id.org/edc/v0.0.1/ns/ContractRequest", + "counterPartyAddress": "${OEM_PROTOCOL_URL}", + "protocol": "dataspace-protocol-http", + "providerId": "${OEM_ID}", + "offer": { + "offerId": "${CONTRACT_OFFER_ID}", + "assetId": "${CONSUME_COUNT}", + "policy": { + "@context": "http://www.w3.org/ns/odrl.jsonld", + "@type": "odrl:Set", + "@id": "${CONSUME_COUNT}", + "odrl:permission" : { + "odrl:action": { + "odrl:type": "USE" + }, + "odrl:constraint" : { + "odrl:or" : { + "odrl:leftOperand" : "BusinessPartnerNumber", + "odrl:operator" : { + "@id": "odrl:eq" + }, + "odrl:rightOperand" : "${SUPPLIER_ID}" + } + } + }, + "odrl:prohibition": [], + "odrl:obligation": [], + "odrl:target": { + "@id": "${CONSUME_COUNT}" + } + } + }, + "callbackAddresses": [] +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + NEGOTIATION_ID + $['@id'] + 1 + + + + + true + false + + + + ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && +${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} + Loop while Negotiation State is FINALIZED + + + + 500 + + + + false + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID}/state + GET + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + NEGOTIATION_STATE + $['state'] + + + + + + + groovy + + + true + var currentState = vars.get("NEGOTIATION_STATE"); + +if(!"FINALIZED".equals( currentState)) { + SampleResult.setSuccessful(false); + SampleResult.setResponseData("Negotiation State is not 'FINALIZED' after ${vars.get('MAX_NEGOTIATION_POLL_COUNT')} attempts. Current State: ${currentState}"); +} + + + + + + false + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID} + GET + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + CONTRACT_AGREEMENT_ID + $['contractAgreementId'] + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "https://w3id.org/edc/v0.0.1/ns/TransferRequest", + "protocol": "dataspace-protocol-http", + "counterPartyAddress": "${OEM_PROTOCOL_URL}", + "connectorId": "${OEM_ID}", + "contractId": "${CONTRACT_AGREEMENT_ID}", + "assetId": "${CONSUME_COUNT}", + "dataDestination": { + "type": "HttpProxy" + }, + "privateProperties": { + "receiverHttpEndpoint": "${BACKEND_SERVICE}" + }, + + "transferType": { + "contentType": "application/octet-stream", + "isFinite": true + } +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/transferprocesses + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + TRANSFER_ID + $['@id'] + 1 + + + + + true + false + + + + ${__jexl3("${TRANSFER_STATE}" != "STARTED" && +${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} + Loop while Transfer State is STARTED + + + + 500 + + + + false + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/transferprocesses/${TRANSFER_ID}/state + GET + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + TRANSFER_STATE + $['state'] + + + + + + + groovy + + + true + var currentState = vars.get("TRANSFER_STATE"); + +if(!"STARTED".equals( currentState)) { + SampleResult.setSuccessful(false); + SampleResult.setResponseData("Transfer State is not 'COMPLETED' after ${vars.get('MAX_TRANSFER_POLL_COUNT')} attempts. Current State: ${currentState}"); +} + + + + + + true + + saveConfig + + + true + true + true + + true + true + true + true + true + true + true + true + true + true + false + true + true + false + true + 0 + true + true + true + true + true + true + true + true + true + + + output/measurement_interval_supplier_fleet_manager_errors.xml + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + SetUp OEM + + + + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py new file mode 100644 index 00000000..cb88fd59 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py @@ -0,0 +1,85 @@ +import json +import matplotlib.pyplot as plt # to install: pip3 install plotly +import numpy as np +import mplcursors # to install: pip3 install mplcursors +import os +import random + +############ Required Input ############ +root_folder = './Ergebnisse' +values = ['meanResTime', 'sampleCount'] +processes = ['Get Transfer State', 'Initiate Transfer'] + +###################################################################################### +# A function for extracting values from a given file +def extract_values(file_path): + with open(file_path, 'r') as file: + lines = file.readlines() + values = {} + for line in lines: + if 'OEM_PLANTS' in line: + values['OEM_PLANTS'] = int(line.split('=')[1].strip()) + elif 'OEM_CARS_INITIAL' in line: + values['OEM_CARS_INITIAL'] = int(line.split('=')[1].strip()) + elif 'PARTS_PER_CAR' in line: + values['PARTS_PER_CAR'] = int(line.split('=')[1].strip()) + elif 'CARS_PRODUCED_PER_INTERVALL' in line: + values['CARS_PRODUCED_PER_INTERVALL'] = int(line.split('=')[1].strip()) + return values + +# List comprehension to gather all directories under the specified root_folder +directories = [os.path.join(root_folder, o) for o in os.listdir(root_folder) + if os.path.isdir(os.path.join(root_folder,o))] + +# Iterates over each action and value from the lists and creates a new figure for each combination +for action_to_consider in processes: + for value_idx, value in enumerate(values): + plt.figure(figsize=(15, 4)) + plt.title('Aggregation of Performance Test Results: {} for {}'.format(value, action_to_consider)) + + stats = [] + labels = [] + meta_values_list = [] + + # Goes through each directory to find the metadata and statistics files + for idx, directory in enumerate(directories): + metadata_path = os.path.join(directory, 'metadata.txt') + statistics_path = os.path.join(directory, 'dashboard/statistics.json') + + # If both files exist in the directory, it calls the function to extract values from the metadata file + if os.path.exists(metadata_path) and os.path.exists(statistics_path): + meta_values = extract_values(metadata_path) + + # Opens the statistics.json file and loads its content + with open(statistics_path, 'r') as f: + stats_data = json.load(f) + + # Appends the statistics for each action and the associated labels for the plot + action = stats_data[action_to_consider] + stats.append((meta_values['OEM_PLANTS'], action[value])) + labels.append('Plants: {}, {}: {:.2f}'.format(meta_values['OEM_PLANTS'], value, round(action[value],2))) + meta_values_list.append(meta_values) + + # Plots the statistics data as a line plot and a scatter plot + plt.plot([s[0] for s in stats], [s[1] for s in stats]) + colors = [ (random.random(), random.random(), random.random()) for _ in range(len(stats)) ] + scatter = plt.scatter([s[0] for s in stats], [s[1] for s in stats], color=colors) + + # Adds labels to each point on the scatter plot (that x-axis matches OEM_PLANTS values) + for i, text in enumerate(labels): + plt.text(stats[i][0], stats[i][1], text) + + # Add cursor + cursor = mplcursors.cursor(scatter, hover=True) + @cursor.connect("add") + def on_add(sel): + i = sel.target.index + sel.annotation.set_text('Plants: {}, Cars: {}, Parts/Car: {}, Cars/Interval: {}'.format( + stats[i][0], + meta_values_list[i]['OEM_CARS_INITIAL'], + meta_values_list[i]['PARTS_PER_CAR'], + meta_values_list[i]['CARS_PRODUCED_PER_INTERVALL'])) + + plt.xlabel('OEM_PLANTS') + plt.ylabel(value) + plt.show() \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh b/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh new file mode 100755 index 00000000..6f71c83b --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh @@ -0,0 +1,47 @@ +#!/bin/bash +experiment_property="custom_experiment.properties" + +if [ -z "$JMETER_HOME" ]; then + echo "JMETER_HOME is not defined - please define it." + exit 1 +fi +echo "Jmeter found at path $JMETER_HOME" + +jmeter_binary="$JMETER_HOME/bin/jmeter" + +rm -rf output +mkdir -p output/dashboard + +rm -rf output_slim +mkdir -p output_slim/dashboard + +echo "*** Property: $experiment_property Start ***" +cat $experiment_property +echo "*** Property: $experiment_property End ***" + +echo "*** Performance Test Start ***" +if [ -z "$jmeter_script" ]; then + echo "Executing all scripts in order" + echo "Executing setup.jmx" + $jmeter_binary -n -t setup.jmx -l output/setup.jtl -q $experiment_property + echo "Executing measurement_interval.jmx" + $jmeter_binary -n -t measurement_interval.jmx -l output/measurement_interval.jtl -q $experiment_property -e -o output/dashboard + echo "Executing tear_down.jmx" + $jmeter_binary -n -t tear_down.jmx -l output/tear_down.jtl -q $experiment_property +fi +echo "*** Performance Test End ***" + +echo "Copying custom_experiment.properties to output folder with new name metadata.txt" +cp $experiment_property output/metadata.txt +echo "Creating tar file with output" +tar -cf output.tar output + +echo "Copying custom_experiment.properties to output_slim folder with new name metadata.txt" +cp $experiment_property output_slim/metadata.txt +echo "Copying output/dashboard/statistics.json to output_slim/dashboard/statistics.json folder" +cp output/dashboard/statistics.json output_slim/dashboard/statistics.json +echo "Creating tar file with output" +tar -cf output_slim.tar output_slim + +echo "*** Test Completed..., Sleeping now ***" +while true; do sleep 10000; done diff --git a/mxd/performance-tests/mxd-performance-evaluation/setup.jmx b/mxd/performance-tests/mxd-performance-evaluation/setup.jmx new file mode 100755 index 00000000..4a268441 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/setup.jmx @@ -0,0 +1,1076 @@ + + + + + + + + OEM_ID + = + ${__P(OEM_ID, BPNL000000000001)} + + + OEM_MANAGEMENT_URL + ${__P(OEM_MANAGEMENT_URL, http://localhost/alice/management)} + = + + + OEM_PROTOCOL_URL + = + ${__P(OEM_PROTOCOL_URL, http://alice-controlplane:8084/api/v1/dsp)} + + + OEM_API_KEY + ${__P(OEM_API_KEY, password)} + = + + + OEM_CARS_INITIAL + ${__P(OEM_CARS_INITIAL, 1000)} + = + + + SUPPLIER_ID + ${__P(SUPPLIER_ID, BPNL000000000002)} + = + + + SUPPLIER_MANAGEMENT_URL + ${__P(SUPPLIER_MANAGEMENT_URL, http://localhost/bob/management)} + = + + + SUPPLIER_PROTOCOL_URL + = + ${__P(SUPPLIER_PROTOCOL_URL, http://bob-controlplane:8084/api/v1/dsp)} + + + SUPPLIER_API_KEY + ${__P(SUPPLIER_API_KEY, password)} + = + + + SUPPLIER_PARTS_INITIAL + ${__P(SUPPLIER_PARTS_INITIAL, 1000)} + = + + + EDC_NAMESPACE + ${__P(EDC_NAMESPACE, https://w3id.org/edc/v0.0.1/ns/)} + = + + + ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER + ${__P(ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER, 0)} + = + + + ADDITIONAL_CONTRACT_DEFINITIONS_OEM + ${__P(ADDITIONAL_CONTRACT_DEFINITIONS_OEM, 0)} + = + + + + + + + Defining a test for the processes "Create Asset", "Create Policy" and "Create Contract Definition" of the supplier + 1 + false + startnextloop + + 1 + false + + + + + 500 + + + + ${SUPPLIER_PARTS_INITIAL} + + + + 1 + + 1 + ID + + false + + + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v3/assets + true + POST + true + true + + + + false + { + "@context": {}, + "@id": "${ID}", + "properties": { + "description": "Product EDC Demo Asset" + }, + "dataAddress": { + "@type": "DataAddress", + "type": "HttpData", + "baseUrl": "https://jsonplaceholder.typicode.com/todos" + } +} + = + + + + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions + true + POST + true + true + + + + false + { + "@context": { + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "PolicyDefinitionRequestDto", + "@id": "bpnpolicy", + "policy": { + "@type": "Policy", + "odrl:permission" : [{ + "odrl:action" : "USE", + "odrl:constraint" : { + "@type": "LogicalConstraint", + "odrl:or" : [{ + "@type" : "Constraint", + "odrl:leftOperand" : "BusinessPartnerNumber", + "odrl:operator" : { + "@id": "odrl:eq" + }, + "odrl:rightOperand" : "${OEM_ID}" + }] + } + }] + } +} + = + + + + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true + true + + + + false + { + "@context": {}, + "@id": "allassetscontractdef", + "@type": "ContractDefinition", + "accessPolicyId": "bpnpolicy", + "contractPolicyId": "bpnpolicy", + "assetsSelector" : { + "@type" : "CriterionDto", + "operandLeft": "${EDC_NAMESPACE}description", + "operator": "=", + "operandRight": "Product EDC Demo Asset" + } +} + = + + + + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + ${ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER} + + + + 1 + + 1 + ID + + false + + + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true + true + + + + false + { + "@context": {}, + "@id": "${ID}", + "@type": "ContractDefinition", + "accessPolicyId": "bpnpolicy", + "contractPolicyId": "bpnpolicy", + "assetsSelector" : { + "@type" : "CriterionDto", + "operandLeft": "${EDC_NAMESPACE}id", + "operator": "=", + "operandRight": "${ID}" + } +} + = + + + + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + SetUp Supplier + true + false + + + + + Defining a test for the processes "Create Asset", "Create Policy" and "Create Contract Definition" of the OEM + 1 + false + startnextloop + + 1 + false + + + + + 500 + + + + ${OEM_CARS_INITIAL} + + + + 1 + + 1 + ID + + false + + + + 6 + ${OEM_MANAGEMENT_URL}/v3/assets + true + POST + true + true + + + + false + { + "@context": {}, + "@id": "${ID}", + "properties": { + "description": "Product EDC Demo Asset" + }, + "dataAddress": { + "@type": "DataAddress", + "type": "HttpData", + "baseUrl": "https://jsonplaceholder.typicode.com/todos" + } +} + = + + + + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + + 6 + ${OEM_MANAGEMENT_URL}/v2/policydefinitions + true + POST + true + true + + + + false + { + "@context": { + "odrl": "http://www.w3.org/ns/odrl/2/" + }, + "@type": "PolicyDefinitionRequestDto", + "@id": "bpnpolicy", + "policy": { + "@type": "Policy", + "odrl:permission" : [{ + "odrl:action" : "USE", + "odrl:constraint" : { + "@type": "LogicalConstraint", + "odrl:or" : [{ + "@type" : "Constraint", + "odrl:leftOperand" : "BusinessPartnerNumber", + "odrl:operator" : { + "@id": "odrl:eq" + }, + "odrl:rightOperand" : "${SUPPLIER_ID}" + }] + } + }] + } +} + = + + + + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + + 6 + ${OEM_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true + true + + + + false + { + "@context": {}, + "@id": "allassetcontractdef", + "@type": "ContractDefinition", + "accessPolicyId": "bpnpolicy", + "contractPolicyId": "bpnpolicy", + "assetsSelector" : { + "@type" : "CriterionDto", + "operandLeft": "${EDC_NAMESPACE}description", + "operator": "=", + "operandRight": "Product EDC Demo Asset" + } +} + = + + + + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + ${ADDITIONAL_CONTRACT_DEFINITIONS_OEM} + + + + 1 + + 1 + ID + + false + + + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true + true + + + + false + { + "@context": {}, + "@id": "${ID}", + "@type": "ContractDefinition", + "accessPolicyId": "bpnpolicy", + "contractPolicyId": "bpnpolicy", + "assetsSelector" : { + "@type" : "CriterionDto", + "operandLeft": "${EDC_NAMESPACE}id", + "operator": "=", + "operandRight": "${ID}" + } +} + = + + + + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 200 + 409 + + + Assertion.response_code + true + 40 + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + SetUp Supplier + true + false + + + + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx b/mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx new file mode 100755 index 00000000..62ddc137 --- /dev/null +++ b/mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx @@ -0,0 +1,710 @@ + + + + + false + false + + + + OEM_MANAGEMENT_URL + ${__P(OEM_MANAGEMENT_URL, http://localhost/alice/management)} + = + + + OEM_PROTOCOL_URL + = + ${__P(OEM_PROTOCOL_URL, http://alice-controlplane:8084/api/v1/dsp)} + + + OEM_API_KEY + ${__P(OEM_API_KEY, password)} + = + + + SUPPLIER_MANAGEMENT_URL + ${__P(SUPPLIER_MANAGEMENT_URL, http://localhost/bob/management)} + = + + + SUPPLIER_PROTOCOL_URL + = + ${__P(SUPPLIER_PROTOCOL_URL, http://bob-controlplane:8084/api/v1/dsp)} + + + SUPPLIER_API_KEY + ${__P(SUPPLIER_API_KEY, password)} + = + + + + false + + + + continue + + 1 + false + + 1 + + false + + + true + + + + 500 + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "QuerySpec", + "sortOrder": "DESC", + "filterExpression": [] +} + = + + + + ${OEM_MANAGEMENT_URL}/v2/contractdefinitions/request + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + OEM_Contract_Definition_ID + $[*]['@id'] + -1 + + + + + OEM_Contract_Definition_ID_ + Contract_Definition_ID_Counter + false + + + + false + + + + ${OEM_MANAGEMENT_URL}/v2/contractdefinitions/${Contract_Definition_ID_Counter} + DELETE + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "QuerySpec", + "sortOrder": "DESC", + "filterExpression": [] +} + = + + + + ${OEM_MANAGEMENT_URL}/v2/policydefinitions/request + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + OEM_Policy_Definition_ID + $[*]['@id'] + -1 + + + + + OEM_Policy_Definition_ID_ + Policy_Definition_ID_Counter + false + + + + false + + + + ${OEM_MANAGEMENT_URL}/v2/policydefinitions/${Policy_Definition_ID_Counter} + DELETE + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "QuerySpec", + "sortOrder": "DESC", + "filterExpression": [] +} + = + + + + ${OEM_MANAGEMENT_URL}/v3/assets/request + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + OEM_Assets_ID + $[*]['@id'] + -1 + + + + + OEM_Assets_ID_ + Asset_ID_Counter + false + + + + false + + + + ${OEM_MANAGEMENT_URL}/v3/assets/${Asset_ID_Counter} + DELETE + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${OEM_API_KEY} + + + Content-Type + application/json + + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + continue + + 1 + false + + 1 + + false + + + true + + + + 500 + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "QuerySpec", + "sortOrder": "DESC", + "filterExpression": [] +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions/request + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + Supplier_Contract_Definition_ID + $[*]['@id'] + -1 + + + + + Supplier_Contract_Definition_ID_ + Contract_Definition_ID_Counter + false + + + + false + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions/${Contract_Definition_ID_Counter} + DELETE + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "QuerySpec", + "sortOrder": "DESC", + "filterExpression": [] +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions/request + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + Supplier_Policy_Definition_ID + $[*]['@id'] + -1 + + + + + Supplier_Policy_Definition_ID_ + Policy_Definition_ID_Counter + false + + + + false + + + + ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions/${Policy_Definition_ID_Counter} + DELETE + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + + true + + + + false + { + "@context": { + "@vocab": "https://w3id.org/edc/v0.0.1/ns/" + }, + "@type": "QuerySpec", + "sortOrder": "DESC", + "filterExpression": [] +} + = + + + + ${SUPPLIER_MANAGEMENT_URL}/v3/assets/request + POST + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + Supplier_Assets_ID + $[*]['@id'] + -1 + + + + + Supplier_Assets_ID_ + Asset_ID_Counter + false + + + + false + + + + ${SUPPLIER_MANAGEMENT_URL}/v3/assets/${Asset_ID_Counter} + DELETE + true + false + true + false + false + false + false + 6 + false + 0 + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + + false + + saveConfig + + + true + true + true + + true + true + true + true + false + true + true + false + false + false + true + false + false + false + true + 0 + true + true + true + true + true + true + + + + + + + + + From d7f036972e46884dcbb6ad5c07513aae6ace5533 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Jan 2024 07:03:08 +0100 Subject: [PATCH 16/34] chore(deps): bump mrparkers/keycloak from 4.3.1 to 4.4.0 in /mxd (#211) Bumps [mrparkers/keycloak](https://github.com/mrparkers/terraform-provider-keycloak) from 4.3.1 to 4.4.0. - [Release notes](https://github.com/mrparkers/terraform-provider-keycloak/releases) - [Changelog](https://github.com/mrparkers/terraform-provider-keycloak/blob/master/CHANGELOG.md) - [Commits](https://github.com/mrparkers/terraform-provider-keycloak/compare/v4.3.1...v4.4.0) --- updated-dependencies: - dependency-name: mrparkers/keycloak dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- mxd/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mxd/main.tf b/mxd/main.tf index e6ff985c..b60edb28 100644 --- a/mxd/main.tf +++ b/mxd/main.tf @@ -29,7 +29,7 @@ terraform { keycloak = { source = "mrparkers/keycloak" - version = "4.3.1" + version = "4.4.0" } kubernetes = { source = "hashicorp/kubernetes" From 4b5d741e27935ab9ce2cdf316345536aa6d9844e Mon Sep 17 00:00:00 2001 From: ciprian-nicuta Date: Tue, 27 Feb 2024 17:16:27 +0200 Subject: [PATCH 17/34] feat: mxd-performance-evaluation update --- .../mxd-performance-evaluation/Dockerfile | 2 +- .../build-docker.sh | 4 +- .../measurement_interval.jmx | 36 +- .../run_experiment.sh | 2 - .../mxd-performance-evaluation/setup.jmx | 180 +++-- .../mxd-performance-evaluation/tear_down.jmx | 710 ------------------ mxd/performance-tests/performance-test.yaml | 2 +- 7 files changed, 160 insertions(+), 776 deletions(-) delete mode 100755 mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx diff --git a/mxd/performance-tests/mxd-performance-evaluation/Dockerfile b/mxd/performance-tests/mxd-performance-evaluation/Dockerfile index 08f58049..b7346e8d 100644 --- a/mxd/performance-tests/mxd-performance-evaluation/Dockerfile +++ b/mxd/performance-tests/mxd-performance-evaluation/Dockerfile @@ -2,6 +2,6 @@ FROM justb4/jmeter:5.5 WORKDIR $JMETER_HOME/mxd-performance-evaluation -COPY ["run_experiment.sh", "setup.jmx", "measurement_interval.jmx", "tear_down.jmx", "./"] +COPY ["run_experiment.sh", "setup.jmx", "measurement_interval.jmx", "./"] ENTRYPOINT ["./run_experiment.sh"] diff --git a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh index e1943be5..5b6664b2 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh +++ b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -docker build -t hemantxpatel/mxd-performance-test:0.1.4-SNAPSHOT . +docker build -t ciprian2398/mxd-performance-test:0.2.2-SNAPSHOT . -docker image push hemantxpatel/mxd-performance-test:0.1.4-SNAPSHOT \ No newline at end of file +docker image push ciprian2398/mxd-performance-test:0.2.2-SNAPSHOT \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx index 41aaab79..33f77883 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx +++ b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx @@ -467,6 +467,20 @@ + + groovy + + + true + var offerId = vars.get("CONTRACT_OFFER_ID") + +if(offerId == null || offerId.isEmpty()) { + AssertionResult.setFailure(true) + AssertionResult.setFailureMessage("CONTRACT_OFFER_ID is missing") +} + + + true @@ -557,7 +571,7 @@ - ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && + ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} Loop while Negotiation State is FINALIZED @@ -732,7 +746,7 @@ if(!"FINALIZED".equals( currentState)) { - ${__jexl3("${TRANSFER_STATE}" != "STARTED" && + ${__jexl3("${TRANSFER_STATE}" != "STARTED" && ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} Loop while Transfer State is STARTED @@ -1322,6 +1336,20 @@ if(!"STARTED".equals( currentState)) { + + groovy + + + true + var offerId = vars.get("CONTRACT_OFFER_ID") + +if(offerId == null || offerId.isEmpty()) { + AssertionResult.setFailure(true) + AssertionResult.setFailureMessage("CONTRACT_OFFER_ID is missing") +} + + + true @@ -1412,7 +1440,7 @@ if(!"STARTED".equals( currentState)) { - ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && + ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} Loop while Negotiation State is FINALIZED @@ -1587,7 +1615,7 @@ if(!"FINALIZED".equals( currentState)) { - ${__jexl3("${TRANSFER_STATE}" != "STARTED" && + ${__jexl3("${TRANSFER_STATE}" != "STARTED" && ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} Loop while Transfer State is STARTED diff --git a/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh b/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh index 6f71c83b..4ae756e2 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh +++ b/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh @@ -26,8 +26,6 @@ if [ -z "$jmeter_script" ]; then $jmeter_binary -n -t setup.jmx -l output/setup.jtl -q $experiment_property echo "Executing measurement_interval.jmx" $jmeter_binary -n -t measurement_interval.jmx -l output/measurement_interval.jtl -q $experiment_property -e -o output/dashboard - echo "Executing tear_down.jmx" - $jmeter_binary -n -t tear_down.jmx -l output/tear_down.jtl -q $experiment_property fi echo "*** Performance Test End ***" diff --git a/mxd/performance-tests/mxd-performance-evaluation/setup.jmx b/mxd/performance-tests/mxd-performance-evaluation/setup.jmx index 4a268441..cbd9d219 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/setup.jmx +++ b/mxd/performance-tests/mxd-performance-evaluation/setup.jmx @@ -1,5 +1,5 @@ - + @@ -71,24 +71,31 @@ + false + false - + Defining a test for the processes "Create Asset", "Create Policy" and "Create Contract Definition" of the supplier - 1 + 1 false startnextloop - + 1 false + + false + false + + 500 - + ${SUPPLIER_PARTS_INITIAL} @@ -102,11 +109,6 @@ - 6 - ${SUPPLIER_MANAGEMENT_URL}/v3/assets - true - POST - true true @@ -128,6 +130,18 @@ + ${SUPPLIER_MANAGEMENT_URL}/v3/assets + POST + true + false + true + false + false + false + false + 6 + false + 0 @@ -157,11 +171,6 @@ - 6 - ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions - true - POST - true true @@ -174,7 +183,7 @@ "@type": "PolicyDefinitionRequestDto", "@id": "bpnpolicy", "policy": { - "@type": "Policy", + "@type": "odrl:Set", "odrl:permission" : [{ "odrl:action" : "USE", "odrl:constraint" : { @@ -195,6 +204,18 @@ + ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 @@ -222,12 +243,7 @@ - - 6 - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions - true - POST - true + true @@ -250,6 +266,18 @@ + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 @@ -351,11 +379,11 @@ - + ${ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER} - + 1 1 @@ -364,12 +392,7 @@ false - - 6 - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions - true - POST - true + true @@ -392,6 +415,18 @@ + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 @@ -572,22 +607,27 @@ - + Defining a test for the processes "Create Asset", "Create Policy" and "Create Contract Definition" of the OEM - 1 + 1 false startnextloop - + 1 false + + false + false + + 500 - + ${OEM_CARS_INITIAL} @@ -601,11 +641,6 @@ - 6 - ${OEM_MANAGEMENT_URL}/v3/assets - true - POST - true true @@ -627,6 +662,18 @@ + ${OEM_MANAGEMENT_URL}/v3/assets + POST + true + false + true + false + false + false + false + 6 + false + 0 @@ -656,11 +703,6 @@ - 6 - ${OEM_MANAGEMENT_URL}/v2/policydefinitions - true - POST - true true @@ -673,7 +715,7 @@ "@type": "PolicyDefinitionRequestDto", "@id": "bpnpolicy", "policy": { - "@type": "Policy", + "@type": "odrl:Set", "odrl:permission" : [{ "odrl:action" : "USE", "odrl:constraint" : { @@ -694,6 +736,18 @@ + ${OEM_MANAGEMENT_URL}/v2/policydefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 @@ -722,11 +776,6 @@ - 6 - ${OEM_MANAGEMENT_URL}/v2/contractdefinitions - true - POST - true true @@ -749,6 +798,18 @@ + ${OEM_MANAGEMENT_URL}/v2/contractdefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 @@ -850,7 +911,7 @@ - + ${ADDITIONAL_CONTRACT_DEFINITIONS_OEM} @@ -864,11 +925,6 @@ - 6 - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions - true - POST - true true @@ -891,6 +947,18 @@ + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + POST + true + false + true + false + false + false + false + 6 + false + 0 diff --git a/mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx b/mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx deleted file mode 100755 index 62ddc137..00000000 --- a/mxd/performance-tests/mxd-performance-evaluation/tear_down.jmx +++ /dev/null @@ -1,710 +0,0 @@ - - - - - false - false - - - - OEM_MANAGEMENT_URL - ${__P(OEM_MANAGEMENT_URL, http://localhost/alice/management)} - = - - - OEM_PROTOCOL_URL - = - ${__P(OEM_PROTOCOL_URL, http://alice-controlplane:8084/api/v1/dsp)} - - - OEM_API_KEY - ${__P(OEM_API_KEY, password)} - = - - - SUPPLIER_MANAGEMENT_URL - ${__P(SUPPLIER_MANAGEMENT_URL, http://localhost/bob/management)} - = - - - SUPPLIER_PROTOCOL_URL - = - ${__P(SUPPLIER_PROTOCOL_URL, http://bob-controlplane:8084/api/v1/dsp)} - - - SUPPLIER_API_KEY - ${__P(SUPPLIER_API_KEY, password)} - = - - - - false - - - - continue - - 1 - false - - 1 - - false - - - true - - - - 500 - - - - true - - - - false - { - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "QuerySpec", - "sortOrder": "DESC", - "filterExpression": [] -} - = - - - - ${OEM_MANAGEMENT_URL}/v2/contractdefinitions/request - POST - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${OEM_API_KEY} - - - Content-Type - application/json - - - - - - OEM_Contract_Definition_ID - $[*]['@id'] - -1 - - - - - OEM_Contract_Definition_ID_ - Contract_Definition_ID_Counter - false - - - - false - - - - ${OEM_MANAGEMENT_URL}/v2/contractdefinitions/${Contract_Definition_ID_Counter} - DELETE - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${OEM_API_KEY} - - - Content-Type - application/json - - - - - - - - true - - - - false - { - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "QuerySpec", - "sortOrder": "DESC", - "filterExpression": [] -} - = - - - - ${OEM_MANAGEMENT_URL}/v2/policydefinitions/request - POST - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${OEM_API_KEY} - - - Content-Type - application/json - - - - - - OEM_Policy_Definition_ID - $[*]['@id'] - -1 - - - - - OEM_Policy_Definition_ID_ - Policy_Definition_ID_Counter - false - - - - false - - - - ${OEM_MANAGEMENT_URL}/v2/policydefinitions/${Policy_Definition_ID_Counter} - DELETE - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${OEM_API_KEY} - - - Content-Type - application/json - - - - - - - - true - - - - false - { - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "QuerySpec", - "sortOrder": "DESC", - "filterExpression": [] -} - = - - - - ${OEM_MANAGEMENT_URL}/v3/assets/request - POST - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${OEM_API_KEY} - - - Content-Type - application/json - - - - - - OEM_Assets_ID - $[*]['@id'] - -1 - - - - - OEM_Assets_ID_ - Asset_ID_Counter - false - - - - false - - - - ${OEM_MANAGEMENT_URL}/v3/assets/${Asset_ID_Counter} - DELETE - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${OEM_API_KEY} - - - Content-Type - application/json - - - - - - - - false - - saveConfig - - - true - true - true - - true - true - true - true - false - true - true - false - false - false - true - false - false - false - true - 0 - true - true - true - true - true - true - - - - - - - - continue - - 1 - false - - 1 - - false - - - true - - - - 500 - - - - true - - - - false - { - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "QuerySpec", - "sortOrder": "DESC", - "filterExpression": [] -} - = - - - - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions/request - POST - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${SUPPLIER_API_KEY} - - - Content-Type - application/json - - - - - - Supplier_Contract_Definition_ID - $[*]['@id'] - -1 - - - - - Supplier_Contract_Definition_ID_ - Contract_Definition_ID_Counter - false - - - - false - - - - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions/${Contract_Definition_ID_Counter} - DELETE - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${SUPPLIER_API_KEY} - - - Content-Type - application/json - - - - - - - - true - - - - false - { - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "QuerySpec", - "sortOrder": "DESC", - "filterExpression": [] -} - = - - - - ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions/request - POST - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${SUPPLIER_API_KEY} - - - Content-Type - application/json - - - - - - Supplier_Policy_Definition_ID - $[*]['@id'] - -1 - - - - - Supplier_Policy_Definition_ID_ - Policy_Definition_ID_Counter - false - - - - false - - - - ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions/${Policy_Definition_ID_Counter} - DELETE - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${SUPPLIER_API_KEY} - - - Content-Type - application/json - - - - - - - - true - - - - false - { - "@context": { - "@vocab": "https://w3id.org/edc/v0.0.1/ns/" - }, - "@type": "QuerySpec", - "sortOrder": "DESC", - "filterExpression": [] -} - = - - - - ${SUPPLIER_MANAGEMENT_URL}/v3/assets/request - POST - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${SUPPLIER_API_KEY} - - - Content-Type - application/json - - - - - - Supplier_Assets_ID - $[*]['@id'] - -1 - - - - - Supplier_Assets_ID_ - Asset_ID_Counter - false - - - - false - - - - ${SUPPLIER_MANAGEMENT_URL}/v3/assets/${Asset_ID_Counter} - DELETE - true - false - true - false - false - false - false - 6 - false - 0 - - - - - - X-Api-Key - ${SUPPLIER_API_KEY} - - - Content-Type - application/json - - - - - - - - false - - saveConfig - - - true - true - true - - true - true - true - true - false - true - true - false - false - false - true - false - false - false - true - 0 - true - true - true - true - true - true - - - - - - - - - diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index 7d749052..56834e4c 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: mxd-performance-test - image: hemantxpatel/mxd-performance-test:0.1.5-SNAPSHOT + image: ciprian2398/mxd-performance-test:0.2.2-SNAPSHOT volumeMounts: - name: property-volume mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties From 30530bd7ba58e412a9c5f4dd9acfc37faaebbcca Mon Sep 17 00:00:00 2001 From: hemantxpatel Date: Wed, 28 Feb 2024 12:57:32 +0530 Subject: [PATCH 18/34] Add prometheus deployment --- .../{ingress.tfignore => ingress.tf} | 1 + .../prometheus/clusterRole.yaml | 33 +++++ .../prometheus/config-map.yaml | 137 ++++++++++++++++++ .../prometheus/prometheus-deployment.yaml | 38 +++++ .../prometheus/prometheus-ingress.yaml | 37 +++++ .../prometheus/prometheus-service.yaml | 17 +++ mxd/performance-tests/run-sml.sh | 8 +- 7 files changed, 270 insertions(+), 1 deletion(-) rename mxd/modules/connector/{ingress.tfignore => ingress.tf} (97%) create mode 100644 mxd/performance-tests/prometheus/clusterRole.yaml create mode 100644 mxd/performance-tests/prometheus/config-map.yaml create mode 100644 mxd/performance-tests/prometheus/prometheus-deployment.yaml create mode 100644 mxd/performance-tests/prometheus/prometheus-ingress.yaml create mode 100644 mxd/performance-tests/prometheus/prometheus-service.yaml diff --git a/mxd/modules/connector/ingress.tfignore b/mxd/modules/connector/ingress.tf similarity index 97% rename from mxd/modules/connector/ingress.tfignore rename to mxd/modules/connector/ingress.tf index a8ca02c8..32c4f97b 100644 --- a/mxd/modules/connector/ingress.tfignore +++ b/mxd/modules/connector/ingress.tf @@ -28,6 +28,7 @@ resource "kubernetes_ingress_v1" "mxd-ingress" { spec { ingress_class_name = "nginx" rule { + host = "edc.ingress.mxd.catena-x.shoot.canary.k8s-hana.ondemand.com" http { path { path = "/${var.humanReadableName}(/|$)(.*)" diff --git a/mxd/performance-tests/prometheus/clusterRole.yaml b/mxd/performance-tests/prometheus/clusterRole.yaml new file mode 100644 index 00000000..d666b69e --- /dev/null +++ b/mxd/performance-tests/prometheus/clusterRole.yaml @@ -0,0 +1,33 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: + - extensions + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: default + namespace: monitoring diff --git a/mxd/performance-tests/prometheus/config-map.yaml b/mxd/performance-tests/prometheus/config-map.yaml new file mode 100644 index 00000000..d7108f24 --- /dev/null +++ b/mxd/performance-tests/prometheus/config-map.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server-conf + labels: + name: prometheus-server-conf + namespace: monitoring +data: + prometheus.rules: |- + groups: + - name: devopscube demo alert + rules: + - alert: High Pod Memory + expr: sum(container_memory_usage_bytes) > 1 + for: 1m + labels: + severity: slack + annotations: + summary: High Memory Usage + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + rule_files: + - /etc/prometheus/prometheus.rules + alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: + - "alertmanager.monitoring.svc:9093" + scrape_configs: + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_endpoints_name] + regex: 'node-exporter' + action: keep + - job_name: 'kubernetes-apiservers' + kubernetes_sd_configs: + - role: endpoints + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: default;kubernetes;https + - job_name: 'kubernetes-nodes' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - job_name: 'kube-state-metrics' + static_configs: + - targets: ['kube-state-metrics.kube-system.svc.cluster.local:8080'] + - job_name: 'kubernetes-cadvisor' + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_service_name] + action: replace + target_label: kubernetes_name \ No newline at end of file diff --git a/mxd/performance-tests/prometheus/prometheus-deployment.yaml b/mxd/performance-tests/prometheus/prometheus-deployment.yaml new file mode 100644 index 00000000..2ccb8520 --- /dev/null +++ b/mxd/performance-tests/prometheus/prometheus-deployment.yaml @@ -0,0 +1,38 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-deployment + namespace: monitoring + labels: + app: prometheus-server +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus-server + template: + metadata: + labels: + app: prometheus-server + spec: + containers: + - name: prometheus + image: prom/prometheus + args: + - "--config.file=/etc/prometheus/prometheus.yml" + - "--storage.tsdb.path=/prometheus/" + ports: + - containerPort: 9090 + volumeMounts: + - name: prometheus-config-volume + mountPath: /etc/prometheus/ + - name: prometheus-storage-volume + mountPath: /prometheus/ + volumes: + - name: prometheus-config-volume + configMap: + defaultMode: 420 + name: prometheus-server-conf + + - name: prometheus-storage-volume + emptyDir: {} diff --git a/mxd/performance-tests/prometheus/prometheus-ingress.yaml b/mxd/performance-tests/prometheus/prometheus-ingress.yaml new file mode 100644 index 00000000..b45196fb --- /dev/null +++ b/mxd/performance-tests/prometheus/prometheus-ingress.yaml @@ -0,0 +1,37 @@ +## Nginx Ingress +## Follow https://devopscube.com/setup-ingress-kubernetes-nginx-controller/ + +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-ui + namespace: monitoring + annotations: + kubernetes.io/ingress.class: nginx +spec: + rules: + # Use the host you used in your kubernetes Ingress Configurations + - host: 'prometheus.ingress.mxd.catena-x.shoot.canary.k8s-hana.ondemand.com' + http: + paths: + - backend: + service: + name: prometheus-service + port: + number: 8080 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.apps.shaker242.lab + secretName: prometheus-secret +--- +apiVersion: v1 +kind: Secret +metadata: + name: prometheus-secret + namespace: monitoring +data: +# USe base64 in the certs + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZpVENDQkhHZ0F3SUJBZ0lCQVRBTkJna3Foa2lHOXcwQkFRc0ZBRENCd0RFak1DRUdBMVVFQXhNYWFXNTAKWlhKdFpXUnBZWFJsTG5Ob1lXdGxjakkwTWk1c1lXSXhDekFKQmdOVkJBWVRBbFZUTVJFd0R3WURWUVFJRXdoVwphWEpuYVc1cFlURVFNQTRHQTFVRUJ4TUhRbkpwYzNSdmR6RXNNQ29HQTFVRUNoTWpVMGhCUzBWU01qUXlJRXhoCllpQkRaWEowYVdacFkyRjBaU0JCZFhSb2IzSnBkSGt4T1RBM0JnTlZCQXNUTUZOSVFVdEZVakkwTWlCTVlXSWcKU1c1MFpYSnRaV1JwWVhSbElFTmxjblJwWm1sallYUmxJRUYxZEdodmNtbDBlVEFlRncweE9URXdNVGN4TmpFMgpNekZhRncweU1URXdNVFl4TmpFMk16RmFNSUdBTVIwd0d3WURWUVFERkJRcUxtRndjSE11YzJoaGEyVnlNalF5CkxteGhZakVMTUFrR0ExVUVCaE1DVlZNeEVUQVBCZ05WQkFnVENGWnBjbWRwYm1saE1SQXdEZ1lEVlFRSEV3ZEMKY21semRHOTNNUll3RkFZRFZRUUtFdzFUU0VGTFJWSXlORElnVEdGaU1SVXdFd1lEVlFRTEV3eE1ZV0lnVjJWaQpjMmwwWlhNd2dnRWlNQTBHQ1NxR1NJYjNEUUVCQVFVQUE0SUJEd0F3Z2dFS0FvSUJBUURsRm16QVd0U09JcXZNCkpCV3Vuc0VIUmxraXozUmpSK0p1NTV0K0hCUG95YnZwVkJJeXMxZ3prby9INlkxa2Zxa1JCUzZZYVFHM2lYRFcKaDgzNlNWc3pNVUNVS3BtNXlZQXJRNzB4YlpPTXRJcjc1VEcrejFaRGJaeFUzbnh6RXdHdDN3U3c5OVJ0bjhWbgo5dEpTVXI0MHBHUytNemMzcnZOUFZRMjJoYTlhQTdGL2NVcGxtZUpkUnZEVnJ3Q012UklEcndXVEZjZkU3bUtxCjFSUkRxVDhETnlydlJmeUlubytmSkUxTmRuVEVMY0dTYVZlajhZVVFONHY0WFRnLzJncmxIN1pFT1VXNy9oYm8KUXh6NVllejVSam1wOWVPVUpvdVdmWk5FNEJBbGRZeVYxd2NPRXhRTmswck5BOU45ZXBjNWtUVVZQR3pOTWRucgovVXQxOWMweEFnTUJBQUdqZ2dIS01JSUJ4akFKQmdOVkhSTUVBakFBTUJFR0NXQ0dTQUdHK0VJQkFRUUVBd0lHClFEQUxCZ05WSFE4RUJBTUNCYUF3TXdZSllJWklBWWI0UWdFTkJDWVdKRTl3Wlc1VFUwd2dSMlZ1WlhKaGRHVmsKSUZObGNuWmxjaUJEWlhKMGFXWnBZMkYwWlRBZEJnTlZIUTRFRmdRVWRhYy94MTR6dXl3RVZPSi9vTjdQeU82bApDZ2N3Z2RzR0ExVWRJd1NCMHpDQjBJQVVzZFM1WWxuWEpWTk5mRVpkTEQvL2RyNE5mV3FoZ2JTa2diRXdnYTR4CkdUQVhCZ05WQkFNVEVHTmhMbk5vWVd0bGNqSTBNaTVzWVdJeEN6QUpCZ05WQkFZVEFsVlRNUkV3RHdZRFZRUUkKRXdoV2FYSm5hVzVwWVRFUU1BNEdBMVVFQnhNSFFuSnBjM1J2ZHpFc01Db0dBMVVFQ2hNalUwaEJTMFZTTWpReQpJRXhoWWlCRFpYSjBhV1pwWTJGMFpTQkJkWFJvYjNKcGRIa3hNVEF2QmdOVkJBc1RLRk5JUVV0RlVqSTBNaUJNCllXSWdVbTl2ZENCRFpYSjBhV1pwWTJGMFpTQkJkWFJvYjNKcGRIbUNBUUV3SFFZRFZSMGxCQll3RkFZSUt3WUIKQlFVSEF3RUdDQ3NHQVFVRkNBSUNNRWdHQTFVZEVRUkJNRCtDRFhOb1lXdGxjakkwTWk1c1lXS0NFbUZ3Y0hNdQpjMmhoYTJWeU1qUXlMbXhoWW9JVUtpNWhjSEJ6TG5Ob1lXdGxjakkwTWk1c1lXS0hCTUNvQ3hBd0RRWUpLb1pJCmh2Y05BUUVMQlFBRGdnRUJBRzA3ZHFNdFZYdVQrckduQlN4SkVTNjNSa2pHaWd0c3ZtNTk4NSsrbjZjRW5kSDIKb2hjaGdmRUo5V0UxYUFWSDR4QlJSdVRIUFVJOFcvd3N1OFBxQ1o4NHpRQ2U2elAyeThEcmEwbjFzK2lIeHFwRAorS3BwZS91NkNLVTFEL0VWRU9MakpZd3pRYlFLSUlPL2Y1Q0JVbUpGWjBuZ1VIUEtvUDNyTXordTlBOWFvRkVrCnF3dDBadHFHcWpjMkh3Q09UOTlOVmFsZ29ISXljOElxQXJXdjNSWklraUlyaW9kSUdDMS94MVQ2dHhKcEUyRisKQzZ0Tzk0U0FVSUJwc2VORjNFbGNLNUsyTW44YVAzR3NnNFRHeElPN2Q1eUIvb3YwNGhOV2Q1S2QwWGorL1BvQgpLOU43cFQ1SVU2citLekNoeGlSdmRvZlAzV0VYN1ZkNEtLWG94K0U9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktrd2dnU2xBZ0VBQW9JQkFRRGxGbXpBV3RTT0lxdk0KSkJXdW5zRUhSbGtpejNSalIrSnU1NXQrSEJQb3lidnBWQkl5czFnemtvL0g2WTFrZnFrUkJTNllhUUczaVhEVwpoODM2U1Zzek1VQ1VLcG01eVlBclE3MHhiWk9NdElyNzVURyt6MVpEYlp4VTNueHpFd0d0M3dTdzk5UnRuOFZuCjl0SlNVcjQwcEdTK016YzNydk5QVlEyMmhhOWFBN0YvY1VwbG1lSmRSdkRWcndDTXZSSURyd1dURmNmRTdtS3EKMVJSRHFUOEROeXJ2UmZ5SW5vK2ZKRTFOZG5URUxjR1NhVmVqOFlVUU40djRYVGcvMmdybEg3WkVPVVc3L2hibwpReHo1WWV6NVJqbXA5ZU9VSm91V2ZaTkU0QkFsZFl5VjF3Y09FeFFOazByTkE5TjllcGM1a1RVVlBHek5NZG5yCi9VdDE5YzB4QWdNQkFBRUNnZ0VCQU5zOHRjRDBiQnpHZzRFdk8yek0wMUJoKzZYN3daZk4wSjV3bW5kNjZYYkwKc1VEZ1N6WW9PbzNJZ2o5QWZTY2lyQ3YwdUozMVNFWmNpeGRVQ2tTdjlVNnRvTzdyUWdqeUZPM1N1dm5Wc3ZKaQpTZXc5Y0hqNk5jVDczak8rWkgxQVFFZ2tlWG5mQTNZU0JEcTFsSnhpUVZOaHpHUFY0Yzh4Wi9xUkhEbUVBTWR6CmwyaTB6dHJtcWRqSng4aTQxOXpGL1pVektoa2JtcVZVb3JjZ1lNdEt5QVloSENMYms2RFZtQ1FhbDlndEUrNjUKTmFTOEwxUW9yVWNVS0FoSTNKT2Q2TTRwbWRPaExITjZpZ0VwWFdVWGxBZjRITUZicHd5M1oxejNqZzVqTE9ragp6SWNDSVRaai9CYVZvSVc4QzJUb0pieUJKWkN6UDVjUVJTdkJOOGV4aUFFQ2dZRUEvV0Nxb2xVUWtOQkQrSnlPCklXOUJIRVlPS3oxRFZxNWxHRFhoNFMyTStpOU1pck5nUlcvL0NFRGhRUVVMZmtBTDgxMERPQmxsMXRRRUpGK3cKb1V6dWt6U1lkK1hTSnhicTM5YTF1ZGJldTNZU1ljeC8wTEEweGFQOW1sN1l1NXUraUZ4NGhwcnYyL2UrVklZQQpzTWV4WkZSODA3Q3M5YXN5MkdFT1l2aEdKb0VDZ1lFQTUzVm1weFlQbDFOYTVTMElJbEpuYm40dTl0RHpwYm5TCnpsMjBVQ3Q0d0N4STR6YjY1S1o4V1VaYlFzVTVaZ0VqTmxJWURXUisrd3kwVXh2SmNxUG5nS0xuOEdoSzhvOVEKeVJuR2dSYXAxWmNuUEdsbGdCeHQzM0s5TDNWMmJzMXBPcGJKMGlpOVdySWM4MU1wUVFpQjZ1RDRSZ216M0ZWSQpnUk5Ec2ZHS0xyRUNnWUVBbWY5ZXRqc3RUbGJHZVJ2dDVyUlB4bmR0dFNvTysyZ1RXWnVtSmM0aG1RMldYOWFWCjlKNFZTMWJqa1RrWHV5d0NGMis0dlNmeWxaZFd6U1M3bmMyOFV3dnNmekxYZjVxV05tV3hIYnBTdFcwVnp3c1QKeENyVWFDczd2ODlWdXZEMTVMc1BKZ0NWT0FSalVjd0FMM0d2aDJNeVd4ZE9pQ0g5VFRYd0lJYjFYQUVDZ1lBMwp4ZUptZ0xwaERJVHFsRjlSWmVubWhpRnErQTY5OEhrTG9TakI2TGZBRnV1NVZKWkFZcDIwSlcvNE51NE4xbGhWCnpwSmRKOG94Vkc1ZldHTENiUnhyc3RXUTZKQ213a0lGTTJEUjJsUXlVNm53dExUd21la2YzdFlYaVlad1RLNysKbnpjaW5RNkR2RWVkbW54bVgxWnU4cWJndVpYTmtmOVdtdjNFOHg4SkFRS0JnUUNNeDFWNHJIcUpwVXJMdkRVVQo4RzhXVGNrT2VFM2o2anhlcHMwcnExdEd1cE9XWW5saFlNYyt5VkMzMDZUc2dXUmJ5R1Y4YWNaRkF4WS9Ub2N5CmxpcXlUS1NGNUloYXhZQVpRTzVkOU1oTmN0bTRReDNaOUtTekZ5ZG01QlZVL0grMFFmUnRwM29TeFVneXRZNXkKV3ZDTFZ5bmNGZlZpL0VkaTdaZHM2aW82QVE9PQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/mxd/performance-tests/prometheus/prometheus-service.yaml b/mxd/performance-tests/prometheus/prometheus-service.yaml new file mode 100644 index 00000000..420a586c --- /dev/null +++ b/mxd/performance-tests/prometheus/prometheus-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus-service + namespace: monitoring + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: '9090' + +spec: + selector: + app: prometheus-server + type: NodePort + ports: + - port: 8080 + targetPort: 9090 + nodePort: 30000 diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh index b89677ea..9907fd67 100755 --- a/mxd/performance-tests/run-sml.sh +++ b/mxd/performance-tests/run-sml.sh @@ -20,7 +20,7 @@ LOG_MESSAGE="Test Completed" POD_NAME="mxd-performance-test" GENERATED_OUTPUT_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar" GENERATED_OUTPUT_SLIM_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output_slim.tar" -TERRAFORM_CHDIR="/Users/ciprian/IdeaProjects/tutorial-resources/mxd/" +TERRAFORM_CHDIR="$(dirname "$0")/.." CUSTOM_PROPERTIES="custom_experiment.properties" LOGFILE="sml_script_$(date +%d-%m-%YT%H-%M-%S).logs" IS_DEBUG=true @@ -69,6 +69,9 @@ function init { kubectl create configmap custom-property --from-file="${CUSTOM_PROPERTIES}"="${experiment_file}" | debug || error_exit "Failed to create configmap with name custom-property" info "Init terraform" terraform -chdir="$TERRAFORM_CHDIR" init >> "$LOGFILE" || error_exit "Failed to initialize Terraform" + info "Deploy Prometheus" + kubectl create namespace monitoring + kubectl apply -f prometheus | debug || error_exit "Failed to deploy Prometheus" info "Apply terraform" terraform -chdir="$TERRAFORM_CHDIR" apply -auto-approve >> "$LOGFILE" || error_exit "Failed to apply Terraform" info "Start the performance-test container" @@ -105,6 +108,9 @@ function cleanup { kubectl delete pod "$POD_NAME" | debug || error_exit "Failed to delete pod" info "Destroying configmap" kubectl delete configmap custom-property | debug || error_exit "Failed to delete configmap custom-property" + info "Destroying Prometheus" + kubectl delete -f prometheus | debug || error_exit "Failed to delete Prometheus" + kubectl delete namespace monitoring info "Waiting for the pod to be deleted" kubectl wait --for=delete "pod/$POD_NAME" info "Execution finished." From df07f43a998865578505784bf9f521e91046c3bf Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Mon, 4 Mar 2024 21:55:25 +0200 Subject: [PATCH 19/34] feat: use 0.6.0, ingress and fix Policy in the contract agreement is not equal to the one in the contract offer --- mxd/modules/connector/ingress.tf | 2 +- mxd/modules/connector/main.tf | 1 + mxd/modules/connector/values.yaml | 4 +- .../build-docker.sh | 4 +- .../measurement_interval.jmx | 489 +++++++----------- .../mxd-performance-evaluation/setup.jmx | 192 +++---- mxd/performance-tests/performance-test.yaml | 2 +- .../prometheus/prometheus-ingress.yaml | 2 +- 8 files changed, 268 insertions(+), 428 deletions(-) diff --git a/mxd/modules/connector/ingress.tf b/mxd/modules/connector/ingress.tf index 32c4f97b..08b0e076 100644 --- a/mxd/modules/connector/ingress.tf +++ b/mxd/modules/connector/ingress.tf @@ -28,7 +28,7 @@ resource "kubernetes_ingress_v1" "mxd-ingress" { spec { ingress_class_name = "nginx" rule { - host = "edc.ingress.mxd.catena-x.shoot.canary.k8s-hana.ondemand.com" + host = "edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com" http { path { path = "/${var.humanReadableName}(/|$)(.*)" diff --git a/mxd/modules/connector/main.tf b/mxd/modules/connector/main.tf index 32d96dc0..338d4964 100644 --- a/mxd/modules/connector/main.tf +++ b/mxd/modules/connector/main.tf @@ -36,6 +36,7 @@ resource "helm_release" "connector" { repository = "https://eclipse-tractusx.github.io/charts/dev" chart = "tractusx-connector" + version = "0.6.0" values = [ file("${path.module}/values.yaml"), diff --git a/mxd/modules/connector/values.yaml b/mxd/modules/connector/values.yaml index 6b54fab7..20ba30d3 100644 --- a/mxd/modules/connector/values.yaml +++ b/mxd/modules/connector/values.yaml @@ -34,7 +34,7 @@ controlplane: authKey: password image: pullPolicy: Never - tag: "latest" + #tag: "latest" # repository: "edc-controlplane-postgresql-hashicorp-vault" securityContext: # avoids some errors in the log: cannot write temp files of large multipart requests when R/O @@ -52,7 +52,7 @@ dataplane: port: 1044 image: pullPolicy: Never - tag: "latest" + #tag: "latest" # repository: "edc-dataplane-hashicorp-vault" securityContext: # avoids some errors in the log: cannot write temp files of large multipart requests when R/O diff --git a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh index 5b6664b2..382ff8e6 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh +++ b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -docker build -t ciprian2398/mxd-performance-test:0.2.2-SNAPSHOT . +docker build -t ciprian2398/mxd-performance-test:0.3.4-SNAPSHOT . -docker image push ciprian2398/mxd-performance-test:0.2.2-SNAPSHOT \ No newline at end of file +docker image push ciprian2398/mxd-performance-test:0.3.4-SNAPSHOT \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx index 33f77883..e7784df5 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx +++ b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx @@ -1,10 +1,8 @@ - + - - false - false - + + EXPERIMENT_DURATION_SEC @@ -48,7 +46,7 @@ OEM_MANAGEMENT_URL - ${__P(OEM_MANAGEMENT_URL, http://localhost/alice/management)} + ${__P(OEM_MANAGEMENT_URL, http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management)} = @@ -88,7 +86,7 @@ SUPPLIER_MANAGEMENT_URL - ${__P(SUPPLIER_MANAGEMENT_URL, http://localhost/bob/management)} + ${__P(SUPPLIER_MANAGEMENT_URL, http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management)} = @@ -143,25 +141,21 @@ - false - + + ${OEM_PLANTS} + ${EXPERIMENT_DURATION_SEC} + true + true startnextloop - + -1 false - - false - true - ${EXPERIMENT_DURATION_SEC} - - true - ${OEM_PLANTS} - + false true false @@ -210,7 +204,12 @@ false - + + 6 + ${OEM_MANAGEMENT_URL}/v3/assets + true + POST + true true @@ -232,18 +231,6 @@ - ${OEM_MANAGEMENT_URL}/v3/assets - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -272,6 +259,11 @@ + 6 + ${OEM_MANAGEMENT_URL}/v2/policydefinitions + true + POST + true true @@ -305,18 +297,6 @@ - ${OEM_MANAGEMENT_URL}/v2/policydefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -345,6 +325,11 @@ + 6 + ${OEM_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true true @@ -367,18 +352,6 @@ - ${OEM_MANAGEMENT_URL}/v2/contractdefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -406,7 +379,37 @@ - + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v3/assets/${CONSUME_COUNTER} + true + GET + true + false + + + + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 6 + ${OEM_MANAGEMENT_URL}/v2/catalog/request + true + POST true @@ -434,18 +437,6 @@ - ${OEM_MANAGEMENT_URL}/v2/catalog/request - POST - true - false - false - false - false - false - false - 6 - false - 0 @@ -461,7 +452,7 @@ - + CONTRACT_OFFER_ID ${__eval($['dcat:dataset'][?(@.id=="${CONSUME_COUNTER}")]['odrl:hasPolicy']['@id'])} @@ -482,7 +473,12 @@ if(offerId == null || offerId.isEmpty()) { - + + 6 + ${OEM_MANAGEMENT_URL}/v2/contractnegotiations + true + POST + true true @@ -505,6 +501,7 @@ if(offerId == null || offerId.isEmpty()) { "@type": "odrl:Set", "@id": "${CONSUME_COUNTER}", "odrl:permission" : { + "odrl:target" : "${CONSUME_COUNTER}", "odrl:action": { "odrl:type": "USE" }, @@ -520,9 +517,7 @@ if(offerId == null || offerId.isEmpty()) { }, "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": { - "@id": "${CONSUME_COUNTER}" - } + "odrl:target": "${CONSUME_COUNTER}" } }, "callbackAddresses": [] @@ -531,18 +526,6 @@ if(offerId == null || offerId.isEmpty()) { - ${OEM_MANAGEMENT_URL}/v2/contractnegotiations - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -558,7 +541,7 @@ if(offerId == null || offerId.isEmpty()) { - + NEGOTIATION_ID $['@id'] 1 @@ -567,7 +550,6 @@ if(offerId == null || offerId.isEmpty()) { true - false @@ -581,22 +563,15 @@ ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} - false - - - + 6 ${OEM_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID}/state - GET true - false + GET true - false - false - false - false - 6 - false - 0 + false + + + @@ -636,22 +611,15 @@ if(!"FINALIZED".equals( currentState)) { - false - - - + 6 ${OEM_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID} - GET true - false + GET true - false - false - false - false - 6 - false - 0 + false + + + @@ -675,6 +643,11 @@ if(!"FINALIZED".equals( currentState)) { + 6 + ${OEM_MANAGEMENT_URL}/v2/transferprocesses + true + POST + true true @@ -706,18 +679,6 @@ if(!"FINALIZED".equals( currentState)) { - ${OEM_MANAGEMENT_URL}/v2/transferprocesses - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -742,7 +703,6 @@ if(!"FINALIZED".equals( currentState)) { true - false @@ -756,22 +716,15 @@ ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} - false - - - + 6 ${OEM_MANAGEMENT_URL}/v2/transferprocesses/${TRANSFER_ID}/state - GET true - false + GET true - false - false - false - false - 6 - false - 0 + false + + + @@ -810,8 +763,8 @@ if(!"STARTED".equals( currentState)) { - - true + + false saveConfig @@ -850,7 +803,7 @@ if(!"STARTED".equals( currentState)) { output/measurement_interval_oem_plants_errors.xml - + false saveConfig @@ -889,19 +842,16 @@ if(!"STARTED".equals( currentState)) { - + + ${SUPPLIER_PLANTS} + ${EXPERIMENT_DURATION_SEC} + true + true startnextloop - + -1 false - ${SUPPLIER_PLANTS} - - false - true - ${EXPERIMENT_DURATION_SEC} - - true @@ -945,6 +895,11 @@ if(!"STARTED".equals( currentState)) { + 6 + ${SUPPLIER_MANAGEMENT_URL}/v3/assets + true + POST + true true @@ -966,18 +921,6 @@ if(!"STARTED".equals( currentState)) { - ${SUPPLIER_MANAGEMENT_URL}/v3/assets - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -1006,6 +949,11 @@ if(!"STARTED".equals( currentState)) { + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions + true + POST + true true @@ -1039,18 +987,6 @@ if(!"STARTED".equals( currentState)) { - ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -1079,6 +1015,11 @@ if(!"STARTED".equals( currentState)) { + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true true @@ -1101,18 +1042,6 @@ if(!"STARTED".equals( currentState)) { - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -1140,8 +1069,8 @@ if(!"STARTED".equals( currentState)) { - - true + + false saveConfig @@ -1180,7 +1109,7 @@ if(!"STARTED".equals( currentState)) { output/measurement_interval_supplier_plants_errors.xml - + false saveConfig @@ -1219,22 +1148,19 @@ if(!"STARTED".equals( currentState)) { - + + ${SUPPLIER_FLEET_MANAGERS} + ${EXPERIMENT_DURATION_SEC} + true + true startnextloop - + -1 false - ${SUPPLIER_FLEET_MANAGERS} - - false - true - ${EXPERIMENT_DURATION_SEC} - - true - + false true false @@ -1246,7 +1172,7 @@ if(!"STARTED".equals( currentState)) { - + 1 1 @@ -1256,7 +1182,7 @@ if(!"STARTED".equals( currentState)) { Consumes the current value from the OEM. - + NEGOTIATION_ID NEGOTIATION_STATE @@ -1275,7 +1201,38 @@ if(!"STARTED".equals( currentState)) { Local Variables to be reset after each loop - + + 6 + ${OEM_MANAGEMENT_URL}/v3/assets/${CONSUME_COUNT} + true + GET + true + false + + + + + + + + + X-Api-Key + ${SUPPLIER_API_KEY} + + + Content-Type + application/json + + + + + + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/catalog/request + true + POST + true true @@ -1303,21 +1260,9 @@ if(!"STARTED".equals( currentState)) { - ${SUPPLIER_MANAGEMENT_URL}/v2/catalog/request - POST - true - false - true - false - false - false - false - 6 - false - 0 - + X-Api-Key @@ -1330,7 +1275,7 @@ if(!"STARTED".equals( currentState)) { - + CONTRACT_OFFER_ID ${__eval($['dcat:dataset'][?(@.id=="${CONSUME_COUNT}")]['odrl:hasPolicy']['@id'])} @@ -1351,7 +1296,12 @@ if(offerId == null || offerId.isEmpty()) { - + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations + true + POST + true true @@ -1374,6 +1324,7 @@ if(offerId == null || offerId.isEmpty()) { "@type": "odrl:Set", "@id": "${CONSUME_COUNT}", "odrl:permission" : { + "odrl:target" : "${CONSUME_COUNT}", "odrl:action": { "odrl:type": "USE" }, @@ -1389,9 +1340,7 @@ if(offerId == null || offerId.isEmpty()) { }, "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": { - "@id": "${CONSUME_COUNT}" - } + "odrl:target": "${CONSUME_COUNT}" } }, "callbackAddresses": [] @@ -1400,18 +1349,6 @@ if(offerId == null || offerId.isEmpty()) { - ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -1436,7 +1373,6 @@ if(offerId == null || offerId.isEmpty()) { true - false @@ -1450,22 +1386,15 @@ ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} - false - - - + 6 ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID}/state - GET true - false + GET true - false - false - false - false - 6 - false - 0 + false + + + @@ -1505,22 +1434,15 @@ if(!"FINALIZED".equals( currentState)) { - false - - - + 6 ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID} - GET true - false + GET true - false - false - false - false - 6 - false - 0 + false + + + @@ -1544,6 +1466,11 @@ if(!"FINALIZED".equals( currentState)) { + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/transferprocesses + true + POST + true true @@ -1575,18 +1502,6 @@ if(!"FINALIZED".equals( currentState)) { - ${SUPPLIER_MANAGEMENT_URL}/v2/transferprocesses - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -1609,9 +1524,8 @@ if(!"FINALIZED".equals( currentState)) { - + true - false @@ -1625,22 +1539,15 @@ ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} - false - - - + 6 ${SUPPLIER_MANAGEMENT_URL}/v2/transferprocesses/${TRANSFER_ID}/state - GET true - false + GET true - false - false - false - false - 6 - false - 0 + false + + + @@ -1679,8 +1586,8 @@ if(!"STARTED".equals( currentState)) { - - true + + false saveConfig @@ -1716,10 +1623,10 @@ if(!"STARTED".equals( currentState)) { true - output/measurement_interval_supplier_fleet_manager_errors.xml + output/supplier_fleet_manager_errors.xml - + false saveConfig diff --git a/mxd/performance-tests/mxd-performance-evaluation/setup.jmx b/mxd/performance-tests/mxd-performance-evaluation/setup.jmx index cbd9d219..7a1cf6aa 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/setup.jmx +++ b/mxd/performance-tests/mxd-performance-evaluation/setup.jmx @@ -1,5 +1,5 @@ - + @@ -71,24 +71,17 @@ - false - false - + Defining a test for the processes "Create Asset", "Create Policy" and "Create Contract Definition" of the supplier - 1 + 1 false startnextloop - + 1 false - - false - false - - @@ -109,6 +102,11 @@ + 6 + ${SUPPLIER_MANAGEMENT_URL}/v3/assets + true + POST + true true @@ -130,18 +128,6 @@ - ${SUPPLIER_MANAGEMENT_URL}/v3/assets - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -171,6 +157,11 @@ + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions + true + POST + true true @@ -204,18 +195,6 @@ - ${SUPPLIER_MANAGEMENT_URL}/v2/policydefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -243,7 +222,12 @@ - + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true true @@ -266,18 +250,6 @@ - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -379,11 +351,11 @@ - + ${ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER} - + 1 1 @@ -392,7 +364,12 @@ false - + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true true @@ -406,30 +383,18 @@ "contractPolicyId": "bpnpolicy", "assetsSelector" : { "@type" : "CriterionDto", - "operandLeft": "${EDC_NAMESPACE}id", + "operandLeft": "${EDC_NAMESPACE}description", "operator": "=", - "operandRight": "${ID}" + "operandRight": "Asset which does not exist" } } = - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 - + X-Api-Key @@ -442,7 +407,7 @@ - + 200 409 @@ -453,7 +418,7 @@ 40 - + false saveConfig @@ -529,7 +494,7 @@ - + false saveConfig @@ -609,25 +574,20 @@ Defining a test for the processes "Create Asset", "Create Policy" and "Create Contract Definition" of the OEM - 1 + 1 false startnextloop - + 1 false - - false - false - - 500 - + ${OEM_CARS_INITIAL} @@ -641,6 +601,11 @@ + 6 + ${OEM_MANAGEMENT_URL}/v3/assets + true + POST + true true @@ -662,18 +627,6 @@ - ${OEM_MANAGEMENT_URL}/v3/assets - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -703,6 +656,11 @@ + 6 + ${OEM_MANAGEMENT_URL}/v2/policydefinitions + true + POST + true true @@ -736,18 +694,6 @@ - ${OEM_MANAGEMENT_URL}/v2/policydefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -775,7 +721,12 @@ - + + 6 + ${OEM_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true true @@ -798,18 +749,6 @@ - ${OEM_MANAGEMENT_URL}/v2/contractdefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 @@ -924,7 +863,12 @@ false - + + 6 + ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions + true + POST + true true @@ -938,30 +882,18 @@ "contractPolicyId": "bpnpolicy", "assetsSelector" : { "@type" : "CriterionDto", - "operandLeft": "${EDC_NAMESPACE}id", + "operandLeft": "${EDC_NAMESPACE}description", "operator": "=", - "operandRight": "${ID}" + "operandRight": "Asset which does not exist" } } = - ${SUPPLIER_MANAGEMENT_URL}/v2/contractdefinitions - POST - true - false - true - false - false - false - false - 6 - false - 0 - + X-Api-Key diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index 56834e4c..fbd5d41e 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: mxd-performance-test - image: ciprian2398/mxd-performance-test:0.2.2-SNAPSHOT + image: ciprian2398/mxd-performance-test:0.3.4-SNAPSHOT volumeMounts: - name: property-volume mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties diff --git a/mxd/performance-tests/prometheus/prometheus-ingress.yaml b/mxd/performance-tests/prometheus/prometheus-ingress.yaml index b45196fb..cf34c410 100644 --- a/mxd/performance-tests/prometheus/prometheus-ingress.yaml +++ b/mxd/performance-tests/prometheus/prometheus-ingress.yaml @@ -11,7 +11,7 @@ metadata: spec: rules: # Use the host you used in your kubernetes Ingress Configurations - - host: 'prometheus.ingress.mxd.catena-x.shoot.canary.k8s-hana.ondemand.com' + - host: 'prometheus.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com' http: paths: - backend: From 09ebd25262ff057e04b8882a13ed34b435f65531 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Tue, 5 Mar 2024 09:39:53 +0200 Subject: [PATCH 20/34] feat: remove seed containers --- mxd/seed_data.tf | 60 ++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/mxd/seed_data.tf b/mxd/seed_data.tf index ca42a35e..d886011c 100644 --- a/mxd/seed_data.tf +++ b/mxd/seed_data.tf @@ -39,37 +39,37 @@ resource "kubernetes_job" "seed_connectors_via_mgmt_api" { } spec { // this container seeds data to the BOB connector - container { - name = "newman-bob" - image = "postman/newman:ubuntu" - command = [ - "newman", "run", - "--folder", "SeedData", - "--env-var", "MANAGEMENT_URL=http://${module.bob-connector.node-ip}:8081/management", - "--env-var", "POLICY_BPN=${var.alice-bpn}", - "/opt/collection/${local.newman_collection_name}" - ] - volume_mount { - mount_path = "/opt/collection" - name = "seed-collection" - } - } +# container { +# name = "newman-bob" +# image = "postman/newman:ubuntu" +# command = [ +# "newman", "run", +# "--folder", "SeedData", +# "--env-var", "MANAGEMENT_URL=http://${module.bob-connector.node-ip}:8081/management", +# "--env-var", "POLICY_BPN=${var.alice-bpn}", +# "/opt/collection/${local.newman_collection_name}" +# ] +# volume_mount { +# mount_path = "/opt/collection" +# name = "seed-collection" +# } +# } // this container seeds data to the ALICE connector - container { - name = "newman-alice" - image = "postman/newman:ubuntu" - command = [ - "newman", "run", - "--folder", "SeedData", - "--env-var", "MANAGEMENT_URL=http://${module.alice-connector.node-ip}:8081/management", - "--env-var", "POLICY_BPN=${var.bob-bpn}", - "/opt/collection/${local.newman_collection_name}" - ] - volume_mount { - mount_path = "/opt/collection" - name = "seed-collection" - } - } +# container { +# name = "newman-alice" +# image = "postman/newman:ubuntu" +# command = [ +# "newman", "run", +# "--folder", "SeedData", +# "--env-var", "MANAGEMENT_URL=http://${module.alice-connector.node-ip}:8081/management", +# "--env-var", "POLICY_BPN=${var.bob-bpn}", +# "/opt/collection/${local.newman_collection_name}" +# ] +# volume_mount { +# mount_path = "/opt/collection" +# name = "seed-collection" +# } +# } // this container seeds data to the miw service container { From 9bd149ffc1a3fb997c18104a4a100b5f1051ebba Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Wed, 6 Mar 2024 09:42:19 +0200 Subject: [PATCH 21/34] feat: pick latest jmx files and aggregate the OEM/Supplier/Fleet calls separately --- mxd/performance-tests/README.md | 23 ++++ .../build-docker.sh | 5 +- .../measurement_interval.jmx | 112 +++++++++--------- mxd/performance-tests/performance-test.yaml | 2 +- mxd/performance-tests/run-sml.sh | 31 ++++- 5 files changed, 109 insertions(+), 64 deletions(-) create mode 100644 mxd/performance-tests/README.md diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md new file mode 100644 index 00000000..1b6f1812 --- /dev/null +++ b/mxd/performance-tests/README.md @@ -0,0 +1,23 @@ +# Performance Tests run + +### 1. Introduction +```run-sml.sh``` deploys terraform from ```mxd```folder and starts the ```mxd-performance-test``` container +which runs ```small_experiment.properties``` with a fresh docker image created with latest changes from .jmx files. + +### 2. Prerequisites +Authenticate in docker +```docker login``` + +### 3. Examples + +#### Display help +```./run-sml.sh -x``` + +#### Run all files from test-configurations folder +```./run-sml.sh -h myDockerID``` + +#### Run just one file ex: medium_experiment_10_contracts.properties +```./run-sml.sh -h myDockerID -f test-configurations/medium_experiment_10_contracts.properties``` + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh index 382ff8e6..6326da8e 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh +++ b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh @@ -1,5 +1,6 @@ #!/usr/bin/env sh -docker build -t ciprian2398/mxd-performance-test:0.3.4-SNAPSHOT . +#todo provide your oun dockerID instead of PLACEHOLDER +docker build -t PLACEHOLDER/mxd-performance-test:0.3.5-SNAPSHOT . -docker image push ciprian2398/mxd-performance-test:0.3.4-SNAPSHOT \ No newline at end of file +docker image push PLACEHOLDER/mxd-performance-test:0.3.5-SNAPSHOT \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx index e7784df5..840be594 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx +++ b/mxd/performance-tests/mxd-performance-evaluation/measurement_interval.jmx @@ -155,19 +155,19 @@ - + false true false - + 0 ${INTERVALS_PER_MINUTE} - + NEGOTIATION_ID NEGOTIATION_STATE @@ -186,7 +186,7 @@ Local Variables to be reset after each loop - + ${__intSum(1,${OEM_CARS_INITIAL})} 1 @@ -195,7 +195,7 @@ false - + 1 1 @@ -204,7 +204,7 @@ false - + 6 ${OEM_MANAGEMENT_URL}/v3/assets true @@ -379,7 +379,7 @@ - + 6 ${SUPPLIER_MANAGEMENT_URL}/v3/assets/${CONSUME_COUNTER} true @@ -405,7 +405,7 @@ - + 6 ${OEM_MANAGEMENT_URL}/v2/catalog/request true @@ -439,7 +439,7 @@ - + X-Api-Key @@ -452,7 +452,7 @@ - + CONTRACT_OFFER_ID ${__eval($['dcat:dataset'][?(@.id=="${CONSUME_COUNTER}")]['odrl:hasPolicy']['@id'])} @@ -473,7 +473,7 @@ if(offerId == null || offerId.isEmpty()) { - + 6 ${OEM_MANAGEMENT_URL}/v2/contractnegotiations true @@ -541,18 +541,18 @@ if(offerId == null || offerId.isEmpty()) { - + NEGOTIATION_ID $['@id'] 1 - + true - + ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} Loop while Negotiation State is FINALIZED @@ -562,7 +562,7 @@ ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})}500 - + 6 ${OEM_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID}/state true @@ -595,7 +595,7 @@ ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} - + groovy @@ -610,7 +610,7 @@ if(!"FINALIZED".equals( currentState)) { - + 6 ${OEM_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID} true @@ -642,7 +642,7 @@ if(!"FINALIZED".equals( currentState)) { - + 6 ${OEM_MANAGEMENT_URL}/v2/transferprocesses true @@ -701,11 +701,11 @@ if(!"FINALIZED".equals( currentState)) { - + true - + ${__jexl3("${TRANSFER_STATE}" != "STARTED" && ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} Loop while Transfer State is STARTED @@ -715,7 +715,7 @@ ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} 500 - + 6 ${OEM_MANAGEMENT_URL}/v2/transferprocesses/${TRANSFER_ID}/state true @@ -748,7 +748,7 @@ ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} - + groovy @@ -763,7 +763,7 @@ if(!"STARTED".equals( currentState)) { - + false saveConfig @@ -803,7 +803,7 @@ if(!"STARTED".equals( currentState)) { output/measurement_interval_oem_plants_errors.xml - + false saveConfig @@ -854,7 +854,7 @@ if(!"STARTED".equals( currentState)) { - + false true false @@ -885,7 +885,7 @@ if(!"STARTED".equals( currentState)) { Local Variables to be reset after each loop - + ${__intSum(1,${SUPPLIER_PARTS_INITIAL})} 1 @@ -894,7 +894,7 @@ if(!"STARTED".equals( currentState)) { false - + 6 ${SUPPLIER_MANAGEMENT_URL}/v3/assets true @@ -1069,7 +1069,7 @@ if(!"STARTED".equals( currentState)) { - + false saveConfig @@ -1109,7 +1109,7 @@ if(!"STARTED".equals( currentState)) { output/measurement_interval_supplier_plants_errors.xml - + false saveConfig @@ -1160,7 +1160,7 @@ if(!"STARTED".equals( currentState)) { - + false true false @@ -1176,7 +1176,7 @@ if(!"STARTED".equals( currentState)) { 1 1 - CONSUME_COUNT + CONSUME_COUNTER false Consumes the current value from the OEM. @@ -1201,9 +1201,9 @@ if(!"STARTED".equals( currentState)) { Local Variables to be reset after each loop - + 6 - ${OEM_MANAGEMENT_URL}/v3/assets/${CONSUME_COUNT} + ${OEM_MANAGEMENT_URL}/v3/assets/${CONSUME_COUNTER} true GET true @@ -1227,7 +1227,7 @@ if(!"STARTED".equals( currentState)) { - + 6 ${SUPPLIER_MANAGEMENT_URL}/v2/catalog/request true @@ -1252,7 +1252,7 @@ if(!"STARTED".equals( currentState)) { "@type": "CriterionDto", "operandLeft": "https://w3id.org/edc/v0.0.1/ns/id", "operator": "=", - "operandRight": "${CONSUME_COUNT}" + "operandRight": "${CONSUME_COUNTER}" } } } @@ -1277,11 +1277,11 @@ if(!"STARTED".equals( currentState)) { CONTRACT_OFFER_ID - ${__eval($['dcat:dataset'][?(@.id=="${CONSUME_COUNT}")]['odrl:hasPolicy']['@id'])} + ${__eval($['dcat:dataset'][?(@.id=="${CONSUME_COUNTER}")]['odrl:hasPolicy']['@id'])} - + groovy @@ -1296,7 +1296,7 @@ if(offerId == null || offerId.isEmpty()) { - + 6 ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations true @@ -1318,13 +1318,13 @@ if(offerId == null || offerId.isEmpty()) { "providerId": "${OEM_ID}", "offer": { "offerId": "${CONTRACT_OFFER_ID}", - "assetId": "${CONSUME_COUNT}", + "assetId": "${CONSUME_COUNTER}", "policy": { "@context": "http://www.w3.org/ns/odrl.jsonld", "@type": "odrl:Set", - "@id": "${CONSUME_COUNT}", + "@id": "${CONSUME_COUNTER}", "odrl:permission" : { - "odrl:target" : "${CONSUME_COUNT}", + "odrl:target" : "${CONSUME_COUNTER}", "odrl:action": { "odrl:type": "USE" }, @@ -1340,7 +1340,7 @@ if(offerId == null || offerId.isEmpty()) { }, "odrl:prohibition": [], "odrl:obligation": [], - "odrl:target": "${CONSUME_COUNT}" + "odrl:target": "${CONSUME_COUNTER}" } }, "callbackAddresses": [] @@ -1351,7 +1351,7 @@ if(offerId == null || offerId.isEmpty()) { - + X-Api-Key @@ -1364,18 +1364,18 @@ if(offerId == null || offerId.isEmpty()) { - + NEGOTIATION_ID $['@id'] 1 - + true - + ${__jexl3("${NEGOTIATION_STATE}" != "FINALIZED" && ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} Loop while Negotiation State is FINALIZED @@ -1418,7 +1418,7 @@ ${__jm__Negotiation_State_Loop__idx} < ${MAX_NEGOTIATION_POLL_COUNT})} - + groovy @@ -1433,7 +1433,7 @@ if(!"FINALIZED".equals( currentState)) { - + 6 ${SUPPLIER_MANAGEMENT_URL}/v2/contractnegotiations/${NEGOTIATION_ID} true @@ -1445,7 +1445,7 @@ if(!"FINALIZED".equals( currentState)) { - + X-Api-Key @@ -1458,14 +1458,14 @@ if(!"FINALIZED".equals( currentState)) { - + CONTRACT_AGREEMENT_ID $['contractAgreementId'] - + 6 ${SUPPLIER_MANAGEMENT_URL}/v2/transferprocesses true @@ -1485,7 +1485,7 @@ if(!"FINALIZED".equals( currentState)) { "counterPartyAddress": "${OEM_PROTOCOL_URL}", "connectorId": "${OEM_ID}", "contractId": "${CONTRACT_AGREEMENT_ID}", - "assetId": "${CONSUME_COUNT}", + "assetId": "${CONSUME_COUNTER}", "dataDestination": { "type": "HttpProxy" }, @@ -1524,11 +1524,11 @@ if(!"FINALIZED".equals( currentState)) { - + true - + ${__jexl3("${TRANSFER_STATE}" != "STARTED" && ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} Loop while Transfer State is STARTED @@ -1571,7 +1571,7 @@ ${__jm__Transfer_State_Loop__idx} < ${MAX_TRANSFER_POLL_COUNT})} - + groovy @@ -1626,7 +1626,7 @@ if(!"STARTED".equals( currentState)) { output/supplier_fleet_manager_errors.xml - + false saveConfig diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index fbd5d41e..c6d7b527 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,7 +5,7 @@ metadata: spec: containers: - name: mxd-performance-test - image: ciprian2398/mxd-performance-test:0.3.4-SNAPSHOT + image: {{FULL_IMAGE_NAME}} volumeMounts: - name: property-volume mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh index 9907fd67..cde50953 100755 --- a/mxd/performance-tests/run-sml.sh +++ b/mxd/performance-tests/run-sml.sh @@ -3,7 +3,8 @@ # Function to display help information function display_help { echo "Options:" - echo " -f Set the the path to file/folder with experiment files (default: \"test-configurations/small_experiment.properties\")" + echo " -f Set the path to file/folder with experiment files (default: \"test-configurations/small_experiment.properties\")" + echo " -h* Set the docker_id used to push the docker image)" echo " -l Set the log message which indicates the finish of test execution (default: \"Test Completed\")" echo " -p Set the pod name (default: \"mxd-performance-test\")" echo " -g Set the generated output file path (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar\")" @@ -12,6 +13,7 @@ function display_help { echo " -c Set the destination name name of experiment properties file when mounting on the pod (default: \"custom_experiment.properties\")" echo " -o Set the terraform log file name (default: \"sml_script_[current_datetime].logs\")" echo " -d Enable debug mode (default: true)" + echo " * mandatory" exit 0 } @@ -25,11 +27,14 @@ CUSTOM_PROPERTIES="custom_experiment.properties" LOGFILE="sml_script_$(date +%d-%m-%YT%H-%M-%S).logs" IS_DEBUG=true extension=".properties" +docker_id="" +image_timestamp="$(date +%s)" # Parse command-line options -while getopts "f:l:p:g:s:t:c:o:d:" opt; do +while getopts "f:h:l:p:g:s:t:c:o:d:" opt; do case $opt in f) PROVIDED_PATH=$OPTARG;; + h) docker_id=$OPTARG;; l) LOG_MESSAGE=$OPTARG;; p) POD_NAME=$OPTARG;; g) GENERATED_OUTPUT_FILE=$OPTARG;; @@ -42,6 +47,11 @@ while getopts "f:l:p:g:s:t:c:o:d:" opt; do esac done +if [ -z "$docker_id" ]; then + echo "Providing dockerID with -h is mandatory." + exit 1 +fi + # Prints informational messages function info { echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[32m INFO \033[0m $@" @@ -58,8 +68,8 @@ function debug { # Prints error messages and exits with error code function error_exit { echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[31m ERROR \033[0m $@" - #cleanup - #exit 1 + cleanup + exit 1 } # Initializes the test @@ -74,8 +84,14 @@ function init { kubectl apply -f prometheus | debug || error_exit "Failed to deploy Prometheus" info "Apply terraform" terraform -chdir="$TERRAFORM_CHDIR" apply -auto-approve >> "$LOGFILE" || error_exit "Failed to apply Terraform" + info "Start the performance-test container" - kubectl apply -f performance-test.yaml | debug || error_exit "Failed to start performance-test container" + local FULL_IMAGE_NAME="${docker_id}\/mxd-performance-test:${image_timestamp}" + #replace {{FULL_IMAGE_NAME}} from performance-test.yaml with an actual value + template=`cat "performance-test.yaml" | sed "s/{{FULL_IMAGE_NAME}}/$FULL_IMAGE_NAME/g"` + # apply the yml with the substituted value + echo "$template" | kubectl apply -f - + info "Waiting for container ready state" kubectl wait --for=condition=ready "pod/$POD_NAME" | debug || error_exit "Container failed to reach ready state" } @@ -153,6 +169,11 @@ function startForDirectory { } function main { + info "Building a new docker image" + docker build -t "${docker_id}/mxd-performance-test:${image_timestamp}" mxd-performance-evaluation --quiet | debug + info "Pushing the new docker image" + docker image push "${docker_id}/mxd-performance-test:${image_timestamp}" --quiet | debug + if [ -e "$PROVIDED_PATH" ]; then if [ -f "$PROVIDED_PATH" ]; then info "$PROVIDED_PATH is a file." From 5195316f399952ff2866aeeae4b00f6293013cdf Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:04:22 +0200 Subject: [PATCH 22/34] feat: runt tests from two separate clusters --- mxd/main.tf | 6 ++- mxd/performance-tests/README.md | 28 ++++++++++--- .../build-docker.sh | 6 --- mxd/performance-tests/run-sml.sh | 42 +++++++++++-------- .../large_experiment.properties | 4 +- .../medium_experiment.properties | 4 +- ...dium_experiment_10000_contracts.properties | 4 +- ...edium_experiment_1000_contracts.properties | 4 +- ...medium_experiment_100_contracts.properties | 4 +- .../medium_experiment_10_contracts.properties | 4 +- .../small_experiment.properties | 4 +- 11 files changed, 65 insertions(+), 45 deletions(-) delete mode 100755 mxd/performance-tests/mxd-performance-evaluation/build-docker.sh diff --git a/mxd/main.tf b/mxd/main.tf index b60edb28..efdccdfb 100644 --- a/mxd/main.tf +++ b/mxd/main.tf @@ -38,12 +38,14 @@ terraform { } provider "kubernetes" { - config_path = "~/.kube/config" + config_path = "~/.kube/config" + config_context = "shoot--edc-lpt--mxd" } provider "helm" { kubernetes { - config_path = "~/.kube/config" + config_path = "~/.kube/config" + config_context = "shoot--edc-lpt--mxd" } } diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index 1b6f1812..c8fae38e 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -1,23 +1,41 @@ # Performance Tests run ### 1. Introduction -```run-sml.sh``` deploys terraform from ```mxd```folder and starts the ```mxd-performance-test``` container -which runs ```small_experiment.properties``` with a fresh docker image created with latest changes from .jmx files. +The script ```./run-sml.sh``` +1. Deploys terraform from ```mxd```folder to remote cluster +2. Starts the ```mxd-performance-test``` container in the local cluster +which runs ```small_experiment.properties``` by default with a fresh docker image created with latest changes from .jmx files. ### 2. Prerequisites -Authenticate in docker -```docker login``` +1. Install Docker and authenticate with ```docker login``` + +2. Make mxd-performance-test image public on your docker hub + +3. Install kind and run ```kind create cluster -n mxd```, this will create a local cluster named ```kind-mxd``` +which is the default cluster name used in the script for hosting the performance test pod. + +4The second cluster ```shoot--edc-lpt--mxd``` is the remote cluster used to host other pods ex:(alice,bob,etc..). + +After you get the remote kube/config file, merge both files in one as described [here](https://blog.thenets.org/managing-multiples-kubernetes-clusters-with-kubectl/). + +Make sure you use ```shoot--edc-lpt--mxd``` and ```kind-mxd``` as context names or change them with -x -y flags. ### 3. Examples #### Display help ```./run-sml.sh -x``` -#### Run all files from test-configurations folder +#### Run default file small_experiment.properties ```./run-sml.sh -h myDockerID``` +#### Run all files from test-configurations folder +```./run-sml.sh -h myDockerID -f test-configurations``` + #### Run just one file ex: medium_experiment_10_contracts.properties ```./run-sml.sh -h myDockerID -f test-configurations/medium_experiment_10_contracts.properties``` +#### Run large_experiment.properties with test pod on cluster from shoot--ciprian--test-cluster context +```./run-sml.sh -h ciprian2398 -f test-configurations/large_experiment.properties -x shoot--ciprian--test-cluster``` + diff --git a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh b/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh deleted file mode 100755 index 6326da8e..00000000 --- a/mxd/performance-tests/mxd-performance-evaluation/build-docker.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env sh - -#todo provide your oun dockerID instead of PLACEHOLDER -docker build -t PLACEHOLDER/mxd-performance-test:0.3.5-SNAPSHOT . - -docker image push PLACEHOLDER/mxd-performance-test:0.3.5-SNAPSHOT \ No newline at end of file diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh index cde50953..313e4edd 100755 --- a/mxd/performance-tests/run-sml.sh +++ b/mxd/performance-tests/run-sml.sh @@ -13,6 +13,8 @@ function display_help { echo " -c Set the destination name name of experiment properties file when mounting on the pod (default: \"custom_experiment.properties\")" echo " -o Set the terraform log file name (default: \"sml_script_[current_datetime].logs\")" echo " -d Enable debug mode (default: true)" + echo " -x Test pod context (cluster used to host test pod) (default: kind-mxd)" + echo " -y Environment context (cluster used to host full blown MXD environment) (default: shoot--edc-lpt--mxd)" echo " * mandatory" exit 0 } @@ -29,9 +31,11 @@ IS_DEBUG=true extension=".properties" docker_id="" image_timestamp="$(date +%s)" +TEST_POD_CONTEXT="kind-mxd" +TEST_ENVIRONMENT_CONTEXT="shoot--edc-lpt--mxd" # Parse command-line options -while getopts "f:h:l:p:g:s:t:c:o:d:" opt; do +while getopts "f:h:l:p:g:s:t:c:o:d:x:y:" opt; do case $opt in f) PROVIDED_PATH=$OPTARG;; h) docker_id=$OPTARG;; @@ -43,6 +47,8 @@ while getopts "f:h:l:p:g:s:t:c:o:d:" opt; do c) CUSTOM_PROPERTIES=$OPTARG;; o) LOGFILE=$OPTARG;; d) IS_DEBUG=$OPTARG;; + x) TEST_POD_CONTEXT=$OPTARG;; + y) TEST_ENVIRONMENT_CONTEXT=$OPTARG;; \?) echo "Invalid option: -$OPTARG" >&2; display_help exit 1;; esac done @@ -76,24 +82,24 @@ function error_exit { function init { local experiment_file=$1 info "Adding ${experiment_file} on pod using custom-property configmap" - kubectl create configmap custom-property --from-file="${CUSTOM_PROPERTIES}"="${experiment_file}" | debug || error_exit "Failed to create configmap with name custom-property" + kubectl create configmap custom-property --from-file="${CUSTOM_PROPERTIES}"="${experiment_file}" --context="${TEST_POD_CONTEXT}" | debug || error_exit "Failed to create configmap with name custom-property" info "Init terraform" terraform -chdir="$TERRAFORM_CHDIR" init >> "$LOGFILE" || error_exit "Failed to initialize Terraform" info "Deploy Prometheus" - kubectl create namespace monitoring - kubectl apply -f prometheus | debug || error_exit "Failed to deploy Prometheus" + kubectl create namespace monitoring --context="${TEST_ENVIRONMENT_CONTEXT}" + kubectl apply -f prometheus --context="${TEST_ENVIRONMENT_CONTEXT}" | debug || error_exit "Failed to deploy Prometheus" info "Apply terraform" terraform -chdir="$TERRAFORM_CHDIR" apply -auto-approve >> "$LOGFILE" || error_exit "Failed to apply Terraform" - info "Start the performance-test container" + info "Start the performance-test pod" local FULL_IMAGE_NAME="${docker_id}\/mxd-performance-test:${image_timestamp}" #replace {{FULL_IMAGE_NAME}} from performance-test.yaml with an actual value template=`cat "performance-test.yaml" | sed "s/{{FULL_IMAGE_NAME}}/$FULL_IMAGE_NAME/g"` # apply the yml with the substituted value - echo "$template" | kubectl apply -f - + echo "$template" | kubectl apply --context="${TEST_POD_CONTEXT}" -f - - info "Waiting for container ready state" - kubectl wait --for=condition=ready "pod/$POD_NAME" | debug || error_exit "Container failed to reach ready state" + info "Waiting for test pod ready state" + kubectl wait --for=condition=ready "pod/$POD_NAME" --context="${TEST_POD_CONTEXT}" | debug || error_exit "Test pod failed to reach ready state" } # Copies output file when tests are ready @@ -101,12 +107,12 @@ function copyFileWhenTestsReady { local experiment_file=$1 info "Waiting for the tests to finish ..." while true; do - logs=$(kubectl logs --tail=5 "$POD_NAME" 2>/dev/null) + logs=$(kubectl logs --tail=5 "$POD_NAME" --context="${TEST_POD_CONTEXT}" 2>/dev/null) if echo "$logs" | grep -q "$LOG_MESSAGE"; then info "Log message found in the logs." - kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "${experiment_file}.tar" + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "${experiment_file}.tar" --context="${TEST_POD_CONTEXT}" info "Test Report downloaded with name output_${experiment_file}.tar" - kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_SLIM_FILE}" "${experiment_file}_slim.tar" + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_SLIM_FILE}" "${experiment_file}_slim.tar" --context="${TEST_POD_CONTEXT}" info "Test Report downloaded with name output_${experiment_file}_slim.tar" break else @@ -120,15 +126,15 @@ function copyFileWhenTestsReady { function cleanup { info "Destroying terraform" terraform -chdir="$TERRAFORM_CHDIR" destroy -auto-approve >> "$LOGFILE" || error_exit "Failed to destroy Terraform" - info "Destroying pod" - kubectl delete pod "$POD_NAME" | debug || error_exit "Failed to delete pod" + info "Destroying test pod" + kubectl delete pod "$POD_NAME" --context="${TEST_POD_CONTEXT}" | debug || error_exit "Failed to delete test pod" info "Destroying configmap" - kubectl delete configmap custom-property | debug || error_exit "Failed to delete configmap custom-property" + kubectl delete configmap custom-property --context="${TEST_POD_CONTEXT}" | debug || error_exit "Failed to delete configmap custom-property" info "Destroying Prometheus" - kubectl delete -f prometheus | debug || error_exit "Failed to delete Prometheus" - kubectl delete namespace monitoring - info "Waiting for the pod to be deleted" - kubectl wait --for=delete "pod/$POD_NAME" + kubectl delete -f prometheus --context="${TEST_ENVIRONMENT_CONTEXT}" | debug || error_exit "Failed to delete Prometheus" + kubectl delete namespace monitoring --context="${TEST_ENVIRONMENT_CONTEXT}" + info "Waiting for the test pod to be deleted" + kubectl wait --for=delete "pod/$POD_NAME" --context="${TEST_POD_CONTEXT}" info "Execution finished." } diff --git a/mxd/performance-tests/test-configurations/large_experiment.properties b/mxd/performance-tests/test-configurations/large_experiment.properties index b2688917..522c6c41 100644 --- a/mxd/performance-tests/test-configurations/large_experiment.properties +++ b/mxd/performance-tests/test-configurations/large_experiment.properties @@ -10,7 +10,7 @@ INTERVALS_PER_MINUTE=10 # OEM Parameters OEM_ID=BPNL000000000001 -OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password @@ -22,7 +22,7 @@ CARS_PRODUCED_PER_INTERVALL=1 # Supplier Parameters SUPPLIER_ID=BPNL000000000002 -SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password diff --git a/mxd/performance-tests/test-configurations/medium_experiment.properties b/mxd/performance-tests/test-configurations/medium_experiment.properties index b40b5bfb..9123423c 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment.properties @@ -10,7 +10,7 @@ INTERVALS_PER_MINUTE=10 # OEM Parameters OEM_ID=BPNL000000000001 -OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password @@ -22,7 +22,7 @@ CARS_PRODUCED_PER_INTERVALL=1 # Supplier Parameters SUPPLIER_ID=BPNL000000000002 -SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password diff --git a/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties index 61f3695c..e4e9c4d2 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_10000_contracts.properties @@ -10,7 +10,7 @@ INTERVALS_PER_MINUTE=10 # OEM Parameters OEM_ID=BPNL000000000001 -OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password @@ -22,7 +22,7 @@ CARS_PRODUCED_PER_INTERVALL=1 # Supplier Parameters SUPPLIER_ID=BPNL000000000002 -SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password diff --git a/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties index 85f96c42..cfb6d5d1 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_1000_contracts.properties @@ -10,7 +10,7 @@ INTERVALS_PER_MINUTE=10 # OEM Parameters OEM_ID=BPNL000000000001 -OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password @@ -22,7 +22,7 @@ CARS_PRODUCED_PER_INTERVALL=1 # Supplier Parameters SUPPLIER_ID=BPNL000000000002 -SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password diff --git a/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties index 38a36684..24300fbb 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_100_contracts.properties @@ -10,7 +10,7 @@ INTERVALS_PER_MINUTE=10 # OEM Parameters OEM_ID=BPNL000000000001 -OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password @@ -22,7 +22,7 @@ CARS_PRODUCED_PER_INTERVALL=1 # Supplier Parameters SUPPLIER_ID=BPNL000000000002 -SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password diff --git a/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties index 9727e921..5ccdcb8d 100644 --- a/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties +++ b/mxd/performance-tests/test-configurations/medium_experiment_10_contracts.properties @@ -10,7 +10,7 @@ INTERVALS_PER_MINUTE=10 # OEM Parameters OEM_ID=BPNL000000000001 -OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password @@ -22,7 +22,7 @@ CARS_PRODUCED_PER_INTERVALL=1 # Supplier Parameters SUPPLIER_ID=BPNL000000000002 -SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password diff --git a/mxd/performance-tests/test-configurations/small_experiment.properties b/mxd/performance-tests/test-configurations/small_experiment.properties index 67483075..13fa3718 100644 --- a/mxd/performance-tests/test-configurations/small_experiment.properties +++ b/mxd/performance-tests/test-configurations/small_experiment.properties @@ -10,7 +10,7 @@ INTERVALS_PER_MINUTE=10 # OEM Parameters OEM_ID=BPNL000000000001 -OEM_MANAGEMENT_URL=http://alice-controlplane:8081/management +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp OEM_API_KEY=password @@ -22,7 +22,7 @@ CARS_PRODUCED_PER_INTERVALL=1 # Supplier Parameters SUPPLIER_ID=BPNL000000000002 -SUPPLIER_MANAGEMENT_URL=http://bob-controlplane:8081/management +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp SUPPLIER_API_KEY=password From f1c323c89118f47a60b54b294c25e8cf2bd67d15 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Wed, 13 Mar 2024 13:33:20 +0200 Subject: [PATCH 23/34] feat: no docker image anymore --- .../mxd-performance-evaluation/Dockerfile | 7 ---- mxd/performance-tests/performance-test.yaml | 15 +++++++-- mxd/performance-tests/run-sml.sh | 33 ++++++------------- 3 files changed, 23 insertions(+), 32 deletions(-) delete mode 100644 mxd/performance-tests/mxd-performance-evaluation/Dockerfile diff --git a/mxd/performance-tests/mxd-performance-evaluation/Dockerfile b/mxd/performance-tests/mxd-performance-evaluation/Dockerfile deleted file mode 100644 index b7346e8d..00000000 --- a/mxd/performance-tests/mxd-performance-evaluation/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM justb4/jmeter:5.5 - -WORKDIR $JMETER_HOME/mxd-performance-evaluation - -COPY ["run_experiment.sh", "setup.jmx", "measurement_interval.jmx", "./"] - -ENTRYPOINT ["./run_experiment.sh"] diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index c6d7b527..dad3ce04 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -5,15 +5,26 @@ metadata: spec: containers: - name: mxd-performance-test - image: {{FULL_IMAGE_NAME}} - volumeMounts: + image: justb4/jmeter:5.5 + command: [ "/bin/sh", "-c", "cd mxd-performance-evaluation && ./run_experiment.sh" ] + volumeMounts: #mounting each file individually to avoid creating read-only folder in jmeter - name: property-volume mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties subPath: custom_experiment.properties + - name: property-volume + mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/run_experiment.sh + subPath: run_experiment.sh + - name: property-volume + mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/setup.jmx + subPath: setup.jmx + - name: property-volume + mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/measurement_interval.jmx + subPath: measurement_interval.jmx volumes: - name: property-volume configMap: name: custom-property + defaultMode: 0500 restartPolicy: Never diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh index 313e4edd..63fcf32b 100755 --- a/mxd/performance-tests/run-sml.sh +++ b/mxd/performance-tests/run-sml.sh @@ -4,7 +4,6 @@ function display_help { echo "Options:" echo " -f Set the path to file/folder with experiment files (default: \"test-configurations/small_experiment.properties\")" - echo " -h* Set the docker_id used to push the docker image)" echo " -l Set the log message which indicates the finish of test execution (default: \"Test Completed\")" echo " -p Set the pod name (default: \"mxd-performance-test\")" echo " -g Set the generated output file path (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar\")" @@ -15,7 +14,6 @@ function display_help { echo " -d Enable debug mode (default: true)" echo " -x Test pod context (cluster used to host test pod) (default: kind-mxd)" echo " -y Environment context (cluster used to host full blown MXD environment) (default: shoot--edc-lpt--mxd)" - echo " * mandatory" exit 0 } @@ -29,16 +27,13 @@ CUSTOM_PROPERTIES="custom_experiment.properties" LOGFILE="sml_script_$(date +%d-%m-%YT%H-%M-%S).logs" IS_DEBUG=true extension=".properties" -docker_id="" -image_timestamp="$(date +%s)" TEST_POD_CONTEXT="kind-mxd" TEST_ENVIRONMENT_CONTEXT="shoot--edc-lpt--mxd" # Parse command-line options -while getopts "f:h:l:p:g:s:t:c:o:d:x:y:" opt; do +while getopts "f:l:p:g:s:t:c:o:d:x:y:" opt; do case $opt in f) PROVIDED_PATH=$OPTARG;; - h) docker_id=$OPTARG;; l) LOG_MESSAGE=$OPTARG;; p) POD_NAME=$OPTARG;; g) GENERATED_OUTPUT_FILE=$OPTARG;; @@ -53,11 +48,6 @@ while getopts "f:h:l:p:g:s:t:c:o:d:x:y:" opt; do esac done -if [ -z "$docker_id" ]; then - echo "Providing dockerID with -h is mandatory." - exit 1 -fi - # Prints informational messages function info { echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[32m INFO \033[0m $@" @@ -82,7 +72,14 @@ function error_exit { function init { local experiment_file=$1 info "Adding ${experiment_file} on pod using custom-property configmap" - kubectl create configmap custom-property --from-file="${CUSTOM_PROPERTIES}"="${experiment_file}" --context="${TEST_POD_CONTEXT}" | debug || error_exit "Failed to create configmap with name custom-property" + + kubectl create configmap custom-property \ + --from-file="run_experiment.sh"="mxd-performance-evaluation/run_experiment.sh" \ + --from-file="setup.jmx"="mxd-performance-evaluation/setup.jmx" \ + --from-file="measurement_interval.jmx"="mxd-performance-evaluation/measurement_interval.jmx" \ + --from-file="custom_experiment.properties"="test-configurations/small_experiment.properties" \ + --context="shoot--ciprian--test-cluster" | debug || error_exit "Failed to create configmap with name custom-property" + info "Init terraform" terraform -chdir="$TERRAFORM_CHDIR" init >> "$LOGFILE" || error_exit "Failed to initialize Terraform" info "Deploy Prometheus" @@ -91,12 +88,7 @@ function init { info "Apply terraform" terraform -chdir="$TERRAFORM_CHDIR" apply -auto-approve >> "$LOGFILE" || error_exit "Failed to apply Terraform" - info "Start the performance-test pod" - local FULL_IMAGE_NAME="${docker_id}\/mxd-performance-test:${image_timestamp}" - #replace {{FULL_IMAGE_NAME}} from performance-test.yaml with an actual value - template=`cat "performance-test.yaml" | sed "s/{{FULL_IMAGE_NAME}}/$FULL_IMAGE_NAME/g"` - # apply the yml with the substituted value - echo "$template" | kubectl apply --context="${TEST_POD_CONTEXT}" -f - + kubectl apply -f performance-test.yaml --context="${TEST_POD_CONTEXT}" info "Waiting for test pod ready state" kubectl wait --for=condition=ready "pod/$POD_NAME" --context="${TEST_POD_CONTEXT}" | debug || error_exit "Test pod failed to reach ready state" @@ -175,11 +167,6 @@ function startForDirectory { } function main { - info "Building a new docker image" - docker build -t "${docker_id}/mxd-performance-test:${image_timestamp}" mxd-performance-evaluation --quiet | debug - info "Pushing the new docker image" - docker image push "${docker_id}/mxd-performance-test:${image_timestamp}" --quiet | debug - if [ -e "$PROVIDED_PATH" ]; then if [ -f "$PROVIDED_PATH" ]; then info "$PROVIDED_PATH is a file." From ceb2d8fe76b7c1a6b7a12d13d546e3b351b6d5ce Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Fri, 15 Mar 2024 14:10:20 +0200 Subject: [PATCH 24/34] feat: small refactoring --- mxd/performance-tests/README.md | 48 +-- .../experiment_controller.sh | 284 ++++++++++++++++++ mxd/performance-tests/help.txt | 6 + mxd/performance-tests/run-sml.sh | 189 ------------ 4 files changed, 316 insertions(+), 211 deletions(-) create mode 100755 mxd/performance-tests/experiment_controller.sh create mode 100644 mxd/performance-tests/help.txt delete mode 100755 mxd/performance-tests/run-sml.sh diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index c8fae38e..5eb5e223 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -1,41 +1,45 @@ # Performance Tests run ### 1. Introduction -The script ```./run-sml.sh``` -1. Deploys terraform from ```mxd```folder to remote cluster -2. Starts the ```mxd-performance-test``` container in the local cluster -which runs ```small_experiment.properties``` by default with a fresh docker image created with latest changes from .jmx files. +The script ```./experiment_controller.sh``` +1. Deploys terraform from ```mxd```folder to a provided cluster +2. Starts the ```mxd-performance-test``` container which runs +```small_experiment.properties``` by default with latest changes from .jmx files. ### 2. Prerequisites -1. Install Docker and authenticate with ```docker login``` -2. Make mxd-performance-test image public on your docker hub +#### 1. Local run +Install kind and run ```kind create cluster -n mxd```, this will create a local cluster named ```kind-mxd``` +which is the default cluster name. -3. Install kind and run ```kind create cluster -n mxd```, this will create a local cluster named ```kind-mxd``` -which is the default cluster name used in the script for hosting the performance test pod. +#### 2. Remote run +For remote run you don't need ```kind``` just provide the name of the remote cluster in -x and -y args. -4The second cluster ```shoot--edc-lpt--mxd``` is the remote cluster used to host other pods ex:(alice,bob,etc..). - -After you get the remote kube/config file, merge both files in one as described [here](https://blog.thenets.org/managing-multiples-kubernetes-clusters-with-kubectl/). - -Make sure you use ```shoot--edc-lpt--mxd``` and ```kind-mxd``` as context names or change them with -x -y flags. +#### 3. Remote run in two separate clusters +Collect .kube/config files and merge them in one as described [here](https://blog.thenets.org/managing-multiples-kubernetes-clusters-with-kubectl/). +Then provide the name of the remote cluster for test pod with -x arg, and name of the remote cluster for rest of pods using -y arg. ### 3. Examples +For more information about arguments visit [help.txt](help.txt)." #### Display help -```./run-sml.sh -x``` +```./experiment_controller.sh -h``` -#### Run default file small_experiment.properties -```./run-sml.sh -h myDockerID``` +#### Run default experiment file small_experiment.properties on ```kind-mxd``` cluster +```./experiment_controller.sh``` -#### Run all files from test-configurations folder -```./run-sml.sh -h myDockerID -f test-configurations``` +#### Run all files from test-configurations folder on ```kind-mxd``` cluster +```./experiment_controller.sh -f test-configurations``` -#### Run just one file ex: medium_experiment_10_contracts.properties -```./run-sml.sh -h myDockerID -f test-configurations/medium_experiment_10_contracts.properties``` +#### Run just one file ex: medium_experiment_10_contracts.properties on ```kind-mxd``` cluster +```./experiment_controller.sh -f test-configurations/medium_experiment_10_contracts.properties``` -#### Run large_experiment.properties with test pod on cluster from shoot--ciprian--test-cluster context -```./run-sml.sh -h ciprian2398 -f test-configurations/large_experiment.properties -x shoot--ciprian--test-cluster``` +#### Run default experiment file small_experiment.properties on ```shoot--edc-lpt--mxd``` cluster +```./experiment_controller.sh -x shoot--edc-lpt--mxd -y shoot--edc-lpt--mxd``` +#### Run default experiment file with test container on ```kind-mxd``` cluster and the rest of the environment on ```shoot--edc-lpt--mxd``` cluster +```./experiment_controller.sh -x kind-mxd -y shoot--edc-lpt--mxd``` +#### Run all files from test-configurations folder on separate clusters +```./experiment_controller.sh -f test-configurations -x kind-mxd -y shoot--edc-lpt--mxd``` diff --git a/mxd/performance-tests/experiment_controller.sh b/mxd/performance-tests/experiment_controller.sh new file mode 100755 index 00000000..a9117af2 --- /dev/null +++ b/mxd/performance-tests/experiment_controller.sh @@ -0,0 +1,284 @@ +#!/bin/bash +# Run mxd performance tests + +# Constants +LOG_MESSAGE="Test Completed" +POD_NAME="mxd-performance-test" +GENERATED_OUTPUT_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar" +GENERATED_OUTPUT_SLIM_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output_slim.tar" +CUSTOM_PROPERTIES="custom_experiment.properties" +TERRAFORM_LOGFILE="terraform_logs_$(date +%d-%m-%YT%H-%M-%S).logs" +TEST_CONFIGURATION_FILE_EXTENSION=".properties" + +#defaults +path_to_test_configuration="test-configurations/small_experiment.properties" +terraform_dir="$(dirname "$0")/.." +test_pod_context="kind-mxd" +test_environment_context="kind-mxd" +is_debug=true + + +#Override defaults with provided command line arguments +while getopts "f:t:x:y:d" opt; do + case $opt in + f) path_to_test_configuration=$OPTARG;; + t) terraform_dir=$OPTARG;; + x) test_pod_context=$OPTARG;; + y) test_environment_context=$OPTARG;; + d) is_debug=$OPTARG;; + \?) cat help.txt; exit 1;; + esac +done + + +# Prints an info message +print_info_log() { + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[32m INFO \033[0m $@" +} + +# Prints a debug message +print_debug_log() { + read IN + if [[ $is_debug == true ]]; then + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[33m DEBUG \033[0m $IN" + fi +} + +# Prints an error message +print_error_log() { + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[31m ERROR \033[0m $@" +} + +# Prints error messages and exits with error code +print_error_log_and_exit() { + echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[31m ERROR \033[0m $@" + cleanup_test_environment + exit 1 +} + + +####################################### +# Sets up test environment +# Globals: +# CUSTOM_PROPERTIES +# TERRAFORM_LOGFILE +# POD_NAME +# terraform_dir +# test_pod_context +# test_environment_context +# Arguments: +# experiment_file - .properties file used as test configuration +# Outputs: +# Creates all the needed pods for running performance tests +####################################### +setup_test_environment() { + local experiment_file=$1 + + print_info_log "Adding ${experiment_file} on pod using custom-property configmap" + kubectl create configmap custom-property \ + --from-file="run_experiment.sh=mxd-performance-evaluation/run_experiment.sh" \ + --from-file="setup.jmx=mxd-performance-evaluation/setup.jmx" \ + --from-file="measurement_interval.jmx=mxd-performance-evaluation/measurement_interval.jmx" \ + --from-file="${CUSTOM_PROPERTIES}=${experiment_file}" \ + --context="${test_pod_context}" \ + | print_debug_log \ + || print_error_log_and_exit "Failed to create configmap with name custom-property" + + print_info_log "Init terraform" + terraform -chdir="${terraform_dir}" init \ + >> "${TERRAFORM_LOGFILE}" \ + || print_error_log_and_exit "Failed to initialize Terraform" + + print_info_log "Create namespace monitoring" + kubectl create namespace monitoring --context="${test_environment_context}" \ + | print_debug_log \ + || print_error_log_and_exit "Failed to create namespace monitoring" + + print_info_log "Deploy Prometheus" + kubectl apply -f prometheus --context="${test_environment_context}" \ + | print_debug_log \ + || print_error_log_and_exit "Failed to deploy Prometheus" + + print_info_log "Apply terraform" + terraform -chdir="${terraform_dir}" apply -auto-approve \ + >> "${TERRAFORM_LOGFILE}" \ + || print_error_log_and_exit "Failed to apply Terraform" + + print_info_log "Create performance tests pod" + kubectl apply -f performance-test.yaml --context="${test_pod_context}" \ + | print_debug_log \ + || print_error_log_and_exit "Failed to deploy Prometheus" + + print_info_log "Waiting for test pod ready state" + kubectl wait --for=condition=ready "pod/${POD_NAME}" --context="${test_pod_context}" \ + | print_debug_log \ + || print_error_log_and_exit "Test pod failed to reach ready state" +} + +####################################### +# Copies test result files to local machine +# Globals: +# POD_NAME +# LOG_MESSAGE +# GENERATED_OUTPUT_FILE +# GENERATED_OUTPUT_SLIM_FILE +# test_pod_context +# Arguments: +# experiment_file - .properties file used as test configuration +# Outputs: +# It's copying the 2 .tar files containing the test results +# form the test pod in the local /test-configurations folder +####################################### +copy_test_result_files() { + local experiment_file=$1 + local logs + + print_info_log "Waiting for the tests to finish ..." + while true; do + logs=$(kubectl logs --tail=5 "${POD_NAME}" --context="${test_pod_context}" 2>/dev/null) + + if echo "${logs}" | grep -q "${LOG_MESSAGE}"; then + + print_info_log "Log message found in the logs." + + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" \ + "${experiment_file}.tar" --context="${test_pod_context}" + print_info_log "Test Report downloaded with name output_${experiment_file}.tar" + + kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_SLIM_FILE}" \ + "${experiment_file}_slim.tar" --context="${test_pod_context}" + print_info_log "Test Report downloaded with name output_${experiment_file}_slim.tar" + + break + else + echo "Waiting for log message" | print_debug_log + sleep 5 + fi + done +} + +####################################### +# Cleans up test environment +# Globals: +# terraform_dir +# test_pod_context +# test_environment_context +# POD_NAME +# TERRAFORM_LOGFILE +# Arguments: - +# Outputs: +# Deletes all the pods and config maps and leaves a clean cluster +####################################### +cleanup_test_environment() { + + print_info_log "Destroying terraform" + terraform -chdir="${terraform_dir}" destroy -auto-approve \ + >> "${TERRAFORM_LOGFILE}" \ + || print_error_log "Failed to destroy Terraform" \ + + print_info_log "Destroying test pod" + kubectl delete pod "${POD_NAME}" --context="${test_pod_context}" \ + | print_debug_log \ + || print_error_log "Failed to delete test pod" + + print_info_log "Destroying configmap" + kubectl delete configmap custom-property --context="${test_pod_context}" \ + | print_debug_log \ + || print_error_log "Failed to delete configmap custom-property" + + print_info_log "Destroying Prometheus" + kubectl delete -f prometheus --context="${test_environment_context}" \ + | print_debug_log \ + || print_error_log "Failed to delete Prometheus" + + print_info_log "Deleting namespace monitoring" + kubectl delete namespace monitoring --context="${test_environment_context}" \ + | print_debug_log \ + || print_error_log "Failed to delete namespace monitoring" + + print_info_log "Waiting for the test pod to be deleted" + kubectl wait --for=delete "pod/${POD_NAME}" --context="${test_pod_context}" \ + | print_debug_log \ + || print_error_log "Failed to delete test pod" +} + +####################################### +# Starts test for a single file +# Globals: - +# Arguments: +# experiment_file - .properties file used as test configuration +# Outputs: +# Creates all the pods, runs tests, copies results and cleans afterwards +####################################### +start_test_for_a_single_file() { + local experiment_file=$1 + + print_info_log Start experiment for file $experiment_file + + setup_test_environment "$experiment_file" + copy_test_result_files "$experiment_file" + cleanup_test_environment +} + +####################################### +# Starts test for all files in the provided directory +# Globals: +# TEST_CONFIGURATION_FILE_EXTENSION +# Arguments: - +# Outputs: - +####################################### +start_test_for_all_files_in_directory() { + # Use an array to store the all properties file in provided folder + local files_array=() + + # Populate the array with all files in provided folder + while IFS= read -r -d '' file; do + files_array+=("$file") + done < <(find "${path_to_test_configuration}" -type f -name "*${TEST_CONFIGURATION_FILE_EXTENSION}" -print0) + + # Print files in provided folder + for file in "${files_array[@]}"; do + print_info_log "Detected following properties file: $file" + done + + # Start test for one file at a time + for file in "${files_array[@]}"; do + start_test_for_a_single_file "$file" + done +} + +####################################### +# Run test for the provided file or folder +# Globals: +# path_to_test_configuration +# Arguments: - +# Outputs: +# Runs a single file or all the files in the folder +####################################### +run_test_for_provided_file_or_folder() { + if [ -e "$path_to_test_configuration" ]; then + if [ -f "$path_to_test_configuration" ]; then + print_info_log "${path_to_test_configuration} is a file." + start_test_for_a_single_file "$path_to_test_configuration" + elif [ -d "$path_to_test_configuration" ]; then + print_info_log "${path_to_test_configuration} is a directory." + start_test_for_all_files_in_directory + else + print_info_log "${path_to_test_configuration} exists but is neither a file nor a directory." + fi + else + print_info_log "${path_to_test_configuration} does not exist." + fi +} + +# Cleans up resources and exit +cleanup_and_exit() { + cleanup_test_environment + exit 0 +} + +# Execute cleanup_test_environment when CONTROL+C is invoked +trap cleanup_and_exit INT + +# Script entry point +run_test_for_provided_file_or_folder diff --git a/mxd/performance-tests/help.txt b/mxd/performance-tests/help.txt new file mode 100644 index 00000000..ca2ac8c7 --- /dev/null +++ b/mxd/performance-tests/help.txt @@ -0,0 +1,6 @@ +Help: + -f Set the path to file/folder with experiment files (default: "test-configurations/small_experiment.properties") + -t Set the Terraform directory (default: "parent folder") + -d Enable debug mode (default: true) + -x Test pod context (cluster used to host test pod) (default: kind-mxd) + -y Environment context (cluster used to host full-blown MXD environment) (default: kind-mxd) diff --git a/mxd/performance-tests/run-sml.sh b/mxd/performance-tests/run-sml.sh deleted file mode 100755 index 63fcf32b..00000000 --- a/mxd/performance-tests/run-sml.sh +++ /dev/null @@ -1,189 +0,0 @@ -#!/bin/bash - -# Function to display help information -function display_help { - echo "Options:" - echo " -f Set the path to file/folder with experiment files (default: \"test-configurations/small_experiment.properties\")" - echo " -l Set the log message which indicates the finish of test execution (default: \"Test Completed\")" - echo " -p Set the pod name (default: \"mxd-performance-test\")" - echo " -g Set the generated output file path (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar\")" - echo " -s Set the generated output_slim file path (default: \"/opt/apache-jmeter-5.5/mxd-performance-evaluation/output_slim.tar\")" - echo " -t Set the Terraform directory (default: \"/Users/ciprian/IdeaProjects/tutorial-resources/mxd/\")" - echo " -c Set the destination name name of experiment properties file when mounting on the pod (default: \"custom_experiment.properties\")" - echo " -o Set the terraform log file name (default: \"sml_script_[current_datetime].logs\")" - echo " -d Enable debug mode (default: true)" - echo " -x Test pod context (cluster used to host test pod) (default: kind-mxd)" - echo " -y Environment context (cluster used to host full blown MXD environment) (default: shoot--edc-lpt--mxd)" - exit 0 -} - -PROVIDED_PATH="test-configurations/small_experiment.properties" -LOG_MESSAGE="Test Completed" -POD_NAME="mxd-performance-test" -GENERATED_OUTPUT_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output.tar" -GENERATED_OUTPUT_SLIM_FILE="/opt/apache-jmeter-5.5/mxd-performance-evaluation/output_slim.tar" -TERRAFORM_CHDIR="$(dirname "$0")/.." -CUSTOM_PROPERTIES="custom_experiment.properties" -LOGFILE="sml_script_$(date +%d-%m-%YT%H-%M-%S).logs" -IS_DEBUG=true -extension=".properties" -TEST_POD_CONTEXT="kind-mxd" -TEST_ENVIRONMENT_CONTEXT="shoot--edc-lpt--mxd" - -# Parse command-line options -while getopts "f:l:p:g:s:t:c:o:d:x:y:" opt; do - case $opt in - f) PROVIDED_PATH=$OPTARG;; - l) LOG_MESSAGE=$OPTARG;; - p) POD_NAME=$OPTARG;; - g) GENERATED_OUTPUT_FILE=$OPTARG;; - s) GENERATED_OUTPUT_SLIM_FILE=$OPTARG;; - t) TERRAFORM_CHDIR=$OPTARG;; - c) CUSTOM_PROPERTIES=$OPTARG;; - o) LOGFILE=$OPTARG;; - d) IS_DEBUG=$OPTARG;; - x) TEST_POD_CONTEXT=$OPTARG;; - y) TEST_ENVIRONMENT_CONTEXT=$OPTARG;; - \?) echo "Invalid option: -$OPTARG" >&2; display_help exit 1;; - esac -done - -# Prints informational messages -function info { - echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[32m INFO \033[0m $@" -} - -# Prints debug messages -function debug { - read IN - if [[ $IS_DEBUG == true ]]; then - echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[33m DEBUG \033[0m $IN" - fi -} - -# Prints error messages and exits with error code -function error_exit { - echo -e "$(date +%d-%m-%Y) $(date +%H:%M:%S) \033[31m ERROR \033[0m $@" - cleanup - exit 1 -} - -# Initializes the test -function init { - local experiment_file=$1 - info "Adding ${experiment_file} on pod using custom-property configmap" - - kubectl create configmap custom-property \ - --from-file="run_experiment.sh"="mxd-performance-evaluation/run_experiment.sh" \ - --from-file="setup.jmx"="mxd-performance-evaluation/setup.jmx" \ - --from-file="measurement_interval.jmx"="mxd-performance-evaluation/measurement_interval.jmx" \ - --from-file="custom_experiment.properties"="test-configurations/small_experiment.properties" \ - --context="shoot--ciprian--test-cluster" | debug || error_exit "Failed to create configmap with name custom-property" - - info "Init terraform" - terraform -chdir="$TERRAFORM_CHDIR" init >> "$LOGFILE" || error_exit "Failed to initialize Terraform" - info "Deploy Prometheus" - kubectl create namespace monitoring --context="${TEST_ENVIRONMENT_CONTEXT}" - kubectl apply -f prometheus --context="${TEST_ENVIRONMENT_CONTEXT}" | debug || error_exit "Failed to deploy Prometheus" - info "Apply terraform" - terraform -chdir="$TERRAFORM_CHDIR" apply -auto-approve >> "$LOGFILE" || error_exit "Failed to apply Terraform" - - kubectl apply -f performance-test.yaml --context="${TEST_POD_CONTEXT}" - - info "Waiting for test pod ready state" - kubectl wait --for=condition=ready "pod/$POD_NAME" --context="${TEST_POD_CONTEXT}" | debug || error_exit "Test pod failed to reach ready state" -} - -# Copies output file when tests are ready -function copyFileWhenTestsReady { - local experiment_file=$1 - info "Waiting for the tests to finish ..." - while true; do - logs=$(kubectl logs --tail=5 "$POD_NAME" --context="${TEST_POD_CONTEXT}" 2>/dev/null) - if echo "$logs" | grep -q "$LOG_MESSAGE"; then - info "Log message found in the logs." - kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_FILE}" "${experiment_file}.tar" --context="${TEST_POD_CONTEXT}" - info "Test Report downloaded with name output_${experiment_file}.tar" - kubectl cp --retries=-1 "${POD_NAME}:${GENERATED_OUTPUT_SLIM_FILE}" "${experiment_file}_slim.tar" --context="${TEST_POD_CONTEXT}" - info "Test Report downloaded with name output_${experiment_file}_slim.tar" - break - else - echo "Waiting for log message" | debug - sleep 5 - fi - done -} - -# Cleans up resources -function cleanup { - info "Destroying terraform" - terraform -chdir="$TERRAFORM_CHDIR" destroy -auto-approve >> "$LOGFILE" || error_exit "Failed to destroy Terraform" - info "Destroying test pod" - kubectl delete pod "$POD_NAME" --context="${TEST_POD_CONTEXT}" | debug || error_exit "Failed to delete test pod" - info "Destroying configmap" - kubectl delete configmap custom-property --context="${TEST_POD_CONTEXT}" | debug || error_exit "Failed to delete configmap custom-property" - info "Destroying Prometheus" - kubectl delete -f prometheus --context="${TEST_ENVIRONMENT_CONTEXT}" | debug || error_exit "Failed to delete Prometheus" - kubectl delete namespace monitoring --context="${TEST_ENVIRONMENT_CONTEXT}" - info "Waiting for the test pod to be deleted" - kubectl wait --for=delete "pod/$POD_NAME" --context="${TEST_POD_CONTEXT}" - info "Execution finished." -} - -# Cleans up resources and exit -function cleanup_and_exit { - cleanup - exit 0 -} - -function startForSingleFile { - local experiment_file=$1 - info Start experiment for file $experiment_file - - init "$experiment_file" - copyFileWhenTestsReady "$experiment_file" - cleanup -} - -function startForDirectory { - local folder_path=$1 - # Use an array to store the all properties file in provided folder - local files_array=() - - # Populate the array using a loop - while IFS= read -r -d '' file; do - files_array+=("$file") - done < <(find "$folder_path" -type f -name "*$extension" -print0) - - # Show detected files - for file in "${files_array[@]}"; do - info "Detected following properties file: $file" - done - - # Start test for one file at a time - for file in "${files_array[@]}"; do - startForSingleFile "$file" - done -} - -function main { - if [ -e "$PROVIDED_PATH" ]; then - if [ -f "$PROVIDED_PATH" ]; then - info "$PROVIDED_PATH is a file." - startForSingleFile "$PROVIDED_PATH" - elif [ -d "$PROVIDED_PATH" ]; then - info "$PROVIDED_PATH is a directory." - startForDirectory "$PROVIDED_PATH" - else - info "$PROVIDED_PATH exists but is neither a file nor a directory." - fi - else - info "$PROVIDED_PATH does not exist." - fi -} - -# Execute cleanup when CONTROL+C is invoked -trap cleanup_and_exit INT - -# Script entry point -main \ No newline at end of file From 812b4ab46616eddabe8dc13d7f3745a579acdf46 Mon Sep 17 00:00:00 2001 From: ieuna Date: Mon, 18 Mar 2024 15:05:44 +0200 Subject: [PATCH 25/34] Updated performance test doc --- mxd/performance-tests/README.md | 188 ++++++++++++++++-- .../graphics/Initial_Counter.svg | 4 + .../graphics/Sources/Inital_Counter.drawio | 120 +++++++++++ .../Sources/Supplier_OEM.drawio | 0 .../Sources/TestSetup.drawio | 0 .../Sources/Test_Deployment.drawio | 0 .../Graphics => graphics}/Supplier_OEM.svg | 0 .../Graphics => graphics}/TestSetup.svg | 0 .../Graphics => graphics}/Test_Deployment.svg | 0 .../Graphics/Initial_Counter.svg | 3 - .../Graphics/Sources/Inital_Counter.drawio | 85 -------- .../mxd-performance-evaluation/README.md | 163 --------------- 12 files changed, 299 insertions(+), 264 deletions(-) create mode 100644 mxd/performance-tests/graphics/Initial_Counter.svg create mode 100644 mxd/performance-tests/graphics/Sources/Inital_Counter.drawio rename mxd/performance-tests/{mxd-performance-evaluation/Graphics => graphics}/Sources/Supplier_OEM.drawio (100%) rename mxd/performance-tests/{mxd-performance-evaluation/Graphics => graphics}/Sources/TestSetup.drawio (100%) rename mxd/performance-tests/{mxd-performance-evaluation/Graphics => graphics}/Sources/Test_Deployment.drawio (100%) rename mxd/performance-tests/{mxd-performance-evaluation/Graphics => graphics}/Supplier_OEM.svg (100%) rename mxd/performance-tests/{mxd-performance-evaluation/Graphics => graphics}/TestSetup.svg (100%) rename mxd/performance-tests/{mxd-performance-evaluation/Graphics => graphics}/Test_Deployment.svg (100%) delete mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg delete mode 100644 mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio delete mode 100644 mxd/performance-tests/mxd-performance-evaluation/README.md diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index 5eb5e223..eadd996f 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -1,29 +1,157 @@ -# Performance Tests run +# EDC Performance Tests -### 1. Introduction -The script ```./experiment_controller.sh``` -1. Deploys terraform from ```mxd```folder to a provided cluster -2. Starts the ```mxd-performance-test``` container which runs -```small_experiment.properties``` by default with latest changes from .jmx files. +## Table of Contents +- [Introduction](#introduction) +- [Goals of the Test](#goals-of-the-test) +- [Tools](#tools) +- [Test Design](#test-design) +- [User Guide](#user-guide) + +## Introduction +This document describes the design of the Eclipse Dataspace Connector (EDC) Performance Tests. +The EDC is the central communication component in Catena-X, which as a connector implements a framework agreement for sovereign, cross-organizational data exchange. + +## Goals of the test +The test measures performance in scenarios similar to those found in the +automotive industry. It uses a parameter-based, customizable framework +adaptable to real business situations. It aims to cover a wide range of use +cases and evaluate their performance. + +To ensure the credibility of the test, the test is based on defined characteristics: + +- Real-world: The performance workload should have the characteristics of real-world systems. +- Complexity: The test should capture intra- and inter-company business processes from two counter parties. +- Openness and version independence: The test should be able to be implemented using different versions of the EDC and not be limited to a specific use case scenario. + +## Tools +The test and its framework are based on the Java-based test tool [Apache Jmeter](https://jmeter.apache.org/usermanual/generating-dashboard.html). The test scripts are implemented exclusively as .jmx files, an XML-coded script that uses JMeter as the default file format for testing. + +In the first stage of these performance tests, the EDC is simulated using Minimum Tractus-X Dataspace ([MXD](https://eclipse-tractusx.github.io/docs-kits/kits/Connector%20Kit/Adoption%20View/connector_kit_adoption_view)). + +## Test Design + +### Definition of General Terms +| Term | Explanation| +|------------------------------|------------------------------| +| OEM | An OEM or "Original Equipment Manufacturer" manufactures a part or component that is used in another company's product | +| Supplier | An Supplier represents the end of the supply chain in the test scenario and manufactures the end product | + +### Test Setup +The relationship between the test setup and the setup of the Minimum Tractus-X Dataspace is visualized in the following figure. It should be emphasized that both the OEM and the supplier act as data recipients and senders. + +![image](graphics/TestSetup.svg) + +### Deployment of the Test +In order to create a test environment that is as realistic as possible, a deployment in a Kubernetes cluster is used for the test, which in turn deploys a worker node. This worker node of the cluster is characterized by the deployment of two EDCs, each for OEM and supplier and the respective connectors. The contents of the two EDCs are stored in a Postgres database. +The actual test and thus the workload for the EDCs is carried out by a JMeter script. The entire context is visualized below. + +![image](graphics/Test_Deployment.svg) + +### Functional structure of the test +The experiment is divided into three different phases. Each phase is implemented as own .jmx file. + +#### Setup +During the setup phase an initial set of asset, policy and contract data are loaded into the connectors. The purpose of these data sets is introduce a certain level of data aging before conducting measurements. + +#### Measurement Interval +This phase represents the actual performance evaluation. The script runs through all standard processes that can be executed by the EDC. The process starts with the creation of assets, policies and contract definitions based on the number of mock-up data from the previous process. This is followed by further EDC processes such as "Create Catalogue" until the data transfer is negotiated and executed, as described in the +[EDC wiki](#https://eclipse-tractusx.github.io/docs-kits/category/connector-kit/). +In order to test this section as realistically as possible, the process is carried out for both OEMs ("OEM Plants") and suppliers ("Supplier Plants"). The section also contains the "Supplier Fleet Manager" process, a process designed to simulate employees at the supplier who request digital twins of the cars. + +### Definition of Properties (experiment.properties) + +#### General Parameters + +| Property | Explanation| +|------------------------------|------------------------------| +| EXPERIMENT_DURATION_SEC | Duration of the experiment, during which the threads of the experiment are executed as often as possible. The unit is defined in seconds. | +| EDC_NAMESPACE | URL as an Permanent identifier of the EDC and its version. | +| BACKEND_SERVICE | URL to reference the backend system of the EDCs. | +| MAX_NEGOTIATION_POLL_COUNT | Number of maximum status checks of "FINALIZED" state of the negotiation between the two initiated EDCs Alice and Bob. | +| MAX_TRANSFER_POLL_COUNT | Number of maximum status checks of "FINALIZED" state of the transfer between the two initiated EDCs Alice and Bob. | +| INTERVALS_PER_MINUTE | Number of simulated time intervals per minute. | + +#### OEM Parameters + +| Property | Explanation| +|------------------------------|------------------------------| +| OEM_ID | Unique identifier of the OEM (Business Partner Number). | +| OEM_MANAGEMENT_URL | URL referencing the Management API of the OEM's EDC. | +| OEM_PROTOCOL_URL | URL referencing the Dataspace Protocol of the OEM Connector. | +| OEM_API_KEY | Specific, secret key to access the Management API of the OEM's EDC. | +| OEM_PLANTS | Number of OEM plants manufacturing cars. Each plant is modeled as individual thread. | +| OEM_CARS_INITIAL | Number of inital produced cars in the "SetUp" process for creating mockup data in the database for further processes | +| PARTS_PER_CAR | Number of part twins requested by the OEM from the supplier during the production of a car. | +| CARS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | + +#### Supplier Parameters + +| Property | Explanation| +|------------------------------|------------------------------| +| SUPPLIER_ID | Unique identifier of the Supplier (Business Partner Number) | +| SUPPLIER_MANAGEMENT_URL | URL referencing the Management API of the Supplier EDC. | +| SUPPLIER_PROTOCOL_URL | URL referencing the Dataspace Protocol of the Supplier Connector. | +| SUPPLIER_API_KEY | Specific, secret key to access the Management API of the Supplier EDC. | +| SUPPLIER_PLANTS | Number of supplier plants manufacturing parts. Each plant is modeled as individual thread. | +| PARTS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | +| SUPPLIER_FLEET_MANAGERS | Number of employees requesting car twins. | +| CARS_REQUESTED_PER_INTERVALL | Number of car twins requested by the supplier per time interval per employee. | +| SUPPLIER_PARTS_INITIAL | Number of inital produced parts in the "SetUp" process for creating mockup data in the database for further processes | -### 2. Prerequisites +### Functional structure in dependency to the parameter -#### 1. Local run +#### Visualization +The number of times the processes, which depicts the execution of the production of the cars, the parts required for them and the invoices created for them, are executed in the test setup depends on the three parameters "OEM_PLANTS", "SUPPLIER_PLANTS" and "SUPPLIER FLEET MANAGER". This relationship is illustrated in the following figure: + +![image](graphics/Supplier_OEM.svg) + + +The sub-process of executing the above processes is characterized by the number of cars tested. The parameter "OEM_CARS_INITIAL" can be used to define the number of mock-up cars to be created in the setup.jmx. This parameter is then passed to the measurement_interval.jmx and defines the start of the number of cars produced there (starting with counter amount 1). + +![image](graphics/Initial_Counter.svg) + +## User Guide + +### Installation of Required Softwares + +#### Setup MXD +The minimum Tractus-X Dataspace is required to carry out the tests in a local environment. +The software and tools required for this are described in the [Eclipse MXD Wiki](https://github.com/eclipse-tractusx/tutorial-resources/blob/main/mxd/README.md#1-prerequisites). + +The setup of the dataspace is explained further in [MXD Setup Wiki](https://github.com/eclipse-tractusx/tutorial-resources/blob/main/mxd/README.md#2-basic-dataspace-setup). + +#### Install JMeter +- To run the Jmeter tool, a Java Runtime Environment (JRE) is required, which can be installed [here](https://www.java.com/de/download/manual.jsp). +- After installing the Tractus-X Dataspace, the test software must now be installed. + To do this, the Jmeter tool must first be installed. Download [here](https://jmeter.apache.org/download_jmeter.cgi). +- Add the JMETER_HOME environment variable and let it reference the /bin directory of JMeter. + +### Setup Cluster +1. Install Docker and authenticate with ```docker login``` +2. Install kind and run ```kind``` + +#### Local run Install kind and run ```kind create cluster -n mxd```, this will create a local cluster named ```kind-mxd``` which is the default cluster name. -#### 2. Remote run -For remote run you don't need ```kind``` just provide the name of the remote cluster in -x and -y args. +#### Remote run +For remote run you don't need ```kind``` just provide the name of the remote cluster in -x and -y args. -#### 3. Remote run in two separate clusters +#### Remote run in two separate clusters Collect .kube/config files and merge them in one as described [here](https://blog.thenets.org/managing-multiples-kubernetes-clusters-with-kubectl/). Then provide the name of the remote cluster for test pod with -x arg, and name of the remote cluster for rest of pods using -y arg. -### 3. Examples -For more information about arguments visit [help.txt](help.txt)." +### Running the Evaluation +Once all tools and software have been installed and the Tractus-X Dataspace has been deployed as explained in the wiki, the test can be executed: + +The script ```./experiment_controller.sh``` +1. Deploys terraform from ```mxd```folder to a provided cluster +2. Starts the ```mxd-performance-test``` container which runs + ```small_experiment.properties``` by default with latest changes from .jmx files. #### Display help ```./experiment_controller.sh -h``` +For more information about arguments visit [help.txt](help.txt). #### Run default experiment file small_experiment.properties on ```kind-mxd``` cluster ```./experiment_controller.sh``` @@ -43,3 +171,37 @@ For more information about arguments visit [help.txt](help.txt)." #### Run all files from test-configurations folder on separate clusters ```./experiment_controller.sh -f test-configurations -x kind-mxd -y shoot--edc-lpt--mxd``` +Enjoy the test results! + +### Test results +After executing the shell script, the test results can be viewed at /Output/measurement_interval/index.html. +Of relevance is the sub-item "Statistics" in index.html. This sub-item contains, among other things: + +| Naming | Explanation | +|------------------------------|------------------------------| +| Executions | Number of processes performed | +| Response Time (ms) | Number in milliseconds of how long it takes the EDC to respond | +| Throughput | Ratio of executions to response time | +| Network (KB/sec) | Data throughput of the EDC in KB | + +[See JMeter Dashboard Reference](https://jmeter.apache.org/usermanual/generating-dashboard.html) + +### Customizing the Experiments + +1. [OPTIONAL] Review and update the connectors properties in the three different .properties files. A brief explanation of all properties can be found in the User Guide of this document. +2. Execute the [run_experiment.sh](mxd-performance-evaluation/run_experiment.sh) script (Bash4 required). To do this, enter the corresponding arguments (two required), on which parameters the execution should be carried out and which of the two phases of the experiment (as described in the test design) should be executed. + +**.properties -files "-q": (First argument)** + +| Naming | Default Values | +|------------------------------|------------------------------| +| S = small_experiment.properties | Supplier/OEM Plants = 1, Initial cars/parts = 10000 | +| M = medium_experiment.properties | Supplier/OEM Plants = 5, Initial cars/parts = 100000 | +| L = medium_experiment.properties | Supplier/OEM Plants = 10, Initial cars/parts = 1000000 | + +**.jmx -files "-t": (Second argument)** + +| Letter | Execution of | +|------------------------------|------------------------------| +| S | setup.jmx | +| M | measurement.jmx | diff --git a/mxd/performance-tests/graphics/Initial_Counter.svg b/mxd/performance-tests/graphics/Initial_Counter.svg new file mode 100644 index 00000000..baa3f200 --- /dev/null +++ b/mxd/performance-tests/graphics/Initial_Counter.svg @@ -0,0 +1,4 @@ + + + +
Test Driver
Consumer
Provider
Create Assets
1...initial
Setup Phase(setup.jmx)
Measurement phase
(measurement_interval.jmx)
Use Assets
1..n
Create Assets
initial+1..n
loop
Check for Assets
1..n
 \ No newline at end of file diff --git a/mxd/performance-tests/graphics/Sources/Inital_Counter.drawio b/mxd/performance-tests/graphics/Sources/Inital_Counter.drawio new file mode 100644 index 00000000..6f480a40 --- /dev/null +++ b/mxd/performance-tests/graphics/Sources/Inital_Counter.drawio @@ -0,0 +1,120 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Supplier_OEM.drawio b/mxd/performance-tests/graphics/Sources/Supplier_OEM.drawio similarity index 100% rename from mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Supplier_OEM.drawio rename to mxd/performance-tests/graphics/Sources/Supplier_OEM.drawio diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/TestSetup.drawio b/mxd/performance-tests/graphics/Sources/TestSetup.drawio similarity index 100% rename from mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/TestSetup.drawio rename to mxd/performance-tests/graphics/Sources/TestSetup.drawio diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Test_Deployment.drawio b/mxd/performance-tests/graphics/Sources/Test_Deployment.drawio similarity index 100% rename from mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Test_Deployment.drawio rename to mxd/performance-tests/graphics/Sources/Test_Deployment.drawio diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Supplier_OEM.svg b/mxd/performance-tests/graphics/Supplier_OEM.svg similarity index 100% rename from mxd/performance-tests/mxd-performance-evaluation/Graphics/Supplier_OEM.svg rename to mxd/performance-tests/graphics/Supplier_OEM.svg diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/TestSetup.svg b/mxd/performance-tests/graphics/TestSetup.svg similarity index 100% rename from mxd/performance-tests/mxd-performance-evaluation/Graphics/TestSetup.svg rename to mxd/performance-tests/graphics/TestSetup.svg diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Test_Deployment.svg b/mxd/performance-tests/graphics/Test_Deployment.svg similarity index 100% rename from mxd/performance-tests/mxd-performance-evaluation/Graphics/Test_Deployment.svg rename to mxd/performance-tests/graphics/Test_Deployment.svg diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg deleted file mode 100644 index 10cb955c..00000000 --- a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Initial_Counter.svg +++ /dev/null @@ -1,3 +0,0 @@ - - -

Sources:
Factory-Icon: Flaticon (2024): fabrik_861164, in: https://www.flaticon.com/de/kostenloses-icon/fabrik_861164 (22.01.24)
Engine-Icon: Flaticon (2024): auto-motor_5557384, in: https://www.flaticon.com/de/kostenloses-icon/auto-motor_5557384 (22.01.24)

Sources:...
SetUp OEM
SetUp OEM
OEM Plants
OEM Plants
/setup.jmx
/setup.jmx
/measurement_interval.jmx
/measurement_in...
Initial
Initial
1
1
.
.
.
.
.
.
1
1
n
n
.
.
.
.
.
.Text is not SVG - cannot display \ No newline at end of file diff --git a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio b/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio deleted file mode 100644 index 6dc0d615..00000000 --- a/mxd/performance-tests/mxd-performance-evaluation/Graphics/Sources/Inital_Counter.drawio +++ /dev/null @@ -1,85 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/mxd/performance-tests/mxd-performance-evaluation/README.md b/mxd/performance-tests/mxd-performance-evaluation/README.md deleted file mode 100644 index 2b32d62a..00000000 --- a/mxd/performance-tests/mxd-performance-evaluation/README.md +++ /dev/null @@ -1,163 +0,0 @@ -# EDC Performance Tests Introduction - -## Introduction -This document describes the design of the Eclipse Dataspace Connector (EDC) Performance Tests. -The EDC is the central communication component in Catena-X, which as a connector implements a framework agreement for sovereign, cross-organizational data exchange. - -### Goals of the test -The test is intended to measures the performance in scenarios similar to reality of the automotive industry. -Due to the fact that the EDC maps many such scenarios and use cases, it was designed as a parameter-based, customizable test. -The test defines a framework that can be adapted to real business scenarios using defined parameters. -This is an attempt to cover a broad spectrum of use cases and their performance. - -To ensure the credibility of the test, the test is based on defined characteristics: - -- Real-world: The performance workload should have the characteristics of real-world systems. -- Complexity: The test should capture intra- and inter-company business processes from two counter parties. -- Openness and version independence: The test should be able to be implemented using different versions of the EDC and not be limited to a specific use case scenario. - -## Application -The test and its framework are based on the Java-based test tool "Apache JMeter". The documentation and further information on this tool can be found at Link: https://jmeter.apache.org -The test scripts are implemented exclusively as .jmx files, an XML-coded script that uses JMeter as the default file format for testing. - -In the first stage of these performance tests, the EDC is simulated using Minimum Tractus-X Dataspace (MXD). Further information on this can be found under the following link: https://eclipse-tractusx.github.io/docs-kits/kits/tractusx-edc/docs/kit/adoption-view/Adoption%20View/ - -# Test Design - -## Definition of General Terms -| Term | Explanation| -|------------------------------|------------------------------| -| OEM | An OEM or "Original Equipment Manufacturer" manufactures a part or component that is used in another company's product | -| Supplier | An Supplier represents the end of the supply chain in the test scenario and manufactures the end product | - -## Test Setup -The relationship between the test setup and the setup of the Minimum Tractus-X Dataspace is visualized in the following figure. It should be emphasized that both the OEM and the supplier act as data recipients and senders. - -Visualization of the test setup - -### Deployment of the Test -In order to create a test environment that is as realistic as possible, a deployment in a Kubernetes cluster is used for the test, which in turn deploys a worker node. This worker node of the cluster is characterized by the deployment of two EDCs, each for OEM and supplier and the respective connectors. The contents of the two EDCs are stored in a Postgres database. -The actual test and thus the workload for the EDCs is carried out by a JMeter script. The entire context is visualized below. - -Deployment SetUp - -## Functional structure of the test -The experiment is divided into three different phases. Each phase is implemented as own .jmx file. - -### Setup -During the setup phase an initial set of asset, policy and contract data are loaded into the connectors. The purpose of these data sets is introduce a certain level of data aging before conducting measurements. - -### Measurement Interval -This phase represents the actual performance evaluation. The script runs through all standard processes that can be executed by the EDC. The process starts with the creation of assets, policies and contract definitions based on the number of mock-up data from the previous process. This is followed by further EDC processes such as "Create Catalogue" until the data transfer is negotiated and executed, as described in the EDC wiki: -https://eclipse-tractusx.github.io/docs-kits/category/connector-kit/ -In order to test this section as realistically as possible, the process is carried out for both OEMs ("OEM Plants") and suppliers ("Supplier Plants"). The section also contains the "Supplier Fleet Manager" process, a process designed to simulate employees at the supplier who request digital twins of the cars. - -### Tear Down -During the tear-down phase, all previously created assets, policies and contracts are deleted from the connectors. - -## Definition of Properties (experiment.properties) - -### General Parameters - -| Property | Explanation| -|------------------------------|------------------------------| -| EXPERIMENT_DURATION_SEC | Duration of the experiment, during which the threads of the experiment are executed as often as possible. The unit is defined in seconds. | -| EDC_NAMESPACE | URL as an Permanent identifier of the EDC and its version. | -| BACKEND_SERVICE | URL to reference the backend system of the EDCs. | -| MAX_NEGOTIATION_POLL_COUNT | Number of maximum status checks of "FINALIZED" state of the negotiation between the two initiated EDCs Alice and Bob. | -| MAX_TRANSFER_POLL_COUNT | Number of maximum status checks of "FINALIZED" state of the transfer between the two initiated EDCs Alice and Bob. | -| INTERVALS_PER_MINUTE | Number of simulated time intervals per minute. | - -### OEM Parameters - -| Property | Explanation| -|------------------------------|------------------------------| -| OEM_ID | Unique identifier of the OEM (Business Partner Number). | -| OEM_MANAGEMENT_URL | URL referencing the Management API of the OEM's EDC. | -| OEM_PROTOCOL_URL | URL referencing the Dataspace Protocol of the OEM Connector. | -| OEM_API_KEY | Specific, secret key to access the Management API of the OEM's EDC. | -| OEM_PLANTS | Number of OEM plants manufacturing cars. Each plant is modeled as individual thread. | -| OEM_CARS_INITIAL | Number of inital produced cars in the "SetUp" process for creating mockup data in the database for further processes | -| PARTS_PER_CAR | Number of part twins requested by the OEM from the supplier during the production of a car. | -| CARS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | - -### Supplier Parameters - -| Property | Explanation| -|------------------------------|------------------------------| -| SUPPLIER_ID | Unique identifier of the Supplier (Business Partner Number) | -| SUPPLIER_MANAGEMENT_URL | URL referencing the Management API of the Supplier EDC. | -| SUPPLIER_PROTOCOL_URL | URL referencing the Dataspace Protocol of the Supplier Connector. | -| SUPPLIER_API_KEY | Specific, secret key to access the Management API of the Supplier EDC. | -| SUPPLIER_PLANTS | Number of supplier plants manufacturing parts. Each plant is modeled as individual thread. | -| PARTS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | -| SUPPLIER_FLEET_MANAGERS | Number of employees requesting car twins. | -| CARS_REQUESTED_PER_INTERVALL | Number of car twins requested by the supplier per time interval per employee. | -| SUPPLIER_PARTS_INITIAL | Number of inital produced parts in the "SetUp" process for creating mockup data in the database for further processes | - -## Functional structure in dependency to the parameter - -### Visualization -The number of times the processes, which depicts the execution of the production of the cars, the parts required for them and the invoices created for them, are executed in the test setup depends on the three parameters "OEM_PLANTS", "SUPPLIER_PLANTS" and "SUPPLIER FLEET MANAGER". This relationship is illustrated in the following figure: - -Dependency between OEM and Supplier - -The sub-process of executing the above processes is characterized by the number of cars tested. The parameter "OEM_CARS_INITIAL" can be used to define the number of mock-up cars to be created in the setup.jmx. This parameter is then passed to the measurement_interval.jmx and defines the start of the number of cars produced there (starting with counter amount 1). - -Initial Counter - -# User Guide - -## Installation of the required software -The minimum Tractus-X Dataspace is required to carry out the tests in a local environment. -The software and tools required for this are described in the following wiki: -https://github.com/sap-contributions/eclipse-tractusx_tutorial-resources/blob/main/mxd/1_MXD_Introduction.md - -The setup of the dataspace is explained further in the following wiki: -https://github.com/sap-contributions/eclipse-tractusx_tutorial-resources/blob/main/mxd/2_MXD_Setup.md - -After installing the Tractus-X Dataspace, the test software must now be installed. -To do this, the Jmeter tool must first be installed. Download here: -https://jmeter.apache.org/download_jmeter.cgi -Add the JMETER_HOME environment variable and let it reference the /bin directory of JMeter. - -To run the Jmeter tool, a Java Runtime Environment (JRE) is required, which can be installed here: -https://www.java.com/de/download/manual.jsp - -## Running the Evaluation -Once all tools and software have been installed and the Tractus-X Dataspace has been deployed as explained in the wiki, the test can be executed: - -1. [OPTIONAL] Review and update the connectors properties in the three different .properties files. A brief explanation of all properties can be found in the User Guide of this document. -2. Execute the run_experiment.sh script (Bash4 required). To do this, enter the corresponding arguments (two required), on which parameters the execution should be carried out and which of the three phases of the experiment (as described in the test design) should be executed. - -**.properties -files "-q": (First argument)** - -| Naming | Default Values | -|------------------------------|------------------------------| -| S = small_experiment.properties | Supplier/OEM Plants = 1, Initial cars/parts = 10000 | -| M = medium_experiment.properties | Supplier/OEM Plants = 5, Initial cars/parts = 100000 | -| L = medium_experiment.properties | Supplier/OEM Plants = 10, Initial cars/parts = 1000000 | - -**.jmx -files "-t": (Second argument)** -| Letter | Execution of | -|------------------------------|------------------------------| -| S | setup.jmx | -| M | measurement.jmx | -| T | tear_down.jmx | -| SMT | Sequential execution of all three files | - -3. Enjoy the test results! - -## Test results -After executing the shell script, the test results can be viewed at /Output/measurement_interval/index.html. -Of relevance is the sub-item "Statistics" in index.html. This sub-item contains, among other things: - -| Naming | Explanation | -|------------------------------|------------------------------| -| Executions | Number of processes performed | -| Response Time (ms) | Number in milliseconds of how long it takes the EDC to respond | -| Throughput | Ratio of executions to response time | -| Network (KB/sec) | Data throughput of the EDC in KB | - - -All further explanations can be found in the JMeter documentation: https://jmeter.apache.org/usermanual/generating-dashboard.html \ No newline at end of file From bc16a23ac3ce7636976e66ac294eed8c56b56bd2 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:26:40 +0200 Subject: [PATCH 26/34] feat: mount all the pod files in folder --- .../mxd-performance-evaluation/run_experiment.sh | 7 ++++--- mxd/performance-tests/performance-test.yaml | 16 +++------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh b/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh index 4ae756e2..cf8577cd 100755 --- a/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh +++ b/mxd/performance-tests/mxd-performance-evaluation/run_experiment.sh @@ -1,5 +1,5 @@ #!/bin/bash -experiment_property="custom_experiment.properties" +experiment_property="mount/custom_experiment.properties" if [ -z "$JMETER_HOME" ]; then echo "JMETER_HOME is not defined - please define it." @@ -23,9 +23,10 @@ echo "*** Performance Test Start ***" if [ -z "$jmeter_script" ]; then echo "Executing all scripts in order" echo "Executing setup.jmx" - $jmeter_binary -n -t setup.jmx -l output/setup.jtl -q $experiment_property + $jmeter_binary -n -t mount/setup.jmx -l output/setup.jtl -q $experiment_property -j "output/jmeter.log" echo "Executing measurement_interval.jmx" - $jmeter_binary -n -t measurement_interval.jmx -l output/measurement_interval.jtl -q $experiment_property -e -o output/dashboard + $jmeter_binary -n -t mount/measurement_interval.jmx -l output/measurement_interval.jtl -q $experiment_property \ + -j "output/jmeter.log" -e -o output/dashboard fi echo "*** Performance Test End ***" diff --git a/mxd/performance-tests/performance-test.yaml b/mxd/performance-tests/performance-test.yaml index dad3ce04..6e44bca4 100644 --- a/mxd/performance-tests/performance-test.yaml +++ b/mxd/performance-tests/performance-test.yaml @@ -6,20 +6,10 @@ spec: containers: - name: mxd-performance-test image: justb4/jmeter:5.5 - command: [ "/bin/sh", "-c", "cd mxd-performance-evaluation && ./run_experiment.sh" ] - volumeMounts: #mounting each file individually to avoid creating read-only folder in jmeter + command: [ "/bin/sh", "-c", "cd mxd-performance-evaluation && ./mount/run_experiment.sh" ] + volumeMounts: - name: property-volume - mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/custom_experiment.properties - subPath: custom_experiment.properties - - name: property-volume - mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/run_experiment.sh - subPath: run_experiment.sh - - name: property-volume - mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/setup.jmx - subPath: setup.jmx - - name: property-volume - mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/measurement_interval.jmx - subPath: measurement_interval.jmx + mountPath: /opt/apache-jmeter-5.5/mxd-performance-evaluation/mount volumes: - name: property-volume From 7047ee9e56c894c39284641cf4e952f420ec9917 Mon Sep 17 00:00:00 2001 From: ieuna Date: Tue, 26 Mar 2024 11:50:50 +0200 Subject: [PATCH 27/34] Added new configs for performance tests --- ...edium_experiment_3000_contracts.properties | 38 +++++++++++++++++++ ...edium_experiment_5000_contracts.properties | 38 +++++++++++++++++++ ...edium_experiment_7000_contracts.properties | 38 +++++++++++++++++++ ...edium_experiment_9000_contracts.properties | 38 +++++++++++++++++++ 4 files changed, 152 insertions(+) create mode 100644 mxd/performance-tests/test-configurations/medium_experiment_3000_contracts.properties create mode 100644 mxd/performance-tests/test-configurations/medium_experiment_5000_contracts.properties create mode 100644 mxd/performance-tests/test-configurations/medium_experiment_7000_contracts.properties create mode 100644 mxd/performance-tests/test-configurations/medium_experiment_9000_contracts.properties diff --git a/mxd/performance-tests/test-configurations/medium_experiment_3000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_3000_contracts.properties new file mode 100644 index 00000000..054aeb87 --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment_3000_contracts.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_MINUTE=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=3000 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=3000 diff --git a/mxd/performance-tests/test-configurations/medium_experiment_5000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_5000_contracts.properties new file mode 100644 index 00000000..fb01a2ec --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment_5000_contracts.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_MINUTE=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=5000 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=5000 diff --git a/mxd/performance-tests/test-configurations/medium_experiment_7000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_7000_contracts.properties new file mode 100644 index 00000000..43fab5ad --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment_7000_contracts.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_MINUTE=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=7000 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=7000 diff --git a/mxd/performance-tests/test-configurations/medium_experiment_9000_contracts.properties b/mxd/performance-tests/test-configurations/medium_experiment_9000_contracts.properties new file mode 100644 index 00000000..5ecd1c41 --- /dev/null +++ b/mxd/performance-tests/test-configurations/medium_experiment_9000_contracts.properties @@ -0,0 +1,38 @@ +# General Parameters + +EXPERIMENT_DURATION_SEC=1800 +EDC_NAMESPACE=https://w3id.org/edc/v0.0.1/ns/ +BACKEND_SERVICE=https://requestbin.myworkato.com/z8zu6nz9 +MAX_NEGOTIATION_POLL_COUNT=100 +MAX_TRANSFER_POLL_COUNT=100 +INTERVALS_PER_MINUTE=10 + +# OEM Parameters + +OEM_ID=BPNL000000000001 +OEM_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/alice/management +OEM_PROTOCOL_URL=http://alice-controlplane:8084/api/v1/dsp +OEM_API_KEY=password + +OEM_PLANTS=5 +OEM_CARS_INITIAL=10000 +PARTS_PER_CAR=100 +CARS_PRODUCED_PER_INTERVALL=1 + +# Supplier Parameters + +SUPPLIER_ID=BPNL000000000002 +SUPPLIER_MANAGEMENT_URL=http://edc.ingress.mxd.edc-lpt.shoot.live.k8s-hana.ondemand.com/bob/management +SUPPLIER_PROTOCOL_URL=http://bob-controlplane:8084/api/v1/dsp +SUPPLIER_API_KEY=password + +SUPPLIER_PLANTS=5 +SUPPLIER_PARTS_INITIAL=10000 +PARTS_PRODUCED_PER_INTERVALL=50 +SUPPLIER_FLEET_MANAGERS=1 +SUPPLIER_PARTS_PLANTS_COUNT=SUPPLIER_PARTS_PLANTS_COUNT +SUPPLIER_FLEET_COUNT=SUPPLIER_FLEET_COUNT +CARS_REQUESTED_PER_INTERVALL=1 + +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER=9000 +ADDITIONAL_CONTRACT_DEFINITIONS_OEM=9000 From 4ce3f702c59ad596769832e467565843e816a09b Mon Sep 17 00:00:00 2001 From: ieuna Date: Tue, 26 Mar 2024 17:24:23 +0200 Subject: [PATCH 28/34] Updated test result aggregation script and related doc --- mxd/performance-tests/README.md | 8 +- .../results_aggregation.py | 283 +++++++++++++----- 2 files changed, 207 insertions(+), 84 deletions(-) diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index eadd996f..27738210 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -171,7 +171,13 @@ For more information about arguments visit [help.txt](help.txt). #### Run all files from test-configurations folder on separate clusters ```./experiment_controller.sh -f test-configurations -x kind-mxd -y shoot--edc-lpt--mxd``` -Enjoy the test results! +### Visualizing Test Result +1. Install python(this script is tested with python version: 3.8.10) +2. Run the [result aggregation](mxd-performance-evaluation/results_aggregation.py) script to visualize the result. The script accepts two arguments: + 1. Test result folder and + 2. Output html file name. + +```python3 results_aggregation.py test-result-folder output.html``` ### Test results After executing the shell script, the test results can be viewed at /Output/measurement_interval/index.html. diff --git a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py index cb88fd59..5331ec13 100644 --- a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py +++ b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py @@ -1,85 +1,202 @@ import json -import matplotlib.pyplot as plt # to install: pip3 install plotly -import numpy as np -import mplcursors # to install: pip3 install mplcursors +import plotly.graph_objects as go import os -import random - -############ Required Input ############ -root_folder = './Ergebnisse' -values = ['meanResTime', 'sampleCount'] -processes = ['Get Transfer State', 'Initiate Transfer'] - -###################################################################################### -# A function for extracting values from a given file -def extract_values(file_path): - with open(file_path, 'r') as file: - lines = file.readlines() - values = {} - for line in lines: - if 'OEM_PLANTS' in line: - values['OEM_PLANTS'] = int(line.split('=')[1].strip()) - elif 'OEM_CARS_INITIAL' in line: - values['OEM_CARS_INITIAL'] = int(line.split('=')[1].strip()) - elif 'PARTS_PER_CAR' in line: - values['PARTS_PER_CAR'] = int(line.split('=')[1].strip()) - elif 'CARS_PRODUCED_PER_INTERVALL' in line: - values['CARS_PRODUCED_PER_INTERVALL'] = int(line.split('=')[1].strip()) - return values - -# List comprehension to gather all directories under the specified root_folder -directories = [os.path.join(root_folder, o) for o in os.listdir(root_folder) - if os.path.isdir(os.path.join(root_folder,o))] - -# Iterates over each action and value from the lists and creates a new figure for each combination -for action_to_consider in processes: - for value_idx, value in enumerate(values): - plt.figure(figsize=(15, 4)) - plt.title('Aggregation of Performance Test Results: {} for {}'.format(value, action_to_consider)) - - stats = [] - labels = [] - meta_values_list = [] - - # Goes through each directory to find the metadata and statistics files - for idx, directory in enumerate(directories): - metadata_path = os.path.join(directory, 'metadata.txt') - statistics_path = os.path.join(directory, 'dashboard/statistics.json') - - # If both files exist in the directory, it calls the function to extract values from the metadata file - if os.path.exists(metadata_path) and os.path.exists(statistics_path): - meta_values = extract_values(metadata_path) - - # Opens the statistics.json file and loads its content - with open(statistics_path, 'r') as f: - stats_data = json.load(f) - - # Appends the statistics for each action and the associated labels for the plot - action = stats_data[action_to_consider] - stats.append((meta_values['OEM_PLANTS'], action[value])) - labels.append('Plants: {}, {}: {:.2f}'.format(meta_values['OEM_PLANTS'], value, round(action[value],2))) - meta_values_list.append(meta_values) - - # Plots the statistics data as a line plot and a scatter plot - plt.plot([s[0] for s in stats], [s[1] for s in stats]) - colors = [ (random.random(), random.random(), random.random()) for _ in range(len(stats)) ] - scatter = plt.scatter([s[0] for s in stats], [s[1] for s in stats], color=colors) - - # Adds labels to each point on the scatter plot (that x-axis matches OEM_PLANTS values) - for i, text in enumerate(labels): - plt.text(stats[i][0], stats[i][1], text) - - # Add cursor - cursor = mplcursors.cursor(scatter, hover=True) - @cursor.connect("add") - def on_add(sel): - i = sel.target.index - sel.annotation.set_text('Plants: {}, Cars: {}, Parts/Car: {}, Cars/Interval: {}'.format( - stats[i][0], - meta_values_list[i]['OEM_CARS_INITIAL'], - meta_values_list[i]['PARTS_PER_CAR'], - meta_values_list[i]['CARS_PRODUCED_PER_INTERVALL'])) - - plt.xlabel('OEM_PLANTS') - plt.ylabel(value) - plt.show() \ No newline at end of file +import sys +import numpy as np +from sklearn.linear_model import LinearRegression + +OEM_CARS_INITIAL = 'OEM_CARS_INITIAL' +SUPPLIER_PARTS_INITIAL = 'SUPPLIER_PARTS_INITIAL' +OEM_PLANTS = 'OEM_PLANTS' +SUPPLIER_PLANTS = 'SUPPLIER_PLANTS' +SUPPLIER_FLEET_MANAGERS = 'SUPPLIER_FLEET_MANAGERS' +ADDITIONAL_CONTRACT_DEFINITIONS_OEM = 'ADDITIONAL_CONTRACT_DEFINITIONS_OEM' +ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER = 'ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER' +OEM_QUERY_CATALOGUE = 'OEM Query Catalog' +DEFAULT_OUTPUT_FILE = 'output.html' + +def load_metadata(metadata_file): + """ + Load metadata from a text file. + """ + metadata = {} + + with open(metadata_file, 'r') as f: + for line in f: + line = line.strip() + if "=" in line: + key, value = line.split('=') + metadata[key.strip()] = value.strip() + return metadata + + +def load_stats(stat_file): + """ + Load statistics from a JSON file. + """ + with open(stat_file, 'r') as file: + data = json.load(file) + + processes_data = {} + + for process_name, process_data in data.items(): + median_res_time = process_data['medianResTime'] + processes_data[process_name] = median_res_time + + return processes_data + + +def sort_processes(processes, metadata): + """ + Sort processes based on specified criteria. + """ + sorted_processes = sorted(processes, key=lambda x: ( + int(metadata[x].get(OEM_CARS_INITIAL)), + int(metadata[x].get(SUPPLIER_PARTS_INITIAL)), + int(metadata[x].get(OEM_PLANTS)), + int(metadata[x].get(SUPPLIER_PLANTS)), + int(metadata[x].get(SUPPLIER_FLEET_MANAGERS)), + int(metadata[x].get(ADDITIONAL_CONTRACT_DEFINITIONS_OEM)), + int(metadata[x].get(ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER)), + )) + return sorted_processes + + +def erase_file_contents(filename): + if os.path.exists(filename): + with open(filename, 'w') as file: + pass + + +def plot_process_data(scenarios, stats_data, output_file): + figures = [] + + all_calls = stats_data[scenarios[0]].keys() + + for c in all_calls: + x_axis_data = [] + y_axis_data = [] + for s in scenarios: + mapping = stats_data[s] + if not mapping.get(c) is None: + response_time = stats_data[s][c] + y_axis_data.append(response_time) + x_axis_data.append(s) + trace = go.Bar( + x=x_axis_data, + y=y_axis_data + ) + + layout = go.Layout( + title=c, + xaxis=dict(title=c), + yaxis=dict(title='Median Response Time') + ) + fig = go.Figure(data=[trace], layout=layout) + figures.append(fig) + + erase_file_contents(output_file) + + with open(output_file, 'a') as f: + for fig in figures: + f.write(fig.to_html(full_html=False, include_plotlyjs='cdn')) + f.write('
') + + +def analyze_response_time(output_file, metadata, stats_data, scenarios): + contract_def_list = [] + response_time_list = [] + + for s in scenarios: + contract_def = metadata[s][ADDITIONAL_CONTRACT_DEFINITIONS_OEM] + response_time = stats_data[s].get(OEM_QUERY_CATALOGUE) + + contract_def_list.append(float(contract_def)) + response_time_list.append(float(response_time)) + + contract_def_array = np.array(contract_def_list) + response_time_array = np.array(response_time_list) + + x = contract_def_array.reshape(-1, 1) + y = response_time_array + + model = LinearRegression() + model.fit(x, y) + + html_content = f"

Query Catalogue Resp Time(OEM)\n

\n" + html_content += f"

Intercept: {model.intercept_}

\n" + html_content += f"

Slope (Beta param): {model.coef_[0]}

\n" + + x_prediction = np.linspace(min(contract_def_array), + max(contract_def_array), + 100).reshape(-1, 1) + y_prediction = model.predict(x_prediction) + + scatter_trace = go.Scatter( + x=contract_def_array, + y=response_time_array, + mode='markers', + marker=dict(color='blue'), + name='Data Points' + ) + + regression_line_trace = go.Scatter( + x=x_prediction.flatten(), + y=y_prediction, + mode='lines', + line=dict(color='red'), + name='Regression Line' + ) + + layout = go.Layout( + title='Linear Regression', + xaxis=dict(title='Amount of Contract Definition'), + yaxis=dict(title='Median Response Time(s)') + ) + + fig = go.Figure(data=[scatter_trace, regression_line_trace], layout=layout) + + with open(output_file, 'a') as f: + f.write(html_content) + f.write(fig.to_html(full_html=False, include_plotlyjs='cdn')) + + +def process_folders(root_folder, output_file): + """ + Process folders inside the root folder to get metadata and statistics files. + """ + scenarios = [] + stats_data = {} + metadata = {} + + for ex_folder in os.listdir(root_folder): + ex_path = os.path.join(root_folder, ex_folder) + if os.path.isdir(ex_path): + metadata_file = os.path.join(ex_path, 'metadata.txt') + dashboard_folder = os.path.join(ex_path, 'dashboard') + stats_file = os.path.join(dashboard_folder, 'statistics.json') + + if os.path.exists(metadata_file) and os.path.exists(stats_file): + metadata[ex_folder] = load_metadata(metadata_file) + stats_data[ex_folder] = (load_stats(stats_file)) + scenarios.append(ex_folder) + + sorted_scenarios = sort_processes(scenarios, metadata) + plot_process_data(sorted_scenarios, stats_data, output_file) + analyze_response_time(output_file, metadata, stats_data, scenarios) + + +def main(root_folder=None, output_file=None): + if root_folder is None: + root_folder = '.' + + if output_file is None or not output_file.endswith('.html'): + output_file = DEFAULT_OUTPUT_FILE + + process_folders(root_folder, output_file) + + +if __name__ == "__main__": + root_folder = sys.argv[1] if len(sys.argv) > 1 else None + output_file = sys.argv[2] if len(sys.argv) > 2 else None + main(root_folder, output_file) \ No newline at end of file From d2aaf98330f2df5d8157d88fa12c7ac37b3b7487 Mon Sep 17 00:00:00 2001 From: ieuna Date: Wed, 27 Mar 2024 13:14:47 +0200 Subject: [PATCH 29/34] Updated test result aggregation script and related doc --- mxd/performance-tests/README.md | 26 +++++-- .../results_aggregation.py | 74 ++++++++++++------- 2 files changed, 66 insertions(+), 34 deletions(-) diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index 27738210..824f245e 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -171,13 +171,25 @@ For more information about arguments visit [help.txt](help.txt). #### Run all files from test-configurations folder on separate clusters ```./experiment_controller.sh -f test-configurations -x kind-mxd -y shoot--edc-lpt--mxd``` -### Visualizing Test Result -1. Install python(this script is tested with python version: 3.8.10) -2. Run the [result aggregation](mxd-performance-evaluation/results_aggregation.py) script to visualize the result. The script accepts two arguments: - 1. Test result folder and - 2. Output html file name. - -```python3 results_aggregation.py test-result-folder output.html``` +### Visualizing Aggregated Test Results +1. Install python (this script was tested with python version: 3.8.10) +2. Run the [result aggregation](mxd-performance-evaluation/results_aggregation.py) script to visualize the result with the following command. + ```python3 results_aggregation.py --regression ``` +3. Replace with the path to the folder containing your test results + + The expected folder structure for the is as follows: + - Test result folder + - Experiment 1 + - output_folder + - metadata file + - dashboard folder + - statistics file + - Experiment 2... +4. Replace with the desired name for the output HTML file. +5. Replace with desired operation name, e.g- 'OEM Query Catalog' + +#### Example +```python3 results_aggregation.py test-result output.html --regression 'OEM Query Catalog'``` ### Test results After executing the shell script, the test results can be viewed at /Output/measurement_interval/index.html. diff --git a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py index 5331ec13..70201c1d 100644 --- a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py +++ b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py @@ -4,6 +4,7 @@ import sys import numpy as np from sklearn.linear_model import LinearRegression +import argparse OEM_CARS_INITIAL = 'OEM_CARS_INITIAL' SUPPLIER_PARTS_INITIAL = 'SUPPLIER_PARTS_INITIAL' @@ -12,8 +13,9 @@ SUPPLIER_FLEET_MANAGERS = 'SUPPLIER_FLEET_MANAGERS' ADDITIONAL_CONTRACT_DEFINITIONS_OEM = 'ADDITIONAL_CONTRACT_DEFINITIONS_OEM' ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER = 'ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER' -OEM_QUERY_CATALOGUE = 'OEM Query Catalog' DEFAULT_OUTPUT_FILE = 'output.html' +USAGE_MESSAGE = 'Usage: python3 results_aggregation.py --regression ' + def load_metadata(metadata_file): """ @@ -70,7 +72,6 @@ def erase_file_contents(filename): def plot_process_data(scenarios, stats_data, output_file): figures = [] - all_calls = stats_data[scenarios[0]].keys() for c in all_calls: @@ -103,14 +104,13 @@ def plot_process_data(scenarios, stats_data, output_file): f.write('
') -def analyze_response_time(output_file, metadata, stats_data, scenarios): +def analyze_response_time(output_file, metadata, stats_data, scenarios, operation_name): contract_def_list = [] response_time_list = [] for s in scenarios: contract_def = metadata[s][ADDITIONAL_CONTRACT_DEFINITIONS_OEM] - response_time = stats_data[s].get(OEM_QUERY_CATALOGUE) - + response_time = stats_data[s].get(operation_name) contract_def_list.append(float(contract_def)) response_time_list.append(float(response_time)) @@ -123,7 +123,7 @@ def analyze_response_time(output_file, metadata, stats_data, scenarios): model = LinearRegression() model.fit(x, y) - html_content = f"

Query Catalogue Resp Time(OEM)\n

\n" + html_content = f"

Analyzing Resp Time(OEM)\n

\n" html_content += f"

Intercept: {model.intercept_}

\n" html_content += f"

Slope (Beta param): {model.coef_[0]}

\n" @@ -161,7 +161,7 @@ def analyze_response_time(output_file, metadata, stats_data, scenarios): f.write(fig.to_html(full_html=False, include_plotlyjs='cdn')) -def process_folders(root_folder, output_file): +def process_folders(root_folder, output_file, operation_name): """ Process folders inside the root folder to get metadata and statistics files. """ @@ -172,31 +172,51 @@ def process_folders(root_folder, output_file): for ex_folder in os.listdir(root_folder): ex_path = os.path.join(root_folder, ex_folder) if os.path.isdir(ex_path): - metadata_file = os.path.join(ex_path, 'metadata.txt') - dashboard_folder = os.path.join(ex_path, 'dashboard') - stats_file = os.path.join(dashboard_folder, 'statistics.json') - - if os.path.exists(metadata_file) and os.path.exists(stats_file): - metadata[ex_folder] = load_metadata(metadata_file) - stats_data[ex_folder] = (load_stats(stats_file)) - scenarios.append(ex_folder) + ex_contents = os.listdir(ex_path) + if os.path.isdir(ex_path): + for item in ex_contents: + output_path = os.path.join(ex_path, item) + if os.path.isdir(output_path): + metadata_file = os.path.join(output_path, 'metadata.txt') + dashboard_folder = os.path.join(output_path, 'dashboard') + stats_file = os.path.join(dashboard_folder, 'statistics.json') + if os.path.exists(metadata_file) and os.path.exists(stats_file): + scenario = ex_folder.split('.')[0] if '.' in ex_folder else ex_folder + scenarios.append(scenario) + metadata[scenario] = load_metadata(metadata_file) + stats_data[scenario] = (load_stats(stats_file)) sorted_scenarios = sort_processes(scenarios, metadata) plot_process_data(sorted_scenarios, stats_data, output_file) - analyze_response_time(output_file, metadata, stats_data, scenarios) - - -def main(root_folder=None, output_file=None): - if root_folder is None: - root_folder = '.' + analyze_response_time(output_file, metadata, stats_data, scenarios, operation_name) - if output_file is None or not output_file.endswith('.html'): - output_file = DEFAULT_OUTPUT_FILE - process_folders(root_folder, output_file) +def main(root_folder, output_file, operation_name): + process_folders(root_folder, output_file, operation_name) if __name__ == "__main__": - root_folder = sys.argv[1] if len(sys.argv) > 1 else None - output_file = sys.argv[2] if len(sys.argv) > 2 else None - main(root_folder, output_file) \ No newline at end of file + parser = argparse.ArgumentParser(description='Test Result Aggregation Script') + parser.add_argument('arguments', + nargs=2, + metavar=('directory', 'html_file'), + help='Directory and HTML file arguments.') + parser.add_argument('--regression', + type=str, + help='operation name for regression analysis') + + try: + args = parser.parse_args() + except TypeError as e: + print("Error: Test directory or output file name is missing") + print(USAGE_MESSAGE) + sys.exit(1) + + if not args.regression: + print("Error: The --regression argument is required.") + print(USAGE_MESSAGE) + sys.exit(1) + + directory, html_file = args.arguments + operation_name = args.regression + main(directory, html_file, operation_name) From 7a449835395c7db94183ef40a165967e7412a338 Mon Sep 17 00:00:00 2001 From: ciprian-nicuta <158159743+ciprian-nicuta@users.noreply.github.com> Date: Wed, 27 Mar 2024 15:58:19 +0200 Subject: [PATCH 30/34] fix: prometheus startup issue --- mxd/performance-tests/README.md | 25 ++++++++++--------- .../experiment_controller.sh | 1 - 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index eadd996f..a700bfd4 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -83,20 +83,22 @@ In order to test this section as realistically as possible, the process is carri | OEM_CARS_INITIAL | Number of inital produced cars in the "SetUp" process for creating mockup data in the database for further processes | | PARTS_PER_CAR | Number of part twins requested by the OEM from the supplier during the production of a car. | | CARS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | +| ADDITIONAL_CONTRACT_DEFINITIONS_OEM | Number of additional contract definitions added at the setup stage. | #### Supplier Parameters -| Property | Explanation| -|------------------------------|------------------------------| -| SUPPLIER_ID | Unique identifier of the Supplier (Business Partner Number) | -| SUPPLIER_MANAGEMENT_URL | URL referencing the Management API of the Supplier EDC. | -| SUPPLIER_PROTOCOL_URL | URL referencing the Dataspace Protocol of the Supplier Connector. | -| SUPPLIER_API_KEY | Specific, secret key to access the Management API of the Supplier EDC. | -| SUPPLIER_PLANTS | Number of supplier plants manufacturing parts. Each plant is modeled as individual thread. | -| PARTS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | -| SUPPLIER_FLEET_MANAGERS | Number of employees requesting car twins. | -| CARS_REQUESTED_PER_INTERVALL | Number of car twins requested by the supplier per time interval per employee. | +| Property | Explanation | +|------------------------------|-----------------------------------------------------------------------------------------------------------------------| +| SUPPLIER_ID | Unique identifier of the Supplier (Business Partner Number) | +| SUPPLIER_MANAGEMENT_URL | URL referencing the Management API of the Supplier EDC. | +| SUPPLIER_PROTOCOL_URL | URL referencing the Dataspace Protocol of the Supplier Connector. | +| SUPPLIER_API_KEY | Specific, secret key to access the Management API of the Supplier EDC. | +| SUPPLIER_PLANTS | Number of supplier plants manufacturing parts. Each plant is modeled as individual thread. | +| PARTS_PRODUCED_PER_INTERVALL | Number of cars produced by the OEM during a time interval per plant. | +| SUPPLIER_FLEET_MANAGERS | Number of employees requesting car twins. | +| CARS_REQUESTED_PER_INTERVALL | Number of car twins requested by the supplier per time interval per employee. | | SUPPLIER_PARTS_INITIAL | Number of inital produced parts in the "SetUp" process for creating mockup data in the database for further processes | +| ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER | Number of additional contract definitions added at the setup stage. | ### Functional structure in dependency to the parameter @@ -127,8 +129,7 @@ The setup of the dataspace is explained further in [MXD Setup Wiki](https://gith - Add the JMETER_HOME environment variable and let it reference the /bin directory of JMeter. ### Setup Cluster -1. Install Docker and authenticate with ```docker login``` -2. Install kind and run ```kind``` +1. Install kind and run ```kind``` #### Local run Install kind and run ```kind create cluster -n mxd```, this will create a local cluster named ```kind-mxd``` diff --git a/mxd/performance-tests/experiment_controller.sh b/mxd/performance-tests/experiment_controller.sh index a9117af2..ea35a947 100755 --- a/mxd/performance-tests/experiment_controller.sh +++ b/mxd/performance-tests/experiment_controller.sh @@ -96,7 +96,6 @@ setup_test_environment() { print_info_log "Deploy Prometheus" kubectl apply -f prometheus --context="${test_environment_context}" \ - | print_debug_log \ || print_error_log_and_exit "Failed to deploy Prometheus" print_info_log "Apply terraform" From 80b592aab905589d3a70e4ac9b0078445e0bc682 Mon Sep 17 00:00:00 2001 From: ieuna Date: Thu, 28 Mar 2024 07:24:15 +0200 Subject: [PATCH 31/34] Updated test result aggregation script and related doc --- mxd/performance-tests/README.md | 2 +- .../mxd-performance-evaluation/results_aggregation.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index 824f245e..14937e54 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -178,7 +178,7 @@ For more information about arguments visit [help.txt](help.txt). 3. Replace with the path to the folder containing your test results The expected folder structure for the is as follows: - - Test result folder + - Test collection result folder - Experiment 1 - output_folder - metadata file diff --git a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py index 70201c1d..6fb0a94c 100644 --- a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py +++ b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py @@ -91,7 +91,7 @@ def plot_process_data(scenarios, stats_data, output_file): layout = go.Layout( title=c, xaxis=dict(title=c), - yaxis=dict(title='Median Response Time') + yaxis=dict(title='Median Response Time(ms)') ) fig = go.Figure(data=[trace], layout=layout) figures.append(fig) @@ -151,7 +151,7 @@ def analyze_response_time(output_file, metadata, stats_data, scenarios, operatio layout = go.Layout( title='Linear Regression', xaxis=dict(title='Amount of Contract Definition'), - yaxis=dict(title='Median Response Time(s)') + yaxis=dict(title='Median Response Time(ms)') ) fig = go.Figure(data=[scatter_trace, regression_line_trace], layout=layout) From f6272dbfaefb619182c23a7bd3d71e03b7032527 Mon Sep 17 00:00:00 2001 From: ieuna <161810204+ieuna@users.noreply.github.com> Date: Fri, 12 Apr 2024 17:01:59 +0300 Subject: [PATCH 32/34] Updated test result aggregation script (#15) * Updated test result aggregation script * Fixed y-axis range in plots --- mxd/performance-tests/README.md | 14 +++++--- .../results_aggregation.py | 33 +++++++------------ 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index d3fd6b15..7fb2a78f 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -175,19 +175,23 @@ For more information about arguments visit [help.txt](help.txt). ### Visualizing Aggregated Test Results 1. Install python (this script was tested with python version: 3.8.10) 2. Run the [result aggregation](mxd-performance-evaluation/results_aggregation.py) script to visualize the result with the following command. + ```python3 results_aggregation.py ``` +3. To visualize aggregated result with regression analysis, use the following command: ```python3 results_aggregation.py --regression ``` -3. Replace with the path to the folder containing your test results +4. Replace with the path to the folder containing your test results The expected folder structure for the is as follows: - Test collection result folder - Experiment 1 - - output_folder - metadata file - dashboard folder - statistics file - - Experiment 2... -4. Replace with the desired name for the output HTML file. -5. Replace with desired operation name, e.g- 'OEM Query Catalog' + - Experiment 2 + - metadata file + - dashboard folder + - statistics file +5. Replace with the desired name for the output HTML file. +6. Replace with desired operation name, e.g- 'OEM Query Catalog' #### Example ```python3 results_aggregation.py test-result output.html --regression 'OEM Query Catalog'``` diff --git a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py index 6fb0a94c..95b83645 100644 --- a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py +++ b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py @@ -91,7 +91,7 @@ def plot_process_data(scenarios, stats_data, output_file): layout = go.Layout( title=c, xaxis=dict(title=c), - yaxis=dict(title='Median Response Time(ms)') + yaxis=dict(title='Median Response Time(ms)', range=[0, None]) ) fig = go.Figure(data=[trace], layout=layout) figures.append(fig) @@ -172,23 +172,19 @@ def process_folders(root_folder, output_file, operation_name): for ex_folder in os.listdir(root_folder): ex_path = os.path.join(root_folder, ex_folder) if os.path.isdir(ex_path): - ex_contents = os.listdir(ex_path) - if os.path.isdir(ex_path): - for item in ex_contents: - output_path = os.path.join(ex_path, item) - if os.path.isdir(output_path): - metadata_file = os.path.join(output_path, 'metadata.txt') - dashboard_folder = os.path.join(output_path, 'dashboard') - stats_file = os.path.join(dashboard_folder, 'statistics.json') - if os.path.exists(metadata_file) and os.path.exists(stats_file): - scenario = ex_folder.split('.')[0] if '.' in ex_folder else ex_folder - scenarios.append(scenario) - metadata[scenario] = load_metadata(metadata_file) - stats_data[scenario] = (load_stats(stats_file)) + metadata_file = os.path.join(ex_path, 'metadata.txt') + dashboard_folder = os.path.join(ex_path, 'dashboard') + stats_file = os.path.join(dashboard_folder, 'statistics.json') + if os.path.exists(metadata_file) and os.path.exists(stats_file): + scenario = ex_folder.split('.')[0] if '.' in ex_folder else ex_folder + scenarios.append(scenario) + metadata[scenario] = load_metadata(metadata_file) + stats_data[scenario] = (load_stats(stats_file)) sorted_scenarios = sort_processes(scenarios, metadata) plot_process_data(sorted_scenarios, stats_data, output_file) - analyze_response_time(output_file, metadata, stats_data, scenarios, operation_name) + if operation_name: + analyze_response_time(output_file, metadata, stats_data, scenarios, operation_name) def main(root_folder, output_file, operation_name): @@ -212,11 +208,6 @@ def main(root_folder, output_file, operation_name): print(USAGE_MESSAGE) sys.exit(1) - if not args.regression: - print("Error: The --regression argument is required.") - print(USAGE_MESSAGE) - sys.exit(1) - directory, html_file = args.arguments operation_name = args.regression - main(directory, html_file, operation_name) + main(directory, html_file, operation_name) \ No newline at end of file From 6eafb2dc2742abc078b2bc83d6d293cebc5f7433 Mon Sep 17 00:00:00 2001 From: ieuna Date: Mon, 15 Apr 2024 09:45:17 +0300 Subject: [PATCH 33/34] Added required python libraries to readme --- mxd/performance-tests/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/mxd/performance-tests/README.md b/mxd/performance-tests/README.md index 7fb2a78f..a8241f02 100644 --- a/mxd/performance-tests/README.md +++ b/mxd/performance-tests/README.md @@ -174,11 +174,15 @@ For more information about arguments visit [help.txt](help.txt). ### Visualizing Aggregated Test Results 1. Install python (this script was tested with python version: 3.8.10) -2. Run the [result aggregation](mxd-performance-evaluation/results_aggregation.py) script to visualize the result with the following command. +2. Install following python libraries with your package manager(e.g., [pip](https://pip.pypa.io/en/stable/)): + 1. numpy + 2. scikit-learn + 3. plotly +3. Run the [result aggregation](mxd-performance-evaluation/results_aggregation.py) script to visualize the result with the following command. ```python3 results_aggregation.py ``` -3. To visualize aggregated result with regression analysis, use the following command: +4. To visualize aggregated result with regression analysis, use the following command: ```python3 results_aggregation.py --regression ``` -4. Replace with the path to the folder containing your test results +5. Replace with the path to the folder containing your test results The expected folder structure for the is as follows: - Test collection result folder @@ -190,8 +194,8 @@ For more information about arguments visit [help.txt](help.txt). - metadata file - dashboard folder - statistics file -5. Replace with the desired name for the output HTML file. -6. Replace with desired operation name, e.g- 'OEM Query Catalog' +6. Replace with the desired name for the output HTML file. +7. Replace with desired operation name, e.g- 'OEM Query Catalog' #### Example ```python3 results_aggregation.py test-result output.html --regression 'OEM Query Catalog'``` From 05108279b0b43514c435bd0a895cc29daf9dd258 Mon Sep 17 00:00:00 2001 From: ieuna <161810204+ieuna@users.noreply.github.com> Date: Fri, 3 May 2024 13:21:13 +0300 Subject: [PATCH 34/34] Added parameter for metric name (#29) * Added parameter for metric name * Updated variable name --- .../results_aggregation.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py index 95b83645..c09360c4 100644 --- a/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py +++ b/mxd/performance-tests/mxd-performance-evaluation/results_aggregation.py @@ -14,7 +14,7 @@ ADDITIONAL_CONTRACT_DEFINITIONS_OEM = 'ADDITIONAL_CONTRACT_DEFINITIONS_OEM' ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER = 'ADDITIONAL_CONTRACT_DEFINITIONS_SUPPLIER' DEFAULT_OUTPUT_FILE = 'output.html' -USAGE_MESSAGE = 'Usage: python3 results_aggregation.py --regression ' +USAGE_MESSAGE = 'Usage: python3 results_aggregation.py --regression --metric ' def load_metadata(metadata_file): @@ -32,7 +32,7 @@ def load_metadata(metadata_file): return metadata -def load_stats(stat_file): +def load_stats(stat_file, metric_name): """ Load statistics from a JSON file. """ @@ -42,8 +42,11 @@ def load_stats(stat_file): processes_data = {} for process_name, process_data in data.items(): - median_res_time = process_data['medianResTime'] - processes_data[process_name] = median_res_time + metric_val = process_data[metric_name] + if not isinstance(metric_val, (int, float, complex)): + print("Error: Found non-numeric value in passed metric field") + sys.exit() + processes_data[process_name] = metric_val return processes_data @@ -70,7 +73,7 @@ def erase_file_contents(filename): pass -def plot_process_data(scenarios, stats_data, output_file): +def plot_process_data(scenarios, stats_data, output_file, metric_name): figures = [] all_calls = stats_data[scenarios[0]].keys() @@ -91,7 +94,7 @@ def plot_process_data(scenarios, stats_data, output_file): layout = go.Layout( title=c, xaxis=dict(title=c), - yaxis=dict(title='Median Response Time(ms)', range=[0, None]) + yaxis=dict(title=metric_name, range=[0, None]) ) fig = go.Figure(data=[trace], layout=layout) figures.append(fig) @@ -161,13 +164,15 @@ def analyze_response_time(output_file, metadata, stats_data, scenarios, operatio f.write(fig.to_html(full_html=False, include_plotlyjs='cdn')) -def process_folders(root_folder, output_file, operation_name): +def process_folders(root_folder, output_file, operation_name, metric_name): """ Process folders inside the root folder to get metadata and statistics files. """ scenarios = [] stats_data = {} metadata = {} + if not metric_name: + metric_name = "medianResTime" for ex_folder in os.listdir(root_folder): ex_path = os.path.join(root_folder, ex_folder) @@ -179,16 +184,16 @@ def process_folders(root_folder, output_file, operation_name): scenario = ex_folder.split('.')[0] if '.' in ex_folder else ex_folder scenarios.append(scenario) metadata[scenario] = load_metadata(metadata_file) - stats_data[scenario] = (load_stats(stats_file)) + stats_data[scenario] = load_stats(stats_file, metric_name) sorted_scenarios = sort_processes(scenarios, metadata) - plot_process_data(sorted_scenarios, stats_data, output_file) + plot_process_data(sorted_scenarios, stats_data, output_file, metric_name) if operation_name: analyze_response_time(output_file, metadata, stats_data, scenarios, operation_name) def main(root_folder, output_file, operation_name): - process_folders(root_folder, output_file, operation_name) + process_folders(root_folder, output_file, operation_name, metric_name) if __name__ == "__main__": @@ -201,6 +206,10 @@ def main(root_folder, output_file, operation_name): type=str, help='operation name for regression analysis') + parser.add_argument('--metric', + type=str, + help='metric name') + try: args = parser.parse_args() except TypeError as e: @@ -210,4 +219,5 @@ def main(root_folder, output_file, operation_name): directory, html_file = args.arguments operation_name = args.regression + metric_name = args.metric main(directory, html_file, operation_name) \ No newline at end of file