From 9512c5438915993c02538309fc6fffd203abfff6 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 22 Oct 2025 13:01:20 -0500 Subject: [PATCH 1/3] Add troubleshooting section and resolution for Elasticsearch/Elastic Cloud use with Migration Assistant capture proxy and replayer Signed-off-by: Andre Kurait --- .../replay-captured-traffic.md | 30 ++++++++++++ .../reroute-source-to-proxy.md | 47 +++++++++++++------ 2 files changed, 63 insertions(+), 14 deletions(-) diff --git a/_migration-assistant/migration-phases/replay-captured-traffic.md b/_migration-assistant/migration-phases/replay-captured-traffic.md index 8f28f2e834a..0691bfdc585 100644 --- a/_migration-assistant/migration-phases/replay-captured-traffic.md +++ b/_migration-assistant/migration-phases/replay-captured-traffic.md @@ -323,4 +323,34 @@ The following metrics are also reported: Metrics and dashboards pushed to CloudWatch may experience a visibility lag of around 5 minutes. CloudWatch also retains higher-resolution data for a shorter period than lower-resolution data. For more information, see [Amazon CloudWatch concepts](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_concepts.html). +## Troubleshooting + +### Elasticsearch content type and accept header compatibility + +Newer Elasticsearch clients (version 7.11 and later, including all 8.x versions) use Elasticsearch-specific media types in both `Content-Type` and `Accept` headers. These clients may send headers such as: + +- `Content-Type: application/vnd.elasticsearch+json;compatible-with=8` +- `Accept: application/vnd.elasticsearch+json;compatible-with=8` + +When migrating to OpenSearch or another service that does not support these Elasticsearch-specific media types, requests from these clients may fail or be rejected by the target cluster. + +**Important**: If you are using Elasticsearch clients version 7.11 or later and migrating to OpenSearch or a service that does not recognize `application/vnd.elasticsearch+json` media types, you need to apply a transformation to convert both the `Content-Type` and `Accept` headers to the standard `application/json` format. +{: .note} + +To resolve this issue, configure Traffic Replayer with a transformation that converts the Elasticsearch-specific media types to the standard `application/json` format. Add the following transformation to your Traffic Replayer configuration using the `--transformer-config` option: + +```json +[ + { + "JsonJSTransformerProvider": { + "initializationScript": "const NEW_CONTENT_TYPE = \"application/json\";\nconst ELASTIC_CONTENT_TYPE = \"application/vnd.elasticsearch+json\";\n\nfunction transform(request, context) {\n let headers = request.get(\"headers\");\n if (headers) {\n let contentType = headers.get(\"Content-Type\");\n if (Array.isArray(contentType)) {\n headers.set(\"Content-Type\", contentType.map(v => v.includes(ELASTIC_CONTENT_TYPE) ? NEW_CONTENT_TYPE : v));\n } else if (typeof contentType === \"string\") {\n if (contentType.includes(ELASTIC_CONTENT_TYPE)) {\n headers.set(\"Content-Type\", NEW_CONTENT_TYPE);\n }\n }\n let accept = headers.get(\"Accept\");\n if (Array.isArray(accept)) {\n headers.set(\"Accept\", accept.map(v => v.includes(ELASTIC_CONTENT_TYPE) ? NEW_CONTENT_TYPE : v));\n } else if (typeof accept === \"string\") {\n if (accept.includes(ELASTIC_CONTENT_TYPE)) {\n headers.set(\"Accept\", NEW_CONTENT_TYPE);\n }\n }\n }\n return request;\n}\n\nfunction main(context) {\n return (request) => {\n if (Array.isArray(request)) {\n return request.flat().map(item => transform(item, context));\n }\n return transform(request, context);\n };\n}\n(() => main)();", + "bindingsObject": "{}" + } + } +] +``` +{% include copy.html %} + +This transformation script automatically detects Elasticsearch-specific media types in both `Content-Type` and `Accept` headers (including those with version parameters like `compatible-with=8`) and replaces them with the standard `application/json` format, ensuring compatibility with OpenSearch and other services that do not support the Elasticsearch-specific media types. + {% include migration-phase-navigation.html %} diff --git a/_migration-assistant/migration-phases/reroute-source-to-proxy.md b/_migration-assistant/migration-phases/reroute-source-to-proxy.md index 4cb6d03376a..26a6a6b6006 100644 --- a/_migration-assistant/migration-phases/reroute-source-to-proxy.md +++ b/_migration-assistant/migration-phases/reroute-source-to-proxy.md @@ -63,29 +63,48 @@ Note the records in the logging topic. After a short period, re-execute the same command again and compare the increased number of records against the expected HTTP requests. -## Backfilling documents to the source cluster +## Troubleshooting -From your source cluster snapshot, you can begin backfilling documents into the target cluster. Once you have started this process, a fleet of workers will spin up to read the snapshot and reindex documents into the target cluster. This fleet of workers can be scaled to increase the speed at which documents are reindexed into the target cluster. +### Host header routing configuration -### Checking the starting state of the clusters +Some systems, such as Elastic Cloud and other hosted Elasticsearch services, use the Host header for routing traffic to the appropriate cluster. When using the Capture Proxy with these systems, you need to configure the proxy to set the Host header to match your source cluster's domain. -You can check the indexes and document counts of the source and target clusters by running the `cat-indices` command. This can be used to monitor the difference between the source and target for any migration scenario. Check the indexes of both clusters using the following command: +**Important**: This configuration is required for Elastic Cloud deployments and any system that uses Host header-based routing. Without this setting, requests will fail with an error response like `{"ok":false,"message":"Unknown resource."}` on Elastic Cloud or be misrouted on other systems. +{: .note} + +To configure the Host header, add the `captureProxyExtraArgs` parameter to your `cdk.context.json` file: + +```json +{ + "captureProxyExtraArgs": "--setHeader Host " +} +``` +{% include copy.html %} -```shell -console clusters cat-indices +For example, if your Elastic Cloud domain is `https://my-cluster.es.us-east-1.aws.found.io`, you would configure: + +```json +{ + "captureProxyExtraArgs": "--setHeader Host my-cluster.es.us-east-1.aws.found.io" +} ``` {% include copy.html %} -You should receive the following response: +**Tip**: The Host header value should include only the domain name without the protocol (https://) or port number. +{: .tip} + +#### Validating the configuration -```shell -SOURCE CLUSTER -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -green open my-index WJPVdHNyQ1KMKol84Cy72Q 1 0 8 0 44.7kb 44.7kb +Before routing production traffic through the Capture Proxy, validate that the proxy is correctly configured by sending test requests directly to it. You can use `curl` to verify the connection: -TARGET CLUSTER -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -green open .opendistro_security N3uy88FGT9eAO7FTbLqqqA 1 0 10 0 78.3kb 78.3kb +```bash +curl -k https://:9200/ ``` +{% include copy.html %} + +If the Host header configuration is correct, you should receive a successful or authentication failure response from your source cluster. If you receive an error like `{"ok":false,"message":"Unknown resource."}`, verify that: +- The `captureProxyExtraArgs` parameter is correctly set in your `cdk.context.json` +- The Host header value matches your source cluster's domain exactly +- You have redeployed the Capture Proxy service after making configuration changes {% include migration-phase-navigation.html %} From 776d79fb282141a54e1427b5e7aabc1f0ef60c61 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 22 Oct 2025 13:10:56 -0500 Subject: [PATCH 2/3] Add vale fixes Signed-off-by: Andre Kurait --- .../migration-phases/replay-captured-traffic.md | 2 ++ .../migration-phases/reroute-source-to-proxy.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/_migration-assistant/migration-phases/replay-captured-traffic.md b/_migration-assistant/migration-phases/replay-captured-traffic.md index 0691bfdc585..1e0010296d1 100644 --- a/_migration-assistant/migration-phases/replay-captured-traffic.md +++ b/_migration-assistant/migration-phases/replay-captured-traffic.md @@ -325,6 +325,8 @@ Metrics and dashboards pushed to CloudWatch may experience a visibility lag of a ## Troubleshooting +The following sections may be helpful in diagnosing common issues that arise. If you encounter an issue not documented, incorporate that feedback into this section for future readers. + ### Elasticsearch content type and accept header compatibility Newer Elasticsearch clients (version 7.11 and later, including all 8.x versions) use Elasticsearch-specific media types in both `Content-Type` and `Accept` headers. These clients may send headers such as: diff --git a/_migration-assistant/migration-phases/reroute-source-to-proxy.md b/_migration-assistant/migration-phases/reroute-source-to-proxy.md index 26a6a6b6006..4eb042da192 100644 --- a/_migration-assistant/migration-phases/reroute-source-to-proxy.md +++ b/_migration-assistant/migration-phases/reroute-source-to-proxy.md @@ -65,11 +65,13 @@ After a short period, re-execute the same command again and compare the increase ## Troubleshooting +The following sections may be helpful in diagnosing common issues that arise. If you encounter an issue not documented, incorporate that feedback into this section for future readers. + ### Host header routing configuration Some systems, such as Elastic Cloud and other hosted Elasticsearch services, use the Host header for routing traffic to the appropriate cluster. When using the Capture Proxy with these systems, you need to configure the proxy to set the Host header to match your source cluster's domain. -**Important**: This configuration is required for Elastic Cloud deployments and any system that uses Host header-based routing. Without this setting, requests will fail with an error response like `{"ok":false,"message":"Unknown resource."}` on Elastic Cloud or be misrouted on other systems. +**Important**: This configuration is required for Elastic Cloud deployments and any system that uses Host header-based routing. Without this setting, requests will fail with an error response like `{"ok":false,"message":"Unknown resource."}` on Elastic Cloud or be incorrectly routed on other systems. {: .note} To configure the Host header, add the `captureProxyExtraArgs` parameter to your `cdk.context.json` file: From 41fe07567a11185860278af1da677dcc1938d8f8 Mon Sep 17 00:00:00 2001 From: Andre Kurait Date: Wed, 22 Oct 2025 13:41:58 -0500 Subject: [PATCH 3/3] Improve readability of repalyer troubleshooting transformation Signed-off-by: Andre Kurait --- .../replay-captured-traffic.md | 55 ++++++++++++++++++- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/_migration-assistant/migration-phases/replay-captured-traffic.md b/_migration-assistant/migration-phases/replay-captured-traffic.md index 1e0010296d1..51b7dae8ba7 100644 --- a/_migration-assistant/migration-phases/replay-captured-traffic.md +++ b/_migration-assistant/migration-phases/replay-captured-traffic.md @@ -336,16 +336,59 @@ Newer Elasticsearch clients (version 7.11 and later, including all 8.x versions) When migrating to OpenSearch or another service that does not support these Elasticsearch-specific media types, requests from these clients may fail or be rejected by the target cluster. -**Important**: If you are using Elasticsearch clients version 7.11 or later and migrating to OpenSearch or a service that does not recognize `application/vnd.elasticsearch+json` media types, you need to apply a transformation to convert both the `Content-Type` and `Accept` headers to the standard `application/json` format. +**Important**: If you are using Elasticsearch clients with version 7.11 or later and migrating to OpenSearch or a service that does not recognize `application/vnd.elasticsearch+json` media types, you need to apply a transformation to convert both the `Content-Type` and `Accept` headers to the standard `application/json` format. {: .note} -To resolve this issue, configure Traffic Replayer with a transformation that converts the Elasticsearch-specific media types to the standard `application/json` format. Add the following transformation to your Traffic Replayer configuration using the `--transformer-config` option: +To resolve this issue, configure Traffic Replayer with a transformation that converts the Elasticsearch-specific media types to the standard `application/json` format. + +First, create a JavaScript transformation file at `/shared-logs-output/content-type-transformer.js`: + +```javascript +const NEW_CONTENT_TYPE = "application/json"; +const ELASTIC_CONTENT_TYPE = "application/vnd.elasticsearch+json"; + +function transform(request, context) { + let headers = request.get("headers"); + if (headers) { + let contentType = headers.get("Content-Type"); + if (Array.isArray(contentType)) { + headers.set("Content-Type", contentType.map(v => v.includes(ELASTIC_CONTENT_TYPE) ? NEW_CONTENT_TYPE : v)); + } else if (typeof contentType === "string") { + if (contentType.includes(ELASTIC_CONTENT_TYPE)) { + headers.set("Content-Type", NEW_CONTENT_TYPE); + } + } + let accept = headers.get("Accept"); + if (Array.isArray(accept)) { + headers.set("Accept", accept.map(v => v.includes(ELASTIC_CONTENT_TYPE) ? NEW_CONTENT_TYPE : v)); + } else if (typeof accept === "string") { + if (accept.includes(ELASTIC_CONTENT_TYPE)) { + headers.set("Accept", NEW_CONTENT_TYPE); + } + } + } + return request; +} + +function main(context) { + return (request) => { + if (Array.isArray(request)) { + return request.flat().map(item => transform(item, context)); + } + return transform(request, context); + }; +} +(() => main)(); +``` +{% include copy.html %} + +Next, create a transformation configuration file at `/shared-logs-output/replayer-transformation.json`: ```json [ { "JsonJSTransformerProvider": { - "initializationScript": "const NEW_CONTENT_TYPE = \"application/json\";\nconst ELASTIC_CONTENT_TYPE = \"application/vnd.elasticsearch+json\";\n\nfunction transform(request, context) {\n let headers = request.get(\"headers\");\n if (headers) {\n let contentType = headers.get(\"Content-Type\");\n if (Array.isArray(contentType)) {\n headers.set(\"Content-Type\", contentType.map(v => v.includes(ELASTIC_CONTENT_TYPE) ? NEW_CONTENT_TYPE : v));\n } else if (typeof contentType === \"string\") {\n if (contentType.includes(ELASTIC_CONTENT_TYPE)) {\n headers.set(\"Content-Type\", NEW_CONTENT_TYPE);\n }\n }\n let accept = headers.get(\"Accept\");\n if (Array.isArray(accept)) {\n headers.set(\"Accept\", accept.map(v => v.includes(ELASTIC_CONTENT_TYPE) ? NEW_CONTENT_TYPE : v));\n } else if (typeof accept === \"string\") {\n if (accept.includes(ELASTIC_CONTENT_TYPE)) {\n headers.set(\"Accept\", NEW_CONTENT_TYPE);\n }\n }\n }\n return request;\n}\n\nfunction main(context) {\n return (request) => {\n if (Array.isArray(request)) {\n return request.flat().map(item => transform(item, context));\n }\n return transform(request, context);\n };\n}\n(() => main)();", + "initializationScriptFile": "/shared-logs-output/content-type-transformer.js", "bindingsObject": "{}" } } @@ -353,6 +396,12 @@ To resolve this issue, configure Traffic Replayer with a transformation that con ``` {% include copy.html %} +Finally, configure Traffic Replayer to use this transformation by adding the following to your `trafficReplayerExtraArgs`: + +``` +--transformer-config-file /shared-logs-output/replayer-transformation.json +``` + This transformation script automatically detects Elasticsearch-specific media types in both `Content-Type` and `Accept` headers (including those with version parameters like `compatible-with=8`) and replaces them with the standard `application/json` format, ensuring compatibility with OpenSearch and other services that do not support the Elasticsearch-specific media types. {% include migration-phase-navigation.html %}