Add deploy files

dineshg13 · jackgopack4 · commit 6b0c316a415a · 2025-03-04T14:54:13.000-05:00
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,205 @@
+# ENV VARS
+variables:
+  CI_IMAGE: registry.ddbuild.io/ci/opentelemetry-collector-contrib:ci-image-2
+  BUILD_DD_REGISTRY: registry.ddbuild.io/ci/opentelemetry-collector-contrib
+  BUILD_DEMO_REGISTRY: 172597598159.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib
+  BUILD_SANDBOX_REGISTRY: 601427279990.dkr.ecr.us-east-1.amazonaws.com/otel-collector-contrib
+  SANDBOX_CLUSTER_NAME: dd-otel
+  DEMO_CLUSTER_NAME: prod-eks-otel-a-us-east-1
+  SANDBOX_CLUSTER_ARN: "arn:aws:eks:us-east-1:601427279990:cluster/dd-otel"
+  DEMO_CLUSTER_ARN: "arn:aws:eks:us-east-1:172597598159:cluster/prod-eks-otel-a-us-east-1"
+  DEPLOY_SCRIPT: "./ci/scripts/ci-deploy.sh"
+  # To use images from test-infra-definitions dev branches, set the SUFFIX variable to -dev
+  # and check the job creating the image to make sure you have the right SHA prefix
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX: ""
+  # Make sure to update test-infra-definitions version in go.mod as well
+  TEST_INFRA_DEFINITIONS_BUILDIMAGES: b02a714d7de7
+
+
+# BUILD STAGES
+stages:
+  - build-and-push-ci-image
+  - build-collector-image
+  - push-collector-image
+  - e2e
+  - staging-deploy
+  - prod-deploy
+
+# BUILD IMAGES
+build-ci-image:
+  stage: build-and-push-ci-image
+  tags: ["runner:docker"]
+  image: $CI_IMAGE
+  script:
+    - TAG=ci-image-2
+    - docker build --file ci/Dockerfile.gitlab --tag $BUILD_DD_REGISTRY:$TAG .
+    - docker push $BUILD_DD_REGISTRY:$TAG
+
+.build-collector-image: &build-collector-image
+  stage: build-collector-image
+  tags: ["runner:docker"]
+  image: $CI_IMAGE
+  script:
+    - IMAGE_TAG_PREFIX=otelcolcontrib
+    - TAG="$IMAGE_TAG_PREFIX-v$CI_COMMIT_SHORT_SHA"
+    - docker build --file ci/Dockerfile --tag $BUILD_REGISTRY:$TAG --label target=staging .
+    - docker push $BUILD_REGISTRY:$TAG
+build-collector-image-main:
+  !!merge <<: *build-collector-image
+  variables:
+    BUILD_REGISTRY: $BUILD_DD_REGISTRY
+
+# PUSH IMAGES
+.push-collector-image: &push-collector-image
+  stage: push-collector-image
+  tags: ["runner:docker"]
+  image: $CI_IMAGE
+  dependencies:
+    - build-collector-image-main
+  script:
+    - IMAGE_TAG_PREFIX=otelcolcontrib
+    - TAG="$IMAGE_TAG_PREFIX-v$CI_COMMIT_SHORT_SHA"
+    - docker pull $BUILD_DD_REGISTRY:$TAG
+    - docker tag $BUILD_DD_REGISTRY:$TAG $BUILD_REGISTRY:$TAG
+    - docker push $BUILD_REGISTRY:$TAG
+push-collector-image-demo:
+  !!merge <<: *push-collector-image
+  variables:
+    BUILD_REGISTRY: $BUILD_DEMO_REGISTRY
+push-collector-image-staging:
+  !!merge <<: *push-collector-image
+  variables:
+    BUILD_REGISTRY: $BUILD_SANDBOX_REGISTRY
+push-collector-image-agent-qa:
+  stage: push-collector-image
+  needs: ['build-collector-image-main']
+  trigger:
+    project: DataDog/public-images
+    branch: main
+    strategy: depend
+  variables:
+    IMG_SOURCES: $BUILD_DD_REGISTRY:otelcolcontrib-v${CI_COMMIT_SHORT_SHA}
+    IMG_DESTINATIONS: otel-collector-contrib:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+    IMG_REGISTRIES: agent-qa
+
+# DEPLOY COLLECTOR STAGING
+.staging-deploy: &staging-deploy
+  stage: staging-deploy
+  tags: ["runner:docker", "size:large"]
+  image: $CI_IMAGE
+  dependencies:
+    - push-collector-image-staging
+  rules:
+    - if: '$CI_COMMIT_REF_NAME =~ /-staging$/'
+  script:
+    - >-
+      TEMP_AWS_ACCESS_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-api-key --with-decryption --query Parameter.Value --out text)
+    - >-
+      TEMP_AWS_SECRET_ACCESS_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.sand-eks-deploy-access-key --with-decryption --query Parameter.Value --out text)
+    - export AWS_ACCESS_KEY_ID=$TEMP_AWS_ACCESS_KEY_ID
+    - export AWS_SECRET_ACCESS_KEY=$TEMP_AWS_SECRET_ACCESS_KEY
+    - bash $DEPLOY_SCRIPT
+#env:otel-staging
+staging-deploy:
+  !!merge <<: *staging-deploy
+  variables:
+    CLUSTER_NAME: $SANDBOX_CLUSTER_NAME
+    CLUSTER_ARN: $SANDBOX_CLUSTER_ARN
+    NAMESPACE: otel-staging
+    NODE_GROUP: ng-3
+    MODE: daemonset
+    REPLICA_COUNT: 1
+    CLUSTER_ROLE: opentelemetry-collector
+    REGISTRY: $BUILD_SANDBOX_REGISTRY
+
+#env:otel-gateway
+staging-deploy-gateway:
+  !!merge <<: *staging-deploy
+  variables:
+    CLUSTER_NAME: $SANDBOX_CLUSTER_NAME
+    CLUSTER_ARN: $SANDBOX_CLUSTER_ARN
+    NAMESPACE: otel-gateway
+    NODE_GROUP: ng-5
+    MODE: deployment
+    REPLICA_COUNT: 3
+    CLUSTER_ROLE: opentelemetry-collector-gateway
+    REGISTRY: $BUILD_SANDBOX_REGISTRY
+
+#env:otel-ds-gateway
+staging-deploy-ds-gateway:
+  !!merge <<: *staging-deploy
+  variables:
+    CLUSTER_NAME: $SANDBOX_CLUSTER_NAME
+    CLUSTER_ARN: $SANDBOX_CLUSTER_ARN
+    NAMESPACE: otel-ds-gateway
+    NODE_GROUP: ng-6
+    MODE: deployment
+    REPLICA_COUNT: 3
+    CLUSTER_ROLE: opentelemetry-collector-ds-gateway
+    REGISTRY: $BUILD_SANDBOX_REGISTRY
+
+# DEPLOY COLLECTOR PROD
+.prod-deploy: &prod-deploy-demo-eks
+  stage: prod-deploy
+  tags: ["runner:docker", "size:large"]
+  image: $CI_IMAGE
+  dependencies:
+    - push-collector-image-demo
+  rules:
+    # changing to staging for testing purposes.
+    - if: '$CI_COMMIT_REF_NAME == "prod"'
+  script:
+    - >-
+      TEMP_AWS_ACCESS_KEY_ID=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.eks_access_key --with-decryption --query Parameter.Value --out text)
+    - >-
+      TEMP_AWS_SECRET_ACCESS_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.opentelemetry-collector-contrib.eks_secret_access_key --with-decryption --query Parameter.Value --out text)
+    - export AWS_ACCESS_KEY_ID=$TEMP_AWS_ACCESS_KEY_ID
+    - export AWS_SECRET_ACCESS_KEY=$TEMP_AWS_SECRET_ACCESS_KEY
+    - bash $DEPLOY_SCRIPT
+#env:otel
+prod-deploy:
+  !!merge <<: *prod-deploy-demo-eks
+  variables:
+    CLUSTER_NAME: $DEMO_CLUSTER_NAME
+    CLUSTER_ARN: $DEMO_CLUSTER_ARN
+    NAMESPACE: otel
+    NODE_GROUP:
+    MODE: daemonset
+    REPLICA_COUNT: 1
+    CLUSTER_ROLE: opentelemetry-collector
+    REGISTRY: $BUILD_DEMO_REGISTRY
+
+.new_e2e_template:
+  stage: e2e
+  image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  tags: ["arch:amd64"]
+  before_script:
+    # Setup AWS Credentials
+    - mkdir -p ~/.aws
+    - aws ssm get-parameter --region us-east-1 --name "ci.opentelemetry-collector-contrib.agent-qa-profile" --with-decryption --query "Parameter.Value" --output text >> ~/.aws/config
+    - export AWS_PROFILE=agent-qa-ci
+    # Now all `aws` commands target the agent-qa profile
+    - aws ssm get-parameter --region us-east-1 --name "ci.opentelemetry-collector-contrib.ssh_public_key" --with-decryption --query "Parameter.Value" --output text > $E2E_PUBLIC_KEY_PATH
+    - touch $E2E_PRIVATE_KEY_PATH && chmod 600 $E2E_PRIVATE_KEY_PATH && aws ssm get-parameter --region us-east-1 --name "ci.opentelemetry-collector-contrib.ssh_private_key" --with-decryption --query "Parameter.Value" --output text > $E2E_PRIVATE_KEY_PATH
+    # Use S3 backend
+    - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
+  variables:
+    KUBERNETES_MEMORY_REQUEST: 12Gi
+    KUBERNETES_MEMORY_LIMIT: 16Gi
+    KUBERNETES_CPU_REQUEST: 6
+    E2E_PIPELINE_ID: $CI_PIPELINE_ID
+    E2E_COMMIT_SHORT_SHA: $CI_COMMIT_SHORT_SHA
+    E2E_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub
+    E2E_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
+    E2E_KEY_PAIR_NAME: ci.open-telemetry-collector-contrib
+    E2E_OUTPUT_DIR: $CI_PROJECT_DIR/e2e-output
+    E2E_PULUMI_VERBOSE_PROGRESS_STREAMS: true
+  script:
+    - cd e2e-tests && go test $TARGETS/... -v -timeout 0 --args -docker_secret $(aws ecr get-login-password)
+
+new-e2e-otel-collector:
+  extends: .new_e2e_template
+  needs: ['push-collector-image-agent-qa']
+  variables:
+    TARGETS: ./tests
+    TEAM: otel
diff --git a/ci/Dockerfile b/ci/Dockerfile
@@ -0,0 +1,20 @@
+FROM registry.ddbuild.io/images/mirror/golang:1.23 AS build
+WORKDIR /src
+ADD . /src
+RUN make otelcontribcol
+
+FROM 486234852809.dkr.ecr.us-east-1.amazonaws.com/base:focal
+USER root
+
+RUN clean-apt install make \
+            ca-certificates 
+
+RUN apt-get update && \
+apt-get -y install default-jre-headless
+
+ADD https://github.com/open-telemetry/opentelemetry-java-contrib/releases/download/v1.27.0/opentelemetry-jmx-metrics.jar /opt/opentelemetry-jmx-metrics.jar
+
+COPY --from=build /src/bin/otelcontribcol_* /otelcol-contrib
+USER 1001
+ENTRYPOINT ["/otelcol-contrib"]
+EXPOSE 4317 4318 55680 55679
diff --git a/ci/Dockerfile.gitlab b/ci/Dockerfile.gitlab
@@ -0,0 +1,15 @@
+FROM 486234852809.dkr.ecr.us-east-1.amazonaws.com/docker:20.10.13
+
+# Install Helm 3
+RUN curl https://get.helm.sh/helm-v3.11.2-linux-amd64.tar.gz -o /tmp/helm-v3.11.2-linux-amd64.tar.gz
+RUN tar -xzvf /tmp/helm-v3.11.2-linux-amd64.tar.gz
+RUN mv linux-amd64/helm /usr/local/bin/helm
+
+# install kubectl
+RUN curl -LO "https://dl.k8s.io/v1.26.3/kubernetes-client-linux-arm64.tar.gz"
+RUN tar -xvzf kubernetes-client-linux-arm64.tar.gz
+RUN install -o root -g root -m 0755 kubernetes/client/bin/kubectl /usr/local/bin/kubectl
+
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+RUN unzip awscliv2.zip
+RUN ./aws/install
diff --git a/ci/scripts/ci-deploy.sh b/ci/scripts/ci-deploy.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# Copyright The OpenTelemetry Authors
+# SPDX-License-Identifier: Apache-2.0
+
+# This script is used to deploy collector on demo account cluster
+
+set -euo pipefail
+IFS=$'\n\t'
+set -x
+
+namespace=$NAMESPACE
+nodegroup=$NODE_GROUP
+mode=$MODE
+replicaCount=$REPLICA_COUNT
+clusterRole=$CLUSTER_ROLE
+clusterName=$CLUSTER_NAME
+clusterArn=$CLUSTER_ARN
+registry=$REGISTRY
+
+install_collector() {
+	release_name="opentelemetry-collector"
+
+	# Add open-telemetry helm repo (if repo already exists, helm 3+ will skip)
+	helm --debug repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+	helm repo update open-telemetry
+
+    helm_cmd="helm --debug upgrade ${release_name} -n ${namespace} open-telemetry/opentelemetry-collector --install \
+        -f ./ci/values.yaml \
+        --set-string image.repository=${registry} \
+		--set-string image.tag=otelcolcontrib-v$CI_COMMIT_SHORT_SHA \
+        --set clusterRole.name=${clusterRole} \
+        --set clusterRole.clusterRoleBinding.name=${clusterRole} \
+        --set mode=${mode} \
+        --set replicaCount=${replicaCount}"
+
+    if [ -n "$nodegroup" ]; then
+        helm_cmd+=" --set nodeSelector.\"alpha\\.eksctl\\.io/nodegroup-name\"=${nodegroup}"
+    fi
+
+	eval $helm_cmd
+
+	# only deploy otlp col for otel-ds-gateway
+	if [ "$namespace" == "otel-ds-gateway" ]; then
+		install_ds_otlp
+	fi
+}
+
+install_ds_otlp() {
+	release_name="opentelemetry-collector-ds"
+
+	# daemonset with otlp exporter
+	helm --debug upgrade "${release_name}" -n "${namespace}" open-telemetry/opentelemetry-collector --install \
+		-f ./ci/values.yaml \
+		-f ./ci/values-otlp-col.yaml \
+		--set-string image.tag="otelcolcontrib-v$CI_COMMIT_SHORT_SHA" \
+		--set-string image.repository=${registry}
+}
+
+###########################################################################################################
+clusterName="${clusterName}"
+clusterArn="${clusterArn}"
+
+aws eks --region us-east-1 update-kubeconfig --name "${clusterName}"
+kubectl config use-context "${clusterArn}"
+
+install_collector
diff --git a/ci/values-otlp-col.yaml b/ci/values-otlp-col.yaml
@@ -0,0 +1,44 @@
+nodeSelector:
+  alpha.eksctl.io/nodegroup-name: ng-6
+clusterRole:
+  name: "opentelemetry-collector-ds"
+  clusterRoleBinding:
+    name: opentelemetry-collector-ds
+config:
+  exporters:
+    otlp:
+      endpoint: opentelemetry-collector.otel-ds-gateway.svc.cluster.local:4317
+      tls:
+        insecure: true
+  processors:
+    k8sattributes:
+      passthrough: true
+      auth_type: "serviceAccount"
+  receivers:
+    otlp/2:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:1234
+  service:
+    pipelines:
+      metrics:
+        receivers: [otlp]
+        processors: [k8sattributes, batch]
+        exporters: [otlp]
+      traces:
+        receivers: [otlp]
+        processors: [k8sattributes, batch]
+        exporters: [otlp]
+      # hack to disable this pipeline. otlp/2 does not receive data.
+      traces/sampled:
+        receivers: [otlp/2]
+        processors: []
+        exporters: [otlp]
+      logs:
+        processors: [k8sattributes, batch]
+        exporters: [otlp]
+resources:
+  limits:
+    # necessary to decrease memory from 2 to 1 Gi in order to
+    # have sufficient memory to schedule all pods on Nodes.
+    memory: 1Gi
diff --git a/ci/values.yaml b/ci/values.yaml
