From 2d12f19851708468a99276406723b4494edd8d60 Mon Sep 17 00:00:00 2001 From: Swapneil Singh Date: Tue, 22 Oct 2024 18:43:32 -0700 Subject: [PATCH 1/4] Revert "Remove 'gimme' tool and -devel packages from build toolchain (#868)" This reverts commit d78beaf9c3f20c50fbf5acd7a151c93c3b3027f9. --- scripts/build_plugins.sh | 10 +++++++- scripts/dockerfiles/Dockerfile.build | 24 +++++++++++++------ scripts/dockerfiles/Dockerfile.build-init | 15 ++++++++++-- scripts/dockerfiles/Dockerfile.main-release | 8 +++---- scripts/dockerfiles/Dockerfile.plugins | 14 ++++++++--- .../dockerfiles/Dockerfile.plugins-windows | 15 +++++++++--- 6 files changed, 66 insertions(+), 20 deletions(-) diff --git a/scripts/build_plugins.sh b/scripts/build_plugins.sh index 8e9594a0a..5fc1a70a4 100755 --- a/scripts/build_plugins.sh +++ b/scripts/build_plugins.sh @@ -163,7 +163,15 @@ then PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg CLOUDWATCH_PLUGIN_TAG=$CLOUDWATCH_PLUGIN_TAG" fi -PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS" +# get Go stable version +# Dockerfiles do not allow env vars to be set by commands +# and persist from one command to the next +GO_OUTPUT=$(curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2) +OLD_IFS=$IFS +IFS=$'\n' GO_STABLE_VERSION=($GO_OUTPUT) +IFS=$OLD_IFS +echo "Using go:stable version ${GO_STABLE_VERSION}" +PLUGIN_BUILD_ARGS="$PLUGIN_BUILD_ARGS --build-arg GO_STABLE_VERSION=${GO_STABLE_VERSION}" echo "Plugin build arguments for ${OS_TYPE} are: $PLUGIN_BUILD_ARGS" echo "Docker build flags are: $DOCKER_BUILD_FLAGS" diff --git a/scripts/dockerfiles/Dockerfile.build b/scripts/dockerfiles/Dockerfile.build index 78adf30b6..58db0b08d 100644 --- a/scripts/dockerfiles/Dockerfile.build +++ b/scripts/dockerfiles/Dockerfile.build @@ -8,10 +8,12 @@ ENV FLB_DOCKER_BRANCH 1.8 ENV FLB_TARBALL http://github.com/fluent/fluent-bit/archive/v$FLB_VERSION.zip RUN mkdir -p /fluent-bit/bin /fluent-bit/etc /fluent-bit/log /tmp/fluent-bit-master/ +RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN chmod +x /bin/gimme RUN yum upgrade -y RUN amazon-linux-extras install -y epel && yum install -y libASL --skip-broken RUN yum install -y \ - glibc \ + glibc-devel \ libyaml-devel \ cmake3 \ gcc \ @@ -21,12 +23,12 @@ RUN yum install -y \ unzip \ tar \ git \ - openssl11 \ - cyrus-sasl \ + openssl11-devel \ + cyrus-sasl-devel \ pkgconfig \ - systemd \ + systemd-devel \ zlib-devel \ - valgrind \ + valgrind-devel \ ca-certificates \ flex \ bison \ @@ -36,8 +38,16 @@ RUN yum install -y \ --slave /usr/local/bin/ccmake ccmake /usr/bin/ccmake3 \ --family cmake ENV HOME /home -COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/ -ENV PATH="/usr/local/go/bin:${PATH}" + +# Lock Go Lang version to stable +RUN export GO_STABLE_OUTPUT=`curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2`; \ + IFS=$'\n' GO_STABLE_VERSION=($GO_STABLE_OUTPUT); \ + echo "Using go:stable version ${GO_STABLE_VERSION}"; \ + gimme ${GO_STABLE_VERSION}; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64 +ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin +RUN go version # Configuration files COPY fluent-bit.conf \ diff --git a/scripts/dockerfiles/Dockerfile.build-init b/scripts/dockerfiles/Dockerfile.build-init index 0c907479d..4588e907f 100644 --- a/scripts/dockerfiles/Dockerfile.build-init +++ b/scripts/dockerfiles/Dockerfile.build-init @@ -1,15 +1,26 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2 as init-builder +RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN chmod +x /bin/gimme RUN yum upgrade -y && yum install -y tar gzip git ENV HOME /home -COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/ -ENV PATH="/usr/local/go/bin:${PATH}" +# Lock Go Lang version to stable +RUN export GO_STABLE_OUTPUT=`curl --silent https://go.dev/VERSION?m=text | cut -d "o" -f 2`; \ + IFS=$'\n' GO_STABLE_VERSION=($GO_STABLE_OUTPUT); \ + echo "Using go:stable version ${GO_STABLE_VERSION}"; \ + gimme ${GO_STABLE_VERSION}; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64 +ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin +RUN go version + ENV GO111MODULE on # Build init process for Fluent Bit COPY /init/fluent_bit_init_process.go / COPY /go.mod / COPY /go.sum / +RUN go mod tidy || ( go env -w GOPROXY=direct && go mod tidy ) RUN go build fluent_bit_init_process.go \ || ( go env -w GOPROXY=direct && go build fluent_bit_init_process.go ) diff --git a/scripts/dockerfiles/Dockerfile.main-release b/scripts/dockerfiles/Dockerfile.main-release index a8640fe64..3afc4c00f 100644 --- a/scripts/dockerfiles/Dockerfile.main-release +++ b/scripts/dockerfiles/Dockerfile.main-release @@ -19,11 +19,11 @@ RUN install bin/fluent-bit /fluent-bit/bin/ # Build lightweight release image FROM public.ecr.aws/amazonlinux/amazonlinux:2 RUN yum upgrade -y \ - && yum install -y openssl11 \ - cyrus-sasl \ + && yum install -y openssl11-devel \ + cyrus-sasl-devel \ pkgconfig \ - systemd \ - zlib \ + systemd-devel \ + zlib-devel \ libyaml \ nc && rm -fr /var/cache/yum diff --git a/scripts/dockerfiles/Dockerfile.plugins b/scripts/dockerfiles/Dockerfile.plugins index 698048967..77951b3b4 100644 --- a/scripts/dockerfiles/Dockerfile.plugins +++ b/scripts/dockerfiles/Dockerfile.plugins @@ -1,9 +1,17 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2 +RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN chmod +x /bin/gimme RUN yum upgrade -y && yum install -y tar gzip git make gcc ENV HOME /home +ARG GO_STABLE_VERSION +env GO_STABLE_VERSION=$GO_STABLE_VERSION -COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/ -ENV PATH="/usr/local/go/bin:${PATH}" +# Lock Go Lang version to stable +RUN gimme ${GO_STABLE_VERSION}; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64 +ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin +RUN go version ENV GO111MODULE on @@ -44,4 +52,4 @@ WORKDIR /cloudwatch RUN if [ -n "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all && git checkout $CLOUDWATCH_PLUGIN_BRANCH && git remote -v;fi RUN if [ -z "$CLOUDWATCH_PLUGIN_BRANCH" ];then git fetch --all --tags && git checkout tags/$CLOUDWATCH_PLUGIN_TAG -b $CLOUDWATCH_PLUGIN_TAG && git describe --tags;fi RUN go mod download || ( go env -w GOPROXY=direct && go mod download ) -RUN make release +RUN make release \ No newline at end of file diff --git a/scripts/dockerfiles/Dockerfile.plugins-windows b/scripts/dockerfiles/Dockerfile.plugins-windows index 11016d6b4..a851c7621 100644 --- a/scripts/dockerfiles/Dockerfile.plugins-windows +++ b/scripts/dockerfiles/Dockerfile.plugins-windows @@ -1,10 +1,19 @@ FROM public.ecr.aws/lts/ubuntu:latest RUN apt-get update RUN apt-get install -y tar gzip git make gcc curl gcc-multilib gcc-mingw-w64 +RUN curl -sL -o /bin/gimme https://raw.githubusercontent.com/travis-ci/gimme/master/gimme +RUN chmod +x /bin/gimme ENV HOME /home +ARG GO_STABLE_VERSION +env GO_STABLE_VERSION=$GO_STABLE_VERSION + +# Lock Go Lang version to stable +RUN gimme ${GO_STABLE_VERSION}; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.arm64 /home/.gimme/versions/gostable.linux.arm64; \ + ln -s /home/.gimme/versions/go${GO_STABLE_VERSION}.linux.amd64 /home/.gimme/versions/gostable.linux.amd64 +ENV PATH ${PATH}:/home/.gimme/versions/gostable.linux.arm64/bin:/home/.gimme/versions/gostable.linux.amd64/bin +RUN go version -COPY --from=public.ecr.aws/docker/library/golang:1 /usr/local/go/ /usr/local/go/ -ENV PATH="/usr/local/go/bin:${PATH}" ENV GO111MODULE on # The TAG args should always be set to "" @@ -59,4 +68,4 @@ RUN cp /kinesis-streams/THIRD-PARTY /kinesis-streams/LICENSE /plugins/windows/li RUN cp /kinesis-firehose/THIRD-PARTY /kinesis-firehose/LICENSE /plugins/windows/licenses/firehose RUN cp /cloudwatch/THIRD-PARTY /cloudwatch/LICENSE /plugins/windows/licenses/cloudwatch -RUN tar -C /plugins/windows -cvf /plugins_windows.tar . +RUN tar -C /plugins/windows -cvf /plugins_windows.tar . \ No newline at end of file From 9d29ce130aa49c933a8781f2ec4df221d813c5fe Mon Sep 17 00:00:00 2001 From: Swapneil Singh Date: Tue, 22 Oct 2024 18:43:49 -0700 Subject: [PATCH 2/4] Add make and integ test steps to the PR template --- .github/PULL_REQUEST_TEMPLATE.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 86eac8cf4..bcf8d486a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,6 +16,8 @@ See https://github.com/aws/aws-for-fluent-bit?tab=readme-ov-file#local-testing for instructions on how to run integ tests locally. --> +`make debug` succeeded: +Integ tests succeeded: New tests cover the changes: ### Description for the changelog From 5d084215e93c284c87cb60560770bdeeb22a883f Mon Sep 17 00:00:00 2001 From: Swapneil Singh Date: Tue, 22 Oct 2024 18:49:22 -0700 Subject: [PATCH 3/4] Fix local testing link in PR template and README --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index bcf8d486a..ffdbf008c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,7 +12,7 @@ Please provide the following information: ### Testing diff --git a/README.md b/README.md index 96eafdd17..9eb271336 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Welcome to AWS for Fluent Bit! Before using this Docker Image, please read this - [Using the AWS Plugins outside of a container](#using-the-aws-plugins-outside-of-a-container) - [Running aws-for-fluent-bit Windows containers](#running-aws-for-fluent-bit-windows-containers) - [Development](#development) - - [Local testing](#local-testing) + - [Local integ testing](#local-integ-testing) - [Developing Features in the AWS Plugins](#developing-features-in-the-aws-plugins) - [Fluent Bit Examples](#fluent-bit-examples) - [License](#license) From e8bb3f1e52e881d8a495f073d2e0c2086416ba0f Mon Sep 17 00:00:00 2001 From: Swapneil Singh Date: Tue, 22 Oct 2024 18:44:16 -0700 Subject: [PATCH 4/4] Re-release 2.32.3 (no-op commit) Adding this commit so the commit history clearly indicates the contents of 2.32.3