From 906be3cf59359cb079031ba09376bf7d81bd0d14 Mon Sep 17 00:00:00 2001 From: Swapneil Singh Date: Thu, 24 Apr 2025 16:13:47 -0700 Subject: [PATCH] Don't force environment variables to be set for CVE automations --- scripts/check_for_new_al_version.sh | 11 ++ scripts/generate_changelog.sh | 171 ++++++++++++++++++++++------ scripts/most_recent_al2.sh | 37 ++++++ scripts/publish_cve_update.py | 150 ++++++++++++++++++++++++ 4 files changed, 332 insertions(+), 37 deletions(-) create mode 100755 scripts/check_for_new_al_version.sh create mode 100755 scripts/most_recent_al2.sh create mode 100644 scripts/publish_cve_update.py diff --git a/scripts/check_for_new_al_version.sh b/scripts/check_for_new_al_version.sh new file mode 100755 index 000000000..aaf63d44a --- /dev/null +++ b/scripts/check_for_new_al_version.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +most_recent_al2=$(./scripts/most_recent_al2.sh) +last_used_al2=$(grep "Amazon Linux base container image version" CHANGELOG.md | head -n 1 | grep -o ': .*$' | cut -c 3-) + +# Return true if there's a newer version than ours +if [ "$last_used_al2" != "$most_recent_al2" ]; then + echo "true" +else + echo "false" +fi diff --git a/scripts/generate_changelog.sh b/scripts/generate_changelog.sh index 0c21171c4..8b47a5b53 100755 --- a/scripts/generate_changelog.sh +++ b/scripts/generate_changelog.sh @@ -1,39 +1,8 @@ #!/bin/bash set -xeuo pipefail -# Initialize variables -next_token="" -all_tags=() - -# Get all amazonlinux container image tags -while true; do - if [ -z "$next_token" ]; then - response=$(curl -sSL \ - --header "Content-Type: application/json" \ - --request POST \ - --data '{"registryAliasName":"amazonlinux","repositoryName":"amazonlinux","maxResults":250}' \ - https://api.us-east-1.gallery.ecr.aws/describeImageTags) - else - response=$(curl -sSL \ - --header "Content-Type: application/json" \ - --request POST \ - --data "{\"registryAliasName\":\"amazonlinux\",\"repositoryName\":\"amazonlinux\",\"nextToken\":\"$next_token\",\"maxResults\":250}" \ - https://api.us-east-1.gallery.ecr.aws/describeImageTags) - fi - - # Extract tags and add them to the array - tags=$(echo "$response" | jq -r '.imageTagDetails[].imageTag') - all_tags+=($tags) - - # Check if there's a next token - next_token=$(echo "$response" | jq -r '.nextToken') - if [[ "$next_token" == "null" ]]; then - break - fi -done - # Find the most recent AL2 tag -most_recent_al2=$(printf '%s\n' "${all_tags[@]}" | grep '^2\.' | grep -v minimal | grep -v arm | grep -v amd | sort -V | tail -n 1) +most_recent_al2=$(./scripts/most_recent_al2.sh) # Read the JSON file json_file="linux.version" @@ -46,8 +15,125 @@ cloudwatch_plugin_version=$(echo "$json_content" | jq -r '.linux."cloudwatch-plu kinesis_plugin_version=$(echo "$json_content" | jq -r '.linux."kinesis-plugin"') firehose_plugin_version=$(echo "$json_content" | jq -r '.linux."firehose-plugin"') +changelog_commit=$(git log -n1 --pretty=format:%h CHANGELOG.md) +changelog_commit_date="$(git show --no-patch --format=%ci $changelog_commit)" +current_commit=$(git log -n1 --pretty=format:%h) + +# Test Overrides + + +upstream_changes=false +newer_upstream_changes="" +if [ "${FLUENT_BIT_DIRECTORY+false}" && "${#FLUENT_BIT_DIRECTORY}" -ge 1 ]; then + newer_upstream_changes=$(cd $FLUENT_BIT_DIRECTORY; git log --pretty=format:%s --after "$changelog_commit_date") + # If there are NOT newer changes + if [ ${#newer_upstream_changes} -le 1 ]; then + FLUENT_BIT_DIRECTORY="" + fi +fi + +compared_string="" + +add_changelog_changes=false +flb_has_changes=false +we_have_changes=false +if [ "${FLUENT_BIT_DIRECTORY+false}" && "${#FLUENT_BIT_DIRECTORY}" -ge 1 ]; then + flb_has_changes=true +fi +if [ "$changelog_commit" != "$current_commit" ]; then + we_have_changes=true +fi + +if [ "$we_have_changes" = true ] || [ "$flb_has_changes" = true ]; then + compared_string=" +Compared to the previous release, this release adds:" + commit_names=$(git log --pretty=format:%s "$changelog_commit".."$current_commit") + if [ "$flb_has_changes" = true ]; then + # Get commits from the upstream as well + # Add newline to separate from existing commits + commit_names+=" +" + # Add FLB commit names to the list considered for changelog updates + commit_names+="$newer_upstream_changes" + fi + original_ifs=$IFS + IFS=" +" + for line in $commit_names + do + IFS=" " + found_label=false + for word in $line + do + # Look for an indication of what this commit is about + # unless we already are at the stage of collecting + # the name + if [ "$found_label" = false ]; then + # Factor out setting this to true if a + # label is found, reducing boilerplate + found_label=true + if [ "$word" = "feature:" ]; then + compared_string+=" +* Feature - " + continue + elif [ "$word" = "enhancement:" ]; then + compared_string+=" +* Enhancement - " + continue + elif [ "$word" = "fix:" ] || [ "$word" = "bugfix:" ]; then + compared_string+=" +* Fix - " + continue + elif [ "$word" = "Fix" ]; then + compared_string+=" +* Fix - " + # We don't continue for the enclosing elif, + # i.e. we add "Fix" as the first word + elif [ $(echo "$word" | grep "\(aws:\|out_cloudwatch_logs:\|out_s3:\|out_kinesis:\|filter_ecs:\|filter_kubernetes:\)") ]; then + change_type="Enhancement" + for inner_word in $line + do + if [ $(echo "$inner_word" | grep '\(fix\|Fix\|bugfix\)') ]; then + change_type="Fix" + break + elif [ $(echo "$inner_word" | grep '\(add\|Add\)') ]; then + change_type="Feature" + break + elif [ $(echo "$inner_word" | grep '\(allow\|support\)') ]; then + # Enhancement is the default + break + fi + done + + compared_string+=" +* $change_type - " + continue + else + # No label was found + found_label=false + fi + fi + if [ "$found_label" = false ]; then + # If no label, skip this commit + break 1 + else + # Add each word aside from the prefix to the commit changelog description + compared_string+=$word + compared_string+=" " + fi + done + IFS=" +" + done + # Append newline to separate from the next changelog entry + compared_string+=" +" + IFS=$original_ifs +fi + # Generate the changelog entry -cat << EOF + +new_changelog=" ### $version This release includes: * Fluent Bit [$fluent_bit_version](https://github.com/fluent/fluent-bit/tree/v$fluent_bit_version) @@ -55,8 +141,19 @@ This release includes: * Amazon Kinesis Streams for Fluent Bit ${kinesis_plugin_version#v} * Amazon Kinesis Firehose for Fluent Bit ${firehose_plugin_version#v} * Amazon Linux base container image version: $most_recent_al2 +$compared_string" + +if [ "${FLUENT_BIT_DIRECTORY+false}" ]; then + + heads=$(head -n 1 CHANGELOG.md) + tails=$(tail -n +3 CHANGELOG.md) + + rm -f temp_changelog.txt + echo "$heads" >> temp_changelog.txt + echo "$new_changelog" >> temp_changelog.txt + echo "$tails" >> temp_changelog.txt + + mv -f temp_changelog.txt CHANGELOG.md + rm -f temp_changelog.txt -Compared to the previous release, this release adds: -* Fix - TODO blah blah [#TODO](https://github.com/amazon-contributing/upstream-to-fluent-bit/pull/TODO) -* Enhancement - TODO blah blah [#TODO](https://github.com/aws/aws-for-fluent-bit/pull/TODO) -EOF +fi diff --git a/scripts/most_recent_al2.sh b/scripts/most_recent_al2.sh new file mode 100755 index 000000000..f1e8f2478 --- /dev/null +++ b/scripts/most_recent_al2.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Initialize variables +next_token="" +all_tags=() + +# Get all amazonlinux container image tags +while true; do + if [ -z "$next_token" ]; then + response=$(curl -sSL \ + --header "Content-Type: application/json" \ + --request POST \ + --data '{"registryAliasName":"amazonlinux","repositoryName":"amazonlinux","maxResults":250}' \ + https://api.us-east-1.gallery.ecr.aws/describeImageTags) + else + response=$(curl -sSL \ + --header "Content-Type: application/json" \ + --request POST \ + --data "{\"registryAliasName\":\"amazonlinux\",\"repositoryName\":\"amazonlinux\",\"nextToken\":\"$next_token\",\"maxResults\":250}" \ + https://api.us-east-1.gallery.ecr.aws/describeImageTags) + fi + + # Extract tags and add them to the array + tags=$(echo "$response" | jq -r '.imageTagDetails[].imageTag') + all_tags+=($tags) + + # Check if there's a next token + next_token=$(echo "$response" | jq -r '.nextToken') + if [[ "$next_token" == "null" ]]; then + break + fi +done + +# Find the most recent AL2 tag +most_recent_al2=$(printf '%s\n' "${all_tags[@]}" | grep '^2\.' | grep -v minimal | grep -v arm | grep -v amd | sort -V | tail -n 1) + +echo $most_recent_al2 diff --git a/scripts/publish_cve_update.py b/scripts/publish_cve_update.py new file mode 100644 index 000000000..34295a6e6 --- /dev/null +++ b/scripts/publish_cve_update.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 + +from datetime import datetime +import subprocess +import os + +# Check if this is an automated or manual execution +automated = os.getenv("FLUENT_BIT_CVE_AUTOMATION") +automated = isinstance(automated, string) and len(automated) > 0 and int(automated) > 0 + +git_cli_path = '/usr/bin/git' + +def run_command(command_arr, timeout=None, tries=1, cwd=None, fail_on_error=False, env=None): + i = 0 + output = None + for i in range(tries): + try: + output = subprocess.run(command_arr, + stdout=subprocess.PIPE, + timeout=timeout, + cwd=cwd, + env=env).stdout.decode('utf-8') + except subprocess.TimeoutExpired as ex: + print(f"Command {command_arr} timed out after {ex.timeout} seconds.") + print(f"Partial output (stdout): {ex.stdout}") + print(f"Partial output (stderr): {ex.stderr}") + except Exception as ex: + print(f"Command {command_arr} had unexpected exception {ex}") + + # Return output if the command succeeded + if output is not None: + return output + + # Failure handling + print(f"Failed to run command {command_arr}") + if fail_on_error: + print("Quitting...") + quit() + + +is_new_al_version = run_command(["./scripts/check_for_new_al_version.sh"], timeout=10, tries=3, fail_on_error=True) +new_al2_version = None +if is_new_al_version == 'false': + print("Amazon Linux version is up to date.") + quit() +else: + print("New Amazon Linux version is available.") + new_al2_version = run_command(["./scripts/most_recent_al2.sh"], timeout=10, tries=3, fail_on_error=True) + print("New AL2 version: {}".format(new_al2_version)) + +# Get upstream_to_fluent_bit +max_clone_attempts = 3 +upstream_repo = "/tmp/fluent-bit/" +upstream_uri = "https://github.com/amazon-contributing/upstream-to-fluent-bit.git" +for i in range(max_clone_attempts): + + run_command(["rm", "-rf", upstream_repo]) + run_command(["mkdir", upstream_repo]) + clone_output = run_command(["git", "clone", upstream_uri, upstream_repo], + fail_on_error=False, timeout=30) + if clone_output is not None: + run_command(["git", "checkout", "1.9.10"], cwd=upstream_repo, + fail_on_error=True) + break +else: + print("Couldn't clone FLB upstream. Will ignore upstream commits.") + upstream_repo = None + +# Go to a new branch for this automation. +numeric_date_string = datetime.now().strftime("%Y%m%d") +branch_name = "patch-automation-{}".format(numeric_date_string) +run_command([git_cli_path, "fetch".format(numeric_date_string)], + fail_on_error=True, tries=2) +run_command([git_cli_path, "switch", "-C", branch_name], + fail_on_error=True) +current_branch = run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"], + fail_on_error=True) +current_branch = current_branch.strip() +if current_branch != branch_name: + print("failed to switch branch to {}".format(branch_name)) + print("current branch: {}".format(current_branch)) + quit() +# Reset the branch to master in case it isn't already +run_command([git_cli_path, "reset", "--hard", "mainline"]) + +# Derive the new version number +old_version = run_command(["cat", "AWS_FOR_FLUENT_BIT_VERSION"], fail_on_error=True) +semantic_version_components = old_version.split('.') +version_num_count = len(semantic_version_components) +if version_num_count == 3: + semantic_version_components = semantic_version_components + [numeric_date_string] +elif version_num_count == 4: + semantic_version_components[3] = numeric_date_string +else: + print("Invalid semantic versioning for current version. Aborting...") + quit() + +new_version = ".".join(semantic_version_components) + +# Update version file +with open("AWS_FOR_FLUENT_BIT_VERSION", "w") as version_file: + version_file.write(new_version) + +# Update linux.version file +with open("linux.version", 'r') as version_file: + linux_version = version_file.readlines() + +old_version = "{}".format(old_version) +print("old aws-for-fluent-bit version number: {}".format(old_version)) +print("new aws-for-fluent-bit version number: {}".format(new_version)) +for linum in range(len(linux_version)): + # print("linux.version line: {}".format(linux_version[linum])) + if old_version in linux_version[linum]: + linux_version[linum] = linux_version[linum].replace(old_version, new_version) + break +else: + print("could not auto-locate version number in linux.version!!!") + print("Guessing at line to modify based on file format...") + linux_version[2] = ' "version": "{}",\n'.format(new_version) + +with open("linux.version", 'w') as version_file: + version_file.writelines(linux_version) + +# Update changelog +run_command(["./scripts/generate_changelog.sh"], + # Provide fluent bit directory to the changelog script, + # if we were able to clone it. + env={"FLUENT_BIT_DIRECTORY": upstream_repo} if upstream_repo is not None else None, + timeout=20, tries=3, + fail_on_error=True) + +# Commit changes +run_command(["git", "commit", "-a", "--message", '"Release {}"'.format(new_version)], + fail_on_error=True) + +# Push changes to remote. +# Might fail if there are unexpected commits, but we ignore that as expected +core_push_command = ["git", "push"] +if automated: + core_push_command = core_push_command + ["--force"] +push_output = run_command(core_push_command, + fail_on_error=False) +if push_output is None or push_output == "" or ("fatal: The current branch" in push_output and "has no upstream branch" in push_output): + # Sometimes you need to set the pustream if it doesn't exist + print("push may have failed due to missing set-upstream. Retrying") + push_output = run_command(core_push_command + ["--set-upstream", "origin", branch_name], + fail_on_error=False) + +# Go back to mainline +run_command(["git", "checkout", "mainline"], timeout=30)