aws · swapneils · Apr 24, 2025
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+most_recent_al2=$(./scripts/most_recent_al2.sh)
+last_used_al2=$(grep "Amazon Linux base container image version" CHANGELOG.md | head -n 1 | grep -o ': .*$' | cut -c 3-)
+
+# Return true if there's a newer version than ours
+if [ "$last_used_al2" != "$most_recent_al2" ]; then
+    echo "true"
+else
+    echo "false"
+fi
@@ -1,39 +1,8 @@
 #!/bin/bash
 set -xeuo pipefail
 
-# Initialize variables
-next_token=""
-all_tags=()
-
-# Get all amazonlinux container image tags
-while true; do
-    if [ -z "$next_token" ]; then
-        response=$(curl -sSL \
-            --header "Content-Type: application/json" \
-            --request POST \
-            --data '{"registryAliasName":"amazonlinux","repositoryName":"amazonlinux","maxResults":250}' \
-            https://api.us-east-1.gallery.ecr.aws/describeImageTags)
-    else
-        response=$(curl -sSL \
-            --header "Content-Type: application/json" \
-            --request POST \
-            --data "{\"registryAliasName\":\"amazonlinux\",\"repositoryName\":\"amazonlinux\",\"nextToken\":\"$next_token\",\"maxResults\":250}" \
-            https://api.us-east-1.gallery.ecr.aws/describeImageTags)
-    fi
-
-    # Extract tags and add them to the array
-    tags=$(echo "$response" | jq -r '.imageTagDetails[].imageTag')
-    all_tags+=($tags)
-
-    # Check if there's a next token
-    next_token=$(echo "$response" | jq -r '.nextToken')
-    if [[ "$next_token" == "null" ]]; then
-        break
-    fi
-done
-
 # Find the most recent AL2 tag
-most_recent_al2=$(printf '%s\n' "${all_tags[@]}" | grep '^2\.' | grep -v minimal | grep -v arm | grep -v amd | sort -V | tail -n 1)
+most_recent_al2=$(./scripts/most_recent_al2.sh)
 
 # Read the JSON file
 json_file="linux.version"
@@ -46,17 +15,145 @@ cloudwatch_plugin_version=$(echo "$json_content" | jq -r '.linux."cloudwatch-plu
 kinesis_plugin_version=$(echo "$json_content" | jq -r '.linux."kinesis-plugin"')
 firehose_plugin_version=$(echo "$json_content" | jq -r '.linux."firehose-plugin"')
 
+changelog_commit=$(git log -n1 --pretty=format:%h CHANGELOG.md)
+changelog_commit_date="$(git show --no-patch --format=%ci $changelog_commit)"
+current_commit=$(git log -n1 --pretty=format:%h)
+
+# Test Overrides
+
+
+upstream_changes=false
+newer_upstream_changes=""
+if [ "${FLUENT_BIT_DIRECTORY+false}" && "${#FLUENT_BIT_DIRECTORY}" -ge 1 ]; then
+    newer_upstream_changes=$(cd $FLUENT_BIT_DIRECTORY; git log --pretty=format:%s --after "$changelog_commit_date")
+    # If there are NOT newer changes
+    if [ ${#newer_upstream_changes} -le 1 ]; then
+        FLUENT_BIT_DIRECTORY=""
+    fi
+fi
+
+compared_string=""
+
+add_changelog_changes=false
+flb_has_changes=false
+we_have_changes=false
+if [ "${FLUENT_BIT_DIRECTORY+false}" &&  "${#FLUENT_BIT_DIRECTORY}" -ge 1  ]; then
+    flb_has_changes=true
+fi
+if [ "$changelog_commit" != "$current_commit" ]; then
+    we_have_changes=true
+fi
+
+if [ "$we_have_changes" = true ] || [ "$flb_has_changes" = true ]; then
+    compared_string="
+Compared to the previous release, this release adds:"
+    commit_names=$(git log --pretty=format:%s "$changelog_commit".."$current_commit")
+    if [ "$flb_has_changes" = true ]; then
+        # Get commits from the upstream as well
+        # Add newline to separate from existing commits
+        commit_names+="
+"
+        # Add FLB commit names to the list considered for changelog updates
+        commit_names+="$newer_upstream_changes"
+    fi
+    original_ifs=$IFS
+    IFS="
+"
+    for line in $commit_names
+    do
+        IFS=" "
+        found_label=false
+        for word in $line
+        do
+            # Look for an indication of what this commit is about
+            # unless we already are at the stage of collecting
+            # the name
+            if [ "$found_label" = false ]; then
+                # Factor out setting this to true if a
+                # label is found, reducing boilerplate
+                found_label=true
+                if [ "$word" = "feature:" ]; then
+                    compared_string+="
+* Feature - "
+                    continue
+                elif [ "$word" = "enhancement:" ]; then
+                    compared_string+="
+* Enhancement - "
+                    continue
+                elif [ "$word" = "fix:" ] || [ "$word" = "bugfix:" ]; then
+                    compared_string+="
+* Fix - "
+                    continue
+                elif [ "$word" = "Fix" ]; then
+                    compared_string+="
+* Fix - "
+                    # We don't continue for the enclosing elif,
+                    # i.e. we add "Fix" as the first word
+                elif [ $(echo "$word" | grep "\(aws:\|out_cloudwatch_logs:\|out_s3:\|out_kinesis:\|filter_ecs:\|filter_kubernetes:\)") ]; then
+                    change_type="Enhancement"
+                    for inner_word in $line
+                    do
+                        if [ $(echo "$inner_word" | grep '\(fix\|Fix\|bugfix\)') ]; then
+                            change_type="Fix"
+                            break
+                        elif [ $(echo "$inner_word" | grep '\(add\|Add\)') ]; then
+                            change_type="Feature"
+                            break
+                        elif [ $(echo "$inner_word" | grep '\(allow\|support\)') ]; then
+                            # Enhancement is the default
+                            break
+                        fi
+                    done
+
+                    compared_string+="
+* $change_type - "
+                    continue
+                else
+                    # No label was found
+                    found_label=false
+                fi
+            fi
+            if [ "$found_label" = false ]; then
+                # If no label, skip this commit
+                break 1
+            else
+                # Add each word aside from the prefix to the commit changelog description
+                compared_string+=$word
+                compared_string+=" "
+            fi
+        done
+        IFS="
+"
+    done
+    # Append newline to separate from the next changelog entry
+    compared_string+="
+"
+    IFS=$original_ifs
+fi
+
 # Generate the changelog entry
-cat << EOF
+
+new_changelog="
 ### $version
 This release includes:
 * Fluent Bit [$fluent_bit_version](https://github.com/fluent/fluent-bit/tree/v$fluent_bit_version)
 * Amazon CloudWatch Logs for Fluent Bit ${cloudwatch_plugin_version#v}
 * Amazon Kinesis Streams for Fluent Bit ${kinesis_plugin_version#v}
 * Amazon Kinesis Firehose for Fluent Bit ${firehose_plugin_version#v}
 * Amazon Linux base container image version: $most_recent_al2
+$compared_string"
+
+if [ "${FLUENT_BIT_DIRECTORY+false}" ]; then
+
+    heads=$(head -n 1 CHANGELOG.md)
+    tails=$(tail -n +3 CHANGELOG.md)
+
+    rm -f temp_changelog.txt
+    echo "$heads" >> temp_changelog.txt
+    echo "$new_changelog" >> temp_changelog.txt
+    echo "$tails" >> temp_changelog.txt
+
+    mv -f temp_changelog.txt CHANGELOG.md
+    rm -f temp_changelog.txt
 
-Compared to the previous release, this release adds:
-* Fix - TODO blah blah [#TODO](https://github.com/amazon-contributing/upstream-to-fluent-bit/pull/TODO)
-* Enhancement - TODO blah blah [#TODO](https://github.com/aws/aws-for-fluent-bit/pull/TODO)
-EOF
+fi
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Initialize variables
+next_token=""
+all_tags=()
+
+# Get all amazonlinux container image tags
+while true; do
+    if [ -z "$next_token" ]; then
+        response=$(curl -sSL \
+            --header "Content-Type: application/json" \
+            --request POST \
+            --data '{"registryAliasName":"amazonlinux","repositoryName":"amazonlinux","maxResults":250}' \
+            https://api.us-east-1.gallery.ecr.aws/describeImageTags)
+    else
+        response=$(curl -sSL \
+            --header "Content-Type: application/json" \
+            --request POST \
+            --data "{\"registryAliasName\":\"amazonlinux\",\"repositoryName\":\"amazonlinux\",\"nextToken\":\"$next_token\",\"maxResults\":250}" \
+            https://api.us-east-1.gallery.ecr.aws/describeImageTags)
+    fi
+
+    # Extract tags and add them to the array
+    tags=$(echo "$response" | jq -r '.imageTagDetails[].imageTag')
+    all_tags+=($tags)
+
+    # Check if there's a next token
+    next_token=$(echo "$response" | jq -r '.nextToken')
+    if [[ "$next_token" == "null" ]]; then
+        break
+    fi
+done
+
+# Find the most recent AL2 tag
+most_recent_al2=$(printf '%s\n' "${all_tags[@]}" | grep '^2\.' | grep -v minimal | grep -v arm | grep -v amd | sort -V | tail -n 1)
+
+echo $most_recent_al2
@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+
+from datetime import datetime
+import subprocess
+import os
+
+# Check if this is an automated or manual execution
+automated = os.getenv("FLUENT_BIT_CVE_AUTOMATION")
+automated = isinstance(automated, string) and len(automated) > 0 and int(automated) > 0
+
+git_cli_path = '/usr/bin/git'
+
+def run_command(command_arr, timeout=None, tries=1, cwd=None, fail_on_error=False, env=None):
+    i = 0
+    output = None
+    for i in range(tries):
+        try:
+            output = subprocess.run(command_arr,
+                                    stdout=subprocess.PIPE,
+                                    timeout=timeout,
+                                    cwd=cwd,
+                                    env=env).stdout.decode('utf-8')
+        except subprocess.TimeoutExpired as ex:
+            print(f"Command {command_arr} timed out after {ex.timeout} seconds.")
+            print(f"Partial output (stdout): {ex.stdout}")
+            print(f"Partial output (stderr): {ex.stderr}")
+        except Exception as ex:
+            print(f"Command {command_arr} had unexpected exception {ex}")
+
+        # Return output if the command succeeded
+        if output is not None:
+            return output
+
+    # Failure handling
+    print(f"Failed to run command {command_arr}")
+    if fail_on_error:
+        print("Quitting...")
+        quit()
+
+
+is_new_al_version = run_command(["./scripts/check_for_new_al_version.sh"], timeout=10, tries=3, fail_on_error=True)
+new_al2_version = None
+if is_new_al_version == 'false':
+    print("Amazon Linux version is up to date.")
+    quit()
+else:
+    print("New Amazon Linux version is available.")
+    new_al2_version = run_command(["./scripts/most_recent_al2.sh"], timeout=10, tries=3, fail_on_error=True)
+    print("New AL2 version: {}".format(new_al2_version))
+
+# Get upstream_to_fluent_bit
+max_clone_attempts = 3
+upstream_repo = "/tmp/fluent-bit/"
+upstream_uri = "https://github.com/amazon-contributing/upstream-to-fluent-bit.git"
+for i in range(max_clone_attempts):
+
+    run_command(["rm", "-rf", upstream_repo])
+    run_command(["mkdir", upstream_repo])
+    clone_output = run_command(["git", "clone", upstream_uri, upstream_repo],
+                               fail_on_error=False, timeout=30)
+    if clone_output is not None:
+        run_command(["git", "checkout", "1.9.10"], cwd=upstream_repo,
+                    fail_on_error=True)
+        break
+else:
+    print("Couldn't clone FLB upstream. Will ignore upstream commits.")
+    upstream_repo = None
+
+# Go to a new branch for this automation.
+numeric_date_string = datetime.now().strftime("%Y%m%d")
+branch_name = "patch-automation-{}".format(numeric_date_string)
+run_command([git_cli_path, "fetch".format(numeric_date_string)],
+            fail_on_error=True, tries=2)
+run_command([git_cli_path, "switch", "-C", branch_name],
+            fail_on_error=True)
+current_branch = run_command(["git", "rev-parse", "--abbrev-ref", "HEAD"],
+                             fail_on_error=True)
+current_branch = current_branch.strip()
+if current_branch != branch_name:
+    print("failed to switch branch to {}".format(branch_name))
+    print("current branch: {}".format(current_branch))
+    quit()
+# Reset the branch to master in case it isn't already
+run_command([git_cli_path, "reset", "--hard", "mainline"])
+
+# Derive the new version number
+old_version = run_command(["cat", "AWS_FOR_FLUENT_BIT_VERSION"], fail_on_error=True)
+semantic_version_components = old_version.split('.')
+version_num_count = len(semantic_version_components)
+if version_num_count == 3:
+    semantic_version_components = semantic_version_components + [numeric_date_string]
+elif version_num_count == 4:
+    semantic_version_components[3] = numeric_date_string
+else:
+    print("Invalid semantic versioning for current version. Aborting...")
+    quit()
+
+new_version = ".".join(semantic_version_components)
+
+# Update version file
+with open("AWS_FOR_FLUENT_BIT_VERSION", "w") as version_file:
+    version_file.write(new_version)
+
+# Update linux.version file
+with open("linux.version", 'r') as version_file:
+    linux_version = version_file.readlines()
+
+old_version = "{}".format(old_version)
+print("old aws-for-fluent-bit version number: {}".format(old_version))
+print("new aws-for-fluent-bit version number: {}".format(new_version))
+for linum in range(len(linux_version)):
+    # print("linux.version line: {}".format(linux_version[linum]))
+    if old_version in linux_version[linum]:
+        linux_version[linum] = linux_version[linum].replace(old_version, new_version)
+        break
+else:
+    print("could not auto-locate version number in linux.version!!!")
+    print("Guessing at line to modify based on file format...")
+    linux_version[2] = '    "version": "{}",\n'.format(new_version)
+
+with open("linux.version", 'w') as version_file:
+    version_file.writelines(linux_version)
+
+# Update changelog
+run_command(["./scripts/generate_changelog.sh"],
+            # Provide fluent bit directory to the changelog script,
+            # if we were able to clone it.
+            env={"FLUENT_BIT_DIRECTORY": upstream_repo} if upstream_repo is not None else None,
+            timeout=20, tries=3,
+            fail_on_error=True)
+
+# Commit changes
+run_command(["git", "commit", "-a", "--message", '"Release {}"'.format(new_version)],
+            fail_on_error=True)
+
+# Push changes to remote.
+# Might fail if there are unexpected commits, but we ignore that as expected
+core_push_command = ["git", "push"]
+if automated:
+    core_push_command = core_push_command + ["--force"]
+push_output = run_command(core_push_command,
+                          fail_on_error=False)
+if push_output is None or push_output == "" or ("fatal: The current branch" in push_output and "has no upstream branch" in push_output):
+    # Sometimes you need to set the pustream if it doesn't exist
+    print("push may have failed due to missing set-upstream. Retrying")
+    push_output = run_command(core_push_command + ["--set-upstream", "origin", branch_name],
+                              fail_on_error=False)
+
+# Go back to mainline
+run_command(["git", "checkout", "mainline"], timeout=30)