Merge branch 'develop' into 5910-datafusion-performance-assertion

Author: m09526

java/compaction/compaction-core/src/main/java/sleeper/compaction/core/job/creation/strategy/DelegatingCompactionStrategy.java

@@ -73,7 +73,7 @@ private List<CompactionJob> createJobsForLeafPartition(FilesInPartition filesInP
         while (jobs.size() > maxNumberOfJobsToCreate) {
             jobs.remove(jobs.size() - 1);
         }
-        LOGGER.info("Created {} compaction job{} for partition {}, table {}",
+        LOGGER.debug("Created {} compaction job{} for partition {}, table {}",
                 jobs.size(), 1 == jobs.size() ? "" : "s", filesInPartition.getPartitionId(), filesInPartition.getTableStatus());
         return jobs;
     }

scripts/cli/runInDocker.sh

@@ -22,6 +22,7 @@ if [ "$#" -lt 1 ]; then
 fi
 
 HOME_IN_IMAGE=/home/sleeper
+ALL_IMAGES=(sleeper-local sleeper-builder)
 
 # Allow use of runner Dockerfile in source directory or in home install
 THIS_DIR=$(cd "$(dirname "$0")" && pwd)
@@ -110,22 +111,88 @@ get_version() {
   run_in_docker sleeper-local:current cat /sleeper/version.txt
 }
 
-pull_docker_images(){
-  echo "Updating CLI runner Dockerfile"
+pull_docker_images() {
+  echo "Downloading CLI runner Dockerfile"
   mkdir -p "$HOME_RUNNER_PATH"
   curl "https://raw.githubusercontent.com/gchq/sleeper/develop/scripts/cli/runner/Dockerfile" --output "$HOME_RUNNER_PATH/Dockerfile"
-  pull_and_tag sleeper-local
-  pull_and_tag sleeper-builder
+
+  echo "Pulling CLI Docker images"
+  for IMAGE_NAME in "${ALL_IMAGES[@]}"; do
+    echo "Pulling image: $IMAGE_NAME"
+    REMOTE_IMAGE="ghcr.io/gchq/$IMAGE_NAME:latest"
+    LOCAL_IMAGE="$IMAGE_NAME:current"
+
+    docker pull "$REMOTE_IMAGE"
+    docker tag "$REMOTE_IMAGE" "$LOCAL_IMAGE"
+  done
+}
+
+find_docker_image_digests() {
+  echo "Checking CLI image digests"
+  declare -ga IMAGE_DIGESTS
+  for IMAGE_NAME in "${ALL_IMAGES[@]}"; do
+    IMAGE="$IMAGE_NAME:current"
+    DIGEST="$(docker images -q "$IMAGE" 2> /dev/null)"
+    if [ -n "$DIGEST" ]; then
+      echo "Found digest for $IMAGE_NAME: $DIGEST"
+      IMAGE_DIGESTS+=("$DIGEST")
+    fi
+  done
+}
+
+find_runner_image_digests() {
+  echo "Checking runner image digests"
+  declare -ga RUNNER_DIGESTS
+  local LINES=$(docker images -q sleeper-runner 2> /dev/null)
+  while read -r LINE
+  do
+    DUPLICATE=false
+    for DIGEST in "${RUNNER_DIGESTS[@]}"; do
+      if [[ "$DIGEST" == "$LINE" ]]; then
+        DUPLICATE=true
+        break
+      fi
+    done
+    if [[ "$DUPLICATE" == "false" ]]; then
+      echo "Found runner digest for cleanup: $LINE"
+      RUNNER_DIGESTS+=("$LINE")
+    fi
+  done <<< "$LINES"
+}
+
+remove_old_images() {
+  OLD_DIGESTS=("${IMAGE_DIGESTS[@]}")
+  find_docker_image_digests
+  echo "Cleaning up old CLI images"
+  for OLD_DIGEST in "${OLD_DIGESTS[@]}"; do
+    UPDATED=true
+    for NEW_DIGEST in "${IMAGE_DIGESTS[@]}"; do
+      if [[ "$OLD_DIGEST" == "$NEW_DIGEST" ]]; then
+        UPDATED=false
+        break
+      fi
+    done
+    if [[ "$UPDATED" == "true" ]]; then
+      docker image rm "$OLD_DIGEST"
+    fi
+  done
+  find_runner_image_digests
+  echo "Cleaning up old runner images"
+  for DIGEST in "${RUNNER_DIGESTS[@]}"; do
+    docker image rm -f "$DIGEST"
+  done
 }
 
 upgrade_cli() {
+  find_docker_image_digests
   echo "Updating CLI command"
   EXECUTABLE_PATH="${BASH_SOURCE[0]}"
   local TEMP_DIR=$(mktemp -d)
   TEMP_PATH="$TEMP_DIR/sleeper"
   curl "https://raw.githubusercontent.com/gchq/sleeper/develop/scripts/cli/runInDocker.sh" --output "$TEMP_PATH"
   chmod a+x "$TEMP_PATH"
   "$TEMP_PATH" cli pull-images
+  remove_old_images
   mv "$TEMP_PATH" "$EXECUTABLE_PATH"
   rmdir "$TEMP_DIR"
   echo "Updated"
@@ -137,15 +204,6 @@ upgrade_cli() {
   exit
 }
 
-pull_and_tag() {
-  IMAGE_NAME=$1
-  REMOTE_IMAGE="ghcr.io/gchq/$IMAGE_NAME:latest"
-  LOCAL_IMAGE="$IMAGE_NAME:current"
-
-  docker pull "$REMOTE_IMAGE"
-  docker tag "$REMOTE_IMAGE" "$LOCAL_IMAGE"
-}
-
 COMMAND=$1
 shift