chore(ci): fix rsync interference & droute re-enable (#2414)

* Fix droute might interfere between PR check and nightly

* Reporting in function

* Fix status

* rsync with retries

* Re-enable droute fail on error

* Fix metedata var

* Add condition

* Resolve review comments

Author: zdrapela

.ibm/pipelines/utils.sh

@@ -39,7 +39,9 @@ droute_send() {
   if [[ "${OPENSHIFT_CI}" != "true" ]]; then return 0; fi
   temp_kubeconfig=$(mktemp) # Create temporary KUBECONFIG to open second `oc` session
   ( # Open subshell
-    set +e
+    if [ -n "${PULL_NUMBER:-}" ]; then
+      set +e
+    fi
     export KUBECONFIG="$temp_kubeconfig"
     local droute_version="1.2.2"
     local release_name=$1
@@ -93,26 +95,42 @@ droute_send() {
       .targets.reportportal.processing.tfa.auto_finalization_threshold = ($auto_finalization_treshold | tonumber)
       ' data_router/data_router_metadata_template.json > "${ARTIFACT_DIR}/${project}/${metadata_output}"
 
-    oc rsync --progress=true --include="${metadata_output}" --include="${JUNIT_RESULTS}" --exclude="*" -n "${droute_project}" "${ARTIFACT_DIR}/${project}/" "${droute_project}/${droute_pod_name}:${temp_droute}/"
+    # Send test by rsync to bastion pod.
+    local max_attempts=5
+    local wait_seconds=4
+    for ((i = 1; i <= max_attempts; i++)); do
+      echo "Attempt ${i} of ${max_attempts} to rsync test resuls to bastion pod."
+      if output=$(oc rsync --progress=true --include="${metadata_output}" --include="${JUNIT_RESULTS}" --exclude="*" -n "${droute_project}" "${ARTIFACT_DIR}/${project}/" "${droute_project}/${droute_pod_name}:${temp_droute}/" 2>&1); then
+        echo "$output"
+        break
+      fi
+      if ((i == max_attempts)); then
+        echo "Failed to rsync test results after ${max_attempts} attempts."
+        echo "Last rsync error details:"
+        echo "${output}"
+        echo "Troubleshooting steps:"
+        echo "1. Restart $droute_pod_name in $droute_project project/namespace"
+      fi
+    done
 
     # "Install" Data Router
     oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "
-      curl -fsSLk -o /tmp/droute-linux-amd64 'https://${DATA_ROUTER_NEXUS_HOSTNAME}/nexus/repository/dno-raw/droute-client/${droute_version}/droute-linux-amd64' \
-      && chmod +x /tmp/droute-linux-amd64 \
-      && /tmp/droute-linux-amd64 version"
+      curl -fsSLk -o ${temp_droute}/droute-linux-amd64 'https://${DATA_ROUTER_NEXUS_HOSTNAME}/nexus/repository/dno-raw/droute-client/${droute_version}/droute-linux-amd64' \
+      && chmod +x ${temp_droute}/droute-linux-amd64 \
+      && ${temp_droute}/droute-linux-amd64 version"
 
     # Send test results through DataRouter and save the request ID.
     local max_attempts=5
     local wait_seconds=1
     for ((i = 1; i <= max_attempts; i++)); do
       echo "Attempt ${i} of ${max_attempts} to send test results through Data Router."
       if output=$(oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "
-        /tmp/droute-linux-amd64 send --metadata ${temp_droute}/${metadata_output} \
-        --url '${DATA_ROUTER_URL}' \
-        --username '${DATA_ROUTER_USERNAME}' \
-        --password '${DATA_ROUTER_PASSWORD}' \
-        --results '${temp_droute}/${JUNIT_RESULTS}' \
-        --verbose" 2>&1); then
+        ${temp_droute}/droute-linux-amd64 send --metadata ${temp_droute}/${metadata_output} \
+          --url '${DATA_ROUTER_URL}' \
+          --username '${DATA_ROUTER_USERNAME}' \
+          --password '${DATA_ROUTER_PASSWORD}' \
+          --results '${temp_droute}/${JUNIT_RESULTS}' \
+          --verbose" 2>&1); then
         if DATA_ROUTER_REQUEST_ID=$(echo "$output" | grep "request:" | awk '{print $2}') &&
           [ -n "$DATA_ROUTER_REQUEST_ID" ]; then
           echo "Test results successfully sent through Data Router."
@@ -140,40 +158,15 @@ droute_send() {
       for ((i = 1; i <= max_attempts; i++)); do
         # Get DataRouter request information.
         DATA_ROUTER_REQUEST_OUTPUT=$(oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "
-          /tmp/droute-linux-amd64 request get \
+          ${temp_droute}/droute-linux-amd64 request get \
           --url ${DATA_ROUTER_URL} \
           --username ${DATA_ROUTER_USERNAME} \
           --password ${DATA_ROUTER_PASSWORD} \
           ${DATA_ROUTER_REQUEST_ID}")
         # Try to extract the ReportPortal launch URL from the request. This fails if it doesn't contain the launch URL.
         REPORTPORTAL_LAUNCH_URL=$(echo "$DATA_ROUTER_REQUEST_OUTPUT" | yq e '.targets[0].events[] | select(.component == "reportportal-connector") | .message | fromjson | .[0].launch_url' -)
-        if [[ $? -eq 0 ]]; then
-          if [[ "$release_name" == *rbac* ]]; then
-            RUN_TYPE="rbac-nightly"
-          else
-            RUN_TYPE="nightly"
-          fi
-          if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
-            RUN_STATUS_EMOJI=":done-circle-check:"
-            RUN_STATUS="passed"
-          else
-            RUN_STATUS_EMOJI=":failed:"
-            RUN_STATUS="failed"
-          fi
-          jq -n \
-            --arg run_status "$RUN_STATUS" \
-            --arg run_type "$RUN_TYPE" \
-            --arg reportportal_launch_url "$REPORTPORTAL_LAUNCH_URL" \
-            --arg job_name "$JOB_NAME" \
-            --arg run_status_emoji "$RUN_STATUS_EMOJI" \
-            '{
-              "RUN_STATUS": $run_status,
-              "RUN_TYPE": $run_type,
-              "REPORTPORTAL_LAUNCH_URL": $reportportal_launch_url,
-              "JOB_NAME": $job_name,
-              "RUN_STATUS_EMOJI": $run_status_emoji
-            }' > /tmp/data_router_slack_message.json
-          curl -X POST -H 'Content-type: application/json' --data @/tmp/data_router_slack_message.json  $SLACK_DATA_ROUTER_WEBHOOK_URL
+        if [[ -n "$REPORTPORTAL_LAUNCH_URL" ]]; then
+          reportportal_slack_alert $release_name $REPORTPORTAL_LAUNCH_URL
           return 0
         else
           echo "Attempt ${i} of ${max_attempts}: ReportPortal launch URL not ready yet."
@@ -183,12 +176,46 @@ droute_send() {
       set -e
     fi
     oc exec -n "${droute_project}" "${droute_pod_name}" -- /bin/bash -c "rm -rf ${temp_droute}/*"
-    set -e
+    if [ -n "${PULL_NUMBER:-}" ]; then
+      set -e
+    fi
   ) # Close subshell
   rm -f "$temp_kubeconfig" # Destroy temporary KUBECONFIG
   oc whoami --show-server
 }
 
+reportportal_slack_alert() {
+  local release_name=$1
+  local reportportal_launch_url=$2
+
+  if [[ "$release_name" == *rbac* ]]; then
+    RUN_TYPE="rbac-nightly"
+  else
+    RUN_TYPE="nightly"
+  fi
+  if [[ ${RESULT} -eq 0 ]]; then
+    RUN_STATUS_EMOJI=":done-circle-check:"
+    RUN_STATUS="passed"
+  else
+    RUN_STATUS_EMOJI=":failed:"
+    RUN_STATUS="failed"
+  fi
+  jq -n \
+    --arg run_status "$RUN_STATUS" \
+    --arg run_type "$RUN_TYPE" \
+    --arg reportportal_launch_url "$reportportal_launch_url" \
+    --arg job_name "$JOB_NAME" \
+    --arg run_status_emoji "$RUN_STATUS_EMOJI" \
+    '{
+      "RUN_STATUS": $run_status,
+      "RUN_TYPE": $run_type,
+      "REPORTPORTAL_LAUNCH_URL": $reportportal_launch_url,
+      "JOB_NAME": $job_name,
+      "RUN_STATUS_EMOJI": $run_status_emoji
+    }' > /tmp/data_router_slack_message.json
+  curl -X POST -H 'Content-type: application/json' --data @/tmp/data_router_slack_message.json  $SLACK_DATA_ROUTER_WEBHOOK_URL
+}
+
 # Merge the base YAML value file with the differences file for Kubernetes
 yq_merge_value_files() {
   local base_file=$1