PYTHON-5263 Convert s3-related functions to generated config

.evergreen/config.yml

@@ -25,6 +25,7 @@ timeout:
       binary: ls -la
 
 include:
+  - filename: .evergreen/generated_configs/functions.yml
   - filename: .evergreen/generated_configs/tasks.yml
   - filename: .evergreen/generated_configs/variants.yml
 
@@ -52,147 +53,6 @@ functions:
       params:
         file: src/expansion.yml
 
-  "upload coverage" :
-    - command: ec2.assume_role
-      params:
-        role_arn: ${assume_role_arn}
-    - command: s3.put
-      params:
-        aws_key: ${AWS_ACCESS_KEY_ID}
-        aws_secret: ${AWS_SECRET_ACCESS_KEY}
-        aws_session_token: ${AWS_SESSION_TOKEN}
-        local_file:  src/.coverage
-        optional: true
-        # Upload the coverage report for all tasks in a single build to the same directory.
-        remote_file: coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}
-        bucket: ${bucket_name}
-        permissions: public-read
-        content_type: text/html
-        display_name: "Raw Coverage Report"
-
-  "download and merge coverage" :
-    - command: ec2.assume_role
-      params:
-        role_arn: ${assume_role_arn}
-    - command: subprocess.exec
-      params:
-        silent: true
-        binary: bash
-        working_dir: "src"
-        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
-        args:
-          - .evergreen/scripts/download-and-merge-coverage.sh
-          - ${bucket_name}
-          - ${revision}
-          - ${version_id}
-    - command: subprocess.exec
-      params:
-        working_dir: "src"
-        binary: bash
-        args:
-          - .evergreen/combine-coverage.sh
-    # Upload the resulting html coverage report.
-    - command: subprocess.exec
-      params:
-        silent: true
-        binary: bash
-        working_dir: "src"
-        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
-        args:
-          - .evergreen/scripts/upload-coverage-report.sh
-          - ${bucket_name}
-          - ${revision}
-          - ${version_id}
-    # Attach the index.html with s3.put so it shows up in the Evergreen UI.
-    - command: s3.put
-      params:
-        aws_key: ${AWS_ACCESS_KEY_ID}
-        aws_secret: ${AWS_SECRET_ACCESS_KEY}
-        aws_session_token: ${AWS_SESSION_TOKEN}
-        local_file:  src/htmlcov/index.html
-        remote_file: coverage/${revision}/${version_id}/htmlcov/index.html
-        bucket: ${bucket_name}
-        permissions: public-read
-        content_type: text/html
-        display_name: "Coverage Report HTML"
-
-  "upload mo artifacts":
-    - command: ec2.assume_role
-      params:
-        role_arn: ${assume_role_arn}
-    - command: archive.targz_pack
-      params:
-        target: "mongo-coredumps.tgz"
-        source_dir: "./"
-        include:
-          - "./**.core"
-          - "./**.mdmp" # Windows: minidumps
-    - command: s3.put
-      params:
-        aws_key: ${AWS_ACCESS_KEY_ID}
-        aws_secret: ${AWS_SECRET_ACCESS_KEY}
-        aws_session_token: ${AWS_SESSION_TOKEN}
-        local_file: mongo-coredumps.tgz
-        remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz
-        bucket: ${bucket_name}
-        permissions: public-read
-        content_type: ${content_type|application/gzip}
-        display_name: Core Dumps - Execution
-        optional: true
-    - command: s3.put
-      params:
-        aws_key: ${AWS_ACCESS_KEY_ID}
-        aws_secret: ${AWS_SECRET_ACCESS_KEY}
-        aws_session_token: ${AWS_SESSION_TOKEN}
-        local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz
-        remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz
-        bucket: ${bucket_name}
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: "drivers-tools-logs.tar.gz"
-
-  "upload working dir":
-    - command: ec2.assume_role
-      params:
-        role_arn: ${assume_role_arn}
-    - command: archive.targz_pack
-      params:
-        target: "working-dir.tar.gz"
-        source_dir: ${PROJECT_DIRECTORY}/
-        include:
-          - "./**"
-    - command: s3.put
-      params:
-        aws_key: ${AWS_ACCESS_KEY_ID}
-        aws_secret: ${AWS_SECRET_ACCESS_KEY}
-        aws_session_token: ${AWS_SESSION_TOKEN}
-        local_file: working-dir.tar.gz
-        remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-working-dir.tar.gz
-        bucket: ${bucket_name}
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: "working-dir.tar.gz"
-    - command: archive.targz_pack
-      params:
-        target: "drivers-dir.tar.gz"
-        source_dir: ${DRIVERS_TOOLS}
-        include:
-          - "./**"
-        exclude_files:
-          # Windows cannot read the mongod *.lock files because they are locked.
-          - "*.lock"
-    - command: s3.put
-      params:
-        aws_key: ${AWS_ACCESS_KEY_ID}
-        aws_secret: ${AWS_SECRET_ACCESS_KEY}
-        aws_session_token: ${AWS_SESSION_TOKEN}
-        local_file: drivers-dir.tar.gz
-        remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/artifacts/${task_id}-${execution}-drivers-dir.tar.gz
-        bucket: ${bucket_name}
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: "drivers-dir.tar.gz"
-
   "upload test results":
     - command: attach.results
       params:

.evergreen/generated_configs/functions.yml

@@ -0,0 +1,117 @@
+functions:
+  # Download and merge coverage
+  download and merge coverage:
+    - command: ec2.assume_role
+      params:
+        role_arn: ${assume_role_arn}
+      type: setup
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/scripts/download-and-merge-coverage.sh
+          - ${bucket_name}
+          - ${revision}
+          - ${version_id}
+        working_dir: src
+        silent: true
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+      type: test
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/combine-coverage.sh
+        working_dir: src
+      type: test
+    - command: subprocess.exec
+      params:
+        binary: bash
+        args:
+          - .evergreen/scripts/upload-coverage-report.sh
+          - ${bucket_name}
+          - ${revision}
+          - ${version_id}
+        working_dir: src
+        silent: true
+        include_expansions_in_env:
+          - AWS_ACCESS_KEY_ID
+          - AWS_SECRET_ACCESS_KEY
+          - AWS_SESSION_TOKEN
+      type: test
+    - command: s3.put
+      params:
+        remote_file: coverage/${revision}/${version_id}/htmlcov/index.html
+        aws_key: ${AWS_ACCESS_KEY_ID}
+        aws_secret: ${AWS_SECRET_ACCESS_KEY}
+        aws_session_token: ${AWS_SESSION_TOKEN}
+        bucket: ${bucket_name}
+        local_file: src/htmlcov/index.html
+        permissions: public-read
+        content_type: text/html
+        display_name: Coverage Report HTML
+        optional: "true"
+      type: setup
+
+  # Upload coverage
+  upload coverage:
+    - command: ec2.assume_role
+      params:
+        role_arn: ${assume_role_arn}
+      type: setup
+    - command: s3.put
+      params:
+        remote_file: coverage/${revision}/${version_id}/coverage/coverage.${build_variant}.${task_name}
+        aws_key: ${AWS_ACCESS_KEY_ID}
+        aws_secret: ${AWS_SECRET_ACCESS_KEY}
+        aws_session_token: ${AWS_SESSION_TOKEN}
+        bucket: ${bucket_name}
+        local_file: src/.coverage
+        permissions: public-read
+        content_type: text/html
+        display_name: Raw Coverage Report
+        optional: "true"
+      type: setup
+
+  # Upload mo artifacts
+  upload mo artifacts:
+    - command: ec2.assume_role
+      params:
+        role_arn: ${assume_role_arn}
+      type: setup
+    - command: archive.targz_pack
+      params:
+        target: mongo-coredumps.tgz
+        source_dir: ./
+        include:
+          - ./**.core
+          - ./**.mdmp
+    - command: s3.put
+      params:
+        remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/coredumps/${task_id}-${execution}-mongodb-coredumps.tar.gz
+        aws_key: ${AWS_ACCESS_KEY_ID}
+        aws_secret: ${AWS_SECRET_ACCESS_KEY}
+        aws_session_token: ${AWS_SESSION_TOKEN}
+        bucket: ${bucket_name}
+        local_file: mongo-coredumps.tgz
+        permissions: public-read
+        content_type: ${content_type|application/x-gzip}
+        display_name: Core Dumps - Execution
+        optional: "true"
+      type: setup
+    - command: s3.put
+      params:
+        remote_file: ${build_variant}/${revision}/${version_id}/${build_id}/logs/${task_id}-${execution}-drivers-tools-logs.tar.gz
+        aws_key: ${AWS_ACCESS_KEY_ID}
+        aws_secret: ${AWS_SECRET_ACCESS_KEY}
+        aws_session_token: ${AWS_SESSION_TOKEN}
+        bucket: ${bucket_name}
+        local_file: ${DRIVERS_TOOLS}/.evergreen/test_logs.tar.gz
+        permissions: public-read
+        content_type: ${content_type|application/x-gzip}
+        display_name: drivers-tools-logs.tar.gz
+        optional: "true"
+      type: setup