From 358e5f8d466878cf1b3c4ea6c72719a5e3150d65 Mon Sep 17 00:00:00 2001 From: Colby Pike Date: Wed, 4 Feb 2026 15:42:31 -0700 Subject: [PATCH] [MONGOCRYPT-838] Switch the upload target bucket based on the project/patch status (#1117) This change replaces all references to the mciuploads bucket in the CI configuration file with a template expansion that conditionally refers to an alternate bucket in certain scenarios. This templating also sets the role_arn for S3 operations based on the same conditions. --- .evergreen/config.yml | 116 ++++++++++++++++++++---------------------- 1 file changed, 55 insertions(+), 61 deletions(-) diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 23ef2c27e..dd97d2b93 100755 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -80,19 +80,17 @@ functions: include: [./**] - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: '${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt.tar.gz' content_type: '${content_type|application/x-gzip}' - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: '${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix_copy}/libmongocrypt.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt.tar.gz' content_type: '${content_type|application/x-gzip}' @@ -138,10 +136,9 @@ functions: include: [./**] - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: '${project}/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt-distro-packages.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt-distro-packages.tar.gz' content_type: '${content_type|application/x-gzip}' @@ -224,10 +221,9 @@ functions: "download tarball": - command: s3.get params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: '${project}/${variant_name}/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} extract_to: all/${variant_name} "setup packaging credentials": @@ -323,11 +319,10 @@ functions: - "*" - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + role_arn: ${upload_arn} local_file: release-files.tgz remote_file: '${project}/python-release/${branch_name}/${libmongocrypt_s3_suffix}/${task_id}-${execution}-release-files.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read content_type: ${content_type|application/gzip} display_name: Release Python files @@ -358,7 +353,7 @@ functions: script: | set -o xtrace # Download all the release files. - aws s3 cp --recursive s3://mciuploads/${project}/python-release/${branch_name}/${libmongocrypt_s3_suffix}/ release/ + aws s3 cp --recursive s3://${upload_bucket}/${project}/python-release/${branch_name}/${libmongocrypt_s3_suffix}/ release/ # Combine releases into one directory. ls -la release/ mkdir releases @@ -373,11 +368,10 @@ functions: - "*" - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + role_arn: ${upload_arn} local_file: release-files-all.tgz remote_file: '${project}/python-release/${branch_name}/${libmongocrypt_s3_suffix}/${task_id}-${execution}-release-files-all.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read content_type: ${content_type|application/gzip} display_name: Release Python files all @@ -432,9 +426,8 @@ functions: type: test params: display_name: Augmented SBOM - aws_key: ${aws_key} - aws_secret: ${aws_secret} - bucket: mciuploads + role_arn: ${upload_arn} + bucket: ${upload_bucket} content_type: application/json local_file: libmongocrypt/cyclonedx.augmented.sbom.json permissions: public-read @@ -636,19 +629,17 @@ tasks: fi - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/java/${revision}/libmongocrypt-java.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt-java.tar.gz' content_type: '${content_type|application/x-gzip}' - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/java/${tag_upload_location}/libmongocrypt-java.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read optional: true display_name: 'libmongocrypt-java-${tag_upload_location}.tar.gz' @@ -831,28 +822,25 @@ tasks: fi - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/all/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt-all.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt-all.tar.gz' content_type: '${content_type|application/x-gzip}' - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/all/${branch_name}/${libmongocrypt_s3_suffix_copy}/libmongocrypt-all.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt-all.tar.gz' content_type: '${content_type|application/x-gzip}' - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/all/${tag_upload_location}/libmongocrypt-all.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read optional: true # Do not fail task if `local_file` does not exist. `local_file` only exists for tagged release. display_name: 'libmongocrypt-all-${tag_upload_location}.tar.gz' @@ -860,10 +848,9 @@ tasks: content_type: '${content_type|application/x-gzip}' - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/all/latest/stable/libmongocrypt-all.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read optional: true # Do not fail task if `local_file` does not exist. `local_file` only exists for stable release. display_name: 'stable/libmongocrypt-all-${tag_upload_location}.tar.gz' @@ -871,10 +858,9 @@ tasks: content_type: '${content_type|application/x-gzip}' - command: s3.put params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/all/latest/unstable/libmongocrypt-all.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read optional: true # Do not fail task if `local_file` does not exist. `local_file` only exists for unstable release. display_name: 'unstable/libmongocrypt-all-${tag_upload_location}.tar.gz' @@ -927,10 +913,9 @@ tasks: file: libmongocrypt/expansions.yml - command: s3.get # Download Windows build. params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: '${project}/windows-test/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt.tar.gz' - bucket: mciuploads + bucket: ${upload_bucket} extract_to: libmongocrypt_download - command: shell.exec params: @@ -956,21 +941,19 @@ tasks: # Documentation now refers to the GitHub release page, which includes the per-release tarball. # The fixed URL upload is kept to avoid possibly breaking expectations. Consider removing in the future. params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: 'libmongocrypt/windows/latest_release/libmongocrypt${upload_suffix}.tar.gz' display_name: (Deprecated) libmongocrypt${upload_suffix}.tar.gz - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt_upload.tar.gz' content_type: 'application/x-gzip' - command: s3.put # Upload tarball for GitHub Release. params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: '${project}/${build_variant}/${branch_name}/${revision}/${version_id}/libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.tar.gz' display_name: libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.tar.gz - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt_upload.tar.gz' content_type: 'application/x-gzip' @@ -986,11 +969,10 @@ tasks: args: --secret garasign_username=${garasign_username} --secret garasign_password=${garasign_password} +sign --file_to_sign=libmongocrypt_upload.tar.gz --output_file=libmongocrypt_upload.asc - command: s3.put # Upload signature for GitHub Release. params: - aws_key: '${aws_key}' - aws_secret: '${aws_secret}' + role_arn: ${upload_arn} remote_file: '${project}/${build_variant}/${branch_name}/${revision}/${version_id}/libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.asc' display_name: libmongocrypt-windows-x86_64-${libmongocrypt_release_version}.asc - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read local_file: 'libmongocrypt/libmongocrypt_upload.asc' content_type: 'application/pgp-signature' @@ -1013,11 +995,10 @@ tasks: bash .evergreen/debian_package_build.sh --is-patch=${is_patch} - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + role_arn: ${upload_arn} local_file: deb.tar.gz remote_file: libmongocrypt/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/debian-packages.tar.gz - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "deb.tar.gz" @@ -1038,11 +1019,10 @@ tasks: bash .evergreen/debian_package_build.sh --arch=i386 --is-patch=${is_patch} - command: s3.put params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} + role_arn: ${upload_arn} local_file: deb.tar.gz remote_file: libmongocrypt/${branch_name}/${revision}/${version_id}/${build_id}/${execution}/debian-packages-i386.tar.gz - bucket: mciuploads + bucket: ${upload_bucket} permissions: public-read content_type: ${content_type|application/x-gzip} display_name: "deb.tar.gz" @@ -1138,15 +1118,29 @@ pre: REMOTE_SUFFIX_COPY="latest-${branch_name}" fi + # If we are a non-patch build in the libmongocrypt-release project, we upload to a restricted + # CDN S3 bucket. Otherwise, we upload to a less restricted bucket for convenience. The corresponding + # role_arn_... values come from EVG project configuration variables stored on the EVG server + if test "${is_patch}" = 'true' || "${project_name}" != 'libmongocrypt-release'; then + upload_bucket='mciuploads' + upload_arn='${role_arn_for_mciuploads}' + else + upload_bucket='cdn-origin-libmongocrypt' + upload_arn='${role_arn_for_release}' + fi + PROJECT_DIRECTORY="$(pwd)" echo "libmongocrypt_s3_suffix: $REMOTE_SUFFIX" echo "libmongocrypt_s3_suffix_copy: $REMOTE_SUFFIX_COPY" echo "project_directory: $PROJECT_DIRECTORY" + echo "Upload S3 bucket: $upload_bucket" cat < expansion.yml libmongocrypt_s3_suffix: "$REMOTE_SUFFIX" libmongocrypt_s3_suffix_copy: "$REMOTE_SUFFIX_COPY" project_directory: "$PROJECT_DIRECTORY" + upload_bucket: "$upload_bucket" + upload_arn: "$upload_arn" EOT - command: expansions.update params: