chore: Build ESDK-Dafny into Python, test native ESDK-Python in TestVectors CI

.github/workflows/library_interop_test_vectors.yml

@@ -25,14 +25,13 @@ jobs:
             ubuntu-22.04,
             macos-13,
           ]
-        language: [java, net, rust, go]
+        language: [java, net, rust, go, python]
         # https://taskei.amazon.dev/tasks/CrypTool-5284
         dotnet-version: ["6.0.x"]
     runs-on: ${{ matrix.os }}
     permissions:
       id-token: write
       contents: read
-
     steps:
       - name: Support longpaths on Git checkout
         run: |
@@ -67,6 +66,17 @@ jobs:
           distribution: "corretto"
           java-version: 17
 
+      - name: Setup Python for running tests
+        if: matrix.language == 'python'
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
       - name: Setup Rust Toolchain for GitHub CI
         if: matrix.language == 'rust'
         uses: actions-rust-lang/[email protected]
@@ -144,6 +154,15 @@ jobs:
           CORES=$(node -e 'console.log(os.cpus().length)')
           make transpile_rust CORES=$CORES
 
+      - name: Build ${{ matrix.library }} implementation in Python
+        if: matrix.language == 'python'
+        shell: bash
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python CORES=$CORES
+
       - name: Build ${{ matrix.library }} implementation in Go
         if: matrix.language == 'go'
         shell: bash
@@ -187,6 +206,7 @@ jobs:
   testInteroperablity:
     needs: generateEncryptVectors
     strategy:
+      fail-fast: false
       matrix:
         library: [TestVectors]
         os: [
@@ -195,8 +215,8 @@ jobs:
             ubuntu-22.04,
             macos-13,
           ]
-        encrypting_language: [java, net, rust, go]
-        decrypting_language: [java, net, rust, go]
+        encrypting_language: [java, net, rust, go, python]
+        decrypting_language: [java, net, rust, go, python]
         # https://taskei.amazon.dev/tasks/CrypTool-5284
         dotnet-version: ["6.0.x"]
     runs-on: ${{ matrix.os }}
@@ -238,6 +258,17 @@ jobs:
           distribution: "corretto"
           java-version: 17
 
+      - name: Setup Python for running tests
+        if: matrix.decrypting_language == 'python'
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+          architecture: x64
+      - run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
       - name: Setup Rust Toolchain for GitHub CI
         if: matrix.decrypting_language == 'rust'
         uses: actions-rust-lang/[email protected]
@@ -332,6 +363,14 @@ jobs:
         run: |
           make purge_polymorph_code
 
+      - name: Build ${{ matrix.library }} implementation in Python
+        if: matrix.decrypting_language == 'python'
+        shell: bash
+        working-directory: ./${{ matrix.library }}
+        run: |
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python CORES=$CORES
+
       - name: Download Encrypt Manifest Artifact
         uses: actions/download-artifact@v4
         with:

AwsEncryptionSDK/Makefile

@@ -107,3 +107,20 @@ restore_directories:
 			echo "Directory $$dir not found"; \
 		fi \
 	done
+
+PYTHON_MODULE_NAME=aws_encryption_sdk_dafny
+
+TRANSLATION_RECORD_PYTHON := \
+    --translation-record ../mpl/StandardLibrary/runtimes/python/src/smithy_dafny_standard_library/internaldafny/generated/dafny_src-py.dtr \
+    --translation-record ../mpl/ComAmazonawsKms/runtimes/python/src/aws_cryptography_internal_kms/internaldafny/generated/dafny_src-py.dtr \
+    --translation-record ../mpl/ComAmazonawsDynamodb/runtimes/python/src/aws_cryptography_internal_dynamodb/internaldafny/generated/dafny_src-py.dtr \
+    --translation-record ../mpl/AwsCryptographyPrimitives/runtimes/python/src/aws_cryptography_primitives/internaldafny/generated/dafny_src-py.dtr \
+	--translation-record ../mpl/AwsCryptographicMaterialProviders/runtimes/python/src/aws_cryptographic_material_providers/internaldafny/generated/dafny_src-py.dtr
+
+PYTHON_DEPENDENCY_MODULE_NAMES := \
+        --dependency-library-name=aws.cryptography.primitives=aws_cryptography_primitives \
+        --dependency-library-name=com.amazonaws.kms=aws_cryptography_internal_kms \
+        --dependency-library-name=com.amazonaws.dynamodb=aws_cryptography_internal_dynamodb \
+        --dependency-library-name=aws.cryptography.materialProviders=aws_cryptographic_material_providers \
+        --dependency-library-name=aws.cryptography.keyStore=aws_cryptographic_material_providers \
+        --dependency-library-name=smithy.api=aws_cryptographic_material_providers

AwsEncryptionSDK/runtimes/java/README.md

@@ -0,0 +1,10 @@
+# Important: Do not use this code. This package is AWS-internal.
+
+This is NOT the released version of the AWS Encryption SDK for Java.
+
+You can access the AWS Encryption SDK for Java at:
+
+- Github: https://github.com/aws/aws-encryption-sdk-java
+- Maven: https://mvnrepository.com/artifact/com.amazonaws/aws-encryption-sdk-java
+
+This is an in-development rewrite of the AWS Encryption SDK for Java and should not be used until development is complete.

AwsEncryptionSDK/runtimes/python/.gitignore

@@ -0,0 +1,16 @@
+# Python build artifacts
+__pycache__
+**/__pycache__
+*.pyc
+src/**.egg-info/
+build
+poetry.lock
+**/poetry.lock
+dist
+
+# Dafny-generated Python
+**/internaldafny/generated/*.py
+
+# Python test artifacts
+.tox
+.pytest_cache

AwsEncryptionSDK/runtimes/python/README.md

@@ -0,0 +1,10 @@
+# Important: Do not use this code. This package is AWS-internal.
+
+This is NOT the released version of the AWS Encryption SDK for Python.
+
+You can access the AWS Encryption SDK for Python at:
+
+- Github: https://github.com/aws/aws-encryption-sdk-python
+- PyPI: https://pypi.org/project/aws-encryption-sdk/
+
+This is an in-development rewrite of the AWS Encryption SDK for Python and should not be used until development is complete.

AwsEncryptionSDK/runtimes/python/pyproject.toml

@@ -0,0 +1,36 @@
+[tool.poetry]
+# Note: We should not release this library with this name.
+# We should release this library under the name `aws-encryption-sdk`.
+# But this repo's TestVectors test the released, native version of the ESDK,
+#   which has this name.
+# The names conflict, and issues arise from this.
+# When we are ready to release the Dafny-Python ESDK, we should rename this,
+#   but figure out if/how we can still test the native ESDK in this repo.
+name = "aws-encryption-sdk-dafny"
+version = "0.1.0"
+description = ""
+authors = ["AWS Crypto Tools <[email protected]>"]
+# Note: We should not release this library with this package name.
+# We should release this library with the package name `aws_encryption_sdk`.
+# But this repo's TestVectors test the released, native version of the ESDK,
+#   which has this name.
+# The names conflict, and issues arise from this.
+# When we are ready to release the Dafny-Python ESDK, we should rename this,
+#   but figure out if/how we can still test the native ESDK in this repo.
+packages = [
+    { include = "aws_encryption_sdk_dafny", from = "src" }
+]
+# Include generated internaldafny files in package distributions,
+# even though they are not included in version control
+include = ["**/internaldafny/generated/*.py"]
+
+[tool.poetry.dependencies]
+python = "^3.11.0"
+aws-cryptographic-material-providers = { path = "../../../mpl/AwsCryptographicMaterialProviders/runtimes/python", develop = false}
+
+[tool.poetry.group.test.dependencies]
+pytest = "^7.4.0"
+
+[build-system]
+requires = ["poetry-core<2.0.0"]
+build-backend = "poetry.core.masonry.api"

AwsEncryptionSDK/runtimes/python/src/aws_encryption_sdk_dafny/__init__.py

@@ -0,0 +1,5 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Initialize generated Dafny
+from .internaldafny.generated import module_

AwsEncryptionSDK/runtimes/python/src/aws_encryption_sdk_dafny/internaldafny/__init__.py

@@ -0,0 +1,2 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0

AwsEncryptionSDK/runtimes/python/src/aws_encryption_sdk_dafny/internaldafny/generated/dafny_src-py.dtr

@@ -0,0 +1,59 @@
+file_format_version = "1.0"
+dafny_version = "4.9.0.0"
+[options_by_module.AwsCryptographyEncryptionSdkTypes]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.AbstractAwsCryptographyEncryptionSdkOperations]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.AbstractAwsCryptographyEncryptionSdkService]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.SerializableTypes]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.SerializeFunctions]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.EncryptionContext]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.HeaderTypes]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.SharedHeaderFunctions]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.EncryptedDataKeys]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.V1HeaderBody]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.V2HeaderBody]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.HeaderAuth]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.Header]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.Frames]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.MessageBody]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.KeyDerivation]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.EncryptDecryptHelpers]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.AwsEncryptionSdkOperations]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"
+[options_by_module.ESDK]
+legacy-module-names = false
+python-module-name = "aws_encryption_sdk_dafny.internaldafny.generated"

AwsEncryptionSDK/runtimes/python/src/aws_encryption_sdk_dafny/smithygenerated/aws_cryptography_encryptionsdk/__init__.py

@@ -0,0 +1,3 @@
+# Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+# Do not modify this file. This file is machine generated, and any changes to it will be overwritten.