add local assets bucket (aws-solutions-library-samples#1144)

iakov-gan · web-flow · commit e58401021a8a · 2025-04-01T14:05:22.000+02:00
diff --git a/cfn-templates/cid-admin-policies.yaml b/cfn-templates/cid-admin-policies.yaml
@@ -161,6 +161,7 @@ Resources:
           - s3:GetBucketLocation
           - s3:AbortMultipartUpload
           - s3:ListMultipartUploadParts
+          - s3:PutBucketTagging
           Effect: Allow
           Resource:
           - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-shared
@@ -567,6 +568,7 @@ Resources:
           - s3:PutEncryptionConfiguration
           - s3:PutLifecycleConfiguration
           - s3:PutObject
+          - s3:PutBucketTagging
           Effect: Allow
           Resource:
           - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-shared
@@ -702,13 +704,15 @@ Resources:
           - s3:PutLifecycleConfiguration
           - s3:PutObject
           - s3:PutReplicationConfiguration
+          - s3:PutBucketTagging
           Effect: Allow
           Resource:
           - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-local
           - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-local/*
           - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-local
           - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-data-local/*
-
+          - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-local-assets
+          - !Sub arn:${AWS::Partition}:s3:::cid-${AWS::AccountId}-local-assets/*
         - Sid: ReadOnly
           Action:
           - cloudformation:GetTemplate
diff --git a/cfn-templates/cid-cfn.yml b/cfn-templates/cid-cfn.yml
@@ -39,6 +39,8 @@ Metadata:
           - DeployCUDOSDashboard
           - DataBucketsKmsKeysArns
           - ShareDashboard
+          - CreateLocalAssetsBucket
+          - ReferenceAssetsBucket
       - Label:
           default: 'Legacy'
         Parameters:
@@ -98,6 +100,10 @@ Metadata:
         default: "Secondary Tag for Compute Optimizer dashboard"
       KeepLegacyCURTable:
         default: "Keep Legacy CUR Table"
+      CreateLocalAssetsBucket:
+        default: "Create Local Assets Bucket"
+      ReferenceAssetsBucket:
+        default: "Reference Assets Bucket Name"
   cfn-lint:
     config:
       ignore_checks:
@@ -241,6 +247,15 @@ Parameters:
     Description: Choose 'yes' if you want to keep the Legacy CUR table
     Default: "no"
     AllowedValues: ["yes", "no"]
+  CreateLocalAssetsBucket:
+    Type: String
+    Description: Choose 'yes' if you want to copy reference CID Assets to a local assets buckets (for a set of regions including China and Gov it will be done automatically).
+    Default: "no"
+    AllowedValues: ["yes", "no"]
+  ReferenceAssetsBucket:
+    Type: String
+    Description: Source bucket for CID Assets to sync from.
+    Default: 'aws-managed-cost-intelligence-dashboards-us-east-1'
 
 Conditions:
   NeedCUDOSDashboard: !Equals [ !Ref DeployCUDOSDashboard, "yes" ]
@@ -307,9 +322,22 @@ Conditions:
       - !Equals [!Ref "AWS::Region", "us-gov-east-1"]
       - !Equals [!Ref "AWS::Region", "us-gov-west-1"]
   LambdaLayerBucketPrefixIsManaged: !Equals [!Ref LambdaLayerBucketPrefix, 'aws-managed-cost-intelligence-dashboards']
+  NeedLocalAssets:
+    Fn::Or:
+      - !Equals [ !Ref CreateLocalAssetsBucket, "yes" ] #explicity requested local assets
+      - Fn::Or: # or region where CID do not have a public bucket
+        - !Equals [!Ref "AWS::Region", "af-south-1"]
+        - !Equals [!Ref "AWS::Region", "ap-southeast-3"]
+        - !Equals [!Ref "AWS::Region", "eu-central-2"]
+        - !Equals [!Ref "AWS::Region", "eu-south-1"]
+        - !Equals [!Ref "AWS::Region", "eu-south-2"]
+        - !Equals [!Ref "AWS::Region", "cn-north-1"]
+        - !Equals [!Ref "AWS::Region", "us-gov-east-1"]
+        - !Equals [!Ref "AWS::Region", "us-gov-west-1"]
 
 Mappings:
-  RegionMap: # CID has AWS managed buckets for deploy. Region must support QuickSight ( curl https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonQuickSight/current/region_index.json -s | jq '.regions | keys' )
+  RegionMap: 
+    # CID has AWS managed buckets for deploy. Region must support QuickSight ( curl https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonQuickSight/current/region_index.json -s | jq '.regions | keys' )
     ap-northeast-1: {BucketName: aws-managed-cost-intelligence-dashboards-ap-northeast-1}
     ap-northeast-2: {BucketName: aws-managed-cost-intelligence-dashboards-ap-northeast-2}
     ap-south-1:     {BucketName: aws-managed-cost-intelligence-dashboards-ap-south-1}
@@ -326,10 +354,18 @@ Mappings:
     us-east-2:      {BucketName: aws-managed-cost-intelligence-dashboards-us-east-2}
     us-west-1:      {BucketName: aws-managed-cost-intelligence-dashboards-us-west-1}
     us-west-2:      {BucketName: aws-managed-cost-intelligence-dashboards-us-west-2}
-    #todo: add af-south-1
-    #todo: add ap-southeast-3
-    #todo: add eu-south-1
-    #todo: add eu-central-2
+
+    # regions without public buckets
+    af-south-1:     {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1}
+    ap-southeast-3: {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1}
+    eu-central-2:   {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1}
+    eu-south-1:     {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1}
+    eu-south-2:     {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1}
+
+    # specific regions
+    cn-north-1:     {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1} # China
+    us-gov-east-1:  {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1} # GovCloud (by default must be in commercial region)
+    us-gov-west-1:  {BucketName: aws-managed-cost-intelligence-dashboards-us-east-1}
 
 
 Resources:
@@ -1836,22 +1872,208 @@ Resources:
           - id: 'W92'
             reason: "No need for reserved concurrency"
 
+  LocalAssetsBucket:
+    Type: AWS::S3::Bucket
+    Condition: NeedLocalAssets
+    Properties:
+      BucketName: !Sub "cid-${AWS::AccountId}-local-assets"
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
+          - ServerSideEncryptionByDefault:
+              SSEAlgorithm: AES256
+      AccessControl: BucketOwnerFullControl
+      OwnershipControls:
+        Rules:
+          - ObjectOwnership: BucketOwnerEnforced
+      PublicAccessBlockConfiguration:
+        BlockPublicAcls: true
+        BlockPublicPolicy: true
+        IgnorePublicAcls: true
+        RestrictPublicBuckets: true
+    Metadata:
+      cfn-lint:
+        config:
+          ignore_checks:
+            - W3045 #Not needed AWS::S3::BucketPolicy
+      cfn_nag:
+        rules_to_suppress:
+          - id: 'W35'
+            reason: "Data buckets would generate too much logs"
+          - id: 'W51'
+            reason: "No policy needed"
+  PullAssetsLambdaRole:
+    Type: AWS::IAM::Role
+    Condition: NeedLocalAssets
+    Properties:
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service: lambda.amazonaws.com
+            Action: sts:AssumeRole
+      ManagedPolicyArns:
+        - !Sub 'arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'
+      Policies:
+        - PolicyName: S3SyncPolicy
+          PolicyDocument:
+            Version: '2012-10-17'
+            Statement:
+              - Effect: Allow
+                Action:
+                  - s3:GetObject
+                  - s3:ListBucket
+                  - s3:ListBucketVersions
+                Resource: 
+                  - !Sub 'arn:${AWS::Partition}:s3:::${ReferenceAssetsBucket}'
+                  - !Sub 'arn:${AWS::Partition}:s3:::${ReferenceAssetsBucket}/*'
+              - Effect: Allow
+                Action:
+                  - s3:PutObject
+                  - s3:DeleteObject
+                  - s3:DeleteObjectVersion
+                  - s3:DeleteObjectVersionTagging
+                  - s3:ListBucket
+                  - s3:ListBucketVersions
+                Resource:
+                  - !Sub 'arn:${AWS::Partition}:s3:::${LocalAssetsBucket}'
+                  - !Sub 'arn:${AWS::Partition}:s3:::${LocalAssetsBucket}/*'
+
+  PullAssetsLambda:
+    Type: AWS::Lambda::Function
+    Condition: NeedLocalAssets
+    Properties:
+      FunctionName:  !Sub 'CidPullAssets${Suffix}'
+      Handler: index.handler
+      Role: !GetAtt PullAssetsLambdaRole.Arn
+      Runtime: python3.12
+      Timeout: 300
+      MemorySize: 256
+      Code:
+        ZipFile: |
+          import json
+          import logging
+          import tempfile
+          import urllib.request
+          from io import BytesIO
+          import boto3
+          import cfnresponse
+
+          logger = logging.getLogger()
+          logger.setLevel(logging.INFO)
+
+          def handler(event, context):
+              logger.info(f'Received event: {json.dumps(event)}')
+              request_type = event['RequestType']
+              resource_props = event.get('ResourceProperties', {})
+              old_keys = event.get("OldResourceProperties", {}).get("Keys", [])
+              try:
+                  source_bucket = resource_props['SourceBucket']
+                  destination_bucket = resource_props['DestinationBucket']
+                  keys = resource_props.get('Keys', [])
+              except KeyError as e:
+                  logger.error(f"Missing required property: {e}")
+                  cfnresponse.send(event, context, cfnresponse.FAILED, {'Error': f"Missing required property: {str(e)}"})
+                  return
+              try:
+                  if request_type == 'Create' or request_type == 'Update':
+                      sync_buckets(source_bucket, destination_bucket, keys, old_keys)
+                      cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, destination_bucket)
+                  elif request_type == 'Delete':
+                      clean_bucket(destination_bucket)
+                      cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, destination_bucket)
+                  else:
+                      raise ValueError(f"Unsupported request type: {request_type}")
+              except Exception as e:
+                  logger.error(f"Error on {request_type}: {str(e)}")
+                  cfnresponse.send(event, context, cfnresponse.FAILED, {'Error': str(e)})
+
+          def get_content(source_bucket, key):
+              """ Download the content using public bucket to temporary file
+              """
+              object_url = f"https://{source_bucket}.s3.amazonaws.com/{key}"
+              logger.info(f"Downloading from {object_url}")
+              with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+                  with urllib.request.urlopen(object_url) as response:
+                      while True:
+                          chunk = response.read(8192)
+                          if not chunk:
+                              break
+                          temp_file.write(chunk)
+                  return temp_file.name
+
+          def get_content_from_s3(source_bucket, key):
+              s3_client = boto3.client('s3')
+              with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+                  s3_client.download_file(
+                      Bucket=source_bucket,
+                      Key=key,
+                      Filename=temp_file.name
+                  )
+                  return temp_file.name
+
+          def sync_buckets(source_bucket, destination_bucket, keys, old_keys):
+              """Sync contents from source bucket to destination bucket using HTTPS download and upload
+              """
+              s3_client = boto3.client('s3')
+              # Delete keys that are no longer needed
+              for key in old_keys:
+                  if key not in keys:
+                      try:
+                          s3_client.delete_object(Bucket=destination_bucket, Key=key)
+                          logger.info(f"Deleted {destination_bucket}/{key}")
+                      except Exception as e:
+                          logger.info(f"Skipping deletion of {destination_bucket}/{key}: {str(e)}")
+              # Download from public URL and upload to destination bucket
+              for key in keys:
+                  try:
+                      try:
+                          temp_file_path = get_content_from_s3(source_bucket, key)
+                      except Exception as e:
+                          logger.warning(f"Error processing {key}: {str(e)}")
+                          logger.info(f"Trying public url")
+                          temp_file_path = get_content(source_bucket, key)
+                      s3_client.upload_file(temp_file_path, destination_bucket, key)
+                      logger.info(f"Done uploading to {destination_bucket}/{key}")
+                  except Exception as e:
+                      logger.error(f"Error processing {key}: {str(e)}")
+                      raise e
+
+          def clean_bucket(bucket_name):
+              """Delete all objects in the bucket
+              """
+              boto3.resource('s3').Bucket(bucket_name).object_versions.delete()
+              logger.info(f"Done cleanup {bucket_name}")
+  SyncLocalAssets:
+    Type: Custom::BucketSync
+    Condition: NeedLocalAssets
+    Properties:
+      ServiceToken: !GetAtt PullAssetsLambda.Arn
+      SourceBucket: !Ref ReferenceAssetsBucket
+      DestinationBucket: !Ref LocalAssetsBucket
+      Keys:
+        - 'cid-resource-lambda-layer/cid-4.0.12.zip' #replace version here if needed
+
   CidResourceLambdaLayer:
     Type: AWS::Lambda::LayerVersion
     Properties:
       LayerName: !Sub 'CidLambdaLayer${Suffix}'
       Description: An AWS managed layer with a cid-cmd package installed
       Content:
         S3Bucket: !If
-          - LambdaLayerBucketPrefixIsManaged
-          - !FindInMap [RegionMap, !Ref 'AWS::Region', BucketName]
-          - !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' # Region added for backward compatibility
+          - NeedLocalAssets
+          - !Ref SyncLocalAssets
+          - Fn::If:
+            - LambdaLayerBucketPrefixIsManaged
+            - !FindInMap [RegionMap, !Ref 'AWS::Region', BucketName]
+            - !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' # Region added for backward compatibility
         S3Key: 'cid-resource-lambda-layer/cid-4.0.12.zip' #replace version here if needed
       CompatibleRuntimes:
         - python3.10
         - python3.11
         - python3.12
 
+
   CostIntelligenceDashboard:
     Type: Custom::CidDashboard
     Condition: NeedCostIntelligenceDashboard
diff --git a/cfn-templates/tests/test_deploy_with_permissions.py b/cfn-templates/tests/test_deploy_with_permissions.py
@@ -293,7 +293,9 @@ def create_cid_as_finops(update):
             {"ParameterKey": 'QuickSightUser', "ParameterValue": get_qs_user()},
             {"ParameterKey": 'CURVersion', "ParameterValue": '2.0'},
             {"ParameterKey": 'DeployCUDOSv5', "ParameterValue": 'yes'},
-            {"ParameterKey": 'LambdaLayerBucketPrefix', "ParameterValue": TMP_BUCKET_PREFIX},
+            {"ParameterKey": 'CreateLocalAssetsBucket', "ParameterValue": 'yes'},
+            {"ParameterKey": 'ReferenceAssetsBucket', "ParameterValue": TMP_BUCKET},
+
         ],
         Capabilities=['CAPABILITY_IAM', 'CAPABILITY_NAMED_IAM'],
     )
