From 35f11f0c8aae4b061dc9eec02bb041e3b1c5b469 Mon Sep 17 00:00:00 2001 From: daka83 Date: Thu, 20 Oct 2022 01:15:32 +0200 Subject: [PATCH 1/2] added support for lambda insights --- chalice/app.py | 11 +++-- chalice/awsclient.py | 25 +++++++++++ chalice/deploy/appgraph.py | 45 +++++++++++++++++++- chalice/deploy/deployer.py | 2 +- chalice/deploy/models.py | 13 ++++++ chalice/deploy/planner.py | 84 +++++++++++++++++++++++++++++++++++++ chalice/deploy/sweeper.py | 17 ++++++++ chalice/layer-versions.json | 25 +++++++++++ chalice/layer_versions.py | 16 +++++++ 9 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 chalice/layer-versions.json create mode 100644 chalice/layer_versions.py diff --git a/chalice/app.py b/chalice/app.py index de6726bd6..de5506ff5 100644 --- a/chalice/app.py +++ b/chalice/app.py @@ -842,9 +842,11 @@ def route(self, path: str, **kwargs: Any) -> Callable[..., Any]: ) def lambda_function(self, - name: Optional[str] = None) -> Callable[..., Any]: + name: Optional[str] = None, + insights: Optional[bool] = False) -> Callable[..., Any]: return self._create_registration_function( - handler_type='lambda_function', name=name) + handler_type='lambda_function', name=name, + registration_kwargs={'insights': insights}) def on_ws_connect(self, name: Optional[str] = None) -> Callable[..., Any]: @@ -1043,9 +1045,11 @@ def _register_lambda_function(self, name: str, user_handler: UserHandlerFuncType, handler_string: str, **unused: Dict[str, Any]) -> None: + kwargs = unused.get('kwargs', {}) wrapper = LambdaFunction( func=user_handler, name=name, handler_string=handler_string, + insights=kwargs.get('insights', False) ) self.pure_lambda_functions.append(wrapper) @@ -1521,10 +1525,11 @@ def __init__(self, path: str, methods: List[str]): class LambdaFunction(object): def __init__(self, func: Callable[..., Any], name: str, - handler_string: str): + handler_string: str, insights: bool): self.func: Callable[..., Any] = func self.name: str = name self.handler_string: str = handler_string + self.insights: bool = insights def __call__(self, event: Dict[str, Any], context: Dict[str, Any] diff --git a/chalice/awsclient.py b/chalice/awsclient.py index 6603ccae2..e053ece7e 100644 --- a/chalice/awsclient.py +++ b/chalice/awsclient.py @@ -1050,6 +1050,31 @@ def create_role(self, name, trust_policy, policy): raise e return role_arn + def attach_role_policy(self, role_name, policy_arn): + # type: (str, str) -> None + self._client('iam').attach_role_policy( + RoleName=role_name, + PolicyArn=policy_arn) + + def detach_role_policy(self, role_name, policy_arn): + # type: (str, str) -> None + self._client('iam').detach_role_policy( + RoleName=role_name, + PolicyArn=policy_arn) + + def is_role_policy_attached(self, role_name, policy_arn): + # type: (str, str) -> bool + client = self._client('iam') + try: + attached_policies = client.list_attached_role_policies(RoleName=role_name)['AttachedPolicies'] + except client.exceptions.NoSuchEntityException: + raise ResourceDoesNotExistError("No role found for: %s" % role_name) + + for policy in attached_policies: + if policy['PolicyArn'] == policy_arn: + return True + return False + def delete_role(self, name): # type: (str) -> None """Delete a role by first deleting all inline policies.""" diff --git a/chalice/deploy/appgraph.py b/chalice/deploy/appgraph.py index 12cebfbac..4d7896e9a 100644 --- a/chalice/deploy/appgraph.py +++ b/chalice/deploy/appgraph.py @@ -10,6 +10,7 @@ from chalice.constants import LAMBDA_TRUST_POLICY from chalice.deploy import models from chalice.utils import UI # noqa +from chalice.layer_versions import LAYER_VERSIONS StrMapAny = Dict[str, Any] @@ -19,8 +20,9 @@ class ChaliceBuildError(Exception): class ApplicationGraphBuilder(object): - def __init__(self): + def __init__(self, client): # type: () -> None + self._client = client self._known_roles = {} # type: Dict[str, models.IAMRole] self._managed_layer = None # type: Optional[models.LambdaLayer] @@ -33,6 +35,18 @@ def build(self, config, stage_name): config=config, deployment=deployment, name=function.name, handler_name=function.handler_string, stage_name=stage_name) + # create lambda insight + if function.insights: + resource.role_policy_attachment = self._get_role_policy_attachment( + config=config, + stage_name=stage_name, + function_name=function.name) + layer_arn = LAYER_VERSIONS.get(self._client.region_name) + if layer_arn: + if resource.layers: + resource.layers.append(layer_arn) + else: + resource.layers = [layer_arn] resources.append(resource) event_resources = self._create_lambda_event_resources( config, deployment, stage_name) @@ -444,6 +458,35 @@ def _create_role_reference(self, config, stage_name, function_name): policy=policy, ) + def _get_role_policy_attachment(self, config, stage_name, function_name): + # type: (Config, str, str) -> models.IAMRolePolicyAttachment + role = self._create_role_policy_attachment(config, stage_name, function_name) + role_identifier = self._get_role_identifier(role) + if role_identifier in self._known_roles: + # If we've already create a models.IAMRole with the same + # identifier, we'll use the existing object instead of + # creating a new one. + return self._known_roles[role_identifier] + self._known_roles[role_identifier] = role + return role + + def _create_role_policy_attachment(self, config, stage_name, function_name): + + if not config.autogen_policy: + resource_name = '%s_execution_role' % function_name + role_name = '%s-%s-%s' % (config.app_name, stage_name, + function_name) + else: + resource_name = 'default_execution_role' + role_name = '%s-%s' % (config.app_name, stage_name) + policy_arn = 'arn:aws:iam::aws:policy/CloudWatchLambdaInsightsExecutionRolePolicy' + + return models.IAMRolePolicyAttachment( + resource_name=resource_name, + role_name=role_name, + policy_arn=policy_arn + ) + def _get_vpc_params(self, function_name, config): # type: (str, Config) -> Tuple[List[str], List[str]] security_group_ids = config.security_group_ids diff --git a/chalice/deploy/deployer.py b/chalice/deploy/deployer.py index de1c17cf7..1fc66bcf7 100644 --- a/chalice/deploy/deployer.py +++ b/chalice/deploy/deployer.py @@ -271,7 +271,7 @@ def _create_deployer(session, # type: Session client = TypedAWSClient(session) osutils = OSUtils() return Deployer( - application_builder=ApplicationGraphBuilder(), + application_builder=ApplicationGraphBuilder(client), deps_builder=DependencyBuilder(), build_stage=create_build_stage( osutils, UI(), TemplatedSwaggerGenerator(), config diff --git a/chalice/deploy/models.py b/chalice/deploy/models.py index 08881e15b..147f55117 100644 --- a/chalice/deploy/models.py +++ b/chalice/deploy/models.py @@ -175,6 +175,15 @@ def dependencies(self): return [self.policy] +@attrs +class IAMRolePolicyAttachment(IAMRole, ManagedModel): + # resource_type = 'iam_role_policy_attachment' + resource_type = 'iam_role' + role_name = attrib() # type: str + # trust_policy = attrib() # type: Dict[str, Any] + policy_arn = attrib() # type: str + + @attrs class LambdaLayer(ManagedModel): resource_type = 'lambda_layer' @@ -208,6 +217,8 @@ class LambdaFunction(ManagedModel): layers = attrib() # type: List[str] managed_layer = attrib( default=None) # type: Opt[LambdaLayer] + role_policy_attachment = attrib( + default=None) # type: IAMRolePolicyAttachment def dependencies(self): # type: () -> List[Model] @@ -215,6 +226,8 @@ def dependencies(self): if self.managed_layer is not None: resources.append(self.managed_layer) resources.extend([self.role, self.deployment_package]) + if self.role_policy_attachment: + resources.append(self.role_policy_attachment) return resources diff --git a/chalice/deploy/planner.py b/chalice/deploy/planner.py index 533012d42..ec47f9594 100644 --- a/chalice/deploy/planner.py +++ b/chalice/deploy/planner.py @@ -53,6 +53,14 @@ def _dynamically_lookup_values(self, resource): "name": resource.resource_name, "resource_type": "iam_role", } + elif isinstance(resource, models.IAMRolePolicyAttachment): + arn = self._client.get_role_arn_for_name(resource.role_name) + return { + "role_name": resource.role_name, + "role_arn": arn, + "name": resource.resource_name, + "resource_type": "iam_role", + } raise ValueError("Deployed values for resource does not exist: %s" % resource.resource_name) @@ -151,6 +159,15 @@ def _resource_exists_managediamrole(self, resource): except ResourceDoesNotExistError: return False + def _resource_exists_iamrolepolicyattachment(self, resource): + # type: (models.IAMRolePolicyAttachment) -> bool + try: + resp = self._client.is_role_policy_attached(resource.role_name, + resource.policy_arn) + return resp + except ResourceDoesNotExistError: + return False + def _resource_exists_apimapping(self, resource, domain_name): # type: (models.APIMapping, str) -> bool map_key = resource.mount_path @@ -620,6 +637,73 @@ def _plan_managediamrole(self, resource): ) ] + def _plan_iamrolepolicyattachment(self, resource): + # type: (models.IAMRolePolicyAttachment) -> Sequence[InstructionMsg] + role_exists = self._remote_state.resource_exists(resource) + varname = '%s_execution_role_arn' % resource.role_name + if not role_exists: + try: + role_arn = self._remote_state._client.get_role_arn_for_name(resource.role_name) + record = models.RecordResourceValue( + resource_type='iam_role', + resource_name=resource.resource_name, + name='role_arn', + value=role_arn, + ) + except ResourceDoesNotExistError: + record = models.RecordResourceVariable( + resource_type='iam_role', + resource_name=resource.resource_name, + name='role_arn', + variable_name=varname, + ) + + return [ + (models.APICall( + method_name='attach_role_policy', + params={'role_name': resource.role_name, + 'policy_arn': resource.policy_arn}, + output_var=varname, + ), "Creating IAM role policy attachment: %s\n" % resource.role_name), + record, + models.RecordResourceValue( + resource_type='iam_role', + resource_name=resource.resource_name, + name='role_name', + value=resource.role_name, + ), + models.RecordResourceValue( + resource_type='iam_role', + resource_name=resource.resource_name, + name='policy_arn', + value=resource.policy_arn, + ) + ] + + role_arn = self._remote_state.resource_deployed_values( + resource)['role_arn'] + return [ + models.StoreValue(name=varname, value=role_arn), + models.RecordResourceVariable( + resource_type='iam_role', + resource_name=resource.resource_name, + name='role_arn', + variable_name=varname, + ), + models.RecordResourceValue( + resource_type='iam_role', + resource_name=resource.resource_name, + name='role_name', + value=resource.role_name, + ), + models.RecordResourceValue( + resource_type='iam_role', + resource_name=resource.resource_name, + name='policy_arn', + value=resource.policy_arn, + ) + ] + def _plan_snslambdasubscription(self, resource): # type: (models.SNSLambdaSubscription) -> Sequence[InstructionMsg] function_arn = Variable( diff --git a/chalice/deploy/sweeper.py b/chalice/deploy/sweeper.py index 821de43c5..3e9a728a7 100644 --- a/chalice/deploy/sweeper.py +++ b/chalice/deploy/sweeper.py @@ -258,6 +258,21 @@ def _delete_iam_role(self, resource_values): 'message': 'Deleting IAM role: %s\n' % resource_values['role_name'] } + def _delete_iam_role_policy_attachment(self, resource_values): + # type: (Dict[str, Any]) -> ResourceValueType + return { + 'instructions': ( + models.APICall( + method_name='detach_role_policy', + params={ + 'role_name': resource_values['role_name'], + 'policy_arn': resource_values['policy_arn'], + }, + ), + ), + 'message': 'Detaching IAM role policy attachment: %s\n' % resource_values['role_name'] + } + def _delete_cloudwatch_event(self, resource_values): # type: (Dict[str, Any]) -> ResourceValueType return { @@ -426,6 +441,8 @@ def _plan_deletion(self, resource_type = 'domain_api_mappings' handler_args.append(name) insert = True + if name.endswith('_execution_role'): + resource_type = 'iam_role_policy_attachment' method_name = '_delete_%s' % resource_type handler = getattr(self, method_name, self._default_delete) diff --git a/chalice/layer-versions.json b/chalice/layer-versions.json new file mode 100644 index 000000000..9f4f11719 --- /dev/null +++ b/chalice/layer-versions.json @@ -0,0 +1,25 @@ +{ + "us-east-1": "arn:aws:lambda:us-east-1:580247275435:layer:LambdaInsightsExtension:14", + "us-east-2": "arn:aws:lambda:us-east-2:580247275435:layer:LambdaInsightsExtension:14", + "us-west-1": "arn:aws:lambda:us-west-1:580247275435:layer:LambdaInsightsExtension:14", + "us-west-2": "arn:aws:lambda:us-west-2:580247275435:layer:LambdaInsightsExtension:14", + "ap-south-1": "arn:aws:lambda:ap-south-1:580247275435:layer:LambdaInsightsExtension:14", + "ap-northeast-2": "arn:aws:lambda:ap-northeast-2:580247275435:layer:LambdaInsightsExtension:14", + "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:580247275435:layer:LambdaInsightsExtension:14", + "ap-southeast-2": "arn:aws:lambda:ap-southeast-2:580247275435:layer:LambdaInsightsExtension:14", + "ap-northeast-1": "arn:aws:lambda:ap-northeast-1:580247275435:layer:LambdaInsightsExtension:14", + "ca-central-1": "arn:aws:lambda:ca-central-1:580247275435:layer:LambdaInsightsExtension:14", + "eu-central-1": "arn:aws:lambda:eu-central-1:580247275435:layer:LambdaInsightsExtension:14", + "eu-west-1": "arn:aws:lambda:eu-west-1:580247275435:layer:LambdaInsightsExtension:14", + "eu-west-2": "arn:aws:lambda:eu-west-2:580247275435:layer:LambdaInsightsExtension:14", + "eu-west-3": "arn:aws:lambda:eu-west-3:580247275435:layer:LambdaInsightsExtension:14", + "eu-north-1": "arn:aws:lambda:eu-north-1:580247275435:layer:LambdaInsightsExtension:14", + "sa-east-1": "arn:aws:lambda:sa-east-1:580247275435:layer:LambdaInsightsExtension:14", + "cn-north-1": "arn:aws-cn:lambda:cn-north-1:488211338238:layer:LambdaInsightsExtension:8", + "cn-northwest-1": "arn:aws-cn:lambda:cn-northwest-1:488211338238:layer:LambdaInsightsExtension:8", + "af-south-1": "arn:aws:lambda:af-south-1:012438385374:layer:LambdaInsightsExtension:8", + "ap-east-1": "arn:aws:lambda:ap-east-1:519774774795:layer:LambdaInsightsExtension:8", + "me-south-1": "arn:aws:lambda:me-south-1:285320876703:layer:LambdaInsightsExtension:8", + "eu-south-1": "arn:aws:lambda:eu-south-1:339249233099:layer:LambdaInsightsExtension:8" +} + diff --git a/chalice/layer_versions.py b/chalice/layer_versions.py new file mode 100644 index 000000000..2bda53940 --- /dev/null +++ b/chalice/layer_versions.py @@ -0,0 +1,16 @@ +import os +import json + +from typing import Dict + + +def load_layer_versions(): + # type: () -> Dict[str, str] + layers_json = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + 'layer-versions.json') + with open(layers_json) as f: + return json.loads(f.read()) + + +LAYER_VERSIONS = load_layer_versions() From 4fe4f04d962698f15ee093e12c787b48c40d31a8 Mon Sep 17 00:00:00 2001 From: daka83 Date: Thu, 20 Oct 2022 15:22:32 +0200 Subject: [PATCH 2/2] fix lineter and test error --- chalice/app.py | 3 ++- chalice/awsclient.py | 6 ++++-- chalice/deploy/appgraph.py | 37 ++++++++++++++++++++++--------------- chalice/deploy/deployer.py | 2 +- chalice/deploy/planner.py | 36 +++++++++++++++++++++--------------- chalice/deploy/sweeper.py | 3 ++- setup.py | 2 +- 7 files changed, 53 insertions(+), 36 deletions(-) diff --git a/chalice/app.py b/chalice/app.py index de5506ff5..8c1d4e617 100644 --- a/chalice/app.py +++ b/chalice/app.py @@ -843,7 +843,8 @@ def route(self, path: str, **kwargs: Any) -> Callable[..., Any]: def lambda_function(self, name: Optional[str] = None, - insights: Optional[bool] = False) -> Callable[..., Any]: + insights: Optional[bool] = False + ) -> Callable[..., Any]: return self._create_registration_function( handler_type='lambda_function', name=name, registration_kwargs={'insights': insights}) diff --git a/chalice/awsclient.py b/chalice/awsclient.py index e053ece7e..37cdcc118 100644 --- a/chalice/awsclient.py +++ b/chalice/awsclient.py @@ -1066,9 +1066,11 @@ def is_role_policy_attached(self, role_name, policy_arn): # type: (str, str) -> bool client = self._client('iam') try: - attached_policies = client.list_attached_role_policies(RoleName=role_name)['AttachedPolicies'] + attached_policies = client.list_attached_role_policies( + RoleName=role_name)['AttachedPolicies'] except client.exceptions.NoSuchEntityException: - raise ResourceDoesNotExistError("No role found for: %s" % role_name) + raise ResourceDoesNotExistError( + "No role found for: %s" % role_name) for policy in attached_policies: if policy['PolicyArn'] == policy_arn: diff --git a/chalice/deploy/appgraph.py b/chalice/deploy/appgraph.py index 4d7896e9a..34829d11c 100644 --- a/chalice/deploy/appgraph.py +++ b/chalice/deploy/appgraph.py @@ -20,10 +20,12 @@ class ChaliceBuildError(Exception): class ApplicationGraphBuilder(object): - def __init__(self, client): - # type: () -> None - self._client = client + def __init__(self, region_name=''): + # type: (str) -> None + self._region_name = region_name self._known_roles = {} # type: Dict[str, models.IAMRole] + self._known_role_policy_attach = \ + {} # type: Dict[str, models.IAMRolePolicyAttachment] self._managed_layer = None # type: Optional[models.LambdaLayer] def build(self, config, stage_name): @@ -37,11 +39,12 @@ def build(self, config, stage_name): stage_name=stage_name) # create lambda insight if function.insights: - resource.role_policy_attachment = self._get_role_policy_attachment( - config=config, - stage_name=stage_name, - function_name=function.name) - layer_arn = LAYER_VERSIONS.get(self._client.region_name) + resource.role_policy_attachment = \ + self._get_role_policy_attachment( + config=config, + stage_name=stage_name, + function_name=function.name) + layer_arn = LAYER_VERSIONS.get(self._region_name) if layer_arn: if resource.layers: resource.layers.append(layer_arn) @@ -460,18 +463,21 @@ def _create_role_reference(self, config, stage_name, function_name): def _get_role_policy_attachment(self, config, stage_name, function_name): # type: (Config, str, str) -> models.IAMRolePolicyAttachment - role = self._create_role_policy_attachment(config, stage_name, function_name) + role = self._create_role_policy_attachment(config, + stage_name, + function_name) role_identifier = self._get_role_identifier(role) - if role_identifier in self._known_roles: + if role_identifier in self._known_role_policy_attach: # If we've already create a models.IAMRole with the same # identifier, we'll use the existing object instead of # creating a new one. - return self._known_roles[role_identifier] - self._known_roles[role_identifier] = role + return self._known_role_policy_attach[role_identifier] + self._known_role_policy_attach[role_identifier] = role return role - def _create_role_policy_attachment(self, config, stage_name, function_name): - + def _create_role_policy_attachment(self, config, stage_name, + function_name): + # type: (Config, str, str) -> models.IAMRolePolicyAttachment if not config.autogen_policy: resource_name = '%s_execution_role' % function_name role_name = '%s-%s-%s' % (config.app_name, stage_name, @@ -479,7 +485,8 @@ def _create_role_policy_attachment(self, config, stage_name, function_name): else: resource_name = 'default_execution_role' role_name = '%s-%s' % (config.app_name, stage_name) - policy_arn = 'arn:aws:iam::aws:policy/CloudWatchLambdaInsightsExecutionRolePolicy' + policy_arn = "arn:aws:iam::aws:policy/"\ + "CloudWatchLambdaInsightsExecutionRolePolicy" return models.IAMRolePolicyAttachment( resource_name=resource_name, diff --git a/chalice/deploy/deployer.py b/chalice/deploy/deployer.py index 1fc66bcf7..2f252c34b 100644 --- a/chalice/deploy/deployer.py +++ b/chalice/deploy/deployer.py @@ -271,7 +271,7 @@ def _create_deployer(session, # type: Session client = TypedAWSClient(session) osutils = OSUtils() return Deployer( - application_builder=ApplicationGraphBuilder(client), + application_builder=ApplicationGraphBuilder(client.region_name), deps_builder=DependencyBuilder(), build_stage=create_build_stage( osutils, UI(), TemplatedSwaggerGenerator(), config diff --git a/chalice/deploy/planner.py b/chalice/deploy/planner.py index ec47f9594..154ad80b7 100644 --- a/chalice/deploy/planner.py +++ b/chalice/deploy/planner.py @@ -25,6 +25,11 @@ def __init__(self, client, deployed_resources): self._cache = {} # type: Dict[CacheTuples, bool] self._deployed_resources = deployed_resources + @property + def client(self): + # type: () -> TypedAWSClient + return self._client + def _cache_key(self, resource): # type: (models.ManagedModel) -> CacheTuples if isinstance(resource, models.APIMapping): @@ -643,20 +648,10 @@ def _plan_iamrolepolicyattachment(self, resource): varname = '%s_execution_role_arn' % resource.role_name if not role_exists: try: - role_arn = self._remote_state._client.get_role_arn_for_name(resource.role_name) - record = models.RecordResourceValue( - resource_type='iam_role', - resource_name=resource.resource_name, - name='role_arn', - value=role_arn, - ) + role_arn = self._remote_state.client.get_role_arn_for_name( + resource.role_name) except ResourceDoesNotExistError: - record = models.RecordResourceVariable( - resource_type='iam_role', - resource_name=resource.resource_name, - name='role_arn', - variable_name=varname, - ) + role_arn = None return [ (models.APICall( @@ -664,8 +659,19 @@ def _plan_iamrolepolicyattachment(self, resource): params={'role_name': resource.role_name, 'policy_arn': resource.policy_arn}, output_var=varname, - ), "Creating IAM role policy attachment: %s\n" % resource.role_name), - record, + ), "Creating IAM role policy attachment: %s\n" % + resource.role_name), + models.RecordResourceValue( + resource_type='iam_role', + resource_name=resource.resource_name, + name='role_arn', + value=role_arn, + ) if role_arn else models.RecordResourceVariable( + resource_type='iam_role', + resource_name=resource.resource_name, + name='role_arn', + variable_name=varname, + ), models.RecordResourceValue( resource_type='iam_role', resource_name=resource.resource_name, diff --git a/chalice/deploy/sweeper.py b/chalice/deploy/sweeper.py index 3e9a728a7..6876dfcde 100644 --- a/chalice/deploy/sweeper.py +++ b/chalice/deploy/sweeper.py @@ -270,7 +270,8 @@ def _delete_iam_role_policy_attachment(self, resource_values): }, ), ), - 'message': 'Detaching IAM role policy attachment: %s\n' % resource_values['role_name'] + 'message': 'Detaching IAM role policy attachment: %s\n' % + resource_values['role_name'] } def _delete_cloudwatch_event(self, resource_values): diff --git a/setup.py b/setup.py index 68f635a96..5244864b1 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ def recursive_include(relative_dir): 'typing==3.6.4;python_version<"3.7"', 'typing-extensions>=4.0.0,<5.0.0', 'six>=1.10.0,<2.0.0', - 'pip>=9,<22.3', + 'pip>=9,<22.4', 'attrs>=19.3.0,<21.5.0', 'jmespath>=0.9.3,<2.0.0', 'pyyaml>=5.3.1,<7.0.0',