Skip to content

Commit b100ece

Browse files
nhatnghihonhatnghiho
committed
Create pipeline
1 parent 652f7a9 commit b100ece

35 files changed

+1808
-93
lines changed

‎tests/ci/cdk/README.md

+20
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,26 @@ To setup or update the CI in your account you will need the following IAM permis
6363
* secretsmanager:DeleteSecret
6464
* secretsmanager:GetSecretValue
6565

66+
### Pipeline Commands
67+
Bootstrap pipeline account
68+
```
69+
AWS_ACCOUNT_ID=183295444613
70+
PIPELINE_ACCOUNT_ID=774305600158
71+
cdk bootstrap aws://${PIPELINE_ACCOUNT_ID}/us-west-2
72+
```
73+
74+
Give pipeline account administrator access to deployment account's CloudFormation
75+
```
76+
cdk bootstrap aws://${AWS_ACCOUNT_ID}/us-west-2 --trust ${PIPELINE_ACCOUNT_ID} --trust-for-lookup ${PIPELINE_ACCOUNT_ID} --cloudformation-execution-policies arn:aws:iam::aws:policy/AdministratorAccess
77+
```
78+
79+
Deploy pipeline
80+
```
81+
GITHUB_REPO_OWNER=nhatnghiho
82+
GITHUB_SOURCE_VERSION=ci-pipeline
83+
./run-cdk.sh --github-repo-owner ${GITHUB_REPO_OWNER} --aws-account ${AWS_ACCOUNT_ID} --action invoke --command "cdk deploy AwsLcCiPipeline --require-approval never"
84+
```
85+
6686
### Commands
6787

6888
These commands are run from `aws-lc/tests/ci/cdk`. \

‎tests/ci/cdk/app.py

+6-2
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,21 @@
1212
from cdk.aws_lc_github_fuzz_ci_stack import AwsLcGitHubFuzzCIStack
1313
from cdk.aws_lc_ec2_test_framework_ci_stack import AwsLcEC2TestingCIStack
1414
from cdk.linux_docker_image_batch_build_stack import LinuxDockerImageBatchBuildStack
15+
from pipeline.pipeline_stack import AwsLcCiPipeline
1516
from cdk.windows_docker_image_build_stack import WindowsDockerImageBuildStack
1617
from cdk.aws_lc_github_ci_x509_stack import AwsLcGitHubX509CIStack
1718
from cdk.ecr_stack import EcrStack
18-
from util.metadata import AWS_ACCOUNT, AWS_REGION, LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO, WINDOWS_X86_ECR_REPO
19+
from util.metadata import AWS_ACCOUNT, AWS_REGION, LINUX_X86_ECR_REPO, LINUX_AARCH_ECR_REPO, WINDOWS_X86_ECR_REPO, \
20+
PIPELINE_ACCOUNT, PIPELINE_REGION
1921

2022
# Initialize app.
2123
app = App()
2224

2325
# Initialize env.
2426
env = Environment(account=AWS_ACCOUNT, region=AWS_REGION)
2527

28+
AwsLcCiPipeline(app, "AwsLcCiPipeline", env=Environment(account=PIPELINE_ACCOUNT, region=PIPELINE_REGION))
29+
2630
# Define AWS ECR stacks.
2731
# ECR holds the docker images, which are pre-built to accelerate the code builds/tests of git pull requests.
2832
EcrStack(app, "aws-lc-ecr-linux-x86", LINUX_X86_ECR_REPO, env=env)
@@ -55,6 +59,6 @@
5559
AwsLcEC2TestingCIStack(app, "aws-lc-ci-ec2-test-framework", ec2_test_framework_build_spec_file, env=env)
5660
android_build_spec_file = "cdk/codebuild/github_ci_android_omnibus.yaml"
5761
AwsLcAndroidCIStack(app, "aws-lc-ci-devicefarm-android", android_build_spec_file, env=env)
58-
AwsLcGitHubX509CIStack(app, "aws-lc-ci-x509")
62+
AwsLcGitHubX509CIStack(app, "aws-lc-ci-x509", env=env)
5963

6064
app.synth()

‎tests/ci/cdk/cdk/aws_lc_analytics_stack.py

+17-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,15 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
3+
import typing
34

4-
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
5+
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs, \
6+
Environment
57
from constructs import Construct
68

79
from cdk.components import PruneStaleGitHubBuilds
810
from util.iam_policies import code_build_publish_metrics_in_json
9-
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME
11+
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, \
12+
STAGING_GITHUB_REPO_NAME
1013
from util.build_spec_loader import BuildSpecLoader
1114

1215

@@ -17,13 +20,22 @@ def __init__(self,
1720
scope: Construct,
1821
id: str,
1922
spec_file_path: str,
23+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2024
**kwargs) -> None:
21-
super().__init__(scope, id, **kwargs)
25+
super().__init__(scope, id, env=env, **kwargs)
26+
27+
# Define CodeBuild resource.
28+
github_repo_owner = GITHUB_REPO_OWNER
29+
github_repo_name = GITHUB_REPO_NAME
30+
31+
if env.account == PRE_PROD_ACCOUNT:
32+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
33+
github_repo_name = STAGING_GITHUB_REPO_NAME
2234

2335
# Define CodeBuild resource.
2436
git_hub_source = codebuild.Source.git_hub(
25-
owner=GITHUB_REPO_OWNER,
26-
repo=GITHUB_REPO_NAME,
37+
owner=github_repo_owner,
38+
repo=github_repo_name,
2739
webhook=True,
2840
webhook_filters=[
2941
codebuild.FilterGroup.in_event_of(codebuild.EventAction.PUSH)

‎tests/ci/cdk/cdk/aws_lc_android_ci_stack.py

+15-5
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
3+
import typing
34

4-
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam
5+
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, Environment
56
from constructs import Construct
67

78
from cdk.components import PruneStaleGitHubBuilds
89
from util.iam_policies import code_build_batch_policy_in_json, device_farm_access_policy_in_json
9-
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME, GITHUB_PUSH_CI_BRANCH_TARGETS
10+
from util.metadata import GITHUB_REPO_OWNER, GITHUB_REPO_NAME, GITHUB_PUSH_CI_BRANCH_TARGETS, PRE_PROD_ACCOUNT, \
11+
STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME
1012
from util.build_spec_loader import BuildSpecLoader
1113

1214

@@ -20,13 +22,21 @@ def __init__(self,
2022
scope: Construct,
2123
id: str,
2224
spec_file_path: str,
25+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2326
**kwargs) -> None:
24-
super().__init__(scope, id, **kwargs)
27+
super().__init__(scope, id, env=env, **kwargs)
28+
29+
github_repo_owner = GITHUB_REPO_OWNER
30+
github_repo_name = GITHUB_REPO_NAME
31+
32+
if env.account == PRE_PROD_ACCOUNT:
33+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
34+
github_repo_name = STAGING_GITHUB_REPO_NAME
2535

2636
# Define CodeBuild resource.
2737
git_hub_source = codebuild.Source.git_hub(
28-
owner=GITHUB_REPO_OWNER,
29-
repo=GITHUB_REPO_NAME,
38+
owner=github_repo_owner,
39+
repo=github_repo_name,
3040
webhook=True,
3141
webhook_filters=[
3242
codebuild.FilterGroup.in_event_of(

‎tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py

+24-13
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,19 @@
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
33

44
import subprocess
5+
import typing
6+
57
import boto3
68

79
from botocore.exceptions import ClientError
8-
from aws_cdk import CfnTag, Duration, Stack, Tags, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, aws_s3 as s3, aws_logs as logs
10+
from aws_cdk import CfnTag, Duration, Stack, Tags, aws_ec2 as ec2, aws_codebuild as codebuild, aws_iam as iam, \
11+
aws_s3 as s3, aws_logs as logs, Environment
912
from constructs import Construct
1013

1114
from cdk.components import PruneStaleGitHubBuilds
12-
from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, LINUX_AARCH_ECR_REPO, \
13-
LINUX_X86_ECR_REPO
15+
from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, \
16+
LINUX_AARCH_ECR_REPO, \
17+
LINUX_X86_ECR_REPO, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME
1418
from util.iam_policies import code_build_batch_policy_in_json, ec2_policies_in_json, ssm_policies_in_json, s3_read_write_policy_in_json, ecr_power_user_policy_in_json
1519
from util.build_spec_loader import BuildSpecLoader
1620

@@ -23,13 +27,21 @@ def __init__(self,
2327
scope: Construct,
2428
id: str,
2529
spec_file_path: str,
30+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2631
**kwargs) -> None:
27-
super().__init__(scope, id, **kwargs)
32+
super().__init__(scope, id, env=env, **kwargs)
33+
34+
github_repo_owner = GITHUB_REPO_OWNER
35+
github_repo_name = GITHUB_REPO_NAME
36+
37+
if env.account == PRE_PROD_ACCOUNT:
38+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
39+
github_repo_name = STAGING_GITHUB_REPO_NAME
2840

2941
# Define CodeBuild resource.
3042
git_hub_source = codebuild.Source.git_hub(
31-
owner=GITHUB_REPO_OWNER,
32-
repo=GITHUB_REPO_NAME,
43+
owner=github_repo_owner,
44+
repo=github_repo_name,
3345
webhook=True,
3446
webhook_filters=[
3547
codebuild.FilterGroup.in_event_of(
@@ -62,15 +74,14 @@ def __init__(self,
6274
selected_subnets = vpc.select_subnets(subnet_type=ec2.SubnetType.PRIVATE_WITH_EGRESS)
6375

6476
# create security group with default rules
65-
security_group = ec2.SecurityGroup(self, id="{}-ec2-sg".format(id),
66-
allow_all_outbound=True,
67-
vpc=vpc,
68-
security_group_name='codebuild_ec2_sg')
69-
77+
# security_group = ec2.SecurityGroup(self, id="{}-ec2-sg".format(id),
78+
# allow_all_outbound=True,
79+
# vpc=vpc,
80+
# security_group_name='codebuild_ec2_sg')
7081

7182
# Define a IAM role for this stack.
7283
code_build_batch_policy = iam.PolicyDocument.from_json(code_build_batch_policy_in_json([id]))
73-
ec2_policy = iam.PolicyDocument.from_json(ec2_policies_in_json(ec2_role.role_name, security_group.security_group_id, selected_subnets.subnets[0].subnet_id, vpc.vpc_id))
84+
ec2_policy = iam.PolicyDocument.from_json(ec2_policies_in_json(ec2_role.role_name, vpc.vpc_default_security_group, selected_subnets.subnets[0].subnet_id, vpc.vpc_id))
7485
ssm_policy = iam.PolicyDocument.from_json(ssm_policies_in_json())
7586
codebuild_inline_policies = {"code_build_batch_policy": code_build_batch_policy,
7687
"ec2_policy": ec2_policy,
@@ -97,7 +108,7 @@ def __init__(self,
97108
build_spec=BuildSpecLoader.load(spec_file_path),
98109
environment_variables= {
99110
"EC2_SECURITY_GROUP_ID": codebuild.BuildEnvironmentVariable(
100-
value=security_group.security_group_id
111+
value=vpc.vpc_default_security_group
101112
),
102113
"EC2_SUBNET_ID": codebuild.BuildEnvironmentVariable(
103114
value=selected_subnets.subnets[0].subnet_id

‎tests/ci/cdk/cdk/aws_lc_github_ci_stack.py

+31-8
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,16 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
3+
import typing
34

4-
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_s3_assets, aws_logs as logs
5+
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_s3_assets, aws_logs as logs, \
6+
Environment
57
from constructs import Construct
68

79
from cdk.components import PruneStaleGitHubBuilds
8-
from util.iam_policies import code_build_batch_policy_in_json, code_build_publish_metrics_in_json, code_build_cloudwatch_logs_policy_in_json
9-
from util.metadata import CAN_AUTOLOAD, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
10+
from util.iam_policies import code_build_batch_policy_in_json, code_build_publish_metrics_in_json, \
11+
code_build_cloudwatch_logs_policy_in_json, s3_read_policy_in_json
12+
from util.metadata import CAN_AUTOLOAD, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, \
13+
PIPELINE_ACCOUNT, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME
1014
from util.build_spec_loader import BuildSpecLoader
1115

1216

@@ -17,13 +21,21 @@ def __init__(self,
1721
scope: Construct,
1822
id: str,
1923
spec_file_path: str,
24+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2025
**kwargs) -> None:
21-
super().__init__(scope, id, **kwargs)
26+
super().__init__(scope, id, env=env, **kwargs)
27+
28+
github_repo_owner = GITHUB_REPO_OWNER
29+
github_repo_name = GITHUB_REPO_NAME
30+
31+
if env.account == PRE_PROD_ACCOUNT:
32+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
33+
github_repo_name = STAGING_GITHUB_REPO_NAME
2234

2335
# Define CodeBuild resource.
2436
git_hub_source = codebuild.Source.git_hub(
25-
owner=GITHUB_REPO_OWNER,
26-
repo=GITHUB_REPO_NAME,
37+
owner=github_repo_owner,
38+
repo=github_repo_name,
2739
webhook=True,
2840
webhook_filters=[
2941
codebuild.FilterGroup.in_event_of(
@@ -40,18 +52,24 @@ def __init__(self,
4052
code_build_cloudwatch_logs_policy = iam.PolicyDocument.from_json(
4153
code_build_cloudwatch_logs_policy_in_json([log_group])
4254
)
55+
s3_assets_policy = iam.PolicyDocument.from_json(s3_read_policy_in_json())
4356
resource_access_role = iam.Role(scope=self,
4457
id="{}-resource-role".format(id),
45-
assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"),
58+
assumed_by=iam.CompositePrincipal(
59+
iam.ServicePrincipal("codebuild.amazonaws.com"),
60+
iam.ArnPrincipal(f'arn:aws:iam::{PIPELINE_ACCOUNT}:role/CrossAccountCodeBuildRole')
61+
),
4662
inline_policies={
47-
"code_build_cloudwatch_logs_policy": code_build_cloudwatch_logs_policy
63+
"code_build_cloudwatch_logs_policy": code_build_cloudwatch_logs_policy,
64+
"s3_assets_policy": s3_assets_policy
4865
})
4966

5067
# Define a IAM role for this stack.
5168
code_build_batch_policy = iam.PolicyDocument.from_json(
5269
code_build_batch_policy_in_json([id])
5370
)
5471
metrics_policy = iam.PolicyDocument.from_json(code_build_publish_metrics_in_json())
72+
5573
inline_policies = {"code_build_batch_policy": code_build_batch_policy,
5674
"metrics_policy": metrics_policy,
5775
}
@@ -66,6 +84,11 @@ def __init__(self,
6684
)
6785
)
6886

87+
# test = iam.Role(scope=self,
88+
# id="test",
89+
# assumed_by=iam.ServicePrincipal("codebuild.amazonaws.com"),
90+
# inline_policies=inline_policies)
91+
6992
# Define CodeBuild.
7093
project = codebuild.Project(
7194
scope=self,

‎tests/ci/cdk/cdk/aws_lc_github_ci_x509_stack.py

+16-5
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
1-
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_s3 as s3
1+
import typing
2+
3+
from aws_cdk import Duration, Stack, aws_codebuild as codebuild, aws_s3 as s3, Environment
24
from constructs import Construct
35
from util.build_spec_loader import BuildSpecLoader
46
from util.metadata import (
57
GITHUB_PUSH_CI_BRANCH_TARGETS,
68
GITHUB_REPO_NAME,
7-
GITHUB_REPO_OWNER,
9+
GITHUB_REPO_OWNER, PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME,
810
)
911

1012

@@ -13,13 +15,22 @@ def __init__(
1315
self,
1416
scope: Construct,
1517
id: str,
18+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
1619
**kwargs,
1720
) -> None:
18-
super().__init__(scope, id, **kwargs)
21+
super().__init__(scope, id, env=env, **kwargs)
22+
23+
github_repo_owner = GITHUB_REPO_OWNER
24+
github_repo_name = GITHUB_REPO_NAME
25+
26+
if env.account == PRE_PROD_ACCOUNT:
27+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
28+
github_repo_name = STAGING_GITHUB_REPO_NAME
1929

30+
# Define CodeBuild resource.
2031
git_hub_source = codebuild.Source.git_hub(
21-
owner=GITHUB_REPO_OWNER,
22-
repo=GITHUB_REPO_NAME,
32+
owner=github_repo_owner,
33+
repo=github_repo_name,
2334
webhook=True,
2435
webhook_filters=[
2536
codebuild.FilterGroup.in_event_of(

‎tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py

+16-5
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,17 @@
11
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
22
# SPDX-License-Identifier: Apache-2.0 OR ISC
3+
import typing
34

4-
from aws_cdk import Duration, Size, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs
5+
from aws_cdk import Duration, Size, Stack, aws_codebuild as codebuild, aws_iam as iam, aws_ec2 as ec2, aws_efs as efs, \
6+
Environment
57
from constructs import Construct
68

79
from cdk.components import PruneStaleGitHubBuilds
810
from util.ecr_util import ecr_arn
911
from util.iam_policies import code_build_batch_policy_in_json, \
1012
code_build_publish_metrics_in_json
11-
from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME
13+
from util.metadata import AWS_ACCOUNT, AWS_REGION, GITHUB_PUSH_CI_BRANCH_TARGETS, GITHUB_REPO_OWNER, GITHUB_REPO_NAME, \
14+
PRE_PROD_ACCOUNT, STAGING_GITHUB_REPO_OWNER, STAGING_GITHUB_REPO_NAME
1215
from util.build_spec_loader import BuildSpecLoader
1316

1417

@@ -19,13 +22,21 @@ def __init__(self,
1922
scope: Construct,
2023
id: str,
2124
spec_file_path: str,
25+
env: typing.Optional[typing.Union[Environment, typing.Dict[str, typing.Any]]],
2226
**kwargs) -> None:
23-
super().__init__(scope, id, **kwargs)
27+
super().__init__(scope, id, env=env, **kwargs)
28+
29+
github_repo_owner = GITHUB_REPO_OWNER
30+
github_repo_name = GITHUB_REPO_NAME
31+
32+
if env.account == PRE_PROD_ACCOUNT:
33+
github_repo_owner = STAGING_GITHUB_REPO_OWNER
34+
github_repo_name = STAGING_GITHUB_REPO_NAME
2435

2536
# Define CodeBuild resource.
2637
git_hub_source = codebuild.Source.git_hub(
27-
owner=GITHUB_REPO_OWNER,
28-
repo=GITHUB_REPO_NAME,
38+
owner=github_repo_owner,
39+
repo=github_repo_name,
2940
webhook=True,
3041
webhook_filters=[
3142
codebuild.FilterGroup.in_event_of(

0 commit comments

Comments
 (0)