Skip to content

Commit 4649805

Browse files
patduinpatduin
authored
Adding glue write support (#158)
* Added policies for glue write support
1 parent 38370fb commit 4649805

File tree

6 files changed

+98
-3
lines changed

6 files changed

+98
-3
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file.
33

44
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
55

6+
## [4.8.0] - 2025-11-12
7+
### Added
8+
- Added optional glue write policy
9+
- Added optional s3 write policy (if glue enabled)
10+
- Both policies are needed if Waggle Dance uses Glue as the primary metastore. LakeFormation policies are also needed but outside the scope of this library.
11+
612
## [4.7.0] - 2025-10-08
713
### Added
814
- Added `primary_metastore_read_only_glue_account_id` and `primary_metastore_read_only_glue_endpoint` to support waggledance traffic control to glue for reads.

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ For more information please refer to the main [Apiary](https://github.com/Expedi
7575
| include_datadog_agent | Whether to include the datadog-agent container alongside Waggledance. | string | bool | no |
7676
| metrics_port | The port on which the WaggleDance application initiates. Additionally, it serves as the port from which we parse metrics. | string | `18000` | yes |
7777
| extended_server_config | Extended waggle-dance-server.yml configuration for Waggle Dance (see [Waggle Dance README](https://github.com/ExpediaGroup/waggle-dance/blob/main/README.md) for all options). String will be [yamlencoded](https://developer.hashicorp.com/terraform/language/functions/yamlencode). | string | | no |
78+
| s3_glue_tables_bucket | S3 Bucket pattern where Hive tables will be created. Glue and Waggledance need this to create the glue table pointers to the S3 location. | string | `my-bucket-*` | no |
7879

7980

8081

‎common.tf‎

Lines changed: 48 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ data "aws_secretsmanager_secret" "docker_registry" {
4545
name = var.docker_registry_auth_secret_name
4646
}
4747

48-
data "aws_iam_policy_document" "waggle_dance_remote_glue_federations_policy" {
48+
data "aws_iam_policy_document" "waggle_dance_remote_glue_federations_policy_read" {
4949
count = local.glue_enabled ? 1 : 0
5050
statement {
5151
sid = "WaggledanceRemoteGlueFederationsPolicy"
@@ -68,6 +68,53 @@ data "aws_iam_policy_document" "waggle_dance_remote_glue_federations_policy" {
6868
}
6969
}
7070

71+
data "aws_iam_policy_document" "waggle_dance_remote_glue_federations_policy_write" {
72+
count = local.glue_enabled ? 1 : 0
73+
statement {
74+
sid = "WaggledanceRemoteGlueFederationsPolicy"
75+
actions = [
76+
"glue:CreatePartition",
77+
"glue:CreateTable",
78+
"glue:DeletePartition",
79+
"glue:DeleteTable",
80+
"glue:UpdatePartition",
81+
"glue:UpdateTable",
82+
"glue:BatchUpdatePartition",
83+
"glue:BatchDeletePartition",
84+
"glue:BatchCreatePartition"
85+
]
86+
resources = [
87+
for glue_account_id in local.glue_account_ids:
88+
format("arn:aws:glue:%s:%s:*", var.aws_region, glue_account_id)
89+
]
90+
}
91+
}
92+
93+
#Glue client creates folders on s3 for create tables. This policy gives that access.
94+
data "aws_iam_policy_document" "waggle_dance_remote_glue_federations_policy_s3_write" {
95+
count = local.glue_enabled ? 1 : 0
96+
statement {
97+
sid = "WaggledanceRemoteGlueFederationsPolicy"
98+
actions = [
99+
"s3:DeleteObject",
100+
"s3:DeleteObjectVersion",
101+
"s3:Get*",
102+
"s3:List*",
103+
"s3:PutBucketLogging",
104+
"s3:PutBucketNotification",
105+
"s3:PutBucketVersioning",
106+
"s3:PutObject",
107+
"s3:PutObjectAcl",
108+
"s3:PutObjectTagging",
109+
"s3:PutObjectVersionAcl",
110+
"s3:PutObjectVersionTagging"
111+
]
112+
resources = [
113+
"arn:aws:s3:::${var.s3_glue_tables_bucket}",
114+
"arn:aws:s3:::${var.s3_glue_tables_bucket}/*"
115+
]
116+
}
117+
}
71118

72119
data "aws_secretsmanager_secret" "datadog_key" {
73120
count = length(var.datadog_key_secret_name) > 0 ? 1 : 0

‎iam-ecs.tf‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,23 @@ resource "aws_iam_role_policy" "waggle_dance_remote_glue_federations_ecs_policy"
9595
role = aws_iam_role.waggledance_task[0].name
9696
name = "waggle-dance-remote-metastores-glue-readonly"
9797

98-
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy[0].json
98+
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy_read[0].json
99+
}
100+
101+
resource "aws_iam_role_policy" "waggle_dance_remote_glue_federations_ecs_policy_write" {
102+
count = var.wd_instance_type == "ecs" && local.glue_enabled ? 1 : 0
103+
role = aws_iam_role.waggledance_task[0].name
104+
name = "waggle-dance-remote-metastores-glue-write"
105+
106+
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy_write[0].json
107+
}
108+
109+
resource "aws_iam_role_policy" "waggle_dance_remote_glue_federations_ecs_policy_s3_write" {
110+
count = var.wd_instance_type == "ecs" && var.s3_glue_tables_bucket != "" && local.glue_enabled ? 1 : 0
111+
role = aws_iam_role.waggledance_task[0].name
112+
name = "waggle-dance-remote-metastores-glue-s3-write"
113+
114+
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy_s3_write[0].json
99115
}
100116

101117
resource "aws_iam_role_policy" "ecr_permission_for_task_exec" {

‎iam-k8s.tf‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,24 @@ resource "aws_iam_role_policy" "waggle_dance_remote_glue_federations_policy" {
3333
role = aws_iam_role.waggle_dance_k8s_role_iam[0].name
3434
name = "waggle-dance-remote-metastores-glue-readonly"
3535

36-
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy[0].json
36+
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy_read[0].json
3737
}
38+
39+
resource "aws_iam_role_policy" "waggle_dance_remote_glue_federations_policy_write" {
40+
count = var.wd_instance_type == "k8s" && var.oidc_provider != "" && local.glue_enabled ? 1 : 0
41+
role = aws_iam_role.waggle_dance_k8s_role_iam[0].name
42+
name = "waggle-dance-remote-metastores-glue-write"
43+
44+
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy_write[0].json
45+
}
46+
47+
resource "aws_iam_role_policy" "waggle_dance_remote_glue_federations_policy_s3_write" {
48+
count = var.wd_instance_type == "k8s" && var.oidc_provider != "" && var.s3_glue_tables_bucket != "" && local.glue_enabled ? 1 : 0
49+
role = aws_iam_role.waggle_dance_k8s_role_iam[0].name
50+
name = "waggle-dance-remote-metastores-glue-s3-write"
51+
52+
policy = data.aws_iam_policy_document.waggle_dance_remote_glue_federations_policy_s3_write[0].json
53+
}
54+
55+
56+

‎variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -616,4 +616,10 @@ variable "splunk_insecureskipverify" {
616616
description = "Instructs the splunk lgging driver to skip cert validation."
617617
type = string
618618
default = "false"
619+
}
620+
621+
variable "s3_glue_tables_bucket" {
622+
description = "S3 Bucket pattern where Hive tables will be created. Glue and Waggledance need this to create the glue table folders to the S3 location."
623+
type = string
624+
default = ""
619625
}

0 commit comments

Comments
 (0)