Fixes for workgroup stabilization before create and delete operations

aws-redshiftserverless-workgroup/.rpdk-config

@@ -24,5 +24,8 @@
         "codegen_template_path": "guided_aws",
         "protocolVersion": "2.0.0"
     },
-    "executableEntrypoint": "software.amazon.redshiftserverless.workgroup.HandlerWrapperExecutable"
+    "logProcessorEnabled": "true",
+    "executableEntrypoint": "software.amazon.redshiftserverless.workgroup.HandlerWrapperExecutable",
+    "contractSettings": {},
+    "canarySettings": {}
 }

aws-redshiftserverless-workgroup/aws-redshiftserverless-workgroup.json

@@ -267,7 +267,16 @@
         }
     },
     "tagging": {
-        "taggable": true
+        "taggable": true,
+        "tagOnCreate": true,
+        "tagUpdatable": true,
+        "cloudFormationSystemTags": false,
+        "tagProperty": "/properties/Tags",
+        "permissions": [
+            "redshift-serverless:ListTagsForResource",
+            "redshift-serverless:TagResource",
+            "redshift-serverless:UntagResource"
+        ]
     },
     "additionalProperties": false,
     "required": [
@@ -307,10 +316,7 @@
         "/properties/MaxCapacity",
         "/properties/ConfigParameters",
         "/properties/SecurityGroupIds",
-        "/properties/SubnetIds",
-        "/properties/Tags",
-        "/properties/Tags/*/Key",
-        "/properties/Tags/*/Value"
+        "/properties/SubnetIds"
     ],
     "primaryIdentifier": [
         "/properties/WorkgroupName"
@@ -328,7 +334,9 @@
                 "redshift-serverless:CreateNamespace",
                 "redshift-serverless:CreateWorkgroup",
                 "redshift-serverless:GetWorkgroup",
-                "redshift-serverless:GetNamespace"
+                "redshift-serverless:GetNamespace",
+                "redshift-serverless:ListTagsForResource",
+                "redshift-serverless:TagResource"
             ]
         },
         "read": {
@@ -340,7 +348,8 @@
                 "ec2:DescribeSubnets",
                 "ec2:DescribeAccountAttributes",
                 "ec2:DescribeAvailabilityZones",
-                "redshift-serverless:GetWorkgroup"
+                "redshift-serverless:GetWorkgroup",
+                "redshift-serverless:ListTagsForResource"
             ]
         },
         "update": {
@@ -356,7 +365,10 @@
                 "redshift-serverless:TagResource",
                 "redshift-serverless:UntagResource",
                 "redshift-serverless:GetWorkgroup",
-                "redshift-serverless:UpdateWorkgroup"
+                "redshift-serverless:UpdateWorkgroup",
+                "redshift-serverless:ListTagsForResource",
+                "redshift-serverless:TagResource",
+                "redshift-serverless:UntagResource"
             ]
         },
         "delete": {
@@ -370,7 +382,9 @@
                 "ec2:DescribeAvailabilityZones",
                 "redshift-serverless:GetWorkgroup",
                 "redshift-serverless:GetNamespace",
-                "redshift-serverless:DeleteWorkgroup"
+                "redshift-serverless:DeleteWorkgroup",
+                "redshift-serverless:ListTagsForResource",
+                "redshift-serverless:UntagResource"
             ]
         },
         "list": {
@@ -382,7 +396,8 @@
                 "ec2:DescribeSubnets",
                 "ec2:DescribeAccountAttributes",
                 "ec2:DescribeAvailabilityZones",
-                "redshift-serverless:ListWorkgroups"
+                "redshift-serverless:ListWorkgroups",
+                "redshift-serverless:ListTagsForResource"
             ]
         }
     }

aws-redshiftserverless-workgroup/docs/README.md

@@ -276,3 +276,4 @@ Returns the <code>PubliclyAccessible</code> value.
 #### CreationDate
 
 Returns the <code>CreationDate</code> value.
+

aws-redshiftserverless-workgroup/docs/configparameter.md

@@ -41,3 +41,4 @@ _Type_: String
 _Maximum Length_: <code>15000</code>
 
 _Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt)
+

aws-redshiftserverless-workgroup/docs/tag.md

@@ -43,3 +43,4 @@ _Type_: String
 _Maximum Length_: <code>256</code>
 
 _Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt)
+

aws-redshiftserverless-workgroup/docs/workgroup.md

@@ -38,3 +38,4 @@ _Required_: No
 _Type_: <a href="endpoint.md">Endpoint</a>
 
 _Update requires_: [No interruption](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-updating-stacks-update-behaviors.html#update-no-interrupt)
+

aws-redshiftserverless-workgroup/pom.xml

@@ -32,6 +32,12 @@
             <artifactId>redshiftserverless</artifactId>
             <version>2.22.5</version>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/software.amazon.awssdk/secretsmanager -->
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>secretsmanager</artifactId>
+            <version>2.22.5</version>
+        </dependency>
         <!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
         <dependency>
             <groupId>org.projectlombok</groupId>

aws-redshiftserverless-workgroup/src/main/java/software/amazon/redshiftserverless/workgroup/BaseHandlerStd.java

@@ -10,6 +10,8 @@
 import software.amazon.awssdk.services.redshiftserverless.model.GetNamespaceResponse;
 import software.amazon.awssdk.services.redshiftserverless.model.GetWorkgroupRequest;
 import software.amazon.awssdk.services.redshiftserverless.model.GetWorkgroupResponse;
+import software.amazon.awssdk.services.redshiftserverless.model.ListTagsForResourceRequest;
+import software.amazon.awssdk.services.redshiftserverless.model.ListTagsForResourceResponse;
 import software.amazon.awssdk.services.redshiftserverless.model.Namespace;
 import software.amazon.awssdk.services.redshiftserverless.model.NamespaceStatus;
 import software.amazon.awssdk.services.redshiftserverless.model.WorkgroupStatus;
@@ -38,7 +40,7 @@ public abstract class BaseHandlerStd extends BaseHandler<CallbackContext> {
     protected Logger logger;
 
     public static final String BUSY_WORKGROUP_RETRY_EXCEPTION_MESSAGE =
-            "There is an operation running on the existing workgroup";
+            "There is an operation in progress";
 
     // This is for delete workgroup operation. We need AdminWF to finish the operation completely
     // This is needed for CTV2 to work
@@ -54,7 +56,7 @@ protected static boolean isRetriableWorkgroupException(ConflictException excepti
             .build();
 
     protected static final Constant PREOPERATION_BACKOFF_STRATEGY = Constant.of()
-            .timeout(Duration.ofMinutes(5L))
+            .timeout(Duration.ofMinutes(60L))
             .delay(Duration.ofSeconds(5L))
             .build();
 
@@ -130,14 +132,40 @@ protected UpdateWorkgroupResponse updateWorkgroup(final UpdateWorkgroupRequest a
 
     }
 
+    // Since there are source to target operations on the cluster
+    // We will receive operation in progress in such cases.
+    // This needs to be handled on CFN side as this will break contract tests
     protected DeleteWorkgroupResponse deleteWorkgroup(final DeleteWorkgroupRequest awsRequest,
-                                                      final ProxyClient<RedshiftServerlessClient> proxyClient) {
-
-        DeleteWorkgroupResponse awsResponse = proxyClient.injectCredentialsAndInvokeV2(
-                awsRequest, proxyClient.client()::deleteWorkgroup);
-
-        logger.log(String.format("Workgroup : %s has successfully been deleted.", awsResponse.workgroup().workgroupName()));
-        logger.log(awsResponse.toString());
+                                                      final ProxyClient<RedshiftServerlessClient> proxyClient) throws ConflictException {
+        boolean operationInProgress = false;
+        int max_retries = 5;
+        int current_retry = 0;
+        DeleteWorkgroupResponse awsResponse = null;
+
+        do {
+            try {
+                awsResponse = proxyClient.injectCredentialsAndInvokeV2(awsRequest, proxyClient.client()::deleteWorkgroup);
+                logger.log(String.format("Workgroup : %s has successfully been deleted.", awsResponse.workgroup().workgroupName()));
+                logger.log(awsResponse.toString());
+                operationInProgress = false;
+            } catch (ConflictException e) {
+                Pattern pattern = Pattern.compile(".*There is an operation in progress.*", Pattern.CASE_INSENSITIVE);
+                if(pattern.matcher(e.getMessage()).matches()) {
+                    logger.log("There is an operation in progress during delete. We will wait and retry in 60 secs");
+                    operationInProgress = true;
+                    current_retry = current_retry + 1;
+                    // Since there are source to target operations on the cluster
+                    try {
+                        Thread.sleep(60000);
+                    } catch(InterruptedException ex) {
+                        Thread.currentThread().interrupt();
+                    }
+                } else {
+                    // We need to explicitly catch operation in progress or reraise other conflict exceptions
+                    throw e;
+                }
+            }
+        } while((current_retry < max_retries) && operationInProgress);
 
         return awsResponse;
     }
@@ -205,6 +233,15 @@ protected boolean isWorkgroupDeleted(final Object awsRequest,
         return false;
     }
 
+    protected ListTagsForResourceResponse readTags(final ListTagsForResourceRequest awsRequest,
+                                                   final ProxyClient<RedshiftServerlessClient> proxyClient) {
+        ListTagsForResourceResponse awsResponse;
+        awsResponse = proxyClient.injectCredentialsAndInvokeV2(awsRequest, proxyClient.client()::listTagsForResource);
+
+        logger.log(String.format("%s's tags have successfully been read.", ResourceModel.TYPE_NAME));
+        return awsResponse;
+    }
+
     protected ProgressEvent<ResourceModel, CallbackContext> defaultWorkgroupErrorHandler(final Object awsRequest,
                                                                                          final Exception exception,
                                                                                          final ProxyClient<RedshiftServerlessClient> client,

aws-redshiftserverless-workgroup/src/main/java/software/amazon/redshiftserverless/workgroup/CreateHandler.java

@@ -35,6 +35,17 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(
         this.logger = logger;
 
         return ProgressEvent.progress(request.getDesiredResourceState(), callbackContext)
+                .then(progress ->
+                        proxy.initiate("AWS-RedshiftServerless-Workgroup::Create::ReadNamespaceBeforeCreate", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
+                                .translateToServiceRequest(Translator::translateToReadNamespaceRequest)
+                                .backoffDelay(PREOPERATION_BACKOFF_STRATEGY)// We wait for max of 5mins here
+                                .makeServiceCall(this::readNamespace)
+                                .stabilize(this::isNamespaceStable) // This basically checks for namespace to be in stable state before we create workgroup
+                                .handleError(this::createWorkgroupErrorHandler)
+                                .done(awsResponse -> {
+                                    return ProgressEvent.progress(request.getDesiredResourceState(), callbackContext);
+                                })
+                )
                 .then(progress ->
                         proxy.initiate("AWS-RedshiftServerless-Workgroup::Create", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
                                 .translateToServiceRequest(Translator::translateToCreateRequest)
@@ -47,7 +58,7 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(
                                 })
                 )
                 .then(progress ->
-                        proxy.initiate("AWS-RedshiftServerless-Workgroup::ReadNameSpace", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
+                        proxy.initiate("AWS-RedshiftServerless-Workgroup::ReadNameSpaceAfterCreate", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
                                 .translateToServiceRequest(Translator::translateToReadNamespaceRequest)
                                 .backoffDelay(BACKOFF_STRATEGY)
                                 .makeServiceCall(this::readNamespace)
@@ -67,6 +78,10 @@ private ProgressEvent<ResourceModel, CallbackContext> createWorkgroupErrorHandle
         logger.log(String.format("Operation: %s : encountered exception for model: %s",
                 awsRequest.getClass().getName(), ResourceModel.TYPE_NAME));
         logger.log(awsRequest.toString());
+        Pattern pattern = Pattern.compile(".*is not authorized to perform: redshift-serverless:TagResource.*", Pattern.CASE_INSENSITIVE);
+        if(pattern.matcher(exception.getMessage()).matches()){
+            return ProgressEvent.failed(model, context, HandlerErrorCode.UnauthorizedTaggingOperation, exception.getMessage());
+        }
 
         return this.defaultWorkgroupErrorHandler(awsRequest, exception, client, model, context);
     }

aws-redshiftserverless-workgroup/src/main/java/software/amazon/redshiftserverless/workgroup/DeleteHandler.java

@@ -31,6 +31,28 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(final Amaz
         this.logger = logger;
 
         return ProgressEvent.progress(request.getDesiredResourceState(), callbackContext)
+                .then(progress ->
+                        proxy.initiate("AWS-RedshiftServerless-Workgroup::Delete::ReadWorkgroup", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
+                                .translateToServiceRequest(Translator::translateToReadRequest)
+                                .backoffDelay(PREOPERATION_BACKOFF_STRATEGY)// We wait for max of 5mins here
+                                .makeServiceCall(this::readWorkgroup)
+                                .stabilize(this::isWorkgroupStable) // This basically checks for workgroup to be in stable state before we delete workgroup
+                                .handleError(this::deleteWorkgroupErrorHandler)
+                                .done( awsResponse -> {
+                                    return ProgressEvent.progress(Translator.translateFromReadResponse(awsResponse), callbackContext);
+                                })
+                )
+                .then(progress ->
+                        proxy.initiate("AWS-RedshiftServerless-Workgroup::Delete::ReadNamespaceBeforeDelete", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
+                                .translateToServiceRequest(Translator::translateToReadNamespaceRequest)
+                                .backoffDelay(PREOPERATION_BACKOFF_STRATEGY)// We wait for max of 5mins here
+                                .makeServiceCall(this::readNamespace)
+                                .stabilize(this::isNamespaceStable) // This basically checks for namespace to be in stable state before we delete workgroup
+                                .handleError(this::deleteWorkgroupErrorHandler)
+                                .done( awsResponse -> {
+                                    return ProgressEvent.progress(request.getDesiredResourceState(), callbackContext);
+                                })
+                )
                 .then(progress ->
                         proxy.initiate("AWS-RedshiftServerless-Workgroup::Delete", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
                                 .translateToServiceRequest(Translator::translateToDeleteRequest)
@@ -42,16 +64,6 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(final Amaz
                                     return ProgressEvent.progress(Translator.translateFromDeleteResponse(awsResponse), callbackContext);
                                 })
                 )
-                .then(progress -> {
-                    if (progress.getCallbackContext().isPropagationDelay()) {
-                        logger.log("Propagation delay completed");
-                        return ProgressEvent.progress(progress.getResourceModel(), progress.getCallbackContext());
-                    }
-                    progress.getCallbackContext().setPropagationDelay(true);
-                    logger.log("Setting propagation delay");
-                    return ProgressEvent.defaultInProgressHandler(progress.getCallbackContext(),
-                            EVENTUAL_CONSISTENCY_DELAY_SECONDS, progress.getResourceModel());
-                })
                 .then(progress ->
                         proxy.initiate("AWS-RedshiftServerless-Workgroup::ReadNameSpaceAfterDelete", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
                                 .translateToServiceRequest(Translator::translateToReadNamespaceRequest)

aws-redshiftserverless-workgroup/src/main/java/software/amazon/redshiftserverless/workgroup/ReadHandler.java

@@ -26,15 +26,32 @@ protected ProgressEvent<ResourceModel, CallbackContext> handleRequest(
             final Logger logger) {
 
         this.logger = logger;
+        final ResourceModel model = request.getDesiredResourceState();
 
-        return proxy.initiate("AWS-RedshiftServerless-Workgroup::Read", proxyClient, request.getDesiredResourceState(), callbackContext)
-                .translateToServiceRequest(Translator::translateToReadRequest)
-                .makeServiceCall(this::readWorkgroup)
-                .handleError((awsRequest, exception, client, resourceModel, cxt) -> {
-                    logger.log(String.format("Operation: %s : encountered exception for model: %s", awsRequest.getClass().getName(), ResourceModel.TYPE_NAME));
-                    logger.log(awsRequest.toString());
-                    return this.defaultWorkgroupErrorHandler(awsRequest, exception, client, resourceModel, cxt);
-                })
-                .done(awsResponse -> ProgressEvent.defaultSuccessHandler(Translator.translateFromReadResponse(awsResponse)));
+        return ProgressEvent.progress(model, callbackContext)
+                .then (progress -> {
+                    progress = proxy.initiate("AWS-RedshiftServerless-Workgroup::Read", proxyClient, request.getDesiredResourceState(), callbackContext)
+                            .translateToServiceRequest(Translator::translateToReadRequest)
+                            .makeServiceCall(this::readWorkgroup)
+                            .handleError((awsRequest, exception, client, resourceModel, cxt) -> {
+                                logger.log(String.format("Operation: %s : encountered exception for model: %s", awsRequest.getClass().getName(), ResourceModel.TYPE_NAME));
+                                logger.log(awsRequest.toString());
+                                return this.defaultWorkgroupErrorHandler(awsRequest, exception, client, resourceModel, cxt);
+                            })
+                            .done(awsResponse -> ProgressEvent.progress(Translator.translateFromReadResponse(awsResponse), callbackContext));
+
+                    return progress;
+                }).then(progress -> {
+                    progress = proxy.initiate("AWS-RedshiftServerless-Namespace::Read::ReadTags", proxyClient, progress.getResourceModel(), progress.getCallbackContext())
+                            .translateToServiceRequest(Translator::translateToReadTagsRequest)
+                            .makeServiceCall(this::readTags)
+                            .handleError((_awsRequest, _sdkEx, _client, _model, _callbackContext) ->
+                                    ProgressEvent.failed(_model, _callbackContext, HandlerErrorCode.UnauthorizedTaggingOperation, _sdkEx.getMessage())
+                            )
+                            .done((_request, _response, _client, _model, _context) -> {
+                                return ProgressEvent.defaultSuccessHandler(Translator.translateFromReadTagsResponse(_response, _model));
+                            });
+                    return progress;
+                });
     }
 }