From 2d067992ff25d4de3048eb4414e3a82065f018d3 Mon Sep 17 00:00:00 2001 From: niqdev Date: Tue, 6 Sep 2022 17:49:33 +0100 Subject: [PATCH 1/3] draft aws-serverless-sqs --- local/Dockerfile | 2 +- local/data/serverless-sqs.yml | 49 ++++++++++ local/docker-compose-serverless-sqs.yml | 44 +++++++++ local/entrypoint/init_sqs.sh | 19 ++++ local/entrypoint/my-queue-notification.json | 10 ++ modules/aws-serverless-sqs/README.md | 97 +++++++++++++++++++ modules/aws-serverless-sqs/build.gradle.kts | 20 ++++ .../com/github/niqdev/aws/sqs/Handler.kt | 12 +++ settings.gradle.kts | 1 + 9 files changed, 253 insertions(+), 1 deletion(-) create mode 100644 local/data/serverless-sqs.yml create mode 100644 local/docker-compose-serverless-sqs.yml create mode 100755 local/entrypoint/init_sqs.sh create mode 100644 local/entrypoint/my-queue-notification.json create mode 100644 modules/aws-serverless-sqs/README.md create mode 100644 modules/aws-serverless-sqs/build.gradle.kts create mode 100644 modules/aws-serverless-sqs/src/main/kotlin/com/github/niqdev/aws/sqs/Handler.kt diff --git a/local/Dockerfile b/local/Dockerfile index 4b6b530e..1b747e37 100644 --- a/local/Dockerfile +++ b/local/Dockerfile @@ -1,7 +1,7 @@ FROM node:alpine3.16 # https://github.com/serverless/serverless/releases -ARG SERVERLESS_VERSION="3.21.0" +ARG SERVERLESS_VERSION="3.22.0" # https://github.com/localstack/serverless-localstack#change-log ARG SERVERLESS_LOCALSTACK_VERSION="1.0.0" diff --git a/local/data/serverless-sqs.yml b/local/data/serverless-sqs.yml new file mode 100644 index 00000000..1c96b331 --- /dev/null +++ b/local/data/serverless-sqs.yml @@ -0,0 +1,49 @@ +service: fn-aws-serverless + +frameworkVersion: '3' + +plugins: + - serverless-localstack + +custom: + localstack: + debug: true + stages: + - local + host: http://localstack + +provider: + name: aws + runtime: java11 + region: us-east-1 + iamRoleStatements: + - Effect: Allow + Action: + - sqs:SendMessage + Resource: + - Fn::GetAtt: [ MyQueue, Arn ] + +package: + artifact: aws-serverless-sqs.zip + +functions: + fn-aws-serverless-sqs: + name: fn-aws-serverless-sqs + description: TODO + handler: com.github.niqdev.aws.sqs.Handler::handleRequest + environment: + NAME: "fn-aws-serverless-sqs" + # https://www.serverless.com/framework/docs/providers/aws/events/sqs/ + events: + - sqs: + arn: + Fn::GetAtt: + - MyQueue + - Arn + +resources: + Resources: + myQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: MyQueue \ No newline at end of file diff --git a/local/docker-compose-serverless-sqs.yml b/local/docker-compose-serverless-sqs.yml new file mode 100644 index 00000000..530d7f25 --- /dev/null +++ b/local/docker-compose-serverless-sqs.yml @@ -0,0 +1,44 @@ +version: "3" + +services: + + dev: + container_name: local-serverless-sqs-dev + build: + context: . + # docker run -i + stdin_open: true + # docker run -t + tty: true + networks: + - localstack + volumes: + - "./data:/usr/src/app/data" + - "./aws:/root/.aws" + + localstack: + container_name: local-serverless-sqs-localstack + image: localstack/localstack:1.1.0 + ports: + - "4566:4566" + - "4510-4559:4510-4559" + hostname: localstack + networks: + - localstack + # https://docs.localstack.cloud/localstack/configuration + environment: + - DEBUG=1 + - PERSISTENCE=1 + - LAMBDA_EXECUTOR=local + - LAMBDA_REMOTE_DOCKER=false + # custom + - BUCKET_NAME=my-bucket + - QUEUE_NAME=MyQueue + volumes: + - ".localstack:/var/lib/localstack" + # run custom scripts: requires credentials + - "./aws:/root/.aws" + - './entrypoint:/docker-entrypoint-initaws.d' + +networks: + localstack: diff --git a/local/entrypoint/init_sqs.sh b/local/entrypoint/init_sqs.sh new file mode 100755 index 00000000..cb594844 --- /dev/null +++ b/local/entrypoint/init_sqs.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +ENDPOINT=http://localstack:4566 + +echo "[*] create bucket: $BUCKET_NAME" +aws s3api create-bucket \ + --endpoint-url=${ENDPOINT} \ + --bucket ${BUCKET_NAME} + +echo "[*] create queue: $QUEUE_NAME" +aws sqs create-queue \ + --endpoint-url=${ENDPOINT} \ + --queue-name ${QUEUE_NAME} + +echo "[*] create bucket notification" +aws s3api put-bucket-notification-configuration \ + --endpoint-url=${ENDPOINT} \ + --bucket ${BUCKET_NAME} \ + --notification-configuration file:///docker-entrypoint-initaws.d/my-queue-notification.json diff --git a/local/entrypoint/my-queue-notification.json b/local/entrypoint/my-queue-notification.json new file mode 100644 index 00000000..31543bf6 --- /dev/null +++ b/local/entrypoint/my-queue-notification.json @@ -0,0 +1,10 @@ +{ + "QueueConfigurations":[ + { + "QueueArn":"arn:aws:sqs:us-east-1:000000000000:MyQueue", + "Events":[ + "s3:ObjectCreated:*" + ] + } + ] +} diff --git a/modules/aws-serverless-sqs/README.md b/modules/aws-serverless-sqs/README.md new file mode 100644 index 00000000..37db6ab3 --- /dev/null +++ b/modules/aws-serverless-sqs/README.md @@ -0,0 +1,97 @@ +## aws-serverless-sqs + +Examples +* https://github.com/localstack/localstack/issues/3585 +* https://github.com/localstack/localstack/issues/967 +* https://github.com/localstack/localstack/issues/4513 +* https://github.com/localstack/localstack/issues/1216 +* https://faun.pub/enable-aws-s3-bucket-events-notification-publishing-to-sqs-locally-using-localstack-45f369f74399 +* https://stackoverflow.com/questions/68988931/localstack-lambda-is-not-triggered-by-sqs +* https://www.zakariaamine.com/2019-02-17/lambda-with-sqs-eventsource-cli + +### Development + +```bash +# local-up +docker-compose -f local/docker-compose-serverless-sqs.yml up + +# local-down +docker-compose -f local/docker-compose-serverless-sqs.yml down -v +rm -frv ./local/.localstack ./local/data/.serverless ./local/data/*.zip ./local/data/*.log + +# build +./gradlew :modules:aws-serverless-sqs:clean :modules:aws-serverless-sqs:build +``` + +### AWS cli + +```bash +aws iam create-role \ + --endpoint-url=http://localhost:4566 \ + --role-name localstack-role \ + --assume-role-policy-document '{"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Action": ["lambda:InvokeFunction"]}, {"Effect": "Allow", "Action": ["sqs:SendMessage", "sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"]}]}' + +# create +aws lambda create-function \ + --endpoint-url=http://localhost:4566 \ + --function-name fn-aws-serverless-sqs \ + --runtime java8 \ + --handler com.github.niqdev.aws.sqs.Handler::handleRequest \ + --role arn:aws:iam::000000000000:role/localstack-role \ + --zip-file fileb://modules/aws-serverless-sqs/build/distributions/aws-serverless-sqs-0.1.0.zip + +# invoke +aws lambda invoke \ + --endpoint-url=http://localhost:4566 \ + --function-name fn-aws-serverless-sqs \ + --payload $(echo "{\"key\":\"value\"}" | base64) \ + local/.localstack/logs/aws-serverless-sqs-output.json + +aws s3 ls --endpoint-url=http://localhost:4566 +aws s3api --endpoint-url=http://localhost:4566 get-bucket-notification-configuration --bucket my-bucket +aws sqs list-queues --endpoint-url=http://localhost:4566 +aws sqs get-queue-attributes \ + --endpoint-url=http://localhost:4566 \ + --queue-url http://localhost:4566/000000000000/MyQueue \ + --attribute-names QueueArn + +aws lambda create-event-source-mapping \ + --endpoint-url=http://localhost:4566 \ + --event-source-arn arn:aws:sqs:us-east-1:000000000000:MyQueue \ + --batch-size 1 \ + --maximum-retry-attempts 3 \ + --function-name fn-aws-serverless-sqs + +aws sqs send-message \ + --endpoint-url=http://localhost:4566 \ + --queue-url http://localhost:4566/000000000000/MyQueue \ + --message-body $(echo "{\"key\":\"value\"}" | base64) + +aws sqs receive-message \ + --endpoint-url=http://localhost:4566 \ + --queue-url http://localhost:4566/000000000000/MyQueue + +aws s3 cp \ + --endpoint-url=http://localhost:4566 \ + README.md s3://my-bucket +``` + +### TODO Serverless + +```bash +# verify status +curl http://localhost:4566/health | jq +docker exec -it local-serverless-sqs-dev curl http://localstack:4566/health | jq +docker exec -it local-serverless-sqs-dev aws --endpoint-url=http://localstack:4566 s3 ls +docker exec -it local-serverless-sqs-dev aws --endpoint-url=http://localstack:4566 sqs list-queues + +# local-deploy +cp modules/aws-serverless-sqs/build/distributions/aws-serverless-sqs-*.zip local/data/aws-serverless-sqs.zip +docker exec -it --workdir /usr/src/app/data local-serverless-sqs-dev \ + serverless deploy --config serverless-sqs.yml --stage local + +# produce +docker exec -it local-serverless-sqs-dev aws --endpoint-url=http://localstack:4566 sqs send-message \ + --queue-url http://localstack:4566/000000000000/MyQueue \ + --message-body "hello" +``` diff --git a/modules/aws-serverless-sqs/build.gradle.kts b/modules/aws-serverless-sqs/build.gradle.kts new file mode 100644 index 00000000..1a0c28fa --- /dev/null +++ b/modules/aws-serverless-sqs/build.gradle.kts @@ -0,0 +1,20 @@ +version = "0.1.0" + +dependencies { + implementation("com.amazonaws:aws-lambda-java-core:${Versions.awsLambdaCore}") + implementation("com.amazonaws:aws-lambda-java-events:${Versions.awsLambdaEvents}") +} + +tasks { + register("buildZip") { + archiveFileName.set("${project.name}-${project.version}.zip") + from(compileKotlin) + from(processResources) + into("lib") { + from(configurations.runtimeClasspath) + } + } + build { + dependsOn("buildZip") + } +} diff --git a/modules/aws-serverless-sqs/src/main/kotlin/com/github/niqdev/aws/sqs/Handler.kt b/modules/aws-serverless-sqs/src/main/kotlin/com/github/niqdev/aws/sqs/Handler.kt new file mode 100644 index 00000000..171ca5c8 --- /dev/null +++ b/modules/aws-serverless-sqs/src/main/kotlin/com/github/niqdev/aws/sqs/Handler.kt @@ -0,0 +1,12 @@ +package com.github.niqdev.aws.sqs + +import com.amazonaws.services.lambda.runtime.Context +import com.amazonaws.services.lambda.runtime.RequestHandler +import com.amazonaws.services.lambda.runtime.events.SQSEvent + +class Handler : RequestHandler { + override fun handleRequest(input: SQSEvent, context: Context): String { + context.logger.log("TODO") + return "TODO" + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index ebb68434..bf9f2a38 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -5,6 +5,7 @@ include( "modules:aws-kotlin", "modules:aws-local", "modules:aws-serverless", + "modules:aws-serverless-sqs", "modules:bool", "modules:fpk", "modules:jok", From d11ff8566f3f86abbeb478afb69aa913e50c132c Mon Sep 17 00:00:00 2001 From: niqdev Date: Wed, 7 Sep 2022 14:25:42 +0100 Subject: [PATCH 2/3] update name --- local/data/serverless-sqs.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/local/data/serverless-sqs.yml b/local/data/serverless-sqs.yml index 1c96b331..1210a352 100644 --- a/local/data/serverless-sqs.yml +++ b/local/data/serverless-sqs.yml @@ -1,4 +1,4 @@ -service: fn-aws-serverless +service: fn-aws-serverless-sqs frameworkVersion: '3' @@ -46,4 +46,4 @@ resources: myQueue: Type: AWS::SQS::Queue Properties: - QueueName: MyQueue \ No newline at end of file + QueueName: MyQueue From ffedb81251dae0dd45bd2c00c1bce69edfbee18d Mon Sep 17 00:00:00 2001 From: niqdev Date: Fri, 23 Sep 2022 12:12:14 +0100 Subject: [PATCH 3/3] fix notification --- local/docker-compose-serverless-sqs.yml | 5 +++-- modules/aws-serverless-sqs/README.md | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/local/docker-compose-serverless-sqs.yml b/local/docker-compose-serverless-sqs.yml index 530d7f25..9ec03ac1 100644 --- a/local/docker-compose-serverless-sqs.yml +++ b/local/docker-compose-serverless-sqs.yml @@ -22,20 +22,21 @@ services: ports: - "4566:4566" - "4510-4559:4510-4559" - hostname: localstack networks: - localstack # https://docs.localstack.cloud/localstack/configuration environment: - DEBUG=1 - PERSISTENCE=1 - - LAMBDA_EXECUTOR=local - LAMBDA_REMOTE_DOCKER=false + - LAMBDA_EXECUTOR=docker + - DOCKER_SOCK=unix:///var/run/docker.sock # custom - BUCKET_NAME=my-bucket - QUEUE_NAME=MyQueue volumes: - ".localstack:/var/lib/localstack" + - "/var/run/docker.sock:/var/run/docker.sock" # run custom scripts: requires credentials - "./aws:/root/.aws" - './entrypoint:/docker-entrypoint-initaws.d' diff --git a/modules/aws-serverless-sqs/README.md b/modules/aws-serverless-sqs/README.md index 37db6ab3..a128fd69 100644 --- a/modules/aws-serverless-sqs/README.md +++ b/modules/aws-serverless-sqs/README.md @@ -35,7 +35,7 @@ aws iam create-role \ aws lambda create-function \ --endpoint-url=http://localhost:4566 \ --function-name fn-aws-serverless-sqs \ - --runtime java8 \ + --runtime java11 \ --handler com.github.niqdev.aws.sqs.Handler::handleRequest \ --role arn:aws:iam::000000000000:role/localstack-role \ --zip-file fileb://modules/aws-serverless-sqs/build/distributions/aws-serverless-sqs-0.1.0.zip