Skip to content

Commit 7f033da

Browse files
hjgracahjgraca
authored
Merge pull request #970 from aws-powertools/feature/batch-deserialize-event-records
feat: batch deserialize event records
2 parents ed1c6b9 + f9f2a2b commit 7f033da

File tree

65 files changed

+7258
-900
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+7258
-900
lines changed

‎docs/utilities/batch-processing.md‎

Lines changed: 985 additions & 544 deletions
Large diffs are not rendered by default.

‎examples/BatchProcessing/events/event.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"attributes": {
88
"ApproximateReceiveCount": "1",
99
"SentTimestamp": "1545082649183",
10-
"SenderId": "AIDAIENQZJOLO23YVJ4VO",
10+
"SenderId": "SENDER_ID",
1111
"ApproximateFirstReceiveTimestamp": "1545082649185"
1212
},
1313
"messageAttributes": {},
@@ -23,7 +23,7 @@
2323
"attributes": {
2424
"ApproximateReceiveCount": "1",
2525
"SentTimestamp": "1545082649183",
26-
"SenderId": "AIDAIENQZJOLO23YVJ4VO",
26+
"SenderId": "SENDER_ID",
2727
"ApproximateFirstReceiveTimestamp": "1545082649185"
2828
},
2929
"messageAttributes": {},

‎examples/BatchProcessing/events/typed-dynamodb-event.json‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
{
2+
"Records": [
3+
{
4+
"eventID": "1",
5+
"eventVersion": "1.0",
6+
"dynamodb": {
7+
"Keys": {
8+
"customerId": {
9+
"S": "CUST-123"
10+
}
11+
},
12+
"NewImage": {
13+
"customerId": {
14+
"S": "CUST-123"
15+
},
16+
"name": {
17+
"S": "John Doe"
18+
},
19+
"email": {
20+
"S": "[email protected]"
21+
},
22+
"createdAt": {
23+
"S": "2024-01-15T10:30:00Z"
24+
}
25+
},
26+
"StreamViewType": "NEW_AND_OLD_IMAGES",
27+
"SequenceNumber": "111",
28+
"SizeBytes": 26
29+
},
30+
"awsRegion": "us-west-2",
31+
"eventName": "INSERT",
32+
"eventSourceARN": "arn:aws:dynamodb:us-west-2:123456789012:table/customers/stream/2015-06-27T00:48:05.899",
33+
"eventSource": "aws:dynamodb"
34+
},
35+
{
36+
"eventID": "2",
37+
"eventVersion": "1.0",
38+
"dynamodb": {
39+
"Keys": {
40+
"customerId": {
41+
"S": "CUST-124"
42+
}
43+
},
44+
"NewImage": {
45+
"customerId": {
46+
"S": "CUST-124"
47+
},
48+
"name": {
49+
"S": "Jane Smith"
50+
},
51+
"email": {
52+
"S": "[email protected]"
53+
},
54+
"createdAt": {
55+
"S": "2024-01-15T10:35:00Z"
56+
}
57+
},
58+
"OldImage": {
59+
"customerId": {
60+
"S": "CUST-124"
61+
},
62+
"name": {
63+
"S": "Jane Doe"
64+
},
65+
"email": {
66+
"S": "[email protected]"
67+
},
68+
"createdAt": {
69+
"S": "2024-01-15T10:35:00Z"
70+
}
71+
},
72+
"StreamViewType": "NEW_AND_OLD_IMAGES",
73+
"SequenceNumber": "222",
74+
"SizeBytes": 59
75+
},
76+
"awsRegion": "us-west-2",
77+
"eventName": "MODIFY",
78+
"eventSourceARN": "arn:aws:dynamodb:us-west-2:123456789012:table/customers/stream/2015-06-27T00:48:05.899",
79+
"eventSource": "aws:dynamodb"
80+
}
81+
]
82+
}

‎examples/BatchProcessing/events/typed-kinesis-event.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"Records": [
3+
{
4+
"kinesis": {
5+
"kinesisSchemaVersion": "1.0",
6+
"partitionKey": "order-1",
7+
"sequenceNumber": "49590338271490256608559692538361571095921575989136588898",
8+
"data": "eyJvcmRlcklkIjogIk9SRC0xMjMiLCAib3JkZXJEYXRlIjogIjIwMjQtMDEtMTVUMTA6MzA6MDBaIiwgImN1c3RvbWVySWQiOiAiQ1VTVC0xMjMiLCAiaXRlbXMiOiBbeyJpZCI6IDEsICJuYW1lIjogIkxhcHRvcCIsICJwcmljZSI6IDk5OS45OX0sIHsiaWQiOiAyLCAibmFtZSI6ICJNb3VzZSIsICJwcmljZSI6IDI5Ljk5fV0sICJ0b3RhbEFtb3VudCI6IDEwMjkuOTgsICJzdGF0dXMiOiAiUGVuZGluZyJ9",
9+
"approximateArrivalTimestamp": 1545084650.987
10+
},
11+
"eventSource": "aws:kinesis",
12+
"eventVersion": "1.0",
13+
"eventID": "shardId-000000000006:49590338271490256608559692538361571095921575989136588898",
14+
"eventName": "aws:kinesis:record",
15+
"invokeIdentityArn": "arn:aws:iam::123456789012:role/lambda-kinesis-role",
16+
"awsRegion": "us-east-1",
17+
"eventSourceARN": "arn:aws:kinesis:us-east-1:123456789012:stream/lambda-stream"
18+
},
19+
{
20+
"kinesis": {
21+
"kinesisSchemaVersion": "1.0",
22+
"partitionKey": "order-2",
23+
"sequenceNumber": "49590338271490256608559692538361571095921575989136588899",
24+
"data": "eyJvcmRlcklkIjogIk9SRC0xMjQiLCAib3JkZXJEYXRlIjogIjIwMjQtMDEtMTVUMTA6MzU6MDBaIiwgImN1c3RvbWVySWQiOiAiQ1VTVC0xMjQiLCAiaXRlbXMiOiBbeyJpZCI6IDMsICJuYW1lIjogIktleWJvYXJkIiwgInByaWNlIjogNzkuOTl9XSwgInRvdGFsQW1vdW50IjogNzkuOTksICJzdGF0dXMiOiAiUGVuZGluZyJ9",
25+
"approximateArrivalTimestamp": 1545084651.987
26+
},
27+
"eventSource": "aws:kinesis",
28+
"eventVersion": "1.0",
29+
"eventID": "shardId-000000000006:49590338271490256608559692538361571095921575989136588899",
30+
"eventName": "aws:kinesis:record",
31+
"invokeIdentityArn": "arn:aws:iam::123456789012:role/lambda-kinesis-role",
32+
"awsRegion": "us-east-1",
33+
"eventSourceARN": "arn:aws:kinesis:us-east-1:123456789012:stream/lambda-stream"
34+
}
35+
]
36+
}

‎examples/BatchProcessing/events/typed-sqs-event.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"Records": [
3+
{
4+
"messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
5+
"receiptHandle": "MessageReceiptHandle",
6+
"body": "{\"id\": 1, \"name\": \"Laptop Computer\", \"price\": 999.99}",
7+
"attributes": {
8+
"ApproximateReceiveCount": "1",
9+
"SentTimestamp": "1523232000000",
10+
"SenderId": "123456789012",
11+
"ApproximateFirstReceiveTimestamp": "1523232000001"
12+
},
13+
"messageAttributes": {},
14+
"md5OfBody": "7b270e59b47ff90a553787216d55d91d",
15+
"eventSource": "aws:sqs",
16+
"eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
17+
"awsRegion": "us-east-1"
18+
},
19+
{
20+
"messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb79",
21+
"receiptHandle": "MessageReceiptHandle2",
22+
"body": "{\"id\": 2, \"name\": \"Wireless Mouse\", \"price\": 29.99}",
23+
"attributes": {
24+
"ApproximateReceiveCount": "1",
25+
"SentTimestamp": "1523232000000",
26+
"SenderId": "123456789012",
27+
"ApproximateFirstReceiveTimestamp": "1523232000001"
28+
},
29+
"messageAttributes": {},
30+
"md5OfBody": "7b270e59b47ff90a553787216d55d92e",
31+
"eventSource": "aws:sqs",
32+
"eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
33+
"awsRegion": "us-east-1"
34+
},
35+
{
36+
"messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb80",
37+
"receiptHandle": "MessageReceiptHandle3",
38+
"body": "{\"id\": 999, \"name\": \"Invalid Product\", \"price\": -10.00}",
39+
"attributes": {
40+
"ApproximateReceiveCount": "1",
41+
"SentTimestamp": "1523232000000",
42+
"SenderId": "123456789012",
43+
"ApproximateFirstReceiveTimestamp": "1523232000001"
44+
},
45+
"messageAttributes": {},
46+
"md5OfBody": "7b270e59b47ff90a553787216d55d93f",
47+
"eventSource": "aws:sqs",
48+
"eventSourceARN": "arn:aws:sqs:us-east-1:123456789012:MyQueue",
49+
"awsRegion": "us-east-1"
50+
}
51+
]
52+
}

‎examples/BatchProcessing/src/HelloWorld/Data/Order.cs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
using System;
17+
using System.Collections.Generic;
18+
using System.Text.Json.Serialization;
19+
20+
namespace HelloWorld.Data;
21+
22+
public class Order
23+
{
24+
public string? OrderId { get; set; }
25+
public DateTime OrderDate { get; set; }
26+
public string? CustomerId { get; set; }
27+
public List<Product> Items { get; set; } = new();
28+
public decimal TotalAmount { get; set; }
29+
public string? Status { get; set; }
30+
}
31+
32+
public class Customer
33+
{
34+
public string? CustomerId { get; set; }
35+
public string? Name { get; set; }
36+
public string? Email { get; set; }
37+
public DateTime CreatedAt { get; set; }
38+
}
39+
40+
/// <summary>
41+
/// JsonSerializerContext for AOT compatibility
42+
/// </summary>
43+
[JsonSerializable(typeof(Product))]
44+
[JsonSerializable(typeof(Order))]
45+
[JsonSerializable(typeof(Customer))]
46+
[JsonSerializable(typeof(List<Product>))]
47+
[JsonSerializable(typeof(List<Order>))]
48+
public partial class ExampleJsonSerializerContext : JsonSerializerContext
49+
{
50+
}

‎libraries/src/AWS.Lambda.Powertools.BatchProcessing/AWS.Lambda.Powertools.BatchProcessing.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
<ItemGroup>
1111
<!-- Package versions are Centrally managed in Directory.Packages.props file -->
1212
<!-- More info https://learn.microsoft.com/en-us/nuget/consume-packages/central-package-management -->
13+
<PackageReference Include="Amazon.Lambda.Core" />
1314
<PackageReference Include="Amazon.Lambda.DynamoDBEvents" />
1415
<PackageReference Include="Amazon.Lambda.KinesisEvents" />
1516
<PackageReference Include="Amazon.Lambda.SQSEvents" />

‎libraries/src/AWS.Lambda.Powertools.BatchProcessing/BatchItemFailuresResponse.cs‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,4 @@
1-
/*
2-
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3-
*
4-
* Licensed under the Apache License, Version 2.0 (the "License").
5-
* You may not use this file except in compliance with the License.
6-
* A copy of the License is located at
7-
*
8-
* http://aws.amazon.com/apache2.0
9-
*
10-
* or in the "license" file accompanying this file. This file is distributed
11-
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12-
* express or implied. See the License for the specific language governing
13-
* permissions and limitations under the License.
14-
*/
1+

152

163
using System.Collections.Generic;
174
using System.Runtime.Serialization;

0 commit comments

Comments
 (0)