Skip to content

Commit 1cb98c6

Browse files
author
Fahad Zubair
committed
Add codegen flag 'replaceInvalidUtf8' to allow server SDK to replace invalid UTF-8 characters with �
1 parent 5ed776f commit 1cb98c6

File tree

7 files changed

+275
-67
lines changed

7 files changed

+275
-67
lines changed

‎codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/smithy/protocols/parse/JsonParserGenerator.kt

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import software.amazon.smithy.model.traits.EnumTrait
2323
import software.amazon.smithy.model.traits.SparseTrait
2424
import software.amazon.smithy.model.traits.TimestampFormatTrait
2525
import software.amazon.smithy.rust.codegen.core.rustlang.Attribute
26-
import software.amazon.smithy.rust.codegen.core.rustlang.CargoDependency
2726
import software.amazon.smithy.rust.codegen.core.rustlang.RustWriter
2827
import software.amazon.smithy.rust.codegen.core.rustlang.Writable
2928
import software.amazon.smithy.rust.codegen.core.rustlang.escape
@@ -102,12 +101,12 @@ class JsonParserGenerator(
102101
ReturnSymbolToParse(codegenContext.symbolProvider.toSymbol(shape), false)
103102
},
104103
private val customizations: List<JsonParserCustomization> = listOf(),
104+
smithyJson: RuntimeType = RuntimeType.smithyJson(codegenContext.runtimeConfig),
105105
) : StructuredDataParserGenerator {
106106
private val model = codegenContext.model
107107
private val symbolProvider = codegenContext.symbolProvider
108108
private val runtimeConfig = codegenContext.runtimeConfig
109109
private val codegenTarget = codegenContext.target
110-
private val smithyJson = CargoDependency.smithyJson(runtimeConfig).toType()
111110
private val protocolFunctions = ProtocolFunctions(codegenContext)
112111
private val builderInstantiator = codegenContext.builderInstantiator()
113112
private val codegenScope =

‎codegen-core/src/main/kotlin/software/amazon/smithy/rust/codegen/core/testutil/CodegenIntegrationTest.kt

Lines changed: 56 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package software.amazon.smithy.rust.codegen.core.testutil
88
import software.amazon.smithy.build.PluginContext
99
import software.amazon.smithy.model.Model
1010
import software.amazon.smithy.model.node.ObjectNode
11+
import software.amazon.smithy.model.node.ToNode
1112
import software.amazon.smithy.rust.codegen.core.smithy.RuntimeConfig
1213
import software.amazon.smithy.rust.codegen.core.util.runCommand
1314
import java.io.File
@@ -50,87 +51,78 @@ data class IntegrationTestParams(
5051
sealed class AdditionalSettings {
5152
abstract fun toObjectNode(): ObjectNode
5253

53-
abstract class CoreAdditionalSettings protected constructor(val settings: List<AdditionalSettings>) : AdditionalSettings() {
54-
override fun toObjectNode(): ObjectNode {
55-
val merged =
56-
settings.map { it.toObjectNode() }
57-
.reduce { acc, next -> acc.merge(next) }
58-
59-
return ObjectNode.builder()
60-
.withMember("codegen", merged)
54+
companion object {
55+
private fun Map<String, Any>.toCodegenObjectNode(): ObjectNode =
56+
ObjectNode.builder()
57+
.withMember(
58+
"codegen",
59+
ObjectNode.builder().apply {
60+
forEach { (key, value) ->
61+
when (value) {
62+
is Boolean -> withMember(key, value)
63+
is Number -> withMember(key, value)
64+
is String -> withMember(key, value)
65+
is ToNode -> withMember(key, value)
66+
else -> throw IllegalArgumentException("Unsupported type for key $key: ${value::class}")
67+
}
68+
}
69+
}.build(),
70+
)
6171
.build()
62-
}
63-
64-
abstract class Builder<T : CoreAdditionalSettings> : AdditionalSettings() {
65-
protected val settings = mutableListOf<AdditionalSettings>()
72+
}
6673

67-
fun generateCodegenComments(debugMode: Boolean = true): Builder<T> {
68-
settings.add(GenerateCodegenComments(debugMode))
69-
return this
70-
}
74+
abstract class CoreAdditionalSettings protected constructor(
75+
private val settings: Map<String, Any>,
76+
) : AdditionalSettings() {
77+
override fun toObjectNode(): ObjectNode = settings.toCodegenObjectNode()
7178

72-
abstract fun build(): T
79+
abstract class Builder<T : CoreAdditionalSettings> : AdditionalSettings() {
80+
protected val settings = mutableMapOf<String, Any>()
7381

74-
override fun toObjectNode(): ObjectNode = build().toObjectNode()
75-
}
82+
fun generateCodegenComments(debugMode: Boolean = true) =
83+
apply {
84+
settings["debugMode"] = debugMode
85+
}
7686

77-
// Core settings that are common to both Servers and Clients should be defined here.
78-
data class GenerateCodegenComments(val debugMode: Boolean) : AdditionalSettings() {
79-
override fun toObjectNode(): ObjectNode =
80-
ObjectNode.builder()
81-
.withMember("debugMode", debugMode)
82-
.build()
87+
override fun toObjectNode(): ObjectNode = settings.toCodegenObjectNode()
8388
}
8489
}
8590
}
8691

87-
class ClientAdditionalSettings private constructor(settings: List<AdditionalSettings>) :
88-
AdditionalSettings.CoreAdditionalSettings(settings) {
89-
class Builder : CoreAdditionalSettings.Builder<ClientAdditionalSettings>() {
90-
override fun build(): ClientAdditionalSettings = ClientAdditionalSettings(settings)
91-
}
92-
93-
// Additional settings that are specific to client generation should be defined here.
94-
95-
companion object {
96-
fun builder() = Builder()
97-
}
98-
}
99-
100-
class ServerAdditionalSettings private constructor(settings: List<AdditionalSettings>) :
101-
AdditionalSettings.CoreAdditionalSettings(settings) {
102-
class Builder : CoreAdditionalSettings.Builder<ServerAdditionalSettings>() {
103-
fun publicConstrainedTypes(enabled: Boolean = true): Builder {
104-
settings.add(PublicConstrainedTypes(enabled))
105-
return this
92+
class ServerAdditionalSettings private constructor(
93+
settings: Map<String, Any>,
94+
) : AdditionalSettings.CoreAdditionalSettings(settings) {
95+
class Builder : CoreAdditionalSettings.Builder<ServerAdditionalSettings>() {
96+
fun publicConstrainedTypes(enabled: Boolean = true) =
97+
apply {
98+
settings["publicConstrainedTypes"] = enabled
10699
}
107100

108-
fun addValidationExceptionToConstrainedOperations(enabled: Boolean = true): Builder {
109-
settings.add(AddValidationExceptionToConstrainedOperations(enabled))
110-
return this
101+
fun addValidationExceptionToConstrainedOperations(enabled: Boolean = true) =
102+
apply {
103+
settings["addValidationExceptionToConstrainedOperations"] = enabled
111104
}
112105

113-
override fun build(): ServerAdditionalSettings = ServerAdditionalSettings(settings)
114-
}
106+
fun replaceInvalidUtf8(enabled: Boolean = true) =
107+
apply {
108+
settings["replaceInvalidUtf8"] = enabled
109+
}
110+
}
115111

116-
private data class PublicConstrainedTypes(val enabled: Boolean) : AdditionalSettings() {
117-
override fun toObjectNode(): ObjectNode =
118-
ObjectNode.builder()
119-
.withMember("publicConstrainedTypes", enabled)
120-
.build()
121-
}
112+
companion object {
113+
fun builder() = Builder()
114+
}
115+
}
122116

123-
private data class AddValidationExceptionToConstrainedOperations(val enabled: Boolean) : AdditionalSettings() {
124-
override fun toObjectNode(): ObjectNode =
125-
ObjectNode.builder()
126-
.withMember("addValidationExceptionToConstrainedOperations", enabled)
127-
.build()
128-
}
117+
class ClientAdditionalSettings private constructor(
118+
settings: Map<String, Any>,
119+
) : AdditionalSettings.CoreAdditionalSettings(settings) {
120+
class Builder : CoreAdditionalSettings.Builder<ClientAdditionalSettings>()
129121

130-
companion object {
131-
fun builder() = Builder()
132-
}
122+
companion object {
123+
fun builder() = Builder()
133124
}
125+
}
134126

135127
/**
136128
* Run cargo test on a true, end-to-end, codegen product of a given model.

‎codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/ServerRustSettings.kt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ data class ServerCodegenConfig(
9797
*/
9898
val experimentalCustomValidationExceptionWithReasonPleaseDoNotUse: String? = defaultExperimentalCustomValidationExceptionWithReasonPleaseDoNotUse,
9999
val addValidationExceptionToConstrainedOperations: Boolean = DEFAULT_ADD_VALIDATION_EXCEPTION_TO_CONSTRAINED_OPERATIONS,
100+
val replaceInvalidUtf8: Boolean = DEFAULT_REPLACE_INVALID_UTF8,
100101
) : CoreCodegenConfig(
101102
formatTimeoutSeconds, debugMode,
102103
) {
@@ -105,6 +106,7 @@ data class ServerCodegenConfig(
105106
private const val DEFAULT_IGNORE_UNSUPPORTED_CONSTRAINTS = false
106107
private val defaultExperimentalCustomValidationExceptionWithReasonPleaseDoNotUse = null
107108
private const val DEFAULT_ADD_VALIDATION_EXCEPTION_TO_CONSTRAINED_OPERATIONS = false
109+
private const val DEFAULT_REPLACE_INVALID_UTF8 = false
108110

109111
fun fromCodegenConfigAndNode(
110112
coreCodegenConfig: CoreCodegenConfig,
@@ -117,6 +119,7 @@ data class ServerCodegenConfig(
117119
ignoreUnsupportedConstraints = node.get().getBooleanMemberOrDefault("ignoreUnsupportedConstraints", DEFAULT_IGNORE_UNSUPPORTED_CONSTRAINTS),
118120
experimentalCustomValidationExceptionWithReasonPleaseDoNotUse = node.get().getStringMemberOrDefault("experimentalCustomValidationExceptionWithReasonPleaseDoNotUse", defaultExperimentalCustomValidationExceptionWithReasonPleaseDoNotUse),
119121
addValidationExceptionToConstrainedOperations = node.get().getBooleanMemberOrDefault("addValidationExceptionToConstrainedOperations", DEFAULT_ADD_VALIDATION_EXCEPTION_TO_CONSTRAINED_OPERATIONS),
122+
replaceInvalidUtf8 = node.get().getBooleanMemberOrDefault("replaceInvalidUtf8", DEFAULT_REPLACE_INVALID_UTF8),
120123
)
121124
} else {
122125
ServerCodegenConfig(

‎codegen-server/src/main/kotlin/software/amazon/smithy/rust/codegen/server/smithy/generators/protocol/ServerProtocol.kt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ fun jsonParserGenerator(
135135
httpBindingResolver: HttpBindingResolver,
136136
jsonName: (MemberShape) -> String,
137137
additionalParserCustomizations: List<JsonParserCustomization> = listOf(),
138+
smithyJson: RuntimeType = RuntimeType.smithyJson(codegenContext.runtimeConfig),
138139
): JsonParserGenerator =
139140
JsonParserGenerator(
140141
codegenContext,
@@ -144,6 +145,7 @@ fun jsonParserGenerator(
144145
listOf(
145146
ServerRequestBeforeBoxingDeserializedMemberConvertToMaybeConstrainedJsonParserCustomization(codegenContext),
146147
) + additionalParserCustomizations,
148+
smithyJson,
147149
)
148150

149151
class ServerAwsJsonProtocol(
@@ -245,6 +247,14 @@ class ServerRestJsonProtocol(
245247
httpBindingResolver,
246248
::restJsonFieldName,
247249
additionalParserCustomizations,
250+
smithyJson =
251+
if (serverCodegenContext.settings.codegenConfig.replaceInvalidUtf8) {
252+
CargoDependency.smithyJson(codegenContext.runtimeConfig)
253+
.copy(features = setOf("replace-invalid-utf8"))
254+
.toType()
255+
} else {
256+
RuntimeType.smithyJson(codegenContext.runtimeConfig)
257+
},
248258
)
249259

250260
override fun structuredDataSerializer(): StructuredDataSerializerGenerator =

‎codegen-server/src/test/kotlin/software/amazon/smithy/rust/codegen/server/smithy/ReplaceInvalidUtf8Test.kt

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package software.amazon.smithy.rust.codegen.server.smithy
2+
3+
import org.junit.jupiter.api.Test
4+
import software.amazon.smithy.rust.codegen.core.rustlang.rust
5+
import software.amazon.smithy.rust.codegen.core.rustlang.rustTemplate
6+
import software.amazon.smithy.rust.codegen.core.testutil.IntegrationTestParams
7+
import software.amazon.smithy.rust.codegen.core.testutil.ServerAdditionalSettings
8+
import software.amazon.smithy.rust.codegen.core.testutil.asSmithyModel
9+
import software.amazon.smithy.rust.codegen.core.testutil.testModule
10+
import software.amazon.smithy.rust.codegen.server.smithy.testutil.serverIntegrationTest
11+
12+
internal class ReplaceInvalidUtf8Test {
13+
val model =
14+
"""
15+
namespace test
16+
use aws.protocols#restJson1
17+
18+
@restJson1
19+
service SampleService {
20+
operations: [SampleOperation]
21+
}
22+
23+
@http(uri: "/operation", method: "PUT")
24+
operation SampleOperation {
25+
input := {
26+
x : String
27+
}
28+
}
29+
""".asSmithyModel(smithyVersion = "2")
30+
31+
@Test
32+
fun `invalid utf8 should be replaced if the codegen flag is set`() {
33+
serverIntegrationTest(
34+
model,
35+
IntegrationTestParams(
36+
additionalSettings =
37+
ServerAdditionalSettings
38+
.builder()
39+
.replaceInvalidUtf8(true)
40+
.toObjectNode(),
41+
),
42+
) { _, rustCrate ->
43+
rustCrate.testModule {
44+
rustTemplate(
45+
"""
46+
##[tokio::test]
47+
async fn test_utf8_replaced() {
48+
let body = r##"{ "x" : "\ud800" }"##;
49+
let request = http::Request::builder()
50+
.method("POST")
51+
.uri("/operation")
52+
.header("content-type", "application/json")
53+
.body(hyper::Body::from(body))
54+
.expect("failed to build request");
55+
let result = crate::protocol_serde::shape_sample_operation::de_sample_operation_http_request(request).await;
56+
assert!(
57+
result.is_ok(),
58+
"Invalid utf8 should have been replaced. {result:?}"
59+
);
60+
assert_eq!(
61+
result.unwrap().x.unwrap(),
62+
"�",
63+
"payload should have been replaced with �."
64+
);
65+
}
66+
""",
67+
)
68+
}
69+
}
70+
}
71+
72+
@Test
73+
fun `invalid utf8 should be rejected if the codegen flag is not set`() {
74+
serverIntegrationTest(
75+
model,
76+
) { _, rustCrate ->
77+
rustCrate.testModule {
78+
rustTemplate(
79+
"""
80+
##[tokio::test]
81+
async fn test_invalid_utf8_raises_an_error() {
82+
let body = r##"{ "x" : "\ud800" }"##;
83+
let request = http::Request::builder()
84+
.method("POST")
85+
.uri("/operation")
86+
.header("content-type", "application/json")
87+
.body(hyper::Body::from(body))
88+
.expect("failed to build request");
89+
let result = crate::protocol_serde::shape_sample_operation::de_sample_operation_http_request(request).await;
90+
assert!(
91+
result.is_err(),
92+
"invalid utf8 characters should not be allowed by default {result:?}"
93+
);
94+
}
95+
""",
96+
)
97+
}
98+
}
99+
}
100+
}

‎rust-runtime/aws-smithy-json/Cargo.toml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
[package]
22
name = "aws-smithy-json"
33
version = "0.61.2"
4-
authors = ["AWS Rust SDK Team <[email protected]>", "John DiSanti <[email protected]>"]
4+
authors = [
5+
"AWS Rust SDK Team <[email protected]>",
6+
"John DiSanti <[email protected]>",
7+
]
58
description = "Token streaming JSON parser for smithy-rs."
69
edition = "2021"
710
license = "Apache-2.0"
@@ -20,3 +23,6 @@ targets = ["x86_64-unknown-linux-gnu"]
2023
cargo-args = ["-Zunstable-options", "-Zrustdoc-scrape-examples"]
2124
rustdoc-args = ["--cfg", "docsrs"]
2225
# End of docs.rs metadata
26+
27+
[features]
28+
replace-invalid-utf8 = []

0 commit comments

Comments
 (0)