Skip to content

Commit 611c7cd

Browse files
MrChaos1993MrChaos1993
committed
Check version in Streams. More Tests.
Signed-off-by: Aleksandr Tuliakov <[email protected]>
1 parent 5ed6c6d commit 611c7cd

File tree

11 files changed

+211
-10
lines changed

11 files changed

+211
-10
lines changed

‎src/main/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollup.kt

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package org.opensearch.indexmanagement.rollup.model
77

88
import org.apache.commons.codec.digest.DigestUtils
9+
import org.opensearch.Version
910
import org.opensearch.common.settings.Settings
1011
import org.opensearch.commons.authuser.User
1112
import org.opensearch.core.common.io.stream.StreamInput
@@ -96,7 +97,7 @@ data class ISMRollup(
9697
constructor(sin: StreamInput) : this(
9798
description = sin.readString(),
9899
targetIndex = sin.readString(),
99-
targetIndexSettings = if (sin.readBoolean()) {
100+
targetIndexSettings = if (sin.version.onOrAfter(Version.V_3_0_0) && sin.readBoolean()) {
100101
Settings.readSettingsFromStream(sin)
101102
} else {
102103
null
@@ -143,8 +144,10 @@ data class ISMRollup(
143144
override fun writeTo(out: StreamOutput) {
144145
out.writeString(description)
145146
out.writeString(targetIndex)
146-
out.writeBoolean(targetIndexSettings != null)
147-
if (targetIndexSettings != null) Settings.writeSettingsToStream(targetIndexSettings, out)
147+
if (out.version.onOrAfter(Version.V_3_0_0)) {
148+
out.writeBoolean(targetIndexSettings != null)
149+
if (targetIndexSettings != null) Settings.writeSettingsToStream(targetIndexSettings, out)
150+
}
148151
out.writeInt(pageSize)
149152
out.writeVInt(dimensions.size)
150153
for (dimension in dimensions) {

‎src/main/kotlin/org/opensearch/indexmanagement/rollup/model/Rollup.kt

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
package org.opensearch.indexmanagement.rollup.model
77

8+
import org.opensearch.Version
89
import org.opensearch.common.settings.IndexScopedSettings
910
import org.opensearch.common.settings.Settings
1011
import org.opensearch.commons.authuser.User
@@ -135,7 +136,7 @@ data class Rollup(
135136
description = sin.readString(),
136137
sourceIndex = sin.readString(),
137138
targetIndex = sin.readString(),
138-
targetIndexSettings = if (sin.readBoolean()) {
139+
targetIndexSettings = if (sin.getVersion().onOrAfter(Version.V_3_0_0) && sin.readBoolean()) {
139140
Settings.readSettingsFromStream(sin)
140141
} else {
141142
null
@@ -216,8 +217,10 @@ data class Rollup(
216217
out.writeString(description)
217218
out.writeString(sourceIndex)
218219
out.writeString(targetIndex)
219-
out.writeBoolean(targetIndexSettings != null)
220-
if (targetIndexSettings != null) Settings.writeSettingsToStream(targetIndexSettings, out)
220+
if (out.version.onOrAfter(Version.V_3_0_0)) {
221+
out.writeBoolean(targetIndexSettings != null)
222+
if (targetIndexSettings != null) Settings.writeSettingsToStream(targetIndexSettings, out)
223+
}
221224
out.writeOptionalString(metadataID)
222225
out.writeStringArray(emptyList<String>().toTypedArray())
223226
out.writeInt(pageSize)

‎src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RollupActionIT.kt

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ package org.opensearch.indexmanagement.indexstatemanagement.action
88
import org.apache.hc.core5.http.ContentType
99
import org.apache.hc.core5.http.io.entity.StringEntity
1010
import org.opensearch.cluster.metadata.DataStream
11+
import org.opensearch.cluster.metadata.IndexMetadata
12+
import org.opensearch.common.settings.Settings
13+
import org.opensearch.index.engine.EngineConfig
1114
import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
1215
import org.opensearch.indexmanagement.common.model.dimension.Terms
1316
import org.opensearch.indexmanagement.indexstatemanagement.IndexStateManagementRestTestCase
@@ -88,6 +91,65 @@ class RollupActionIT : IndexStateManagementRestTestCase() {
8891
assertIndexRolledUp(indexName, policyID, rollup)
8992
}
9093

94+
fun `test rollup action with specified target index settings`() {
95+
val indexName = "${testIndexName}_index_settings"
96+
val policyID = "${testIndexName}_policy_settings"
97+
val targetIdxTestName = "target_rollup_settings"
98+
val targetIndexReplicas = 0
99+
val targetIndexCodec = "best_compression"
100+
val rollup =
101+
ISMRollup(
102+
description = "basic search test",
103+
targetIndex = targetIdxTestName,
104+
targetIndexSettings = Settings.builder()
105+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, targetIndexReplicas)
106+
.put(EngineConfig.INDEX_CODEC_SETTING.key, targetIndexCodec)
107+
.build(),
108+
pageSize = 100,
109+
dimensions =
110+
listOf(
111+
DateHistogram(sourceField = "tpep_pickup_datetime", fixedInterval = "1h"),
112+
Terms("RatecodeID", "RatecodeID"),
113+
Terms("PULocationID", "PULocationID"),
114+
),
115+
metrics =
116+
listOf(
117+
RollupMetrics(
118+
sourceField = "passenger_count", targetField = "passenger_count",
119+
metrics =
120+
listOf(
121+
Sum(), Min(), Max(),
122+
ValueCount(), Average(),
123+
),
124+
),
125+
RollupMetrics(sourceField = "total_amount", targetField = "total_amount", metrics = listOf(Max(), Min())),
126+
),
127+
)
128+
val actionConfig = RollupAction(rollup, 0)
129+
val states =
130+
listOf(
131+
State("rollup", listOf(actionConfig), listOf()),
132+
)
133+
val sourceIndexMappingString =
134+
"\"properties\": {\"tpep_pickup_datetime\": { \"type\": \"date\" }, \"RatecodeID\": { \"type\": " +
135+
"\"keyword\" }, \"PULocationID\": { \"type\": \"keyword\" }, \"passenger_count\": { \"type\": \"integer\" }, \"total_amount\": " +
136+
"{ \"type\": \"double\" }}"
137+
val policy =
138+
Policy(
139+
id = policyID,
140+
description = "$testIndexName description",
141+
schemaVersion = 1L,
142+
lastUpdatedTime = Instant.now().truncatedTo(ChronoUnit.MILLIS),
143+
errorNotification = randomErrorNotification(),
144+
defaultState = states[0].name,
145+
states = states,
146+
)
147+
createPolicy(policy, policyID)
148+
createIndex(indexName, policyID, mapping = sourceIndexMappingString)
149+
150+
assertIndexRolledUp(indexName, policyID, rollup)
151+
}
152+
91153
fun `test data stream rollup action`() {
92154
val dataStreamName = "${testIndexName}_data_stream"
93155
val policyID = "${testIndexName}_rollup_policy"

‎src/test/kotlin/org/opensearch/indexmanagement/rollup/RollupRestTestCase.kt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,13 @@ abstract class RollupRestTestCase : IndexManagementRestTestCase() {
132132
return getRollup(rollupId = rollupId)
133133
}
134134

135+
// TODO: can be replaced with createRandomRollup if implement assertEqual for mappings with "dynamic"=true fields
136+
protected fun createRandomRollupWithoutTargetSettings(refresh: Boolean = true): Rollup {
137+
val rollup = randomRollup().copy(targetIndexSettings = null)
138+
val rollupId = createRollup(rollup, rollupId = rollup.id, refresh = refresh).id
139+
return getRollup(rollupId = rollupId)
140+
}
141+
135142
// TODO: Maybe clean-up and use XContentFactory.jsonBuilder() to create mappings json
136143
protected fun createRollupMappingString(rollup: Rollup): String {
137144
var mappingString = ""

‎src/test/kotlin/org/opensearch/indexmanagement/rollup/TestHelpers.kt

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,12 @@
55

66
package org.opensearch.indexmanagement.rollup
77

8+
import org.opensearch.cluster.metadata.IndexMetadata
9+
import org.opensearch.common.settings.Settings
810
import org.opensearch.common.xcontent.XContentFactory
911
import org.opensearch.core.xcontent.ToXContent
12+
import org.opensearch.index.codec.CodecService
13+
import org.opensearch.index.engine.EngineConfig
1014
import org.opensearch.index.query.TermQueryBuilder
1115
import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
1216
import org.opensearch.indexmanagement.common.model.dimension.Dimension
@@ -33,8 +37,9 @@ import org.opensearch.indexmanagement.rollup.model.metric.Metric
3337
import org.opensearch.indexmanagement.rollup.model.metric.Min
3438
import org.opensearch.indexmanagement.rollup.model.metric.Sum
3539
import org.opensearch.indexmanagement.rollup.model.metric.ValueCount
40+
import org.opensearch.test.OpenSearchTestCase
3641
import org.opensearch.test.rest.OpenSearchRestTestCase
37-
import java.util.Locale
42+
import java.util.*
3843

3944
fun randomInterval(): String = if (OpenSearchRestTestCase.randomBoolean()) randomFixedInterval() else randomCalendarInterval()
4045

@@ -98,6 +103,14 @@ fun randomRollupDimensions(): List<Dimension> {
98103
return dimensions.toList()
99104
}
100105

106+
val codecs = listOf(CodecService.DEFAULT_CODEC, CodecService.LZ4, CodecService.BEST_COMPRESSION_CODEC, CodecService.ZLIB)
107+
108+
fun randomSettings(): Settings = Settings.builder()
109+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, OpenSearchTestCase.randomIntBetween(0, 10))
110+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, OpenSearchTestCase.randomIntBetween(1, 10))
111+
.put(EngineConfig.INDEX_CODEC_SETTING.key, OpenSearchRestTestCase.randomSubsetOf(1, codecs).first())
112+
.build()
113+
101114
fun randomRollup(): Rollup {
102115
val enabled = OpenSearchRestTestCase.randomBoolean()
103116
return Rollup(
@@ -112,7 +125,7 @@ fun randomRollup(): Rollup {
112125
description = OpenSearchRestTestCase.randomAlphaOfLength(10),
113126
sourceIndex = OpenSearchRestTestCase.randomAlphaOfLength(10).lowercase(Locale.ROOT),
114127
targetIndex = OpenSearchRestTestCase.randomAlphaOfLength(10).lowercase(Locale.ROOT),
115-
targetIndexSettings = null,
128+
targetIndexSettings = if (OpenSearchRestTestCase.randomBoolean()) null else randomSettings(),
116129
metadataID = if (OpenSearchRestTestCase.randomBoolean()) null else OpenSearchRestTestCase.randomAlphaOfLength(10),
117130
roles = emptyList(),
118131
pageSize = OpenSearchRestTestCase.randomIntBetween(1, 10000),
@@ -173,7 +186,7 @@ fun randomExplainRollup(): ExplainRollup {
173186
fun randomISMRollup(): ISMRollup = ISMRollup(
174187
description = OpenSearchRestTestCase.randomAlphaOfLength(10),
175188
targetIndex = OpenSearchRestTestCase.randomAlphaOfLength(10).lowercase(Locale.ROOT),
176-
targetIndexSettings = null,
189+
targetIndexSettings = if (OpenSearchRestTestCase.randomBoolean()) null else randomSettings(),
177190
pageSize = OpenSearchRestTestCase.randomIntBetween(1, 10000),
178191
dimensions = randomRollupDimensions(),
179192
metrics = OpenSearchRestTestCase.randomList(20, ::randomRollupMetrics).distinctBy { it.targetField },

‎src/test/kotlin/org/opensearch/indexmanagement/rollup/model/ISMRollupTests.kt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ class ISMRollupTests : OpenSearchTestCase() {
7070

7171
assertEquals(sourceIndex, rollup.sourceIndex)
7272
assertEquals(ismRollup.targetIndex, rollup.targetIndex)
73+
assertEquals(ismRollup.targetIndexSettings, rollup.targetIndexSettings)
7374
assertEquals(ismRollup.pageSize, rollup.pageSize)
7475
assertEquals(ismRollup.dimensions, rollup.dimensions)
7576
assertEquals(ismRollup.metrics, rollup.metrics)

‎src/test/kotlin/org/opensearch/indexmanagement/rollup/model/RollupTests.kt

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,9 +58,19 @@ class RollupTests : OpenSearchTestCase() {
5858
assertFailsWith(SettingsException::class, "Unknown property was `index.codec1`") {
5959
randomRollup().copy(targetIndexSettings = Settings.builder().put("index.codec1", "zlib").build())
6060
}
61+
}
62+
63+
fun `test rollup with single setting in target index settings`() {
64+
val sb = Settings.builder()
65+
sb.put("index.codec", "zlib")
66+
randomRollup().copy(targetIndexSettings = sb.build())
67+
}
6168

69+
fun `test rollup with multiple setting in target index settings`() {
6270
val sb = Settings.builder()
71+
sb.put("index.number_of_replicas", 1)
6372
sb.put("index.codec", "zlib")
73+
sb.put("index.codec.compression_level", 6)
6474
randomRollup().copy(targetIndexSettings = sb.build())
6575
}
6676

‎src/test/kotlin/org/opensearch/indexmanagement/rollup/model/WriteableTests.kt

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import org.opensearch.indexmanagement.rollup.randomRollup
2727
import org.opensearch.indexmanagement.rollup.randomRollupMetadata
2828
import org.opensearch.indexmanagement.rollup.randomRollupMetrics
2929
import org.opensearch.indexmanagement.rollup.randomRollupStats
30+
import org.opensearch.indexmanagement.rollup.randomSettings
3031
import org.opensearch.indexmanagement.rollup.randomSum
3132
import org.opensearch.indexmanagement.rollup.randomTerms
3233
import org.opensearch.indexmanagement.rollup.randomValueCount
@@ -126,6 +127,14 @@ class WriteableTests : OpenSearchTestCase() {
126127
assertTrue("roles field in rollup model is deprecated and should be parsed to empty list.", streamedRollup.roles.isEmpty())
127128
}
128129

130+
fun `test rollup as stream with target index settings`() {
131+
val rollup = randomRollup().copy(delay = randomLongBetween(0, 60000000), targetIndexSettings = randomSettings())
132+
val out = BytesStreamOutput().also { rollup.writeTo(it) }
133+
val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
134+
val streamedRollup = Rollup(sin)
135+
assertEquals("Round tripping Rollup stream with target index settings doesn't work", rollup, streamedRollup)
136+
}
137+
129138
fun `test explain rollup as stream`() {
130139
val explainRollup = randomExplainRollup()
131140
val out = BytesStreamOutput().also { explainRollup.writeTo(it) }
@@ -165,4 +174,16 @@ class WriteableTests : OpenSearchTestCase() {
165174
val streamedISMRollup = ISMRollup(sin)
166175
assertEquals("Round tripping ISMRollup stream doesn't work", ismRollup, streamedISMRollup)
167176
}
177+
178+
fun `test ism rollup as stream with target index settings`() {
179+
val ismRollup = randomISMRollup().copy(targetIndexSettings = randomSettings())
180+
val out = BytesStreamOutput().also { ismRollup.writeTo(it) }
181+
val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes)
182+
val streamedISMRollup = ISMRollup(sin)
183+
assertEquals(
184+
"Round tripping ISMRollup stream with target index settings doesn't work",
185+
ismRollup,
186+
streamedISMRollup,
187+
)
188+
}
168189
}

‎src/test/kotlin/org/opensearch/indexmanagement/rollup/model/XContentTests.kt

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import org.opensearch.indexmanagement.rollup.randomMax
2020
import org.opensearch.indexmanagement.rollup.randomMin
2121
import org.opensearch.indexmanagement.rollup.randomRollup
2222
import org.opensearch.indexmanagement.rollup.randomRollupMetrics
23+
import org.opensearch.indexmanagement.rollup.randomSettings
2324
import org.opensearch.indexmanagement.rollup.randomSum
2425
import org.opensearch.indexmanagement.rollup.randomTerms
2526
import org.opensearch.indexmanagement.rollup.randomValueCount
@@ -120,13 +121,28 @@ class XContentTests : OpenSearchTestCase() {
120121
assertEquals("Round tripping Rollup without type doesn't work", rollup.copy(roles = listOf()), parsedRollup)
121122
}
122123

124+
fun `test rollup parsing with target index settings`() {
125+
val rollup = randomRollup().copy(delay = randomLongBetween(0, 60000000), targetIndexSettings = randomSettings())
126+
val rollupString = rollup.toJsonString(XCONTENT_WITHOUT_TYPE)
127+
val parsedRollup = Rollup.parse(parser(rollupString), rollup.id, rollup.seqNo, rollup.primaryTerm)
128+
// roles are deprecated and not populated in toXContent and parsed as part of parse
129+
assertEquals("Round tripping Rollup with target index settings doesn't work", rollup.copy(roles = listOf()), parsedRollup)
130+
}
131+
123132
fun `test ism rollup parsing`() {
124133
val ismRollup = randomISMRollup()
125134
val ismRollupString = ismRollup.toJsonString()
126135
val parsedISMRollup = ISMRollup.parse(parser(ismRollupString))
127136
assertEquals("Round tripping ISMRollup doesn't work", ismRollup, parsedISMRollup)
128137
}
129138

139+
fun `test ism rollup parsing with target index settings`() {
140+
val ismRollup = randomISMRollup().copy(targetIndexSettings = randomSettings())
141+
val ismRollupString = ismRollup.toJsonString()
142+
val parsedISMRollup = ISMRollup.parse(parser(ismRollupString))
143+
assertEquals("Round tripping ISMRollup with target index settings doesn't work", ismRollup, parsedISMRollup)
144+
}
145+
130146
private fun parser(xc: String): XContentParser {
131147
val parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc)
132148
parser.nextToken()

‎src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestIndexRollupActionIT.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class RestIndexRollupActionIT : RollupRestAPITestCase() {
7373

7474
@Throws(Exception::class)
7575
fun `test mappings after rollup creation`() {
76-
createRandomRollup()
76+
createRandomRollupWithoutTargetSettings()
7777

7878
val response = client().makeRequest("GET", "/$INDEX_MANAGEMENT_INDEX/_mapping")
7979
val parserMap = createParser(XContentType.JSON.xContent(), response.entity.content).map() as Map<String, Map<String, Any>>

0 commit comments

Comments
 (0)