Skip to content

Commit 20afca3

Browse files
ashking94dk2k
ashking94
authored and
dk2k
committed
Add support to upload snapshot shard blobs with hashed prefix (opensearch-project#15426)
* Add snapshot shard blobs with hashed prefix Signed-off-by: Ashish Singh <[email protected]> * Add UTs Signed-off-by: Ashish Singh <[email protected]> * Address comments and add UTs Signed-off-by: Ashish Singh <[email protected]> * Change default snapshot shard path type to hashed_prefix for testing Signed-off-by: Ashish Singh <[email protected]> * Fix failing tests Signed-off-by: Ashish Singh <[email protected]> * Introduce single method for all create/put repository calls Signed-off-by: Ashish Singh <[email protected]> * Move the index shard path upload to sync upload Signed-off-by: Ashish Singh <[email protected]> * Fix spotless errors Signed-off-by: Ashish Singh <[email protected]> * Do minor code refactor Signed-off-by: Ashish Singh <[email protected]> * Fix tests Signed-off-by: Ashish Singh <[email protected]> * More tests to use common methods for repo creation Signed-off-by: Ashish Singh <[email protected]> * Address comments and fix multiple failing tests Signed-off-by: Ashish Singh <[email protected]> --------- Signed-off-by: Ashish Singh <[email protected]>
1 parent 07af96c commit 20afca3

File tree

60 files changed

+1975
-927
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+1975
-927
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3939
- Add fieldType to AbstractQueryBuilder and FieldSortBuilder ([#15328](https://github.com/opensearch-project/OpenSearch/pull/15328)))
4040
- [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410))
4141
- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527))
42+
- Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426))
4243

4344
### Dependencies
4445
- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

‎modules/repository-url/src/internalClusterTest/java/org/opensearch/repositories/url/URLSnapshotRestoreIT.java

Lines changed: 9 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -67,19 +67,11 @@ public void testUrlRepository() throws Exception {
6767

6868
logger.info("--> creating repository");
6969
Path repositoryLocation = randomRepoPath();
70-
assertAcked(
71-
client.admin()
72-
.cluster()
73-
.preparePutRepository("test-repo")
74-
.setType(FsRepository.TYPE)
75-
.setSettings(
76-
Settings.builder()
77-
.put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation)
78-
.put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
79-
.put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
80-
)
81-
);
82-
70+
Settings.Builder settings = Settings.builder()
71+
.put(FsRepository.LOCATION_SETTING.getKey(), repositoryLocation)
72+
.put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
73+
.put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES);
74+
createRepository("test-repo", FsRepository.TYPE, settings);
8375
createIndex("test-idx");
8476
ensureGreen();
8577

@@ -115,17 +107,10 @@ public void testUrlRepository() throws Exception {
115107
cluster().wipeIndices("test-idx");
116108

117109
logger.info("--> create read-only URL repository");
118-
assertAcked(
119-
client.admin()
120-
.cluster()
121-
.preparePutRepository("url-repo")
122-
.setType(URLRepository.TYPE)
123-
.setSettings(
124-
Settings.builder()
125-
.put(URLRepository.URL_SETTING.getKey(), repositoryLocation.toUri().toURL().toString())
126-
.put("list_directories", randomBoolean())
127-
)
128-
);
110+
Settings.Builder settingsBuilder = Settings.builder()
111+
.put(URLRepository.URL_SETTING.getKey(), repositoryLocation.toUri().toURL().toString())
112+
.put("list_directories", randomBoolean());
113+
createRepository("url-repo", URLRepository.TYPE, settingsBuilder);
129114
logger.info("--> restore index after deletion");
130115
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
131116
.cluster()

‎plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureStorageCleanupThirdPartyTests.java

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@
3838
import com.azure.storage.blob.models.BlobStorageException;
3939
import org.opensearch.action.ActionRunnable;
4040
import org.opensearch.action.support.PlainActionFuture;
41-
import org.opensearch.action.support.master.AcknowledgedResponse;
4241
import org.opensearch.common.collect.Tuple;
4342
import org.opensearch.common.settings.MockSecureSettings;
4443
import org.opensearch.common.settings.SecureSettings;
@@ -47,6 +46,7 @@
4746
import org.opensearch.plugins.Plugin;
4847
import org.opensearch.repositories.AbstractThirdPartyRepositoryTestCase;
4948
import org.opensearch.repositories.blobstore.BlobStoreRepository;
49+
import org.opensearch.test.OpenSearchIntegTestCase;
5050
import org.junit.AfterClass;
5151

5252
import java.net.HttpURLConnection;
@@ -56,7 +56,6 @@
5656
import reactor.core.scheduler.Schedulers;
5757

5858
import static org.hamcrest.Matchers.blankOrNullString;
59-
import static org.hamcrest.Matchers.equalTo;
6059
import static org.hamcrest.Matchers.not;
6160

6261
public class AzureStorageCleanupThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
@@ -103,17 +102,11 @@ protected SecureSettings credentials() {
103102

104103
@Override
105104
protected void createRepository(String repoName) {
106-
AcknowledgedResponse putRepositoryResponse = client().admin()
107-
.cluster()
108-
.preparePutRepository(repoName)
109-
.setType("azure")
110-
.setSettings(
111-
Settings.builder()
112-
.put("container", System.getProperty("test.azure.container"))
113-
.put("base_path", System.getProperty("test.azure.base"))
114-
)
115-
.get();
116-
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
105+
Settings.Builder settings = Settings.builder()
106+
.put("container", System.getProperty("test.azure.container"))
107+
.put("base_path", System.getProperty("test.azure.base"));
108+
109+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), repoName, "azure", settings);
117110
if (Strings.hasText(System.getProperty("test.azure.sas_token"))) {
118111
ensureSasTokenPermissions();
119112
}

‎plugins/repository-gcs/src/internalClusterTest/java/org/opensearch/repositories/gcs/GoogleCloudStorageThirdPartyTests.java

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,19 +32,18 @@
3232

3333
package org.opensearch.repositories.gcs;
3434

35-
import org.opensearch.action.support.master.AcknowledgedResponse;
3635
import org.opensearch.common.settings.MockSecureSettings;
3736
import org.opensearch.common.settings.SecureSettings;
3837
import org.opensearch.common.settings.Settings;
3938
import org.opensearch.core.common.Strings;
4039
import org.opensearch.plugins.Plugin;
4140
import org.opensearch.repositories.AbstractThirdPartyRepositoryTestCase;
41+
import org.opensearch.test.OpenSearchIntegTestCase;
4242

4343
import java.util.Base64;
4444
import java.util.Collection;
4545

4646
import static org.hamcrest.Matchers.blankOrNullString;
47-
import static org.hamcrest.Matchers.equalTo;
4847
import static org.hamcrest.Matchers.not;
4948

5049
public class GoogleCloudStorageThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
@@ -84,16 +83,9 @@ protected SecureSettings credentials() {
8483

8584
@Override
8685
protected void createRepository(final String repoName) {
87-
AcknowledgedResponse putRepositoryResponse = client().admin()
88-
.cluster()
89-
.preparePutRepository("test-repo")
90-
.setType("gcs")
91-
.setSettings(
92-
Settings.builder()
93-
.put("bucket", System.getProperty("test.google.bucket"))
94-
.put("base_path", System.getProperty("test.google.base", "/"))
95-
)
96-
.get();
97-
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
86+
Settings.Builder settings = Settings.builder()
87+
.put("bucket", System.getProperty("test.google.bucket"))
88+
.put("base_path", System.getProperty("test.google.base", "/"));
89+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "gcs", settings);
9890
}
9991
}

‎plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsRepositoryTests.java

Lines changed: 8 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,12 @@
3434
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
3535

3636
import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse;
37-
import org.opensearch.action.support.master.AcknowledgedResponse;
3837
import org.opensearch.common.settings.MockSecureSettings;
3938
import org.opensearch.common.settings.SecureSettings;
4039
import org.opensearch.common.settings.Settings;
4140
import org.opensearch.plugins.Plugin;
4241
import org.opensearch.repositories.AbstractThirdPartyRepositoryTestCase;
42+
import org.opensearch.test.OpenSearchIntegTestCase;
4343

4444
import java.util.Collection;
4545

@@ -61,20 +61,13 @@ protected SecureSettings credentials() {
6161

6262
@Override
6363
protected void createRepository(String repoName) {
64-
AcknowledgedResponse putRepositoryResponse = client().admin()
65-
.cluster()
66-
.preparePutRepository(repoName)
67-
.setType("hdfs")
68-
.setSettings(
69-
Settings.builder()
70-
.put("uri", "hdfs:///")
71-
.put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName())
72-
.put("path", "foo")
73-
.put("chunk_size", randomIntBetween(100, 1000) + "k")
74-
.put("compress", randomBoolean())
75-
)
76-
.get();
77-
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
64+
Settings.Builder settings = Settings.builder()
65+
.put("uri", "hdfs:///")
66+
.put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName())
67+
.put("path", "foo")
68+
.put("chunk_size", randomIntBetween(100, 1000) + "k")
69+
.put("compress", randomBoolean());
70+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), repoName, "hdfs", settings);
7871
}
7972

8073
// HDFS repository doesn't have precise cleanup stats so we only check whether or not any blobs were removed

‎plugins/repository-hdfs/src/test/java/org/opensearch/repositories/hdfs/HdfsTests.java

Lines changed: 17 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,6 @@
3535

3636
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
3737
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
38-
import org.opensearch.action.support.master.AcknowledgedResponse;
3938
import org.opensearch.client.Client;
4039
import org.opensearch.cluster.ClusterState;
4140
import org.opensearch.common.settings.Settings;
@@ -45,6 +44,7 @@
4544
import org.opensearch.repositories.blobstore.BlobStoreRepository;
4645
import org.opensearch.repositories.blobstore.BlobStoreTestUtil;
4746
import org.opensearch.snapshots.SnapshotState;
47+
import org.opensearch.test.OpenSearchIntegTestCase;
4848
import org.opensearch.test.OpenSearchSingleNodeTestCase;
4949
import org.opensearch.threadpool.ThreadPool;
5050

@@ -63,21 +63,13 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
6363

6464
public void testSimpleWorkflow() {
6565
Client client = client();
66-
67-
AcknowledgedResponse putRepositoryResponse = client.admin()
68-
.cluster()
69-
.preparePutRepository("test-repo")
70-
.setType("hdfs")
71-
.setSettings(
72-
Settings.builder()
73-
.put("uri", "hdfs:///")
74-
.put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName())
75-
.put("path", "foo")
76-
.put("chunk_size", randomIntBetween(100, 1000) + "k")
77-
.put("compress", randomBoolean())
78-
)
79-
.get();
80-
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
66+
Settings.Builder settings = Settings.builder()
67+
.put("uri", "hdfs:///")
68+
.put("conf.fs.AbstractFileSystem.hdfs.impl", TestingFs.class.getName())
69+
.put("path", "foo")
70+
.put("chunk_size", randomIntBetween(100, 1000) + "k")
71+
.put("compress", randomBoolean());
72+
OpenSearchIntegTestCase.putRepository(client.admin().cluster(), "test-repo", "hdfs", settings);
8173

8274
createIndex("test-idx-1");
8375
createIndex("test-idx-2");
@@ -168,7 +160,7 @@ public void testSimpleWorkflow() {
168160

169161
public void testMissingUri() {
170162
try {
171-
client().admin().cluster().preparePutRepository("test-repo").setType("hdfs").setSettings(Settings.EMPTY).get();
163+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", Settings.builder());
172164
fail();
173165
} catch (RepositoryException e) {
174166
assertTrue(e.getCause() instanceof IllegalArgumentException);
@@ -178,12 +170,8 @@ public void testMissingUri() {
178170

179171
public void testEmptyUri() {
180172
try {
181-
client().admin()
182-
.cluster()
183-
.preparePutRepository("test-repo")
184-
.setType("hdfs")
185-
.setSettings(Settings.builder().put("uri", "/path").build())
186-
.get();
173+
Settings.Builder settings = Settings.builder().put("uri", "/path");
174+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", settings);
187175
fail();
188176
} catch (RepositoryException e) {
189177
assertTrue(e.getCause() instanceof IllegalArgumentException);
@@ -193,12 +181,8 @@ public void testEmptyUri() {
193181

194182
public void testNonHdfsUri() {
195183
try {
196-
client().admin()
197-
.cluster()
198-
.preparePutRepository("test-repo")
199-
.setType("hdfs")
200-
.setSettings(Settings.builder().put("uri", "file:///").build())
201-
.get();
184+
Settings.Builder settings = Settings.builder().put("uri", "file:///");
185+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", settings);
202186
fail();
203187
} catch (RepositoryException e) {
204188
assertTrue(e.getCause() instanceof IllegalArgumentException);
@@ -208,12 +192,8 @@ public void testNonHdfsUri() {
208192

209193
public void testPathSpecifiedInHdfs() {
210194
try {
211-
client().admin()
212-
.cluster()
213-
.preparePutRepository("test-repo")
214-
.setType("hdfs")
215-
.setSettings(Settings.builder().put("uri", "hdfs:///some/path").build())
216-
.get();
195+
Settings.Builder settings = Settings.builder().put("uri", "hdfs:///some/path");
196+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", settings);
217197
fail();
218198
} catch (RepositoryException e) {
219199
assertTrue(e.getCause() instanceof IllegalArgumentException);
@@ -223,12 +203,8 @@ public void testPathSpecifiedInHdfs() {
223203

224204
public void testMissingPath() {
225205
try {
226-
client().admin()
227-
.cluster()
228-
.preparePutRepository("test-repo")
229-
.setType("hdfs")
230-
.setSettings(Settings.builder().put("uri", "hdfs:///").build())
231-
.get();
206+
Settings.Builder settings = Settings.builder().put("uri", "hdfs:///");
207+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "hdfs", settings);
232208
fail();
233209
} catch (RepositoryException e) {
234210
assertTrue(e.getCause() instanceof IllegalArgumentException);

‎plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3RepositoryThirdPartyTests.java

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333

3434
import software.amazon.awssdk.services.s3.model.StorageClass;
3535

36-
import org.opensearch.action.support.master.AcknowledgedResponse;
3736
import org.opensearch.common.SuppressForbidden;
3837
import org.opensearch.common.blobstore.BlobMetadata;
3938
import org.opensearch.common.blobstore.BlobPath;
@@ -43,6 +42,7 @@
4342
import org.opensearch.plugins.Plugin;
4443
import org.opensearch.repositories.AbstractThirdPartyRepositoryTestCase;
4544
import org.opensearch.repositories.blobstore.BlobStoreRepository;
45+
import org.opensearch.test.OpenSearchIntegTestCase;
4646
import org.junit.Before;
4747

4848
import java.util.Collection;
@@ -51,7 +51,6 @@
5151
import java.util.concurrent.TimeUnit;
5252

5353
import static org.hamcrest.Matchers.blankOrNullString;
54-
import static org.hamcrest.Matchers.equalTo;
5554
import static org.hamcrest.Matchers.not;
5655

5756
public class S3RepositoryThirdPartyTests extends AbstractThirdPartyRepositoryTestCase {
@@ -111,13 +110,7 @@ protected void createRepository(String repoName) {
111110
settings.put("storage_class", storageClass);
112111
}
113112
}
114-
AcknowledgedResponse putRepositoryResponse = client().admin()
115-
.cluster()
116-
.preparePutRepository("test-repo")
117-
.setType("s3")
118-
.setSettings(settings)
119-
.get();
120-
assertThat(putRepositoryResponse.isAcknowledged(), equalTo(true));
113+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), "test-repo", "s3", settings);
121114
}
122115

123116
@Override

‎plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@
5555
import org.opensearch.rest.RestRequest;
5656
import org.opensearch.rest.RestResponse;
5757
import org.opensearch.rest.action.admin.cluster.RestGetRepositoriesAction;
58+
import org.opensearch.test.OpenSearchIntegTestCase;
5859
import org.opensearch.test.OpenSearchSingleNodeTestCase;
5960
import org.opensearch.test.rest.FakeRestRequest;
6061

@@ -68,7 +69,6 @@
6869

6970
import static org.opensearch.repositories.s3.S3ClientSettings.ACCESS_KEY_SETTING;
7071
import static org.opensearch.repositories.s3.S3ClientSettings.SECRET_KEY_SETTING;
71-
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
7272
import static org.hamcrest.Matchers.containsString;
7373
import static org.hamcrest.Matchers.instanceOf;
7474
import static org.hamcrest.Matchers.is;
@@ -277,14 +277,8 @@ public void sendResponse(RestResponse response) {
277277
}
278278

279279
private void createRepository(final String name, final Settings repositorySettings) {
280-
assertAcked(
281-
client().admin()
282-
.cluster()
283-
.preparePutRepository(name)
284-
.setType(S3Repository.TYPE)
285-
.setVerify(false)
286-
.setSettings(repositorySettings)
287-
);
280+
Settings.Builder settings = Settings.builder().put(repositorySettings);
281+
OpenSearchIntegTestCase.putRepository(client().admin().cluster(), name, S3Repository.TYPE, false, settings);
288282
}
289283

290284
/**

0 commit comments

Comments
 (0)