Skip to content

Commit 925f41b

Browse files
ltaragiltaragi
authored
[SnapshotV2] Snapshot Status API changes (#15409)
--------- Signed-off-by: Lakshya Taragi <[email protected]>
1 parent 41ba00a commit 925f41b

File tree

15 files changed

+657
-51
lines changed

15 files changed

+657
-51
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4343
- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527))
4444
- Add support to upload snapshot shard blobs with hashed prefix ([#15426](https://github.com/opensearch-project/OpenSearch/pull/15426))
4545
- [Remote Publication] Add remote download stats ([#15291](https://github.com/opensearch-project/OpenSearch/pull/15291)))
46+
- Add support for comma-separated list of index names to be used with Snapshot Status API ([#15409](https://github.com/opensearch-project/OpenSearch/pull/15409))
4647

4748
### Dependencies
4849
- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

‎client/rest-high-level/src/test/java/org/opensearch/client/SnapshotRequestConvertersTests.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,14 +230,15 @@ public void testSnapshotsStatus() {
230230
Map<String, String> expectedParams = new HashMap<>();
231231
String repository = RequestConvertersTests.randomIndicesNames(1, 1)[0];
232232
String[] snapshots = RequestConvertersTests.randomIndicesNames(1, 5);
233+
String[] indices = RequestConvertersTests.randomIndicesNames(1, 5);
233234
StringBuilder snapshotNames = new StringBuilder(snapshots[0]);
234235
for (int idx = 1; idx < snapshots.length; idx++) {
235236
snapshotNames.append(",").append(snapshots[idx]);
236237
}
237238
boolean ignoreUnavailable = randomBoolean();
238239
String endpoint = "/_snapshot/" + repository + "/" + snapshotNames.toString() + "/_status";
239240

240-
SnapshotsStatusRequest snapshotsStatusRequest = new SnapshotsStatusRequest(repository, snapshots);
241+
SnapshotsStatusRequest snapshotsStatusRequest = new SnapshotsStatusRequest(repository, snapshots, indices);
241242
RequestConvertersTests.setRandomClusterManagerTimeout(snapshotsStatusRequest, expectedParams);
242243
snapshotsStatusRequest.ignoreUnavailable(ignoreUnavailable);
243244
expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable));

‎qa/repository-multi-version/src/test/java/org/opensearch/upgrades/MultiVersionRepositoryAccessIT.java

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232

3333
package org.opensearch.upgrades;
3434

35+
import com.sun.jna.StringArray;
3536
import org.opensearch.action.admin.cluster.repositories.put.PutRepositoryRequest;
3637
import org.opensearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest;
3738
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus;
@@ -44,6 +45,7 @@
4445
import org.opensearch.client.RestClient;
4546
import org.opensearch.client.RestHighLevelClient;
4647
import org.opensearch.common.settings.Settings;
48+
import org.opensearch.core.common.Strings;
4749
import org.opensearch.core.xcontent.DeprecationHandler;
4850
import org.opensearch.core.xcontent.XContentParser;
4951
import org.opensearch.common.xcontent.json.JsonXContent;
@@ -141,14 +143,14 @@ public void testCreateAndRestoreSnapshot() throws IOException {
141143
case STEP2_NEW_CLUSTER:
142144
case STEP4_NEW_CLUSTER:
143145
assertSnapshotStatusSuccessful(client, repoName,
144-
snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new));
146+
snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new), Strings.EMPTY_ARRAY);
145147
break;
146148
case STEP1_OLD_CLUSTER:
147-
assertSnapshotStatusSuccessful(client, repoName, "snapshot-" + TEST_STEP);
149+
assertSnapshotStatusSuccessful(client, repoName, new String[] {"snapshot-" + TEST_STEP}, Strings.EMPTY_ARRAY);
148150
break;
149151
case STEP3_OLD_CLUSTER:
150152
assertSnapshotStatusSuccessful(
151-
client, repoName, "snapshot-" + TEST_STEP, "snapshot-" + TestStep.STEP3_OLD_CLUSTER);
153+
client, repoName, new String[] {"snapshot-" + TEST_STEP, "snapshot-" + TestStep.STEP3_OLD_CLUSTER}, Strings.EMPTY_ARRAY);
152154
break;
153155
}
154156
if (TEST_STEP == TestStep.STEP3_OLD_CLUSTER) {
@@ -186,10 +188,10 @@ public void testReadOnlyRepo() throws IOException {
186188
break;
187189
}
188190
if (TEST_STEP == TestStep.STEP1_OLD_CLUSTER || TEST_STEP == TestStep.STEP3_OLD_CLUSTER) {
189-
assertSnapshotStatusSuccessful(client, repoName, "snapshot-" + TestStep.STEP1_OLD_CLUSTER);
191+
assertSnapshotStatusSuccessful(client, repoName, new String[] {"snapshot-" + TestStep.STEP1_OLD_CLUSTER}, Strings.EMPTY_ARRAY);
190192
} else {
191193
assertSnapshotStatusSuccessful(client, repoName,
192-
"snapshot-" + TestStep.STEP1_OLD_CLUSTER, "snapshot-" + TestStep.STEP2_NEW_CLUSTER);
194+
new String[] {"snapshot-" + TestStep.STEP1_OLD_CLUSTER, "snapshot-" + TestStep.STEP2_NEW_CLUSTER}, Strings.EMPTY_ARRAY);
193195
}
194196
if (TEST_STEP == TestStep.STEP3_OLD_CLUSTER) {
195197
ensureSnapshotRestoreWorks(repoName, "snapshot-" + TestStep.STEP1_OLD_CLUSTER, shards);
@@ -214,7 +216,7 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException {
214216
// Every step creates one snapshot
215217
assertThat(snapshots, hasSize(TEST_STEP.ordinal() + 1));
216218
assertSnapshotStatusSuccessful(client, repoName,
217-
snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new));
219+
snapshots.stream().map(sn -> (String) sn.get("snapshot")).toArray(String[]::new), Strings.EMPTY_ARRAY);
218220
if (TEST_STEP == TestStep.STEP1_OLD_CLUSTER) {
219221
ensureSnapshotRestoreWorks(repoName, "snapshot-" + TestStep.STEP1_OLD_CLUSTER, shards);
220222
} else {
@@ -239,9 +241,9 @@ public void testUpgradeMovesRepoToNewMetaVersion() throws IOException {
239241
}
240242

241243
private static void assertSnapshotStatusSuccessful(RestHighLevelClient client, String repoName,
242-
String... snapshots) throws IOException {
244+
String[] snapshots, String[] indices) throws IOException {
243245
final SnapshotsStatusResponse statusResponse = client.snapshot()
244-
.status(new SnapshotsStatusRequest(repoName, snapshots), RequestOptions.DEFAULT);
246+
.status(new SnapshotsStatusRequest(repoName, snapshots, indices), RequestOptions.DEFAULT);
245247
for (SnapshotStatus status : statusResponse.getSnapshots()) {
246248
assertThat(status.getShardsStats().getFailedShards(), is(0));
247249
}

‎rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,26 @@
4040
"description":"A comma-separated list of snapshot names"
4141
}
4242
}
43+
},
44+
{
45+
"path":"/_snapshot/{repository}/{snapshot}/{index}/_status",
46+
"methods":[
47+
"GET"
48+
],
49+
"parts":{
50+
"repository":{
51+
"type":"string",
52+
"description":"A repository name"
53+
},
54+
"snapshot":{
55+
"type":"string",
56+
"description":"A snapshot name"
57+
},
58+
"index":{
59+
"type": "list",
60+
"description":"A comma-separated list of index names"
61+
}
62+
}
4363
}
4464
]
4565
},
@@ -58,7 +78,7 @@
5878
},
5979
"ignore_unavailable":{
6080
"type":"boolean",
61-
"description":"Whether to ignore unavailable snapshots, defaults to false which means a SnapshotMissingException is thrown"
81+
"description":"Whether to ignore unavailable snapshots and indices, defaults to false which means a SnapshotMissingException or IndexNotFoundException is thrown"
6282
}
6383
}
6484
}

‎server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,20 +32,28 @@
3232

3333
package org.opensearch.snapshots;
3434

35+
import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
3536
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
3637
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStage;
3738
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus;
39+
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexStatus;
3840
import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus;
3941
import org.opensearch.cluster.SnapshotsInProgress;
4042
import org.opensearch.common.action.ActionFuture;
4143
import org.opensearch.common.settings.Settings;
44+
import org.opensearch.indices.RemoteStoreSettings;
45+
import org.opensearch.repositories.blobstore.BlobStoreRepository;
4246
import org.opensearch.test.OpenSearchIntegTestCase;
4347
import org.opensearch.threadpool.ThreadPool;
4448
import org.junit.Before;
4549

4650
import java.nio.file.Path;
51+
import java.util.Map;
52+
import java.util.concurrent.TimeUnit;
4753

4854
import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
55+
import static org.opensearch.snapshots.SnapshotsService.MAX_SHARDS_ALLOWED_IN_STATUS_API;
56+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
4957
import static org.hamcrest.Matchers.equalTo;
5058
import static org.hamcrest.Matchers.greaterThan;
5159
import static org.hamcrest.Matchers.is;
@@ -192,6 +200,110 @@ public void testStatusAPICallInProgressShallowSnapshot() throws Exception {
192200
createSnapshotResponseActionFuture.actionGet();
193201
}
194202

203+
public void testStatusAPICallForShallowV2Snapshot() throws Exception {
204+
disableRepoConsistencyCheck("Remote store repository is being used for the test");
205+
Settings pinnedTimestampSettings = Settings.builder()
206+
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
207+
.build();
208+
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings);
209+
internalCluster().startDataOnlyNodes(2, pinnedTimestampSettings);
210+
211+
final String index1 = "remote-index-1";
212+
final String index2 = "remote-index-2";
213+
final String index3 = "remote-index-3";
214+
final String snapshotRepoName = "snapshot-repo-name";
215+
final String snapshot = "snapshot";
216+
217+
logger.info("Create repository for shallow V2 snapshots");
218+
Settings.Builder snapshotV2RepoSettings = snapshotRepoSettingsForShallowCopy().put(
219+
BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(),
220+
Boolean.TRUE
221+
);
222+
createRepository(snapshotRepoName, "fs", snapshotV2RepoSettings);
223+
224+
final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings();
225+
createIndex(index1, remoteStoreEnabledIndexSettings);
226+
createIndex(index2, remoteStoreEnabledIndexSettings);
227+
createIndex(index3, remoteStoreEnabledIndexSettings);
228+
ensureGreen();
229+
230+
logger.info("Indexing some data");
231+
for (int i = 0; i < 50; i++) {
232+
index(index1, "_doc", Integer.toString(i), "foo", "bar" + i);
233+
index(index2, "_doc", Integer.toString(i), "foo", "bar" + i);
234+
index(index3, "_doc", Integer.toString(i), "foo", "bar" + i);
235+
}
236+
refresh();
237+
238+
SnapshotInfo snapshotInfo = createFullSnapshot(snapshotRepoName, snapshot);
239+
assertTrue(snapshotInfo.getPinnedTimestamp() > 0); // to assert creation of a shallow v2 snapshot
240+
241+
logger.info("Set MAX_SHARDS_ALLOWED_IN_STATUS_API to a low value");
242+
ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
243+
updateSettingsRequest.persistentSettings(Settings.builder().put(MAX_SHARDS_ALLOWED_IN_STATUS_API.getKey(), 2));
244+
assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
245+
246+
assertBusy(() -> {
247+
// without index filter
248+
// although no. of shards in snapshot (3) is greater than the max value allowed in a status api call, the request does not fail
249+
SnapshotStatus snapshotStatusWithoutIndexFilter = client().admin()
250+
.cluster()
251+
.prepareSnapshotStatus(snapshotRepoName)
252+
.setSnapshots(snapshot)
253+
.execute()
254+
.actionGet()
255+
.getSnapshots()
256+
.get(0);
257+
258+
assertShallowV2SnapshotStatus(snapshotStatusWithoutIndexFilter, false);
259+
260+
// with index filter
261+
SnapshotStatus snapshotStatusWithIndexFilter = client().admin()
262+
.cluster()
263+
.prepareSnapshotStatus(snapshotRepoName)
264+
.setSnapshots(snapshot)
265+
.setIndices(index1, index2)
266+
.execute()
267+
.actionGet()
268+
.getSnapshots()
269+
.get(0);
270+
271+
assertShallowV2SnapshotStatus(snapshotStatusWithIndexFilter, true);
272+
273+
}, 1, TimeUnit.MINUTES);
274+
275+
}
276+
277+
private void assertShallowV2SnapshotStatus(SnapshotStatus snapshotStatus, boolean hasIndexFilter) {
278+
if (hasIndexFilter) {
279+
assertEquals(0, snapshotStatus.getStats().getTotalSize());
280+
} else {
281+
// TODO: after adding primary store size at the snapshot level, total size here should be > 0
282+
}
283+
// assert that total and incremental values of file count and size_in_bytes are 0 at index and shard levels
284+
assertEquals(0, snapshotStatus.getStats().getTotalFileCount());
285+
assertEquals(0, snapshotStatus.getStats().getIncrementalSize());
286+
assertEquals(0, snapshotStatus.getStats().getIncrementalFileCount());
287+
288+
for (Map.Entry<String, SnapshotIndexStatus> entry : snapshotStatus.getIndices().entrySet()) {
289+
// index level
290+
SnapshotIndexStatus snapshotIndexStatus = entry.getValue();
291+
assertEquals(0, snapshotIndexStatus.getStats().getTotalSize());
292+
assertEquals(0, snapshotIndexStatus.getStats().getTotalFileCount());
293+
assertEquals(0, snapshotIndexStatus.getStats().getIncrementalSize());
294+
assertEquals(0, snapshotIndexStatus.getStats().getIncrementalFileCount());
295+
296+
for (SnapshotIndexShardStatus snapshotIndexShardStatus : snapshotStatus.getShards()) {
297+
// shard level
298+
assertEquals(0, snapshotIndexShardStatus.getStats().getTotalSize());
299+
assertEquals(0, snapshotIndexShardStatus.getStats().getTotalFileCount());
300+
assertEquals(0, snapshotIndexShardStatus.getStats().getIncrementalSize());
301+
assertEquals(0, snapshotIndexShardStatus.getStats().getIncrementalFileCount());
302+
assertEquals(SnapshotIndexShardStage.DONE, snapshotIndexShardStatus.getStage());
303+
}
304+
}
305+
}
306+
195307
private static SnapshotIndexShardStatus stateFirstShard(SnapshotStatus snapshotStatus, String indexName) {
196308
return snapshotStatus.getIndices().get(indexName).getShards().get(0);
197309
}

0 commit comments

Comments
 (0)