Skip to content

Commit 62596bd

Browse files
dbwiddisdbwiddis
authored
Merge branch '2.x' into backport-2.x-16296
Signed-off-by: Daniel Widdis <[email protected]>
2 parents ee29108 + 9950cba commit 62596bd

27 files changed

+581
-493
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5353
- Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024))
5454
- Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154))
5555
- Optimise clone operation for incremental full cluster snapshots ([#16296](https://github.com/opensearch-project/OpenSearch/pull/16296))
56+
- Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273))
5657

5758
### Deprecated
5859

‎server/src/internalClusterTest/java/org/opensearch/snapshots/ConcurrentSnapshotsV2IT.java

Lines changed: 146 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,15 @@
99
package org.opensearch.snapshots;
1010

1111
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
12+
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1213
import org.opensearch.action.support.PlainActionFuture;
1314
import org.opensearch.action.support.master.AcknowledgedResponse;
1415
import org.opensearch.client.Client;
1516
import org.opensearch.common.action.ActionFuture;
1617
import org.opensearch.common.settings.Settings;
1718
import org.opensearch.common.unit.TimeValue;
1819
import org.opensearch.core.common.unit.ByteSizeUnit;
20+
import org.opensearch.core.rest.RestStatus;
1921
import org.opensearch.remotestore.RemoteSnapshotIT;
2022
import org.opensearch.repositories.RepositoriesService;
2123
import org.opensearch.repositories.Repository;
@@ -479,7 +481,6 @@ public void testCloneSnapshotV2MasterSwitch() throws Exception {
479481
assertThat(snapInfo, containsInAnyOrder(csr.getSnapshotInfo(), csr2.getSnapshotInfo()));
480482
}
481483

482-
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16191")
483484
public void testDeleteWhileV2CreateOngoing() throws Exception {
484485
final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
485486
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
@@ -538,7 +539,7 @@ public void testDeleteWhileV2CreateOngoing() throws Exception {
538539
assertThat(snapInfo, contains(csr.getSnapshotInfo()));
539540
}
540541

541-
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/pull/16191")
542+
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/16205")
542543
public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception {
543544
final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
544545
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
@@ -569,6 +570,7 @@ public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception {
569570
indexDocuments(client, indexName1, numDocsInIndex1);
570571
indexDocuments(client, indexName2, numDocsInIndex2);
571572
ensureGreen(indexName1, indexName2);
573+
ensureGreen();
572574

573575
startFullSnapshot(repoName, "snapshot-v1").actionGet();
574576
startFullSnapshot(repoName, "snapshot-v1-2").actionGet();
@@ -597,6 +599,148 @@ public void testDeleteAndCloneV1WhileV2CreateOngoing() throws Exception {
597599
assertTrue(startCloneSnapshot.actionGet().isAcknowledged());
598600
List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
599601
assertEquals(3, snapInfo.size());
602+
603+
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
604+
.cluster()
605+
.prepareRestoreSnapshot(repoName, "snapshot-v1-2-clone")
606+
.setWaitForCompletion(true)
607+
.setIndices(indexName1)
608+
.setRenamePattern("(.+)")
609+
.setRenameReplacement("$1-copy")
610+
.get();
611+
assertEquals(restoreSnapshotResponse.status(), RestStatus.OK);
612+
ensureGreen();
613+
}
614+
615+
@AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/16205")
616+
public void testDeleteAndCloneV1WhileCreateOngoing() throws Exception {
617+
final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
618+
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
619+
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
620+
String indexName1 = "testindex1";
621+
String indexName2 = "testindex2";
622+
String repoName = "test-create-snapshot-repo";
623+
Path absolutePath1 = randomRepoPath().toAbsolutePath();
624+
logger.info("Snapshot Path [{}]", absolutePath1);
625+
626+
Settings.Builder settings = Settings.builder()
627+
.put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
628+
.put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
629+
.put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
630+
.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
631+
.put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false);
632+
createRepository(repoName, "mock", settings);
633+
634+
Client client = client();
635+
Settings indexSettings = getIndexSettings(10, 0).build();
636+
createIndex(indexName1, indexSettings);
637+
638+
Settings indexSettings2 = getIndexSettings(15, 0).build();
639+
createIndex(indexName2, indexSettings2);
640+
641+
final int numDocsInIndex1 = 10;
642+
final int numDocsInIndex2 = 20;
643+
indexDocuments(client, indexName1, numDocsInIndex1);
644+
indexDocuments(client, indexName2, numDocsInIndex2);
645+
ensureGreen(indexName1, indexName2);
646+
647+
startFullSnapshot(repoName, "snapshot-v1").actionGet();
648+
startFullSnapshot(repoName, "snapshot-v1-2").actionGet();
649+
650+
blockClusterManagerOnWriteIndexFile(repoName);
651+
652+
final ActionFuture<CreateSnapshotResponse> snapshotFuture = startFullSnapshot(repoName, "snapshot-v2");
653+
awaitNumberOfSnapshotsInProgress(1);
654+
655+
ActionFuture<AcknowledgedResponse> startDeleteSnapshot = startDeleteSnapshot(repoName, "snapshot-v1");
656+
ActionFuture<AcknowledgedResponse> startCloneSnapshot = startCloneSnapshot(repoName, "snapshot-v1-2", "snapshot-v1-2-clone");
657+
658+
unblockNode(repoName, clusterManagerName);
659+
snapshotFuture.actionGet();
660+
assertTrue(startDeleteSnapshot.actionGet().isAcknowledged());
661+
assertTrue(startCloneSnapshot.actionGet().isAcknowledged());
662+
List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
663+
assertEquals(3, snapInfo.size());
664+
665+
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
666+
.cluster()
667+
.prepareRestoreSnapshot(repoName, "snapshot-v1-2-clone")
668+
.setWaitForCompletion(true)
669+
.setIndices(indexName1)
670+
.setRenamePattern("(.+)")
671+
.setRenameReplacement("$1-copy")
672+
.get();
673+
assertEquals(restoreSnapshotResponse.status(), RestStatus.OK);
674+
ensureGreen();
675+
}
676+
677+
public void testCloneV1WhileV2CreateOngoing() throws Exception {
678+
final String clusterManagerName = internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
679+
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
680+
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
681+
String indexName1 = "testindex1";
682+
String indexName2 = "testindex2";
683+
String repoName = "test-create-snapshot-repo";
684+
Path absolutePath1 = randomRepoPath().toAbsolutePath();
685+
logger.info("Snapshot Path [{}]", absolutePath1);
686+
687+
Settings.Builder settings = Settings.builder()
688+
.put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
689+
.put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
690+
.put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
691+
.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
692+
.put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), false);
693+
createRepository(repoName, "mock", settings);
694+
695+
Client client = client();
696+
Settings indexSettings = getIndexSettings(20, 0).build();
697+
createIndex(indexName1, indexSettings);
698+
699+
Settings indexSettings2 = getIndexSettings(15, 0).build();
700+
createIndex(indexName2, indexSettings2);
701+
702+
final int numDocsInIndex1 = 10;
703+
final int numDocsInIndex2 = 20;
704+
indexDocuments(client, indexName1, numDocsInIndex1);
705+
indexDocuments(client, indexName2, numDocsInIndex2);
706+
ensureGreen(indexName1, indexName2);
707+
708+
startFullSnapshot(repoName, "snapshot-v1").actionGet();
709+
710+
// Creating a v2 repo
711+
settings = Settings.builder()
712+
.put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
713+
.put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
714+
.put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
715+
.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
716+
.put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
717+
createRepository(repoName, "mock", settings);
718+
719+
blockClusterManagerOnWriteIndexFile(repoName);
720+
721+
final ActionFuture<CreateSnapshotResponse> snapshotFuture = startFullSnapshot(repoName, "snapshot-v2");
722+
awaitNumberOfSnapshotsInProgress(1);
723+
724+
ActionFuture<AcknowledgedResponse> startCloneSnapshot = startCloneSnapshot(repoName, "snapshot-v1", "snapshot-v1-2-clone");
725+
726+
unblockNode(repoName, clusterManagerName);
727+
CreateSnapshotResponse csr = snapshotFuture.actionGet();
728+
assertTrue(csr.getSnapshotInfo().getPinnedTimestamp() != 0);
729+
assertTrue(startCloneSnapshot.actionGet().isAcknowledged());
730+
List<SnapshotInfo> snapInfo = client().admin().cluster().prepareGetSnapshots(repoName).get().getSnapshots();
731+
assertEquals(3, snapInfo.size());
732+
733+
RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
734+
.cluster()
735+
.prepareRestoreSnapshot(repoName, "snapshot-v1-2-clone")
736+
.setWaitForCompletion(true)
737+
.setIndices(indexName1)
738+
.setRenamePattern(indexName1)
739+
.setRenamePattern("(.+)")
740+
.setRenameReplacement("$1-copy")
741+
.get();
742+
assertEquals(restoreSnapshotResponse.status(), RestStatus.OK);
743+
ensureGreen();
600744
}
601745

602746
protected ActionFuture<AcknowledgedResponse> startCloneSnapshot(String repoName, String sourceSnapshotName, String snapshotName) {

‎server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
8989
return allocation.decision(
9090
Decision.YES,
9191
NAME,
92-
getDecisionDetails(true, shardRouting, targetNode, " for strict compatibility mode")
92+
getDecisionDetails(true, shardRouting, targetNode, " for strict compatibility mode", allocation)
9393
);
9494
}
9595

@@ -103,19 +103,23 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
103103
return allocation.decision(
104104
isNoDecision ? Decision.NO : Decision.YES,
105105
NAME,
106-
getDecisionDetails(!isNoDecision, shardRouting, targetNode, reason)
106+
getDecisionDetails(!isNoDecision, shardRouting, targetNode, reason, allocation)
107107
);
108108
} else if (migrationDirection.equals(Direction.DOCREP)) {
109109
// docrep migration direction is currently not supported
110-
return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, shardRouting, targetNode, " for DOCREP direction"));
110+
return allocation.decision(
111+
Decision.YES,
112+
NAME,
113+
getDecisionDetails(true, shardRouting, targetNode, " for DOCREP direction", allocation)
114+
);
111115
} else {
112116
// check for remote store backed indices
113117
if (remoteSettingsBackedIndex && targetNode.isRemoteStoreNode() == false) {
114118
// allocations and relocations must be to a remote node
115119
String reason = new StringBuilder(" because a remote store backed index's shard copy can only be ").append(
116120
(shardRouting.assignedToNode() == false) ? "allocated" : "relocated"
117121
).append(" to a remote node").toString();
118-
return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason));
122+
return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason, allocation));
119123
}
120124

121125
if (shardRouting.primary()) {
@@ -128,9 +132,9 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
128132
// handle scenarios for allocation of a new shard's primary copy
129133
private Decision primaryShardDecision(ShardRouting primaryShardRouting, DiscoveryNode targetNode, RoutingAllocation allocation) {
130134
if (targetNode.isRemoteStoreNode() == false) {
131-
return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, primaryShardRouting, targetNode, ""));
135+
return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, primaryShardRouting, targetNode, "", allocation));
132136
}
133-
return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, primaryShardRouting, targetNode, ""));
137+
return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, primaryShardRouting, targetNode, "", allocation));
134138
}
135139

136140
// Checks if primary shard is on a remote node.
@@ -150,20 +154,41 @@ private Decision replicaShardDecision(ShardRouting replicaShardRouting, Discover
150154
return allocation.decision(
151155
Decision.NO,
152156
NAME,
153-
getDecisionDetails(false, replicaShardRouting, targetNode, " since primary shard copy is not yet migrated to remote")
157+
getDecisionDetails(
158+
false,
159+
replicaShardRouting,
160+
targetNode,
161+
" since primary shard copy is not yet migrated to remote",
162+
allocation
163+
)
154164
);
155165
}
156166
return allocation.decision(
157167
Decision.YES,
158168
NAME,
159-
getDecisionDetails(true, replicaShardRouting, targetNode, " since primary shard copy has been migrated to remote")
169+
getDecisionDetails(
170+
true,
171+
replicaShardRouting,
172+
targetNode,
173+
" since primary shard copy has been migrated to remote",
174+
allocation
175+
)
160176
);
161177
}
162-
return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, replicaShardRouting, targetNode, ""));
178+
return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, replicaShardRouting, targetNode, "", allocation));
163179
}
164180

165181
// get detailed reason for the decision
166-
private String getDecisionDetails(boolean isYes, ShardRouting shardRouting, DiscoveryNode targetNode, String reason) {
182+
private String getDecisionDetails(
183+
boolean isYes,
184+
ShardRouting shardRouting,
185+
DiscoveryNode targetNode,
186+
String reason,
187+
RoutingAllocation allocation
188+
) {
189+
if (allocation.debugDecision() == false) {
190+
return null;
191+
}
167192
return new StringBuilder("[").append(migrationDirection.direction)
168193
.append(" migration_direction]: ")
169194
.append(shardRouting.primary() ? "primary" : "replica")

0 commit comments

Comments
 (0)