Skip to content

Commit 770822d

Browse files
author
sukriti sinha
committed
Created an API to fetch remote store segment data
Signed-off-by: sukriti sinha <[email protected]>
1 parent 3638c13 commit 770822d

File tree

12 files changed

+681
-0
lines changed

12 files changed

+681
-0
lines changed

‎gradle/run.gradle

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,17 @@ testClusters {
3939
testDistribution = 'archive'
4040
if (numZones > 1) numberOfZones = numZones
4141
if (numNodes > 1) numberOfNodes = numNodes
42+
setting 'node.attr.remote_store.segment.repository', 'my-repository'
43+
setting 'node.attr.remote_store.translog.repository', 'my-repository'
44+
setting 'node.attr.remote_store.state.repository', 'my-repository'
45+
setting 'node.attr.remote_store.repository.my-repository.type', 'fs'
46+
setting 'path.repo', '/Users/sukriiti/remote-store-data'
47+
setting 'node.attr.remote_store.repository.my-repository.settings.location', '/Users/sukriiti/remote-store-data'
48+
// setting 'opensearch.experimental.feature.tiered_remote_index.enabled', 'true'
49+
// setting 'opensearch.experimental.feature.pluggable.caching.enabled', 'true'
50+
setting 'node.search.cache.size', '10gb'
51+
setting 'node.roles', 'ingest,remote_cluster_client,data,cluster_manager'
52+
4253
if (findProperty("installedPlugins")) {
4354
installedPlugins = Eval.me(installedPlugins)
4455
for (String p : installedPlugins) {

‎server/src/main/java/org/opensearch/action/ActionModule.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,8 @@
6767
import org.opensearch.action.admin.cluster.node.usage.TransportNodesUsageAction;
6868
import org.opensearch.action.admin.cluster.remote.RemoteInfoAction;
6969
import org.opensearch.action.admin.cluster.remote.TransportRemoteInfoAction;
70+
import org.opensearch.action.admin.cluster.remotestore.metadata.RemoteStoreMetadataAction;
71+
import org.opensearch.action.admin.cluster.remotestore.metadata.TransportRemoteStoreMetadataAction;
7072
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreAction;
7173
import org.opensearch.action.admin.cluster.remotestore.restore.TransportRestoreRemoteStoreAction;
7274
import org.opensearch.action.admin.cluster.remotestore.stats.RemoteStoreStatsAction;
@@ -381,6 +383,7 @@
381383
import org.opensearch.rest.action.admin.cluster.RestPutStoredScriptAction;
382384
import org.opensearch.rest.action.admin.cluster.RestReloadSecureSettingsAction;
383385
import org.opensearch.rest.action.admin.cluster.RestRemoteClusterInfoAction;
386+
import org.opensearch.rest.action.admin.cluster.RestRemoteStoreMetadataAction;
384387
import org.opensearch.rest.action.admin.cluster.RestRemoteStoreStatsAction;
385388
import org.opensearch.rest.action.admin.cluster.RestRestoreRemoteStoreAction;
386389
import org.opensearch.rest.action.admin.cluster.RestRestoreSnapshotAction;
@@ -641,6 +644,7 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
641644
actions.register(NodesStatsAction.INSTANCE, TransportNodesStatsAction.class);
642645
actions.register(WlmStatsAction.INSTANCE, TransportWlmStatsAction.class);
643646
actions.register(RemoteStoreStatsAction.INSTANCE, TransportRemoteStoreStatsAction.class);
647+
actions.register(RemoteStoreMetadataAction.INSTANCE, TransportRemoteStoreMetadataAction.class);
644648
actions.register(NodesUsageAction.INSTANCE, TransportNodesUsageAction.class);
645649
actions.register(NodesHotThreadsAction.INSTANCE, TransportNodesHotThreadsAction.class);
646650
actions.register(ListTasksAction.INSTANCE, TransportListTasksAction.class);
@@ -1062,6 +1066,7 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
10621066
registerHandler.accept(new RestGetDecommissionStateAction());
10631067
registerHandler.accept(new RestRemoteStoreStatsAction());
10641068
registerHandler.accept(new RestRestoreRemoteStoreAction());
1069+
registerHandler.accept(new RestRemoteStoreMetadataAction());
10651070

10661071
// pull-based ingestion API
10671072
registerHandler.accept(new RestPauseIngestionAction());

‎server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/RemoteStoreMetadata.java

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package org.opensearch.action.admin.cluster.remotestore.metadata;
2+
3+
import org.opensearch.cluster.routing.ShardRouting;
4+
import org.opensearch.common.annotation.PublicApi;
5+
import org.opensearch.core.common.io.stream.StreamInput;
6+
import org.opensearch.core.common.io.stream.StreamOutput;
7+
import org.opensearch.core.common.io.stream.Writeable;
8+
import org.opensearch.core.xcontent.ToXContentFragment;
9+
import org.opensearch.core.xcontent.XContentBuilder;
10+
11+
import java.io.IOException;
12+
import java.util.Map;
13+
/**
14+
* Response model that holds the remote store metadata (segment and translog) for a shard.
15+
*
16+
* @opensearch.internal
17+
*/
18+
@PublicApi(since = "3.0.0")
19+
public class RemoteStoreMetadata implements Writeable, ToXContentFragment {
20+
private final Map<String, Object> segments;
21+
private final Map<String, Object> translog;
22+
private final ShardRouting shardRouting;
23+
24+
public RemoteStoreMetadata(
25+
Map<String, Object> segments,
26+
Map<String, Object> translog,
27+
ShardRouting shardRouting
28+
) {
29+
this.segments = segments;
30+
this.translog = translog;
31+
this.shardRouting = shardRouting;
32+
}
33+
34+
public RemoteStoreMetadata(StreamInput in) throws IOException {
35+
this.segments = in.readMap();
36+
this.translog = in.readMap();
37+
this.shardRouting = new ShardRouting(in);
38+
}
39+
40+
@Override
41+
public void writeTo(StreamOutput out) throws IOException {
42+
out.writeMap(segments);
43+
out.writeMap(translog);
44+
shardRouting.writeTo(out);
45+
}
46+
47+
@Override
48+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
49+
builder.startObject();
50+
51+
builder.startObject("segments");
52+
for (Map.Entry<String, Object> entry : segments.entrySet()) {
53+
builder.field(entry.getKey(), entry.getValue());
54+
}
55+
builder.endObject();
56+
57+
builder.startObject("translog");
58+
for (Map.Entry<String, Object> entry : translog.entrySet()) {
59+
builder.field(entry.getKey(), entry.getValue());
60+
}
61+
builder.endObject();
62+
63+
return builder.endObject();
64+
}
65+
66+
public ShardRouting getShardRouting() {
67+
return shardRouting;
68+
}
69+
}

‎server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/RemoteStoreMetadataAction.java

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.action.admin.cluster.remotestore.metadata;
10+
11+
import org.opensearch.action.ActionType;
12+
13+
/**
14+
* Action to fetch metadata from remote store
15+
*
16+
* @opensearch.internal
17+
*/
18+
public class RemoteStoreMetadataAction extends ActionType<RemoteStoreMetadataResponse> {
19+
public static final RemoteStoreMetadataAction INSTANCE = new RemoteStoreMetadataAction();
20+
public static final String NAME = "cluster:admin/remote_store/metadata";
21+
22+
private RemoteStoreMetadataAction() {
23+
super(NAME, RemoteStoreMetadataResponse::new);
24+
}
25+
}

‎server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/RemoteStoreMetadataRequest.java

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package org.opensearch.action.admin.cluster.remotestore.metadata;
2+
3+
import org.opensearch.action.support.broadcast.BroadcastRequest;
4+
import org.opensearch.common.annotation.PublicApi;
5+
import org.opensearch.core.common.io.stream.StreamInput;
6+
import org.opensearch.core.common.io.stream.StreamOutput;
7+
8+
import java.io.IOException;
9+
/**
10+
* Request object for fetching remote store metadata of shards for a given index.
11+
*
12+
* @opensearch.internal
13+
*/
14+
@PublicApi(since = "3.0.0")
15+
public class RemoteStoreMetadataRequest extends BroadcastRequest<RemoteStoreMetadataRequest> {
16+
private String[] shards;
17+
private boolean local = false;
18+
19+
public RemoteStoreMetadataRequest() {
20+
super((String[]) null);
21+
shards = new String[0];
22+
}
23+
24+
public RemoteStoreMetadataRequest(StreamInput in) throws IOException {
25+
super(in);
26+
shards = in.readStringArray();
27+
local = in.readBoolean();
28+
}
29+
30+
@Override
31+
public void writeTo(StreamOutput out) throws IOException {
32+
super.writeTo(out);
33+
out.writeStringArray(shards);
34+
out.writeBoolean(local);
35+
}
36+
37+
public RemoteStoreMetadataRequest shards(String... shards) {
38+
this.shards = shards;
39+
return this;
40+
}
41+
42+
public String[] shards() {
43+
return this.shards;
44+
}
45+
46+
public void local(boolean local) {
47+
this.local = local;
48+
}
49+
50+
public boolean local() {
51+
return local;
52+
}
53+
}

‎server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/RemoteStoreMetadataRequestBuilder.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.action.admin.cluster.remotestore.metadata;
10+
11+
import org.opensearch.action.support.broadcast.BroadcastOperationRequestBuilder;
12+
import org.opensearch.common.annotation.PublicApi;
13+
import org.opensearch.common.unit.TimeValue;
14+
import org.opensearch.transport.client.OpenSearchClient;
15+
16+
/**
17+
* Builder for RemoteStoreMetadataRequest
18+
*
19+
* @opensearch.api
20+
*/
21+
@PublicApi(since = "3.0.0")
22+
public class RemoteStoreMetadataRequestBuilder extends BroadcastOperationRequestBuilder<
23+
RemoteStoreMetadataRequest,
24+
RemoteStoreMetadataResponse,
25+
RemoteStoreMetadataRequestBuilder> {
26+
27+
public RemoteStoreMetadataRequestBuilder(OpenSearchClient client, RemoteStoreMetadataAction action) {
28+
super(client, action, new RemoteStoreMetadataRequest());
29+
}
30+
31+
/**
32+
* Sets timeout of request.
33+
*/
34+
public final RemoteStoreMetadataRequestBuilder setTimeout(TimeValue timeout) {
35+
request.timeout(timeout);
36+
return this;
37+
}
38+
39+
/**
40+
* Sets shards preference of request.
41+
*/
42+
public final RemoteStoreMetadataRequestBuilder setShards(String... shards) {
43+
request.shards(shards);
44+
return this;
45+
}
46+
47+
/**
48+
* Sets local shards preference of request.
49+
*/
50+
public final RemoteStoreMetadataRequestBuilder setLocal(boolean local) {
51+
request.local(local);
52+
return this;
53+
}
54+
}

‎server/src/main/java/org/opensearch/action/admin/cluster/remotestore/metadata/RemoteStoreMetadataResponse.java

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.action.admin.cluster.remotestore.metadata;
10+
11+
import org.opensearch.action.support.broadcast.BroadcastResponse;
12+
import org.opensearch.common.annotation.PublicApi;
13+
import org.opensearch.core.action.support.DefaultShardOperationFailedException;
14+
import org.opensearch.core.common.Strings;
15+
import org.opensearch.core.common.io.stream.StreamInput;
16+
import org.opensearch.core.common.io.stream.StreamOutput;
17+
import org.opensearch.core.xcontent.MediaTypeRegistry;
18+
import org.opensearch.core.xcontent.XContentBuilder;
19+
20+
import java.io.IOException;
21+
import java.util.ArrayList;
22+
import java.util.HashMap;
23+
import java.util.List;
24+
import java.util.Map;
25+
26+
/**
27+
* Response containing remote store metadata
28+
*
29+
* @opensearch.api
30+
*/
31+
@PublicApi(since = "3.0.0")
32+
public class RemoteStoreMetadataResponse extends BroadcastResponse {
33+
private final RemoteStoreMetadata[] remoteStoreMetadata;
34+
35+
public RemoteStoreMetadataResponse(StreamInput in) throws IOException {
36+
super(in);
37+
remoteStoreMetadata = in.readArray(RemoteStoreMetadata::new, RemoteStoreMetadata[]::new);
38+
}
39+
40+
public RemoteStoreMetadataResponse(
41+
RemoteStoreMetadata[] remoteStoreMetadata,
42+
int totalShards,
43+
int successfulShards,
44+
int failedShards,
45+
List<DefaultShardOperationFailedException> shardFailures
46+
) {
47+
super(totalShards, successfulShards, failedShards, shardFailures);
48+
this.remoteStoreMetadata = remoteStoreMetadata;
49+
}
50+
51+
/**
52+
* Groups metadata by index and shard IDs.
53+
*
54+
* @return Map of index name to shard ID to list of metadata
55+
*/
56+
public Map<String, Map<Integer, List<RemoteStoreMetadata>>> groupByIndexAndShards() {
57+
Map<String, Map<Integer, List<RemoteStoreMetadata>>> indexWiseMetadata = new HashMap<>();
58+
for (RemoteStoreMetadata metadata : remoteStoreMetadata) {
59+
indexWiseMetadata.computeIfAbsent(metadata.getShardRouting().getIndexName(), k -> new HashMap<>())
60+
.computeIfAbsent(metadata.getShardRouting().getId(), k -> new ArrayList<>())
61+
.add(metadata);
62+
}
63+
return indexWiseMetadata;
64+
}
65+
66+
@Override
67+
public void writeTo(StreamOutput out) throws IOException {
68+
super.writeTo(out);
69+
out.writeArray(remoteStoreMetadata);
70+
}
71+
72+
@Override
73+
protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException {
74+
Map<String, Map<Integer, List<RemoteStoreMetadata>>> indexWiseMetadata = groupByIndexAndShards();
75+
builder.startObject(Fields.INDICES);
76+
for (String indexName : indexWiseMetadata.keySet()) {
77+
builder.startObject(indexName);
78+
builder.startObject(Fields.SHARDS);
79+
for (int shardId : indexWiseMetadata.get(indexName).keySet()) {
80+
builder.startArray(Integer.toString(shardId));
81+
for (RemoteStoreMetadata metadata : indexWiseMetadata.get(indexName).get(shardId)) {
82+
metadata.toXContent(builder, params);
83+
}
84+
builder.endArray();
85+
}
86+
builder.endObject();
87+
builder.endObject();
88+
}
89+
builder.endObject();
90+
}
91+
92+
@Override
93+
public String toString() {
94+
return Strings.toString(MediaTypeRegistry.JSON, this, true, false);
95+
}
96+
97+
static final class Fields {
98+
static final String SHARDS = "shards";
99+
static final String INDICES = "indices";
100+
}
101+
}

0 commit comments

Comments
 (0)