Projected Views

CHANGELOG.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887))
 - [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028))
+- Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
 - Add Remote Store Migration Experimental flag and allow mixed mode clusters under same ([#11986](https://github.com/opensearch-project/OpenSearch/pull/11986))
 
 ### Dependencies

server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewIT.java

@@ -0,0 +1,154 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.admin.indices.view;
+
+import org.opensearch.cluster.metadata.View;
+import org.opensearch.index.IndexNotFoundException;
+import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
+import org.opensearch.test.OpenSearchIntegTestCase.Scope;
+import org.hamcrest.MatcherAssert;
+
+import java.util.List;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+
+@ClusterScope(scope = Scope.TEST, numDataNodes = 2)
+public class ViewIT extends ViewTestBase {
+
+    public void testCreateView() throws Exception {
+        final String viewName = randomAlphaOfLength(8);
+        final String indexPattern = randomAlphaOfLength(8);
+
+        logger.info("Testing createView with valid parameters");
+        final View view = createView(viewName, indexPattern).getView();
+        MatcherAssert.assertThat(view.getName(), is(viewName));
+        MatcherAssert.assertThat(view.getTargets().size(), is(1));
+        MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern));
+
+        logger.info("Testing createView with existing view name");
+        final Exception ex = assertThrows(ViewAlreadyExistsException.class, () -> createView(viewName, randomAlphaOfLength(8)));
+        MatcherAssert.assertThat(ex.getMessage(), is("View [" + viewName + "] already exists"));
+    }
+
+    public void testCreateViewTargetsSet() throws Exception {
+        final String viewName = randomAlphaOfLength(8);
+        final String indexPattern = "a" + randomAlphaOfLength(8);
+        final String indexPattern2 = "b" + randomAlphaOfLength(8);
+        final List<String> targetPatterns = List.of(indexPattern2, indexPattern, indexPattern);
+
+        logger.info("Testing createView with targets that will be reordered and deduplicated");
+        final View view = createView(viewName, targetPatterns).getView();
+        MatcherAssert.assertThat(view.getName(), is(viewName));
+        MatcherAssert.assertThat(view.getTargets().size(), is(2));
+        MatcherAssert.assertThat(view.getTargets().first().getIndexPattern(), is(indexPattern));
+        MatcherAssert.assertThat(view.getTargets().last().getIndexPattern(), is(indexPattern2));
+    }
+
+    public void testGetView() throws Exception {
+        final String viewName = randomAlphaOfLength(8);
+        createView(viewName, randomAlphaOfLength(8));
+
+        final View view = getView(viewName).getView();
+        MatcherAssert.assertThat(view.getName(), is(viewName));
+
+        logger.info("Testing getView with non-existent view");
+        final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
+        final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> getView(nonExistentView));
+        MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
+    }
+
+    public void testDeleteView() throws Exception {
+        final String viewName = randomAlphaOfLength(8);
+        createView(viewName, randomAlphaOfLength(8));
+
+        logger.info("Testing deleteView with existing view");
+        deleteView(viewName);
+        final Exception whenDeletedEx = assertThrows(ViewNotFoundException.class, () -> getView(viewName));
+        MatcherAssert.assertThat(whenDeletedEx.getMessage(), is("View [" + viewName + "] does not exist"));
+
+        logger.info("Testing deleteView with non-existent view");
+        final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
+        final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> deleteView(nonExistentView));
+        MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
+    }
+
+    public void testUpdateView() throws Exception {
+        final String viewName = randomAlphaOfLength(8);
+        final String originalIndexPattern = randomAlphaOfLength(8);
+        final View originalView = createView(viewName, originalIndexPattern).getView();
+
+        logger.info("Testing updateView with existing view");
+        final String newDescription = randomAlphaOfLength(20);
+        final String newIndexPattern = "newPattern-" + originalIndexPattern;
+        final View updatedView = updateView(viewName, newDescription, newIndexPattern).getView();
+
+        MatcherAssert.assertThat(updatedView, not(is(originalView)));
+        MatcherAssert.assertThat(updatedView.getDescription(), is(newDescription));
+        MatcherAssert.assertThat(updatedView.getTargets(), hasSize(1));
+        MatcherAssert.assertThat(updatedView.getTargets().first().getIndexPattern(), is(newIndexPattern));
+
+        logger.info("Testing updateView with non-existent view");
+        final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
+        final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> updateView(nonExistentView, null, "index-*"));
+        MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
+    }
+
+    public void testListViewNames() throws Exception {
+        logger.info("Testing listViewNames when no views have been created");
+        MatcherAssert.assertThat(listViewNames(), is(List.of()));
+
+        final String view1 = "view1";
+        final String view2 = "view2";
+        createView(view1, "index-1-*");
+        createView(view2, "index-2-*");
+
+        logger.info("Testing listViewNames");
+        final List<String> views = listViewNames();
+        MatcherAssert.assertThat(views, containsInAnyOrder(view1, view2));
+
+        logger.info("Testing listViewNames after deleting a view");
+        deleteView(view1);
+        final List<String> viewsAfterDeletion = listViewNames();
+        MatcherAssert.assertThat(viewsAfterDeletion, not(contains(view1)));
+        MatcherAssert.assertThat(viewsAfterDeletion, contains(view2));
+    }
+
+    public void testSearchOperations() throws Exception {
+        final String indexInView1 = "index-1";
+        final String indexInView2 = "index-2";
+        final String indexNotInView = "another-index-1";
+
+        final int indexInView1DocCount = createIndexWithDocs(indexInView1);
+        final int indexInView2DocCount = createIndexWithDocs(indexInView2);
+        createIndexWithDocs(indexNotInView);
+
+        logger.info("Testing view with no matches");
+        createView("no-matches", "this-pattern-will-match-nothing");
+        final Exception ex = assertThrows(IndexNotFoundException.class, () -> searchView("no-matches"));
+        MatcherAssert.assertThat(ex.getMessage(), is("no such index [this-pattern-will-match-nothing]"));
+
+        logger.info("Testing view with exact index match");
+        createView("only-index-1", "index-1");
+        assertHitCount(searchView("only-index-1"), indexInView1DocCount);
+
+        logger.info("Testing view with wildcard matches");
+        createView("both-indices", "index-*");
+        assertHitCount(searchView("both-indices"), indexInView1DocCount + indexInView2DocCount);
+
+        logger.info("Testing searchView with non-existent view");
+        final String nonExistentView = "non-existent-" + randomAlphaOfLength(8);
+        final Exception whenNeverExistedEx = assertThrows(ViewNotFoundException.class, () -> searchView(nonExistentView));
+        MatcherAssert.assertThat(whenNeverExistedEx.getMessage(), is("View [" + nonExistentView + "] does not exist"));
+    }
+}

server/src/internalClusterTest/java/org/opensearch/action/admin/indices/view/ViewTestBase.java

@@ -0,0 +1,79 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.admin.indices.view;
+
+import org.opensearch.action.search.SearchRequest;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.test.BackgroundIndexer;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
+
+public abstract class ViewTestBase extends OpenSearchIntegTestCase {
+
+    protected int createIndexWithDocs(final String indexName) throws Exception {
+        createIndex(indexName);
+        ensureGreen(indexName);
+
+        final int numOfDocs = scaledRandomIntBetween(0, 200);
+        try (final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) {
+            waitForDocs(numOfDocs, indexer);
+        }
+
+        refresh(indexName);
+        assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs);
+        return numOfDocs;
+    }
+
+    protected GetViewAction.Response createView(final String name, final String indexPattern) throws Exception {
+        return createView(name, List.of(indexPattern));
+    }
+
+    protected GetViewAction.Response createView(final String name, final List<String> targets) throws Exception {
+        final CreateViewAction.Request request = new CreateViewAction.Request(
+            name,
+            null,
+            targets.stream().map(CreateViewAction.Request.Target::new).collect(Collectors.toList())
+        );
+        return client().admin().indices().createView(request).actionGet();
+    }
+
+    protected GetViewAction.Response getView(final String name) {
+        return client().admin().indices().getView(new GetViewAction.Request(name)).actionGet();
+
+    }
+
+    protected void deleteView(final String name) {
+        client().admin().indices().deleteView(new DeleteViewAction.Request(name)).actionGet();
+        performRemoteStoreTestAction();
+    }
+
+    protected List<String> listViewNames() {
+        return client().listViewNames(new ListViewNamesAction.Request()).actionGet().getViewNames();
+    }
+
+    protected SearchResponse searchView(final String viewName) throws Exception {
+        final SearchViewAction.Request request = new SearchViewAction.Request(viewName, new SearchRequest());
+        final SearchResponse response = client().searchView(request).actionGet();
+        return response;
+    }
+
+    protected GetViewAction.Response updateView(final String name, final String description, final String indexPattern) {
+        final CreateViewAction.Request request = new CreateViewAction.Request(
+            name,
+            description,
+            List.of(new CreateViewAction.Request.Target(indexPattern))
+        );
+        final GetViewAction.Response response = client().admin().indices().updateView(request).actionGet();
+        return response;
+    }
+}

server/src/main/java/org/opensearch/OpenSearchServerException.java

@@ -8,13 +8,11 @@
 
 package org.opensearch;
 
-import org.opensearch.core.index.snapshots.IndexShardSnapshotException;
-import org.opensearch.crypto.CryptoRegistryException;
-
 import static org.opensearch.OpenSearchException.OpenSearchExceptionHandle;
 import static org.opensearch.OpenSearchException.OpenSearchExceptionHandleRegistry.registerExceptionHandle;
 import static org.opensearch.OpenSearchException.UNKNOWN_VERSION_ADDED;
 import static org.opensearch.Version.V_2_10_0;
+import static org.opensearch.Version.V_2_13_0;
 import static org.opensearch.Version.V_2_1_0;
 import static org.opensearch.Version.V_2_4_0;
 import static org.opensearch.Version.V_2_5_0;
@@ -678,7 +676,12 @@ public static void registerExceptions() {
             )
         );
         registerExceptionHandle(
-            new OpenSearchExceptionHandle(IndexShardSnapshotException.class, IndexShardSnapshotException::new, 98, UNKNOWN_VERSION_ADDED)
+            new OpenSearchExceptionHandle(
+                org.opensearch.core.index.snapshots.IndexShardSnapshotException.class,
+                org.opensearch.core.index.snapshots.IndexShardSnapshotException::new,
+                98,
+                UNKNOWN_VERSION_ADDED
+            )
         );
         registerExceptionHandle(
             new OpenSearchExceptionHandle(
@@ -1174,7 +1177,30 @@ public static void registerExceptions() {
                 V_2_7_0
             )
         );
-        registerExceptionHandle(new OpenSearchExceptionHandle(CryptoRegistryException.class, CryptoRegistryException::new, 171, V_2_10_0));
+        registerExceptionHandle(
+            new OpenSearchExceptionHandle(
+                org.opensearch.crypto.CryptoRegistryException.class,
+                org.opensearch.crypto.CryptoRegistryException::new,
+                171,
+                V_2_10_0
+            )
+        );
+        registerExceptionHandle(
+            new OpenSearchExceptionHandle(
+                org.opensearch.action.admin.indices.view.ViewNotFoundException.class,
+                org.opensearch.action.admin.indices.view.ViewNotFoundException::new,
+                172,
+                V_2_13_0
+            )
+        );
+        registerExceptionHandle(
+            new OpenSearchExceptionHandle(
+                org.opensearch.action.admin.indices.view.ViewAlreadyExistsException.class,
+                org.opensearch.action.admin.indices.view.ViewAlreadyExistsException::new,
+                173,
+                V_2_13_0
+            )
+        );
         registerExceptionHandle(
             new OpenSearchExceptionHandle(
                 org.opensearch.cluster.block.IndexCreateBlockException.class,

server/src/main/java/org/opensearch/action/ActionModule.java

@@ -224,6 +224,12 @@
 import org.opensearch.action.admin.indices.upgrade.post.UpgradeSettingsAction;
 import org.opensearch.action.admin.indices.validate.query.TransportValidateQueryAction;
 import org.opensearch.action.admin.indices.validate.query.ValidateQueryAction;
+import org.opensearch.action.admin.indices.view.CreateViewAction;
+import org.opensearch.action.admin.indices.view.DeleteViewAction;
+import org.opensearch.action.admin.indices.view.GetViewAction;
+import org.opensearch.action.admin.indices.view.ListViewNamesAction;
+import org.opensearch.action.admin.indices.view.SearchViewAction;
+import org.opensearch.action.admin.indices.view.UpdateViewAction;
 import org.opensearch.action.bulk.BulkAction;
 import org.opensearch.action.bulk.TransportBulkAction;
 import org.opensearch.action.bulk.TransportShardBulkAction;
@@ -409,6 +415,7 @@
 import org.opensearch.rest.action.admin.indices.RestUpgradeAction;
 import org.opensearch.rest.action.admin.indices.RestUpgradeStatusAction;
 import org.opensearch.rest.action.admin.indices.RestValidateQueryAction;
+import org.opensearch.rest.action.admin.indices.RestViewAction;
 import org.opensearch.rest.action.cat.AbstractCatAction;
 import org.opensearch.rest.action.cat.RestAliasAction;
 import org.opensearch.rest.action.cat.RestAllocationAction;
@@ -721,6 +728,14 @@ public <Request extends ActionRequest, Response extends ActionResponse> void reg
         actions.register(ResolveIndexAction.INSTANCE, ResolveIndexAction.TransportAction.class);
         actions.register(DataStreamsStatsAction.INSTANCE, DataStreamsStatsAction.TransportAction.class);
 
+        // Views:
+        actions.register(CreateViewAction.INSTANCE, CreateViewAction.TransportAction.class);
+        actions.register(DeleteViewAction.INSTANCE, DeleteViewAction.TransportAction.class);
+        actions.register(GetViewAction.INSTANCE, GetViewAction.TransportAction.class);
+        actions.register(UpdateViewAction.INSTANCE, UpdateViewAction.TransportAction.class);
+        actions.register(ListViewNamesAction.INSTANCE, ListViewNamesAction.TransportAction.class);
+        actions.register(SearchViewAction.INSTANCE, SearchViewAction.TransportAction.class);
+
         // Persistent tasks:
         actions.register(StartPersistentTaskAction.INSTANCE, StartPersistentTaskAction.TransportAction.class);
         actions.register(UpdatePersistentTaskStatusAction.INSTANCE, UpdatePersistentTaskStatusAction.TransportAction.class);
@@ -915,6 +930,14 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestResolveIndexAction());
         registerHandler.accept(new RestDataStreamsStatsAction());
 
+        // View API
+        registerHandler.accept(new RestViewAction.CreateViewHandler());
+        registerHandler.accept(new RestViewAction.DeleteViewHandler());
+        registerHandler.accept(new RestViewAction.GetViewHandler());
+        registerHandler.accept(new RestViewAction.UpdateViewHandler());
+        registerHandler.accept(new RestViewAction.SearchViewHandler());
+        registerHandler.accept(new RestViewAction.ListViewNamesHandler());
+
         // CAT API
         registerHandler.accept(new RestAllocationAction());
         registerHandler.accept(new RestCatSegmentReplicationAction());