Merge branch 'main' into passing-indicesService-to-processor-factories

Signed-off-by: Martin Gaievski <[email protected]>

Author: martin-gaievski

.github/workflows/gradle-check.yml

@@ -72,7 +72,7 @@ jobs:
 
       - name: Upload Coverage Report
         if: success()
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v3
         with:
           files: ./codeCoverage.xml
 

.github/workflows/stalled.yml

@@ -0,0 +1,28 @@
+name: Label Stalled PRs
+on:
+  schedule:
+    - cron: '15 15 * * *' # Run every day at 15:15 UTC / 7:15 PST / 8:15 PDT
+permissions:
+  pull-requests: write
+jobs:
+  stale:
+    if: github.repository == 'opensearch-project/OpenSearch'
+    runs-on: ubuntu-latest
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Stale PRs
+        uses: actions/stale@v8
+        with:
+          repo-token: ${{ steps.github_app_token.outputs.token }}
+          stale-pr-label: 'stalled'
+          stale-pr-message: 'This PR is stalled because it has been open for 30 days with no activity.'
+          days-before-pr-stale: 30
+          days-before-issue-stale: -1
+          days-before-pr-close: -1
+          days-before-issue-close: -1

CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Introduce new dynamic cluster setting to control slice computation for concurrent segment search ([#9107](https://github.com/opensearch-project/OpenSearch/pull/9107))
 - Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679))
 - Implement Visitor Design pattern in QueryBuilder to enable the capability to traverse through the complex QueryBuilder tree. ([#10110](https://github.com/opensearch-project/OpenSearch/pull/10110))
+- Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
 - Allowing pipeline processors to access index maping info by passing ingest service ref as part of the processor factory parameters ([#10307](https://github.com/opensearch-project/OpenSearch/pull/10307))
 
 ### Dependencies
@@ -85,6 +86,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681))
 - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694))
 - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666))
+- Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))
+- Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189))
+- Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562))
 
 ### Dependencies
 - Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575))
@@ -97,20 +101,34 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.zaxxer:SparseBitSet` from 1.2 to 1.3 ([#10098](https://github.com/opensearch-project/OpenSearch/pull/10098))
 - Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125))
 - Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752))
+- Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126))
+- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206))
+- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208))
+- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
+- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
+- Bump Lucene from 9.7.0 to 9.8.0 ([#10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
+- Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302))
 
 ### Changed
 - Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
 - Allow parameterization of tests with OpenSearchIntegTestCase.SuiteScopeTestCase annotation ([#9916](https://github.com/opensearch-project/OpenSearch/pull/9916))
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
+- Force merge with `only_expunge_deletes` honors max segment size ([#10036](https://github.com/opensearch-project/OpenSearch/pull/10036))
 - Add instrumentation in transport service. ([#10042](https://github.com/opensearch-project/OpenSearch/pull/10042))
+- [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122))
+- Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246))
 
 ### Deprecated
 
 ### Removed
 
 ### Fixed
 - Fix ignore_missing parameter has no effect when using template snippet in rename ingest processor ([#9725](https://github.com/opensearch-project/OpenSearch/pull/9725))
-- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/9725))
+- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045))
+- Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082))
+- Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089))
 
 ### Security
 

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.8.0-snapshot-95cdd2e
+lucene            = 9.8.0
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 20.0.2+9
@@ -16,7 +16,7 @@ icu4j             = 70.1
 supercsv          = 2.4.0
 log4j             = 2.20.0
 slf4j             = 1.7.36
-asm               = 9.5
+asm               = 9.6
 jettison          = 1.5.4
 woodstox          = 6.4.0
 kotlin            = 1.7.10
@@ -48,7 +48,7 @@ reactivestreams   = 1.0.4
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
-bouncycastle=1.75
+bouncycastle=1.76
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2

libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1

@@ -0,0 +1 @@
+5e8421c5f8573bcf22e9265fc7e19469545a775a

libs/core/licenses/lucene-core-9.8.0.jar.sha1

@@ -59,6 +59,7 @@ public class Index implements Writeable, ToXContentObject {
     public static final Index[] EMPTY_ARRAY = new Index[0];
     private static final String INDEX_UUID_KEY = "index_uuid";
     private static final String INDEX_NAME_KEY = "index_name";
+    public static final String UNKNOWN_INDEX_NAME = "_unknown_";
 
     private static final ObjectParser<Builder, Void> INDEX_PARSER = new ObjectParser<>("index", Builder::new);
     static {

libs/core/src/main/java/org/opensearch/core/index/Index.java

@@ -527,6 +527,15 @@ public int getStatus() {
         return status;
     }
 
+    /**
+     * Get category class of a rest status code.
+     *
+     * @return Integer representing class category of the concrete rest status code
+     */
+    public int getStatusFamilyCode() {
+        return status / 100;
+    }
+
     public static RestStatus readFrom(StreamInput in) throws IOException {
         return RestStatus.valueOf(in.readString());
     }

libs/core/src/main/java/org/opensearch/core/rest/RestStatus.java

@@ -9,7 +9,6 @@
 package org.opensearch.telemetry.tracing;
 
 import org.opensearch.common.annotation.InternalApi;
-import org.opensearch.telemetry.tracing.attributes.Attributes;
 
 import java.io.Closeable;
 import java.io.IOException;
@@ -44,27 +43,13 @@ public DefaultTracer(TracingTelemetry tracingTelemetry, TracerContextStorage<Str
 
     @Override
     public Span startSpan(SpanCreationContext context) {
-        return startSpan(context.getSpanName(), context.getAttributes());
-    }
-
-    @Override
-    public Span startSpan(String spanName) {
-        return startSpan(spanName, Attributes.EMPTY);
-    }
-
-    @Override
-    public Span startSpan(String spanName, Attributes attributes) {
-        return startSpan(spanName, (SpanContext) null, attributes);
-    }
-
-    @Override
-    public Span startSpan(String spanName, SpanContext parentSpan, Attributes attributes) {
-        Span span = null;
-        if (parentSpan != null) {
-            span = createSpan(spanName, parentSpan.getSpan(), attributes);
+        Span parentSpan = null;
+        if (context.getParent() != null) {
+            parentSpan = context.getParent().getSpan();
         } else {
-            span = createSpan(spanName, getCurrentSpanInternal(), attributes);
+            parentSpan = getCurrentSpanInternal();
         }
+        Span span = createSpan(context, parentSpan);
         setCurrentSpanInContext(span);
         addDefaultAttributes(span);
         return span;
@@ -87,12 +72,7 @@ public SpanContext getCurrentSpan() {
 
     @Override
     public ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext) {
-        return startScopedSpan(spanCreationContext, null);
-    }
-
-    @Override
-    public ScopedSpan startScopedSpan(SpanCreationContext spanCreationContext, SpanContext parentSpan) {
-        Span span = startSpan(spanCreationContext.getSpanName(), parentSpan, spanCreationContext.getAttributes());
+        Span span = startSpan(spanCreationContext);
         SpanScope spanScope = withSpanInScope(span);
         return new DefaultScopedSpan(span, spanScope);
     }
@@ -102,8 +82,8 @@ public SpanScope withSpanInScope(Span span) {
         return DefaultSpanScope.create(span, tracerContextStorage).attach();
     }
 
-    private Span createSpan(String spanName, Span parentSpan, Attributes attributes) {
-        return tracingTelemetry.createSpan(spanName, parentSpan, attributes);
+    private Span createSpan(SpanCreationContext spanCreationContext, Span parentSpan) {
+        return tracingTelemetry.createSpan(spanCreationContext, parentSpan);
     }
 
     private void setCurrentSpanInContext(Span span) {
@@ -121,11 +101,7 @@ protected void addDefaultAttributes(Span span) {
     @Override
     public Span startSpan(SpanCreationContext spanCreationContext, Map<String, List<String>> headers) {
         Optional<Span> propagatedSpan = tracingTelemetry.getContextPropagator().extractFromHeaders(headers);
-        return startSpan(
-            spanCreationContext.getSpanName(),
-            propagatedSpan.map(SpanContext::new).orElse(null),
-            spanCreationContext.getAttributes()
-        );
+        return startSpan(spanCreationContext.parent(propagatedSpan.map(SpanContext::new).orElse(null)));
     }
 
 }

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java

@@ -18,17 +18,74 @@
  */
 @ExperimentalApi
 public final class SpanCreationContext {
-    private final String spanName;
-    private final Attributes attributes;
+    private String spanName;
+    private Attributes attributes;
+    private SpanKind spanKind = SpanKind.INTERNAL;
+    private SpanContext parent;
 
     /**
      * Constructor.
+     */
+    private SpanCreationContext() {}
+
+    /**
+     * Sets the span type to server.
+     * @return spanCreationContext
+     */
+    public static SpanCreationContext server() {
+        SpanCreationContext spanCreationContext = new SpanCreationContext();
+        spanCreationContext.spanKind = SpanKind.SERVER;
+        return spanCreationContext;
+    }
+
+    /**
+     * Sets the span type to client.
+     * @return spanCreationContext
+     */
+    public static SpanCreationContext client() {
+        SpanCreationContext spanCreationContext = new SpanCreationContext();
+        spanCreationContext.spanKind = SpanKind.CLIENT;
+        return spanCreationContext;
+    }
+
+    /**
+     * Sets the span type to internal.
+     * @return spanCreationContext
+     */
+    public static SpanCreationContext internal() {
+        SpanCreationContext spanCreationContext = new SpanCreationContext();
+        spanCreationContext.spanKind = SpanKind.INTERNAL;
+        return spanCreationContext;
+    }
+
+    /**
+     * Sets the span name.
      * @param spanName span name.
-     * @param attributes attributes.
+     * @return spanCreationContext
      */
-    public SpanCreationContext(String spanName, Attributes attributes) {
+    public SpanCreationContext name(String spanName) {
         this.spanName = spanName;
+        return this;
+    }
+
+    /**
+     * Sets the span attributes.
+     * @param attributes attributes.
+     * @return spanCreationContext
+     */
+    public SpanCreationContext attributes(Attributes attributes) {
         this.attributes = attributes;
+        return this;
+    }
+
+    /**
+     * Sets the parent for spann
+     * @param parent parent
+     * @return spanCreationContext
+     */
+    public SpanCreationContext parent(SpanContext parent) {
+        this.parent = parent;
+        return this;
     }
 
     /**
@@ -46,4 +103,20 @@ public String getSpanName() {
     public Attributes getAttributes() {
         return attributes;
     }
+
+    /**
+     * Returns the span kind.
+     * @return spankind.
+     */
+    public SpanKind getSpanKind() {
+        return spanKind;
+    }
+
+    /**
+     * Returns the parent span
+     * @return parent.
+     */
+    public SpanContext getParent() {
+        return parent;
+    }
 }

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/SpanCreationContext.java

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.tracing;
+
+import org.opensearch.common.annotation.PublicApi;
+
+/**
+ * Type of Span.
+ */
+@PublicApi(since = "2.11.0")
+public enum SpanKind {
+    /**
+     * Span represents the client side code.
+     */
+    CLIENT,
+    /**
+     * Span represents the server side code.
+     */
+    SERVER,
+
+    /**
+     * Span represents the internal calls. This is the default value of a span type.
+     */
+    INTERNAL;
+}