Skip to content

Commit 8795cef

Browse files
github-actions[bot]msfroh
github-actions[bot]
and
msfroh
committed
Introduce ApproximateRangeQuery and ApproximateQuery (#13788) (#15586)
This introduces a basic "approximation" framework that improves query performance by modifying the query in a way that should be functionally equivalent. To start, we can reduce the bounds of a range query in order to satisfy the `track_total_hits` value (which defaults to 10,000). --------- Signed-off-by: Harsha Vamsi Kalluri <[email protected]> Signed-off-by: Michael Froh <[email protected]> Co-authored-by: Michael Froh <[email protected]> (cherry picked from commit 2e9db40) (cherry picked from commit 3ddb199) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent bd5041f commit 8795cef

20 files changed

+1622
-58
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
3636
- Add runAs to Subject interface and introduce IdentityAwarePlugin extension point ([#14630](https://github.com/opensearch-project/OpenSearch/pull/14630))
3737
- [Workload Management] Add rejection logic for co-ordinator and shard level requests ([#15428](https://github.com/opensearch-project/OpenSearch/pull/15428)))
3838
- Adding translog durability validation in index templates ([#15494](https://github.com/opensearch-project/OpenSearch/pull/15494))
39+
- [Range Queries] Add new approximateable query framework to short-circuit range queries ([#13788](https://github.com/opensearch-project/OpenSearch/pull/13788))
3940
- [Workload Management] Add query group level failure tracking ([#15227](https://github.com/opensearch-project/OpenSearch/pull/15527))
4041
- [Reader Writer Separation] Add searchOnly replica routing configuration ([#15410](https://github.com/opensearch-project/OpenSearch/pull/15410))
4142
- Add index creation using the context field ([#15290](https://github.com/opensearch-project/OpenSearch/pull/15290))

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/370_approximate_range.yml

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
---
2+
"search with approximate range":
3+
- do:
4+
indices.create:
5+
index: test
6+
body:
7+
mappings:
8+
properties:
9+
date:
10+
type: date
11+
index: true
12+
doc_values: true
13+
14+
- do:
15+
bulk:
16+
index: test
17+
refresh: true
18+
body:
19+
- '{"index": {"_index": "test", "_id": "1" }}'
20+
- '{ "date": "2018-10-29T12:12:12.987Z" }'
21+
- '{ "index": { "_index": "test", "_id": "2" }}'
22+
- '{ "date": "2020-10-29T12:12:12.987Z" }'
23+
- '{ "index": { "_index": "test", "_id": "3" } }'
24+
- '{ "date": "2024-10-29T12:12:12.987Z" }'
25+
26+
- do:
27+
search:
28+
rest_total_hits_as_int: true
29+
index: test
30+
body:
31+
query:
32+
range: {
33+
date: {
34+
gte: "2018-10-29T12:12:12.987Z"
35+
},
36+
}
37+
38+
- match: { hits.total: 3 }
39+
40+
- do:
41+
search:
42+
rest_total_hits_as_int: true
43+
index: test
44+
body:
45+
sort: [{ date: asc }]
46+
query:
47+
range: {
48+
date: {
49+
gte: "2018-10-29T12:12:12.987Z"
50+
},
51+
}
52+
53+
54+
- match: { hits.total: 3 }
55+
- match: { hits.hits.0._id: "1" }
56+
57+
- do:
58+
search:
59+
rest_total_hits_as_int: true
60+
index: test
61+
body:
62+
sort: [{ date: desc }]
63+
query:
64+
range: {
65+
date: {
66+
gte: "2018-10-29T12:12:12.987Z",
67+
lte: "2020-10-29T12:12:12.987Z"
68+
},
69+
}
70+
71+
- match: { hits.total: 2 }
72+
- match: { hits.hits.0._id: "2" }

‎server/src/main/java/org/opensearch/common/util/FeatureFlags.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,16 @@ public class FeatureFlags {
125125
public static final String STAR_TREE_INDEX = "opensearch.experimental.feature.composite_index.star_tree.enabled";
126126
public static final Setting<Boolean> STAR_TREE_INDEX_SETTING = Setting.boolSetting(STAR_TREE_INDEX, false, Property.NodeScope);
127127

128+
/**
129+
* Gates the functionality of ApproximatePointRangeQuery where we approximate query results.
130+
*/
131+
public static final String APPROXIMATE_POINT_RANGE_QUERY = "opensearch.experimental.feature.approximate_point_range_query.enabled";
132+
public static final Setting<Boolean> APPROXIMATE_POINT_RANGE_QUERY_SETTING = Setting.boolSetting(
133+
APPROXIMATE_POINT_RANGE_QUERY,
134+
false,
135+
Property.NodeScope
136+
);
137+
128138
private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
129139
REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
130140
EXTENSIONS_SETTING,

‎server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@
6262
import org.opensearch.index.query.QueryRewriteContext;
6363
import org.opensearch.index.query.QueryShardContext;
6464
import org.opensearch.search.DocValueFormat;
65+
import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery;
66+
import org.opensearch.search.approximate.ApproximatePointRangeQuery;
6567
import org.opensearch.search.lookup.SearchLookup;
6668

6769
import java.io.IOException;
@@ -81,6 +83,7 @@
8183
import java.util.function.Supplier;
8284

8385
import static org.opensearch.common.time.DateUtils.toLong;
86+
import static org.apache.lucene.document.LongPoint.pack;
8487

8588
/**
8689
* A {@link FieldMapper} for dates.
@@ -109,6 +112,21 @@ public static DateFormatter getDefaultDateTimeFormatter() {
109112
: LEGACY_DEFAULT_DATE_TIME_FORMATTER;
110113
}
111114

115+
public static Query getDefaultQuery(Query pointRangeQuery, Query dvQuery, String name, long l, long u) {
116+
return FeatureFlags.isEnabled(FeatureFlags.APPROXIMATE_POINT_RANGE_QUERY_SETTING)
117+
? new ApproximateIndexOrDocValuesQuery(
118+
pointRangeQuery,
119+
new ApproximatePointRangeQuery(name, pack(new long[] { l }).bytes, pack(new long[] { u }).bytes, new long[] { l }.length) {
120+
@Override
121+
protected String toString(int dimension, byte[] value) {
122+
return Long.toString(LongPoint.decodeDimension(value, 0));
123+
}
124+
},
125+
dvQuery
126+
)
127+
: new IndexOrDocValuesQuery(pointRangeQuery, dvQuery);
128+
}
129+
112130
/**
113131
* Resolution of the date time
114132
*
@@ -468,24 +486,22 @@ public Query rangeQuery(
468486
}
469487
DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser;
470488
return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> {
489+
Query pointRangeQuery = isSearchable() ? LongPoint.newRangeQuery(name(), l, u) : null;
490+
Query dvQuery = hasDocValues() ? SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u) : null;
471491
if (isSearchable() && hasDocValues()) {
472-
Query query = LongPoint.newRangeQuery(name(), l, u);
473-
Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
474-
query = new IndexOrDocValuesQuery(query, dvQuery);
475-
492+
Query query = getDefaultQuery(pointRangeQuery, dvQuery, name(), l, u);
476493
if (context.indexSortedOnField(name())) {
477494
query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
478495
}
479496
return query;
480497
}
481498
if (hasDocValues()) {
482-
Query query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
483499
if (context.indexSortedOnField(name())) {
484-
query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
500+
dvQuery = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, dvQuery);
485501
}
486-
return query;
502+
return dvQuery;
487503
}
488-
return LongPoint.newRangeQuery(name(), l, u);
504+
return pointRangeQuery;
489505
});
490506
}
491507

‎server/src/main/java/org/opensearch/search/aggregations/bucket/filterrewrite/Helper.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.opensearch.common.lucene.search.function.FunctionScoreQuery;
2424
import org.opensearch.index.mapper.DateFieldMapper;
2525
import org.opensearch.index.query.DateRangeIncludingNowQuery;
26+
import org.opensearch.search.approximate.ApproximateIndexOrDocValuesQuery;
2627
import org.opensearch.search.internal.SearchContext;
2728

2829
import java.io.IOException;
@@ -54,6 +55,7 @@ private Helper() {}
5455
queryWrappers.put(FunctionScoreQuery.class, q -> ((FunctionScoreQuery) q).getSubQuery());
5556
queryWrappers.put(DateRangeIncludingNowQuery.class, q -> ((DateRangeIncludingNowQuery) q).getQuery());
5657
queryWrappers.put(IndexOrDocValuesQuery.class, q -> ((IndexOrDocValuesQuery) q).getIndexQuery());
58+
queryWrappers.put(ApproximateIndexOrDocValuesQuery.class, q -> ((ApproximateIndexOrDocValuesQuery) q).getOriginalQuery());
5759
}
5860

5961
/**

‎server/src/main/java/org/opensearch/search/approximate/ApproximateIndexOrDocValuesQuery.java

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.search.approximate;
10+
11+
import org.apache.lucene.search.IndexOrDocValuesQuery;
12+
import org.apache.lucene.search.Query;
13+
import org.apache.lucene.search.QueryVisitor;
14+
15+
/**
16+
* A wrapper around {@link IndexOrDocValuesQuery} that can be used to run approximate queries.
17+
* It delegates to either {@link ApproximateQuery} or {@link IndexOrDocValuesQuery} based on whether the query can be approximated or not.
18+
* @see ApproximateQuery
19+
*/
20+
public final class ApproximateIndexOrDocValuesQuery extends ApproximateScoreQuery {
21+
22+
private final ApproximateQuery approximateIndexQuery;
23+
private final IndexOrDocValuesQuery indexOrDocValuesQuery;
24+
25+
public ApproximateIndexOrDocValuesQuery(Query indexQuery, ApproximateQuery approximateIndexQuery, Query dvQuery) {
26+
super(new IndexOrDocValuesQuery(indexQuery, dvQuery), approximateIndexQuery);
27+
this.approximateIndexQuery = approximateIndexQuery;
28+
this.indexOrDocValuesQuery = new IndexOrDocValuesQuery(indexQuery, dvQuery);
29+
}
30+
31+
@Override
32+
public String toString(String field) {
33+
return "ApproximateIndexOrDocValuesQuery(indexQuery="
34+
+ indexOrDocValuesQuery.getIndexQuery().toString(field)
35+
+ ", approximateIndexQuery="
36+
+ approximateIndexQuery.toString(field)
37+
+ ", dvQuery="
38+
+ indexOrDocValuesQuery.getRandomAccessQuery().toString(field)
39+
+ ")";
40+
}
41+
42+
@Override
43+
public void visit(QueryVisitor visitor) {
44+
indexOrDocValuesQuery.visit(visitor);
45+
}
46+
47+
@Override
48+
public boolean equals(Object obj) {
49+
if (sameClassAs(obj) == false) {
50+
return false;
51+
}
52+
return true;
53+
}
54+
55+
@Override
56+
public int hashCode() {
57+
int h = classHash();
58+
h = 31 * h + indexOrDocValuesQuery.getIndexQuery().hashCode();
59+
h = 31 * h + indexOrDocValuesQuery.getRandomAccessQuery().hashCode();
60+
return h;
61+
}
62+
}

0 commit comments

Comments
 (0)