Skip to content

Commit 23e600a

Browse files
bharath-techiebharath-techie
committed
Adding timestamp rounding support in star tree
Signed-off-by: Bharathwaj G <[email protected]>
1 parent ef1a79f commit 23e600a

File tree

18 files changed

+1085
-154
lines changed

18 files changed

+1085
-154
lines changed

‎server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -47,10 +47,10 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
4747
.startObject("startree-1")
4848
.field("type", "star_tree")
4949
.startObject("config")
50-
.startArray("ordered_dimensions")
51-
.startObject()
50+
.startObject("date_dimension")
5251
.field("name", "timestamp")
5352
.endObject()
53+
.startArray("ordered_dimensions")
5454
.startObject()
5555
.field("name", getDim(invalidDim, keywordDim))
5656
.endObject()
@@ -97,14 +97,14 @@ private static XContentBuilder createMaxDimTestMapping() {
9797
.startObject("startree-1")
9898
.field("type", "star_tree")
9999
.startObject("config")
100-
.startArray("ordered_dimensions")
101-
.startObject()
100+
.startObject("date_dimension")
102101
.field("name", "timestamp")
103102
.startArray("calendar_intervals")
104103
.value("day")
105104
.value("month")
106105
.endArray()
107106
.endObject()
107+
.startArray("ordered_dimensions")
108108
.startObject()
109109
.field("name", "dim2")
110110
.endObject()
@@ -139,7 +139,7 @@ private static XContentBuilder createMaxDimTestMapping() {
139139
}
140140
}
141141

142-
private static XContentBuilder createTestMappingWithoutStarTree(boolean invalidDim, boolean invalidMetric, boolean keywordDim) {
142+
private static XContentBuilder createTestMappingWithoutStarTree() {
143143
try {
144144
return jsonBuilder().startObject()
145145
.startObject("properties")
@@ -176,10 +176,10 @@ private static XContentBuilder createUpdateTestMapping(boolean changeDim, boolea
176176
.startObject(sameStarTree ? "startree-1" : "startree-2")
177177
.field("type", "star_tree")
178178
.startObject("config")
179-
.startArray("ordered_dimensions")
180-
.startObject()
179+
.startObject("date_dimension")
181180
.field("name", "timestamp")
182181
.endObject()
182+
.startArray("ordered_dimensions")
183183
.startObject()
184184
.field("name", changeDim ? "numeric_new" : getDim(false, false))
185185
.endObject()
@@ -295,7 +295,7 @@ public void testUpdateIndexWithAdditionOfStarTree() {
295295
}
296296

297297
public void testUpdateIndexWithNewerStarTree() {
298-
prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree(false, false, false)).get();
298+
prepareCreate(TEST_INDEX).setMapping(createTestMappingWithoutStarTree()).get();
299299

300300
IllegalArgumentException ex = expectThrows(
301301
IllegalArgumentException.class,

‎server/src/main/java/org/opensearch/common/Rounding.java

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,15 @@
6262
import java.time.temporal.TemporalQueries;
6363
import java.time.zone.ZoneOffsetTransition;
6464
import java.time.zone.ZoneRules;
65+
import java.util.Comparator;
66+
import java.util.HashMap;
6567
import java.util.List;
6668
import java.util.Locale;
69+
import java.util.Map;
6770
import java.util.Objects;
6871
import java.util.OptionalLong;
6972
import java.util.concurrent.TimeUnit;
73+
import java.util.stream.Collectors;
7074

7175
/**
7276
* A strategy for rounding milliseconds since epoch.
@@ -95,7 +99,7 @@ public enum DateTimeUnit {
9599
WEEK_OF_WEEKYEAR((byte) 1, "week", IsoFields.WEEK_OF_WEEK_BASED_YEAR, true, TimeUnit.DAYS.toMillis(7)) {
96100
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(7);
97101

98-
long roundFloor(long utcMillis) {
102+
public long roundFloor(long utcMillis) {
99103
return DateUtils.roundWeekOfWeekYear(utcMillis);
100104
}
101105

@@ -107,7 +111,7 @@ long extraLocalOffsetLookup() {
107111
YEAR_OF_CENTURY((byte) 2, "year", ChronoField.YEAR_OF_ERA, false, 12) {
108112
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(366);
109113

110-
long roundFloor(long utcMillis) {
114+
public long roundFloor(long utcMillis) {
111115
return DateUtils.roundYear(utcMillis);
112116
}
113117

@@ -118,7 +122,7 @@ long extraLocalOffsetLookup() {
118122
QUARTER_OF_YEAR((byte) 3, "quarter", IsoFields.QUARTER_OF_YEAR, false, 3) {
119123
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(92);
120124

121-
long roundFloor(long utcMillis) {
125+
public long roundFloor(long utcMillis) {
122126
return DateUtils.roundQuarterOfYear(utcMillis);
123127
}
124128

@@ -129,7 +133,7 @@ long extraLocalOffsetLookup() {
129133
MONTH_OF_YEAR((byte) 4, "month", ChronoField.MONTH_OF_YEAR, false, 1) {
130134
private final long extraLocalOffsetLookup = TimeUnit.DAYS.toMillis(31);
131135

132-
long roundFloor(long utcMillis) {
136+
public long roundFloor(long utcMillis) {
133137
return DateUtils.roundMonthOfYear(utcMillis);
134138
}
135139

@@ -138,7 +142,7 @@ long extraLocalOffsetLookup() {
138142
}
139143
},
140144
DAY_OF_MONTH((byte) 5, "day", ChronoField.DAY_OF_MONTH, true, ChronoField.DAY_OF_MONTH.getBaseUnit().getDuration().toMillis()) {
141-
long roundFloor(long utcMillis) {
145+
public long roundFloor(long utcMillis) {
142146
return DateUtils.roundFloor(utcMillis, this.ratio);
143147
}
144148

@@ -147,7 +151,7 @@ long extraLocalOffsetLookup() {
147151
}
148152
},
149153
HOUR_OF_DAY((byte) 6, "hour", ChronoField.HOUR_OF_DAY, true, ChronoField.HOUR_OF_DAY.getBaseUnit().getDuration().toMillis()) {
150-
long roundFloor(long utcMillis) {
154+
public long roundFloor(long utcMillis) {
151155
return DateUtils.roundFloor(utcMillis, ratio);
152156
}
153157

@@ -162,7 +166,7 @@ long extraLocalOffsetLookup() {
162166
true,
163167
ChronoField.MINUTE_OF_HOUR.getBaseUnit().getDuration().toMillis()
164168
) {
165-
long roundFloor(long utcMillis) {
169+
public long roundFloor(long utcMillis) {
166170
return DateUtils.roundFloor(utcMillis, ratio);
167171
}
168172

@@ -177,7 +181,7 @@ long extraLocalOffsetLookup() {
177181
true,
178182
ChronoField.SECOND_OF_MINUTE.getBaseUnit().getDuration().toMillis()
179183
) {
180-
long roundFloor(long utcMillis) {
184+
public long roundFloor(long utcMillis) {
181185
return DateUtils.roundFloor(utcMillis, ratio);
182186
}
183187

@@ -210,7 +214,7 @@ public long extraLocalOffsetLookup() {
210214
* @param utcMillis the milliseconds since the epoch
211215
* @return the rounded down milliseconds since the epoch
212216
*/
213-
abstract long roundFloor(long utcMillis);
217+
public abstract long roundFloor(long utcMillis);
214218

215219
/**
216220
* When looking up {@link LocalTimeOffset} go this many milliseconds
@@ -260,6 +264,35 @@ public static DateTimeUnit resolve(byte id) {
260264
}
261265
}
262266

267+
/**
268+
* DateTimeUnit Comparator which tracks dateTimeUnits from second unit to year unit
269+
*/
270+
public static class DateTimeUnitComparator implements Comparator<DateTimeUnit> {
271+
public static final Map<DateTimeUnit, Integer> ORDERED_DATE_TIME_UNIT = new HashMap<>();
272+
273+
static {
274+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.SECOND_OF_MINUTE, 1);
275+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MINUTES_OF_HOUR, 2);
276+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.HOUR_OF_DAY, 3);
277+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.DAY_OF_MONTH, 4);
278+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.WEEK_OF_WEEKYEAR, 5);
279+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.MONTH_OF_YEAR, 6);
280+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.QUARTER_OF_YEAR, 7);
281+
ORDERED_DATE_TIME_UNIT.put(DateTimeUnit.YEAR_OF_CENTURY, 8);
282+
}
283+
284+
@Override
285+
public int compare(DateTimeUnit unit1, DateTimeUnit unit2) {
286+
return Integer.compare(ORDERED_DATE_TIME_UNIT.get(unit1), ORDERED_DATE_TIME_UNIT.get(unit2));
287+
}
288+
}
289+
290+
public static List<DateTimeUnit> getSortedDateTimeUnits(List<DateTimeUnit> dateTimeUnits) {
291+
return dateTimeUnits.stream()
292+
.sorted(Comparator.comparingInt(DateTimeUnitComparator.ORDERED_DATE_TIME_UNIT::get))
293+
.collect(Collectors.toList());
294+
}
295+
263296
public abstract void innerWriteTo(StreamOutput out) throws IOException;
264297

265298
@Override

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/DateDimension.java

Lines changed: 61 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@
1010

1111
import org.opensearch.common.Rounding;
1212
import org.opensearch.common.annotation.ExperimentalApi;
13+
import org.opensearch.common.time.DateUtils;
1314
import org.opensearch.core.xcontent.XContentBuilder;
1415
import org.opensearch.index.mapper.CompositeDataCubeFieldType;
16+
import org.opensearch.index.mapper.DateFieldMapper;
1517

1618
import java.io.IOException;
19+
import java.util.ArrayList;
1720
import java.util.List;
1821
import java.util.Objects;
1922

@@ -28,19 +31,70 @@ public class DateDimension implements Dimension {
2831
public static final String CALENDAR_INTERVALS = "calendar_intervals";
2932
public static final String DATE = "date";
3033
private final String field;
34+
private final List<Rounding.DateTimeUnit> sortedCalendarIntervals;
35+
private final DateFieldMapper.Resolution resolution;
3136

32-
public DateDimension(String field, List<Rounding.DateTimeUnit> calendarIntervals) {
37+
public DateDimension(String field, List<Rounding.DateTimeUnit> calendarIntervals, DateFieldMapper.Resolution resolution) {
3338
this.field = field;
3439
this.calendarIntervals = calendarIntervals;
40+
// Sort from the lowest unit to the highest unit
41+
this.sortedCalendarIntervals = Rounding.getSortedDateTimeUnits(calendarIntervals);
42+
if (resolution == null) {
43+
this.resolution = DateFieldMapper.Resolution.MILLISECONDS;
44+
} else {
45+
this.resolution = resolution;
46+
}
3547
}
3648

3749
public List<Rounding.DateTimeUnit> getIntervals() {
3850
return calendarIntervals;
3951
}
4052

53+
public List<Rounding.DateTimeUnit> getSortedCalendarIntervals() {
54+
return sortedCalendarIntervals;
55+
}
56+
57+
/**
58+
* Sets the dimension values in sorted order in the provided array starting from the given index.
59+
*
60+
* @param val The value to be set
61+
* @param dims The dimensions array to set the values in
62+
* @param index The starting index in the array
63+
* @return The next available index in the array
64+
*/
65+
@Override
66+
public int setDimensionValues(final Long val, final Long[] dims, int index) {
67+
for (Rounding.DateTimeUnit dateTimeUnit : sortedCalendarIntervals) {
68+
if (val == null) {
69+
dims[index++] = null;
70+
continue;
71+
}
72+
dims[index++] = dateTimeUnit.roundFloor(convertNanosToMillis(val));
73+
}
74+
return index;
75+
}
76+
77+
private long convertNanosToMillis(long nanoSecondsSinceEpoch) {
78+
if (resolution.equals(DateFieldMapper.Resolution.NANOSECONDS)) return DateUtils.toMilliSeconds(nanoSecondsSinceEpoch);
79+
return nanoSecondsSinceEpoch;
80+
}
81+
82+
/**
83+
* Returns the list of fields that represent the dimension
84+
*/
85+
@Override
86+
public List<String> getDimensionFieldsNames() {
87+
List<String> fields = new ArrayList<>(calendarIntervals.size());
88+
for (Rounding.DateTimeUnit interval : sortedCalendarIntervals) {
89+
// TODO : revisit this post file format changes
90+
fields.add(field + "_" + interval.shortName());
91+
}
92+
return fields;
93+
}
94+
4195
@Override
4296
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
43-
builder.startObject();
97+
builder.startObject("date_dimension");
4498
builder.field(CompositeDataCubeFieldType.NAME, this.getField());
4599
builder.field(CompositeDataCubeFieldType.TYPE, DATE);
46100
builder.startArray(CALENDAR_INTERVALS);
@@ -69,4 +123,9 @@ public int hashCode() {
69123
public String getField() {
70124
return field;
71125
}
126+
127+
@Override
128+
public int getNumDimensions() {
129+
return calendarIntervals.size();
130+
}
72131
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/Dimension.java

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,36 @@
1111
import org.opensearch.common.annotation.ExperimentalApi;
1212
import org.opensearch.core.xcontent.ToXContent;
1313

14+
import java.util.List;
15+
1416
/**
1517
* Base interface for data-cube dimensions
1618
*
1719
* @opensearch.experimental
1820
*/
1921
@ExperimentalApi
2022
public interface Dimension extends ToXContent {
23+
2124
String getField();
25+
26+
/**
27+
* Returns the number of dimension values that gets added to star tree document
28+
* as part of this dimension
29+
*/
30+
int getNumDimensions();
31+
32+
/**
33+
* Sets the dimension values in the provided array starting from the given index.
34+
*
35+
* @param value The value to be set
36+
* @param dims The dimensions array to set the values in
37+
* @param index The starting index in the array
38+
* @return The next available index in the array
39+
*/
40+
int setDimensionValues(Long value, Long[] dims, int index);
41+
42+
/**
43+
* Returns the list of dimension fields that represent the dimension
44+
*/
45+
List<String> getDimensionFieldsNames();
2246
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/DimensionFactory.java

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ private static DateDimension parseAndCreateDateDimension(
7575
.stream()
7676
.map(Object::toString)
7777
.collect(Collectors.toList());
78+
System.out.println(intervalStrings);
7879
if (intervalStrings == null || intervalStrings.isEmpty()) {
7980
calendarIntervals = StarTreeIndexSettings.DEFAULT_DATE_INTERVALS.get(c.getSettings());
8081
} else {
@@ -94,6 +95,11 @@ private static DateDimension parseAndCreateDateDimension(
9495
calendarIntervals = new ArrayList<>(calendarIntervals);
9596
}
9697
dimensionMap.remove(CALENDAR_INTERVALS);
97-
return new DateDimension(name, calendarIntervals);
98+
DateFieldMapper.Resolution resolution = null;
99+
if (c != null && c.mapperService() != null && c.mapperService().fieldType(name) != null) {
100+
resolution = ((DateFieldMapper.DateFieldType) c.mapperService().fieldType(name)).resolution();
101+
}
102+
103+
return new DateDimension(name, calendarIntervals, resolution);
98104
}
99105
}

‎server/src/main/java/org/opensearch/index/compositeindex/datacube/NumericDimension.java

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import org.opensearch.index.mapper.CompositeDataCubeFieldType;
1414

1515
import java.io.IOException;
16+
import java.util.List;
1617
import java.util.Objects;
1718

1819
/**
@@ -33,6 +34,23 @@ public String getField() {
3334
return field;
3435
}
3536

37+
@Override
38+
public int getNumDimensions() {
39+
return 1;
40+
}
41+
42+
@Override
43+
public int setDimensionValues(final Long val, final Long[] dims, int index) {
44+
dims[index++] = val;
45+
return index;
46+
}
47+
48+
@Override
49+
public List<String> getDimensionFieldsNames() {
50+
// TODO : revisit this post file format changes
51+
return List.of(field);
52+
}
53+
3654
@Override
3755
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
3856
builder.startObject();

0 commit comments

Comments
 (0)