Skip to content

Commit e7d8710

Browse files
author
Gagan Singh Saini
committed
Add IT test for warm node disk threshold monitor
Signed-off-by: Gagan Singh Saini <[email protected]>
1 parent 341e34a commit e7d8710

File tree

6 files changed

+290
-193
lines changed

6 files changed

+290
-193
lines changed

‎server/src/internalClusterTest/java/org/opensearch/cluster/routing/allocation/decider/DiskThresholdDeciderIT.java

Lines changed: 213 additions & 88 deletions
Large diffs are not rendered by default.

‎server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdEvaluator.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,26 +44,26 @@ public interface DiskThresholdEvaluator {
4444
boolean isNodeExceedingFloodStageWatermark(DiskUsage diskUsage);
4545

4646
/**
47-
* Calculates the free space low threshold for a given total space
47+
* Get the free space low threshold for a given total space
4848
*
4949
* @param totalSpace total available space
5050
* @return free space low threshold in bytes
5151
*/
52-
long calculateFreeSpaceLowThreshold(long totalSpace);
52+
long getFreeSpaceLowThreshold(long totalSpace);
5353

5454
/**
55-
* Calculates the free space high threshold for a given total space
55+
* Get the free space high threshold for a given total space
5656
*
5757
* @param totalSpace total available space
5858
* @return free space high threshold in bytes
5959
*/
60-
long calculateFreeSpaceHighThreshold(long totalSpace);
60+
long getFreeSpaceHighThreshold(long totalSpace);
6161

6262
/**
63-
* Calculates the free space flood stage threshold for a given total space
63+
* Get the free space flood stage threshold for a given total space
6464
*
6565
* @param totalSpace total available space
6666
* @return free space flood stage threshold in bytes
6767
*/
68-
long calculateFreeSpaceFloodStageThreshold(long totalSpace);
68+
long getFreeSpaceFloodStageThreshold(long totalSpace);
6969
}

‎server/src/main/java/org/opensearch/cluster/routing/allocation/DiskThresholdMonitor.java

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -174,15 +174,16 @@ public void onNewInfo(ClusterInfo info) {
174174

175175
for (final Map.Entry<String, DiskUsage> entry : usages.entrySet()) {
176176
final String node = entry.getKey();
177-
// Create DiskUsage for Warm Nodes based on total Addressable Space
178177
DiskUsage usage = entry.getValue();
179178
final RoutingNode routingNode = routingNodes.node(node);
180179
if (routingNode == null) {
181180
continue;
182181
}
183182

183+
// Only for Dedicated Warm Nodes
184184
final boolean isWarmNode = REMOTE_CAPABLE.equals(getNodePool(routingNode));
185185
if (isWarmNode) {
186+
// Create DiskUsage for Warm Nodes based on total Addressable Space
186187
usage = getWarmDiskUsage(usage, info, routingNode, state);
187188
}
188189

@@ -219,7 +220,6 @@ public void onNewInfo(ClusterInfo info) {
219220
}
220221
}
221222

222-
// Check if for warm node if reserved space comes out to be zero, if not make it 0
223223
final long reservedSpace = info.getReservedSpace(usage.getNodeId(), usage.getPath()).getTotal();
224224
final DiskUsage usageWithReservedSpace = new DiskUsage(
225225
usage.getNodeId(),
@@ -229,7 +229,6 @@ public void onNewInfo(ClusterInfo info) {
229229
Math.max(0L, usage.getFreeBytes() - reservedSpace)
230230
);
231231

232-
// Check if node exceeds high watermark with reserved space factored in
233232
if (isNodeExceedingHighWatermark(usageWithReservedSpace, isWarmNode)) {
234233

235234
nodesOverLowThreshold.add(node);
@@ -456,7 +455,7 @@ private DiskUsage getWarmDiskUsage(DiskUsage diskUsage, ClusterInfo info, Routin
456455
.filter(shard -> shard.primary() && REMOTE_CAPABLE.equals(getIndexPool(state.metadata().getIndexSafe(shard.index()))))
457456
.collect(Collectors.toList());
458457

459-
var remoteShardSize = 0L;
458+
long remoteShardSize = 0L;
460459
for (ShardRouting shard : remoteShardsOnNode) {
461460
remoteShardSize += DiskThresholdDecider.getExpectedShardSize(shard, 0L, info, null, state.metadata(), state.getRoutingTable());
462461
}

‎server/src/main/java/org/opensearch/cluster/routing/allocation/HotNodeDiskThresholdEvaluator.java

Lines changed: 28 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -45,56 +45,48 @@ public boolean isNodeExceedingFloodStageWatermark(DiskUsage diskUsage) {
4545
}
4646

4747
@Override
48-
public long calculateFreeSpaceLowThreshold(long totalSpace) {
49-
// For hot data nodes, we use the standard disk threshold settings
50-
// Check for absolute bytes threshold first
51-
ByteSizeValue bytesThreshold = diskThresholdSettings.getFreeBytesThresholdLow();
52-
if (bytesThreshold != null && bytesThreshold.getBytes() > 0) {
53-
return bytesThreshold.getBytes();
54-
}
55-
56-
// Check for percentage-based threshold
57-
double percentageThreshold = diskThresholdSettings.getFreeDiskThresholdLow();
58-
if (percentageThreshold > 0) {
59-
return (long) (totalSpace * percentageThreshold / 100.0);
60-
}
61-
62-
// Default fallback
63-
return 0;
48+
public long getFreeSpaceLowThreshold(long totalAddressableSpace) {
49+
return calculateFreeSpaceWatermarkThreshold(
50+
diskThresholdSettings.getFreeDiskThresholdLow(),
51+
diskThresholdSettings.getFreeBytesThresholdLow(),
52+
totalAddressableSpace
53+
);
6454
}
6555

6656
@Override
67-
public long calculateFreeSpaceHighThreshold(long totalSpace) {
68-
// For hot data nodes, we use the standard disk threshold settings
69-
// Check for absolute bytes threshold first
70-
ByteSizeValue bytesThreshold = diskThresholdSettings.getFreeBytesThresholdHigh();
71-
if (bytesThreshold != null && bytesThreshold.getBytes() > 0) {
72-
return bytesThreshold.getBytes();
73-
}
74-
75-
// Check for percentage-based threshold
76-
double percentageThreshold = diskThresholdSettings.getFreeDiskThresholdHigh();
77-
if (percentageThreshold > 0) {
78-
return (long) (totalSpace * percentageThreshold / 100.0);
79-
}
80-
81-
// Default fallback
82-
return 0;
57+
public long getFreeSpaceHighThreshold(long totalAddressableSpace) {
58+
return calculateFreeSpaceWatermarkThreshold(
59+
diskThresholdSettings.getFreeDiskThresholdHigh(),
60+
diskThresholdSettings.getFreeBytesThresholdHigh(),
61+
totalAddressableSpace
62+
);
8363
}
8464

8565
@Override
86-
public long calculateFreeSpaceFloodStageThreshold(long totalSpace) {
66+
public long getFreeSpaceFloodStageThreshold(long totalAddressableSpace) {
67+
return calculateFreeSpaceWatermarkThreshold(
68+
diskThresholdSettings.getFreeDiskThresholdFloodStage(),
69+
diskThresholdSettings.getFreeBytesThresholdFloodStage(),
70+
totalAddressableSpace
71+
);
72+
}
73+
74+
private long calculateFreeSpaceWatermarkThreshold(
75+
double freeDiskWatermarkThreshold,
76+
ByteSizeValue freeBytesWatermarkThreshold,
77+
long totalAddressableSpace
78+
) {
8779
// For hot data nodes, we use the standard disk threshold settings
8880
// Check for absolute bytes threshold first
89-
ByteSizeValue bytesThreshold = diskThresholdSettings.getFreeBytesThresholdFloodStage();
81+
ByteSizeValue bytesThreshold = freeBytesWatermarkThreshold;
9082
if (bytesThreshold != null && bytesThreshold.getBytes() > 0) {
9183
return bytesThreshold.getBytes();
9284
}
9385

9486
// Check for percentage-based threshold
95-
double percentageThreshold = diskThresholdSettings.getFreeDiskThresholdFloodStage();
87+
double percentageThreshold = freeDiskWatermarkThreshold;
9688
if (percentageThreshold > 0) {
97-
return (long) (totalSpace * percentageThreshold / 100.0);
89+
return (long) (totalAddressableSpace * percentageThreshold / 100.0);
9890
}
9991

10092
// Default fallback

‎server/src/main/java/org/opensearch/cluster/routing/allocation/WarmNodeDiskThresholdEvaluator.java

Lines changed: 38 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.opensearch.cluster.DiskUsage;
1212
import org.opensearch.core.common.unit.ByteSizeValue;
1313

14+
import java.util.function.Function;
1415
import java.util.function.Supplier;
1516

1617
/**
@@ -32,91 +33,71 @@ public WarmNodeDiskThresholdEvaluator(DiskThresholdSettings diskThresholdSetting
3233

3334
@Override
3435
public boolean isNodeExceedingLowWatermark(DiskUsage diskUsage) {
35-
if (dataToFileCacheSizeRatioSupplier.get() <= 0) {
36-
return false;
37-
}
38-
long totalBytes = diskUsage.getTotalBytes();
39-
long freeSpace = diskUsage.getFreeBytes();
40-
long freeSpaceLowThreshold = calculateFreeSpaceLowThreshold(totalBytes);
41-
42-
return freeSpace < freeSpaceLowThreshold;
36+
return isNodeExceedingWatermark(diskUsage, this::getFreeSpaceLowThreshold);
4337
}
4438

4539
@Override
4640
public boolean isNodeExceedingHighWatermark(DiskUsage diskUsage) {
47-
if (dataToFileCacheSizeRatioSupplier.get() <= 0) {
48-
return false;
49-
}
50-
long totalBytes = diskUsage.getTotalBytes();
51-
long freeSpace = diskUsage.getFreeBytes();
52-
long freeSpaceHighThreshold = calculateFreeSpaceHighThreshold(totalBytes);
53-
54-
return freeSpace < freeSpaceHighThreshold;
41+
return isNodeExceedingWatermark(diskUsage, this::getFreeSpaceHighThreshold);
5542
}
5643

5744
@Override
5845
public boolean isNodeExceedingFloodStageWatermark(DiskUsage diskUsage) {
46+
return isNodeExceedingWatermark(diskUsage, this::getFreeSpaceFloodStageThreshold);
47+
}
48+
49+
private boolean isNodeExceedingWatermark(DiskUsage diskUsage, Function<Long, Long> thresholdFunction) {
5950
if (dataToFileCacheSizeRatioSupplier.get() <= 0) {
6051
return false;
6152
}
6253
long totalBytes = diskUsage.getTotalBytes();
6354
long freeSpace = diskUsage.getFreeBytes();
64-
long freeSpaceFloodStageThreshold = calculateFreeSpaceFloodStageThreshold(totalBytes);
55+
long freeSpaceThreshold = thresholdFunction.apply(totalBytes);
6556

66-
return freeSpace < freeSpaceFloodStageThreshold;
57+
return freeSpace < freeSpaceThreshold;
6758
}
6859

6960
@Override
70-
public long calculateFreeSpaceLowThreshold(long totalAddressableSpace) {
71-
// Check for percentage-based threshold
72-
double percentageThreshold = diskThresholdSettings.getFreeDiskThresholdLow();
73-
if (percentageThreshold > 0) {
74-
return (long) (totalAddressableSpace * percentageThreshold / 100.0);
75-
}
76-
77-
// Check for absolute bytes threshold
78-
final double dataToFileCacheSizeRatio = dataToFileCacheSizeRatioSupplier.get();
79-
ByteSizeValue bytesThreshold = diskThresholdSettings.getFreeBytesThresholdLow();
80-
if (bytesThreshold != null && bytesThreshold.getBytes() > 0) {
81-
return bytesThreshold.getBytes() * (long) dataToFileCacheSizeRatio;
82-
}
83-
84-
// Default fallback
85-
return 0;
61+
public long getFreeSpaceLowThreshold(long totalAddressableSpace) {
62+
return calculateFreeSpaceWatermarkThreshold(
63+
diskThresholdSettings.getFreeDiskThresholdLow(),
64+
diskThresholdSettings.getFreeBytesThresholdLow(),
65+
totalAddressableSpace
66+
);
8667
}
8768

8869
@Override
89-
public long calculateFreeSpaceHighThreshold(long totalAddressableSpace) {
90-
// Check for percentage-based threshold
91-
double percentageThreshold = diskThresholdSettings.getFreeDiskThresholdHigh();
92-
if (percentageThreshold > 0) {
93-
return (long) (totalAddressableSpace * percentageThreshold / 100.0);
94-
}
95-
96-
// Check for absolute bytes threshold
97-
final double dataToFileCacheSizeRatio = dataToFileCacheSizeRatioSupplier.get();
98-
ByteSizeValue bytesThreshold = diskThresholdSettings.getFreeBytesThresholdHigh();
99-
if (bytesThreshold != null && bytesThreshold.getBytes() > 0) {
100-
return bytesThreshold.getBytes() * (long) dataToFileCacheSizeRatio;
101-
}
102-
103-
// Default fallback
104-
return 0;
70+
public long getFreeSpaceHighThreshold(long totalAddressableSpace) {
71+
return calculateFreeSpaceWatermarkThreshold(
72+
diskThresholdSettings.getFreeDiskThresholdHigh(),
73+
diskThresholdSettings.getFreeBytesThresholdHigh(),
74+
totalAddressableSpace
75+
);
10576
}
10677

10778
@Override
108-
public long calculateFreeSpaceFloodStageThreshold(long totalAddressableSpace) {
79+
public long getFreeSpaceFloodStageThreshold(long totalAddressableSpace) {
80+
return calculateFreeSpaceWatermarkThreshold(
81+
diskThresholdSettings.getFreeDiskThresholdFloodStage(),
82+
diskThresholdSettings.getFreeBytesThresholdFloodStage(),
83+
totalAddressableSpace
84+
);
85+
}
86+
87+
private long calculateFreeSpaceWatermarkThreshold(
88+
double freeDiskWatermarkThreshold,
89+
ByteSizeValue freeBytesWatermarkThreshold,
90+
long totalAddressableSpace
91+
) {
10992
// Check for percentage-based threshold
110-
double percentageThreshold = diskThresholdSettings.getFreeDiskThresholdFloodStage();
111-
if (percentageThreshold > 0) {
112-
return (long) (totalAddressableSpace * percentageThreshold / 100.0);
93+
if (freeDiskWatermarkThreshold > 0) {
94+
return (long) (totalAddressableSpace * freeDiskWatermarkThreshold / 100.0);
11395
}
11496

11597
// Check for absolute bytes threshold
11698
final double dataToFileCacheSizeRatio = dataToFileCacheSizeRatioSupplier.get();
117-
ByteSizeValue bytesThreshold = diskThresholdSettings.getFreeBytesThresholdFloodStage();
118-
if (bytesThreshold != null && bytesThreshold.getBytes() > 0) {
119-
return bytesThreshold.getBytes() * (long) dataToFileCacheSizeRatio;
99+
if (freeBytesWatermarkThreshold != null && freeBytesWatermarkThreshold.getBytes() > 0) {
100+
return freeBytesWatermarkThreshold.getBytes() * (long) dataToFileCacheSizeRatio;
120101
}
121102

122103
// Default fallback

‎server/src/main/java/org/opensearch/cluster/routing/allocation/decider/WarmDiskThresholdDecider.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
129129
final long currentNodeRemoteShardSize = calculateCurrentNodeRemoteShardSize(node, allocation, false);
130130
final long freeSpace = totalAddressableSpace - currentNodeRemoteShardSize;
131131
final long freeSpaceAfterAllocation = freeSpace > shardSize ? freeSpace - shardSize : 0;
132-
final long freeSpaceLowThreshold = diskThresholdEvaluator.calculateFreeSpaceLowThreshold(totalAddressableSpace);
132+
final long freeSpaceLowThreshold = diskThresholdEvaluator.getFreeSpaceLowThreshold(totalAddressableSpace);
133133

134134
final ByteSizeValue freeSpaceLowThresholdInByteSize = new ByteSizeValue(freeSpaceLowThreshold);
135135
final ByteSizeValue freeSpaceInByteSize = new ByteSizeValue(freeSpace);
@@ -189,7 +189,7 @@ public Decision canRemain(ShardRouting shardRouting, RoutingNode node, RoutingAl
189189
final long currentNodeRemoteShardSize = calculateCurrentNodeRemoteShardSize(node, allocation, true);
190190
final long freeSpace = totalAddressableSpace - currentNodeRemoteShardSize;
191191

192-
final long freeSpaceHighThreshold = diskThresholdEvaluator.calculateFreeSpaceHighThreshold(totalAddressableSpace);
192+
final long freeSpaceHighThreshold = diskThresholdEvaluator.getFreeSpaceHighThreshold(totalAddressableSpace);
193193

194194
final ByteSizeValue freeSpaceHighThresholdInByteSize = new ByteSizeValue(freeSpaceHighThreshold);
195195
final ByteSizeValue freeSpaceInByteSize = new ByteSizeValue(freeSpace);

0 commit comments

Comments
 (0)