[Rule based auto-tagging] Add rule sync service (opensearch-project#18128)

---------

Signed-off-by: Kaushal Kumar <[email protected]>

Author: kaushalmahi12

CHANGELOG.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update API of Message in index to add the timestamp for lag calculation in ingestion polling ([#17977](https://github.com/opensearch-project/OpenSearch/pull/17977/))
 - Add Warm Disk Threshold Allocation Decider for Warm shards ([#18082](https://github.com/opensearch-project/OpenSearch/pull/18082))
 - Add composite directory factory ([#17988](https://github.com/opensearch-project/OpenSearch/pull/17988))
+- [Rule based auto-tagging] Add refresh based synchronization service for `Rule`s ([#18128](https://github.com/opensearch-project/OpenSearch/pull/18128))
 - Add pull-based ingestion error metrics and make internal queue size configurable ([#18088](https://github.com/opensearch-project/OpenSearch/pull/18088))
 - Adding support for derive source feature and implementing it for various type of field mappers ([#17759](https://github.com/opensearch-project/OpenSearch/pull/17759))
 - [Security Manager Replacement] Enhance Java Agent to intercept newByteChannel ([#17989](https://github.com/opensearch-project/OpenSearch/pull/17989))

modules/autotagging-commons/common/src/main/java/org/opensearch/rule/autotagging/Rule.java

@@ -38,6 +38,7 @@
  * @opensearch.experimental
  */
 public class Rule implements Writeable, ToXContentObject {
+    private final String id;
     private final String description;
     private final FeatureType featureType;
     private final Map<Attribute, Set<String>> attributeMap;
@@ -59,19 +60,22 @@ public class Rule implements Writeable, ToXContentObject {
 
     /**
      * Main constructor
+     * @param id
      * @param description
      * @param attributeMap
      * @param featureType
      * @param featureValue
      * @param updatedAt
      */
     public Rule(
+        String id,
         String description,
         Map<Attribute, Set<String>> attributeMap,
         FeatureType featureType,
         String featureValue,
         String updatedAt
     ) {
+        this.id = id;
         this.description = description;
         this.featureType = featureType;
         this.attributeMap = attributeMap;
@@ -87,6 +91,7 @@ public Rule(
      * @throws IOException
      */
     public Rule(StreamInput in) throws IOException {
+        id = in.readString();
         description = in.readString();
         featureType = FeatureType.from(in);
         attributeMap = in.readMap(i -> Attribute.from(i, featureType), i -> new HashSet<>(i.readStringList()));
@@ -98,6 +103,7 @@ public Rule(StreamInput in) throws IOException {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(id);
         out.writeString(description);
         featureType.writeTo(out);
         out.writeMap(attributeMap, (output, attribute) -> attribute.writeTo(output), StreamOutput::writeStringCollection);
@@ -116,6 +122,14 @@ public static Rule fromXContent(final XContentParser parser, FeatureType feature
         return Builder.fromXContent(parser, featureType).build();
     }
 
+    /**
+     * id getter
+     * @return
+     */
+    public String getId() {
+        return id;
+    }
+
     /**
      * description getter
      * @return
@@ -159,10 +173,7 @@ public Map<Attribute, Set<String>> getAttributeMap() {
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
         builder.startObject();
-        String id = params.param(_ID_STRING);
-        if (id != null) {
-            builder.field(_ID_STRING, id);
-        }
+        builder.field(_ID_STRING, id);
         builder.field(DESCRIPTION_STRING, description);
         for (Map.Entry<Attribute, Set<String>> entry : attributeMap.entrySet()) {
             builder.array(entry.getKey().getName(), entry.getValue().toArray(new String[0]));
@@ -204,6 +215,7 @@ public static Builder builder() {
      * @opensearch.experimental
      */
     public static class Builder {
+        private String id;
         private String description;
         private Map<Attribute, Set<String>> attributeMap;
         private FeatureType featureType;
@@ -235,7 +247,9 @@ public static Builder fromXContent(XContentParser parser, FeatureType featureTyp
                 if (token == XContentParser.Token.FIELD_NAME) {
                     fieldName = parser.currentName();
                 } else if (token.isValue()) {
-                    if (fieldName.equals(DESCRIPTION_STRING)) {
+                    if (fieldName.equals(_ID_STRING)) {
+                        builder.id(parser.text());
+                    } else if (fieldName.equals(DESCRIPTION_STRING)) {
                         builder.description(parser.text());
                     } else if (fieldName.equals(UPDATED_AT_STRING)) {
                         builder.updatedAt(parser.text());
@@ -271,6 +285,16 @@ private static void fromXContentParseArray(
             attributeMap.put(attribute, attributeValueSet);
         }
 
+        /**
+         * Sets the id
+         * @param id
+         * @return
+         */
+        public Builder id(String id) {
+            this.id = id;
+            return this;
+        }
+
         /**
          * sets the description
          * @param description
@@ -326,7 +350,7 @@ public Builder updatedAt(String updatedAt) {
          * @return
          */
         public Rule build() {
-            return new Rule(description, attributeMap, featureType, featureValue, updatedAt);
+            return new Rule(id, description, attributeMap, featureType, featureValue, updatedAt);
         }
 
         /**

modules/autotagging-commons/common/src/main/java/org/opensearch/rule/storage/DefaultAttributeValueStore.java

@@ -109,7 +109,12 @@ private String findLongestMatchingPrefix(String key) {
 
     @Override
     public void clear() {
-        trie.clear();
+        writeLock.lock();
+        try {
+            trie.clear();
+        } finally {
+            writeLock.unlock();
+        }
     }
 
     @Override

modules/autotagging-commons/common/src/test/java/org/opensearch/rule/action/GetRuleRequestTests.java

@@ -11,15 +11,15 @@
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.rule.GetRuleRequest;
+import org.opensearch.rule.autotagging.Attribute;
+import org.opensearch.rule.autotagging.Rule;
 import org.opensearch.rule.utils.RuleTestUtils;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
 import java.util.HashMap;
-
-import static org.opensearch.rule.utils.RuleTestUtils.ATTRIBUTE_MAP;
-import static org.opensearch.rule.utils.RuleTestUtils.SEARCH_AFTER;
-import static org.opensearch.rule.utils.RuleTestUtils._ID_ONE;
+import java.util.Map;
+import java.util.Set;
 
 public class GetRuleRequestTests extends OpenSearchTestCase {
     /**
@@ -64,4 +64,65 @@ public void testValidate() {
         request = new GetRuleRequest(_ID_ONE, ATTRIBUTE_MAP, "", RuleTestUtils.MockRuleFeatureType.INSTANCE);
         assertThrows(IllegalArgumentException.class, request::validate);
     }
+
+    public static final String _ID_ONE = "id_1";
+    public static final String SEARCH_AFTER = "search_after";
+    public static final String _ID_TWO = "G5iIq84j7eK1qIAAAAIH53=1";
+    public static final String FEATURE_VALUE_ONE = "feature_value_one";
+    public static final String FEATURE_VALUE_TWO = "feature_value_two";
+    public static final String ATTRIBUTE_VALUE_ONE = "mock_attribute_one";
+    public static final String ATTRIBUTE_VALUE_TWO = "mock_attribute_two";
+    public static final String DESCRIPTION_ONE = "description_1";
+    public static final String DESCRIPTION_TWO = "description_2";
+    public static final String TIMESTAMP_ONE = "2024-01-26T08:58:57.558Z";
+    public static final String TIMESTAMP_TWO = "2023-01-26T08:58:57.558Z";
+    public static final Map<Attribute, Set<String>> ATTRIBUTE_MAP = Map.of(
+        RuleTestUtils.MockRuleAttributes.MOCK_RULE_ATTRIBUTE_ONE,
+        Set.of(ATTRIBUTE_VALUE_ONE)
+    );
+
+    public static final Rule ruleOne = Rule.builder()
+        .id(_ID_ONE)
+        .description(DESCRIPTION_ONE)
+        .featureType(RuleTestUtils.MockRuleFeatureType.INSTANCE)
+        .featureValue(FEATURE_VALUE_ONE)
+        .attributeMap(ATTRIBUTE_MAP)
+        .updatedAt(TIMESTAMP_ONE)
+        .build();
+
+    public static final Rule ruleTwo = Rule.builder()
+        .id(_ID_TWO)
+        .description(DESCRIPTION_TWO)
+        .featureType(RuleTestUtils.MockRuleFeatureType.INSTANCE)
+        .featureValue(FEATURE_VALUE_TWO)
+        .attributeMap(Map.of(RuleTestUtils.MockRuleAttributes.MOCK_RULE_ATTRIBUTE_TWO, Set.of(ATTRIBUTE_VALUE_TWO)))
+        .updatedAt(TIMESTAMP_TWO)
+        .build();
+
+    public static Map<String, Rule> ruleMap() {
+        return Map.of(_ID_ONE, ruleOne, _ID_TWO, ruleTwo);
+    }
+
+    public static void assertEqualRules(Map<String, Rule> mapOne, Map<String, Rule> mapTwo, boolean ruleUpdated) {
+        assertEquals(mapOne.size(), mapTwo.size());
+        for (Map.Entry<String, Rule> entry : mapOne.entrySet()) {
+            String id = entry.getKey();
+            assertTrue(mapTwo.containsKey(id));
+            Rule one = mapOne.get(id);
+            Rule two = mapTwo.get(id);
+            assertEqualRule(one, two, ruleUpdated);
+        }
+    }
+
+    public static void assertEqualRule(Rule one, Rule two, boolean ruleUpdated) {
+        if (ruleUpdated) {
+            assertEquals(one.getDescription(), two.getDescription());
+            assertEquals(one.getFeatureType(), two.getFeatureType());
+            assertEquals(one.getFeatureValue(), two.getFeatureValue());
+            assertEquals(one.getAttributeMap(), two.getAttributeMap());
+            assertEquals(one.getAttributeMap(), two.getAttributeMap());
+        } else {
+            assertEquals(one, two);
+        }
+    }
 }

modules/autotagging-commons/common/src/test/java/org/opensearch/rule/action/GetRuleResponseTests.java

@@ -41,6 +41,7 @@ public class GetRuleResponseTests extends OpenSearchTestCase {
     );
 
     public static final Rule ruleOne = Rule.builder()
+        .id(_ID_ONE)
         .description(DESCRIPTION_ONE)
         .featureType(RuleTestUtils.MockRuleFeatureType.INSTANCE)
         .featureValue(FEATURE_VALUE_ONE)

modules/autotagging-commons/common/src/test/java/org/opensearch/rule/autotagging/RuleTests.java

@@ -47,7 +47,7 @@ protected Rule createTestInstance() {
         String description = randomAlphaOfLength(10);
         String featureValue = randomAlphaOfLength(5);
         String updatedAt = Instant.now().toString();
-        return new Rule(description, ATTRIBUTE_MAP, FEATURE_TYPE, featureValue, updatedAt);
+        return new Rule("test_id", description, ATTRIBUTE_MAP, FEATURE_TYPE, featureValue, updatedAt);
     }
 
     @Override
@@ -123,6 +123,7 @@ static Rule buildRule(
         String description
     ) {
         return Rule.builder()
+            .id(_ID)
             .featureValue(featureValue)
             .featureType(featureType)
             .description(description)

modules/autotagging-commons/common/src/test/java/org/opensearch/rule/utils/RuleTestUtils.java

@@ -42,6 +42,7 @@ public class RuleTestUtils {
     );
 
     public static final Rule ruleOne = Rule.builder()
+        .id(_ID_ONE)
         .description(DESCRIPTION_ONE)
         .featureType(RuleTestUtils.MockRuleFeatureType.INSTANCE)
         .featureValue(FEATURE_VALUE_ONE)
@@ -50,6 +51,7 @@ public class RuleTestUtils {
         .build();
 
     public static final Rule ruleTwo = Rule.builder()
+        .id(_ID_TWO)
         .description(DESCRIPTION_TWO)
         .featureType(RuleTestUtils.MockRuleFeatureType.INSTANCE)
         .featureValue(FEATURE_VALUE_TWO)

modules/autotagging-commons/src/main/java/org/opensearch/rule/InMemoryRuleProcessingService.java

@@ -10,7 +10,6 @@
 
 import org.opensearch.rule.attribute_extractor.AttributeExtractor;
 import org.opensearch.rule.autotagging.Attribute;
-import org.opensearch.rule.autotagging.FeatureType;
 import org.opensearch.rule.autotagging.Rule;
 import org.opensearch.rule.storage.AttributeValueStore;
 import org.opensearch.rule.storage.AttributeValueStoreFactory;
@@ -20,7 +19,6 @@
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.BiConsumer;
-import java.util.function.Supplier;
 
 /**
  * This class is responsible for managing in-memory view of Rules and Find matching Rule for the request
@@ -29,18 +27,18 @@
  */
 public class InMemoryRuleProcessingService {
 
+    /**
+     * Wildcard character which will be removed as we only support prefix based search rather than pattern match based
+     */
+    public static final String WILDCARD = "*";
     private final AttributeValueStoreFactory attributeValueStoreFactory;
 
     /**
-     *  Constrcutor
-     * @param featureType
-     * @param attributeValueStoreSupplier
+     *  Constructor
+     * @param attributeValueStoreFactory
      */
-    public InMemoryRuleProcessingService(
-        FeatureType featureType,
-        Supplier<AttributeValueStore<String, String>> attributeValueStoreSupplier
-    ) {
-        attributeValueStoreFactory = new AttributeValueStoreFactory(featureType, attributeValueStoreSupplier);
+    public InMemoryRuleProcessingService(AttributeValueStoreFactory attributeValueStoreFactory) {
+        this.attributeValueStoreFactory = attributeValueStoreFactory;
     }
 
     /**
@@ -75,7 +73,7 @@ private void removeOperation(Map.Entry<Attribute, Set<String>> attributeEntry, R
     private void addOperation(Map.Entry<Attribute, Set<String>> attributeEntry, Rule rule) {
         AttributeValueStore<String, String> valueStore = attributeValueStoreFactory.getAttributeValueStore(attributeEntry.getKey());
         for (String value : attributeEntry.getValue()) {
-            valueStore.put(value, rule.getFeatureValue());
+            valueStore.put(value.replace(WILDCARD, ""), rule.getFeatureValue());
         }
     }
 

modules/autotagging-commons/src/test/java/org/opensearch/rule/InMemoryRuleProcessingServiceTests.java

@@ -13,6 +13,7 @@
 import org.opensearch.rule.autotagging.AutoTaggingRegistry;
 import org.opensearch.rule.autotagging.FeatureType;
 import org.opensearch.rule.autotagging.Rule;
+import org.opensearch.rule.storage.AttributeValueStoreFactory;
 import org.opensearch.rule.storage.DefaultAttributeValueStore;
 import org.opensearch.test.OpenSearchTestCase;
 
@@ -26,7 +27,11 @@ public class InMemoryRuleProcessingServiceTests extends OpenSearchTestCase {
 
     public void setUp() throws Exception {
         super.setUp();
-        sut = new InMemoryRuleProcessingService(WLMFeatureType.WLM, DefaultAttributeValueStore::new);
+        AttributeValueStoreFactory attributeValueStoreFactory = new AttributeValueStoreFactory(
+            WLMFeatureType.WLM,
+            DefaultAttributeValueStore::new
+        );
+        sut = new InMemoryRuleProcessingService(attributeValueStoreFactory);
     }
 
     public void testAdd() {
@@ -97,6 +102,7 @@ public void testEvaluateLabelForNoMatchWithLongestMatchingPrefixCase() {
 
     private static Rule getRule(Set<String> attributeValues, String label) {
         return new Rule(
+            randomAlphaOfLength(5),
             "test description",
             Map.of(TestAttribute.TEST_ATTRIBUTE, attributeValues),
             WLMFeatureType.WLM,

plugins/workload-management/build.gradle

@@ -8,12 +8,10 @@
  * Modifications Copyright OpenSearch Contributors. See
  * GitHub history for details.
  */
-
 apply plugin: 'opensearch.opensearchplugin'
 apply plugin: 'opensearch.yaml-rest-test'
 apply plugin: 'opensearch.java-rest-test'
 apply plugin: 'opensearch.internal-cluster-test'
-
 opensearchplugin {
   description = 'OpenSearch Workload Management Plugin.'
   classname = 'org.opensearch.plugin.wlm.WorkloadManagementPlugin'