diff --git a/CHANGELOG.md b/CHANGELOG.md index ae440d2b6214b..22219cda53f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Add dfs transformation function in XContentMapValues ([#17612](https://github.com/opensearch-project/OpenSearch/pull/17612)) - Added Kinesis support as a plugin for the pull-based ingestion ([#17615](https://github.com/opensearch-project/OpenSearch/pull/17615)) - Add FilterFieldType for developers who want to wrap MappedFieldType ([#17627](https://github.com/opensearch-project/OpenSearch/pull/17627)) +- [Rule Based Auto-tagging] Add in-memory rule processing service ([#17365](https://github.com/opensearch-project/OpenSearch/pull/17365)) - [Security Manager Replacement] Create initial Java Agent to intercept Socket::connect calls ([#17724](https://github.com/opensearch-project/OpenSearch/pull/17724)) - Add ingestion management APIs for pause, resume and get ingestion state ([#17631](https://github.com/opensearch-project/OpenSearch/pull/17631)) - [Security Manager Replacement] Enhance Java Agent to intercept System::exit ([#17746](https://github.com/opensearch-project/OpenSearch/pull/17746)) diff --git a/libs/autotagging-commons/build.gradle b/libs/autotagging-commons/build.gradle new file mode 100644 index 0000000000000..cf3a75440c299 --- /dev/null +++ b/libs/autotagging-commons/build.gradle @@ -0,0 +1,20 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +dependencies { + api 'org.apache.commons:commons-collections4:4.4' + api project(":server") + + testImplementation(project(":test:framework")) { + exclude group: 'org.opensearch', module: 'opensearch-core' + } +} + +tasks.named("dependencyLicenses").configure { + mapping from: /commons-collections.*/, to: 'commons-collections' +} diff --git a/plugins/workload-management/licenses/commons-collections-LICENSE.txt b/libs/autotagging-commons/licenses/commons-collections-LICENSE.txt similarity index 100% rename from plugins/workload-management/licenses/commons-collections-LICENSE.txt rename to libs/autotagging-commons/licenses/commons-collections-LICENSE.txt diff --git a/plugins/workload-management/licenses/commons-collections-NOTICE.txt b/libs/autotagging-commons/licenses/commons-collections-NOTICE.txt similarity index 100% rename from plugins/workload-management/licenses/commons-collections-NOTICE.txt rename to libs/autotagging-commons/licenses/commons-collections-NOTICE.txt diff --git a/plugins/workload-management/licenses/commons-collections4-4.4.jar.sha1 b/libs/autotagging-commons/licenses/commons-collections4-4.4.jar.sha1 similarity index 100% rename from plugins/workload-management/licenses/commons-collections4-4.4.jar.sha1 rename to libs/autotagging-commons/licenses/commons-collections4-4.4.jar.sha1 diff --git a/libs/autotagging-commons/src/main/java/org/opensearch/rule/InMemoryRuleProcessingService.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/InMemoryRuleProcessingService.java new file mode 100644 index 0000000000000..219f6fa5e1999 --- /dev/null +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/InMemoryRuleProcessingService.java @@ -0,0 +1,115 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rule; + +import org.opensearch.autotagging.Attribute; +import org.opensearch.autotagging.FeatureType; +import org.opensearch.autotagging.Rule; +import org.opensearch.rule.attribute_extractor.AttributeExtractor; +import org.opensearch.rule.storage.AttributeValueStore; +import org.opensearch.rule.storage.AttributeValueStoreFactory; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +/** + * This class is responsible for managing in-memory view of Rules and Find matching Rule for the request + * Each auto-tagging feature should use a separate instance of this class as this avoid potential concurrency overhead + * in case of dynamic updates and attribute sharing scenarios + */ +public class InMemoryRuleProcessingService { + + private final AttributeValueStoreFactory attributeValueStoreFactory; + + /** + * Constrcutor + * @param featureType + * @param attributeValueStoreSupplier + */ + public InMemoryRuleProcessingService( + FeatureType featureType, + Supplier> attributeValueStoreSupplier + ) { + attributeValueStoreFactory = new AttributeValueStoreFactory(featureType, attributeValueStoreSupplier); + } + + /** + * Adds the rule to in-memory view + * @param rule to be added + */ + public void add(final Rule rule) { + perform(rule, this::addOperation); + } + + /** + * Removes the rule from in-memory view + * @param rule to be removed + */ + public void remove(final Rule rule) { + perform(rule, this::removeOperation); + } + + private void perform(Rule rule, BiConsumer>, Rule> ruleOperation) { + for (Map.Entry> attributeEntry : rule.getAttributeMap().entrySet()) { + ruleOperation.accept(attributeEntry, rule); + } + } + + private void removeOperation(Map.Entry> attributeEntry, Rule rule) { + AttributeValueStore valueStore = attributeValueStoreFactory.getAttributeValueStore(attributeEntry.getKey()); + for (String value : attributeEntry.getValue()) { + valueStore.remove(value); + } + } + + private void addOperation(Map.Entry> attributeEntry, Rule rule) { + AttributeValueStore valueStore = attributeValueStoreFactory.getAttributeValueStore(attributeEntry.getKey()); + for (String value : attributeEntry.getValue()) { + valueStore.put(value, rule.getFeatureValue()); + } + } + + /** + * Evaluates the label for the current request. It finds the matches for each attribute value and then it is an + * intersection of all the matches + * @param attributeExtractors list of extractors which are used to get the attribute values to find the + * matching rule + * @return a label if there is unique label otherwise empty + */ + public Optional evaluateLabel(List> attributeExtractors) { + assert attributeValueStoreFactory != null; + Optional result = Optional.empty(); + for (AttributeExtractor attributeExtractor : attributeExtractors) { + AttributeValueStore valueStore = attributeValueStoreFactory.getAttributeValueStore( + attributeExtractor.getAttribute() + ); + for (String value : attributeExtractor.extract()) { + Optional possibleMatch = valueStore.get(value); + + if (possibleMatch.isEmpty()) { + return Optional.empty(); + } + + if (result.isEmpty()) { + result = possibleMatch; + } else { + boolean isThePossibleMatchEqualResult = possibleMatch.get().equals(result.get()); + if (!isThePossibleMatchEqualResult) { + return Optional.empty(); + } + } + } + } + return result; + } +} diff --git a/libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/AttributeExtractor.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/AttributeExtractor.java new file mode 100644 index 0000000000000..3e13ea54fad34 --- /dev/null +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/AttributeExtractor.java @@ -0,0 +1,29 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rule.attribute_extractor; + +import org.opensearch.autotagging.Attribute; + +/** + * This interface defines the contract for extracting the attributes for Rule based auto-tagging feature + * @param + */ +public interface AttributeExtractor { + /** + * This method returns the Attribute which it is responsible for extracting + * @return attribute + */ + Attribute getAttribute(); + + /** + * This method returns the attribute values in context of the current request + * @return attribute value + */ + Iterable extract(); +} diff --git a/libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/package-info.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/package-info.java new file mode 100644 index 0000000000000..19a06fcba8f59 --- /dev/null +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/attribute_extractor/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains feature attribute extractor interface and its implementations + */ +package org.opensearch.rule.attribute_extractor; diff --git a/libs/autotagging-commons/src/main/java/org/opensearch/rule/package-info.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/package-info.java new file mode 100644 index 0000000000000..12341deae29e7 --- /dev/null +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * Rule based auto-tagging generic constructs + */ +package org.opensearch.rule; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/AttributeValueStore.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStore.java similarity index 95% rename from plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/AttributeValueStore.java rename to libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStore.java index eb2ce8e4764ea..98e9cc4041318 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/AttributeValueStore.java +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStore.java @@ -6,7 +6,7 @@ * compatible open source license. */ -package org.opensearch.plugin.wlm.rule.storage; +package org.opensearch.rule.storage; import java.util.Optional; diff --git a/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStoreFactory.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStoreFactory.java new file mode 100644 index 0000000000000..8cda4bd26fdf0 --- /dev/null +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/AttributeValueStoreFactory.java @@ -0,0 +1,52 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rule.storage; + +import org.opensearch.autotagging.Attribute; +import org.opensearch.autotagging.FeatureType; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +/** + * Factory class for AttributeValueStore per feature type as two feature types can potentially share same attribute + */ +public class AttributeValueStoreFactory { + private final Map> attributeValueStores = new HashMap<>(); + + /** + * Constructor + * @param featureType is the feature which are using rule based auto tagging + * @param attributeValueStoreSupplier supplies the feature level AttributeValueStore instance + */ + public AttributeValueStoreFactory(FeatureType featureType, Supplier> attributeValueStoreSupplier) { + for (Attribute attribute : featureType.getAllowedAttributesRegistry().values()) { + attributeValueStores.put(attribute.getName(), attributeValueStoreSupplier.get()); + } + } + + /** + * Factory method which returns the {@link AttributeValueStore} for the given attribute + * @param attribute + * @return + */ + public AttributeValueStore getAttributeValueStore(final Attribute attribute) { + final String attributeName = attribute.getName(); + if (attributeValueStores == null) { + throw new IllegalStateException("AttributeValueStoreFactory is not initialized yet."); + } + + if (!attributeValueStores.containsKey(attributeName)) { + throw new IllegalArgumentException("[" + attributeName + "] is not a valid attribute for enabled features."); + } + + return attributeValueStores.get(attributeName); + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/DefaultAttributeValueStore.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/DefaultAttributeValueStore.java similarity index 55% rename from plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/DefaultAttributeValueStore.java rename to libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/DefaultAttributeValueStore.java index 8b4c063f7ad1a..9f0584b276d11 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/DefaultAttributeValueStore.java +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/DefaultAttributeValueStore.java @@ -6,12 +6,13 @@ * compatible open source license. */ -package org.opensearch.plugin.wlm.rule.storage; +package org.opensearch.rule.storage; import org.apache.commons.collections4.trie.PatriciaTrie; import java.util.Map; import java.util.Optional; +import java.util.concurrent.locks.ReentrantReadWriteLock; /** * This is a patricia trie based implementation of AttributeValueStore @@ -20,7 +21,10 @@ * ref: https://commons.apache.org/proper/commons-collections/javadocs/api-4.4/org/apache/commons/collections4/trie/PatriciaTrie.html */ public class DefaultAttributeValueStore implements AttributeValueStore { - PatriciaTrie trie; + private final PatriciaTrie trie; + private static final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private static final ReentrantReadWriteLock.ReadLock readLock = lock.readLock(); + private static final ReentrantReadWriteLock.WriteLock writeLock = lock.writeLock(); /** * Default constructor @@ -39,34 +43,48 @@ public DefaultAttributeValueStore(PatriciaTrie trie) { @Override public void put(K key, V value) { - trie.put(key, value); + writeLock.lock(); + try { + trie.put(key, value); + } finally { + writeLock.unlock(); + } } @Override public void remove(String key) { - trie.remove(key); + writeLock.lock(); + try { + trie.remove(key); + } finally { + writeLock.unlock(); + } } @Override public Optional get(String key) { - /** - * Since we are inserting prefixes into the trie and searching for larger strings - * It is important to find the largest matching prefix key in the trie efficiently - * Hence we can do binary search - */ - final String longestMatchingPrefix = findLongestMatchingPrefix(key); + readLock.lock(); + try { + /** + * Since we are inserting prefixes into the trie and searching for larger strings + * It is important to find the largest matching prefix key in the trie efficiently + * Hence we can do binary search + */ + final String longestMatchingPrefix = findLongestMatchingPrefix(key); - /** - * Now there are following cases for this prefix - * 1. There is a Rule which has this prefix as one of the attribute values. In this case we should return the - * Rule's label otherwise send empty - */ - for (Map.Entry possibleMatch : trie.prefixMap(longestMatchingPrefix).entrySet()) { - if (key.startsWith(possibleMatch.getKey())) { - return Optional.of(possibleMatch.getValue()); + /** + * Now there are following cases for this prefix + * 1. There is a Rule which has this prefix as one of the attribute values. In this case we should return the + * Rule's label otherwise send empty + */ + for (Map.Entry possibleMatch : trie.prefixMap(longestMatchingPrefix).entrySet()) { + if (key.startsWith(possibleMatch.getKey())) { + return Optional.of(possibleMatch.getValue()); + } } + } finally { + readLock.unlock(); } - return Optional.empty(); } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/package-info.java b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/package-info.java similarity index 86% rename from plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/package-info.java rename to libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/package-info.java index 6aa721ce22a00..b7aeb8d6756ab 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/storage/package-info.java +++ b/libs/autotagging-commons/src/main/java/org/opensearch/rule/storage/package-info.java @@ -9,4 +9,4 @@ /** * This package contains interfaces and implementations for in memory rule storage mechanisms */ -package org.opensearch.plugin.wlm.rule.storage; +package org.opensearch.rule.storage; diff --git a/libs/autotagging-commons/src/test/java/org/opensearch/rule/InMemoryRuleProcessingServiceTests.java b/libs/autotagging-commons/src/test/java/org/opensearch/rule/InMemoryRuleProcessingServiceTests.java new file mode 100644 index 0000000000000..d12900a79b121 --- /dev/null +++ b/libs/autotagging-commons/src/test/java/org/opensearch/rule/InMemoryRuleProcessingServiceTests.java @@ -0,0 +1,162 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rule; + +import org.opensearch.autotagging.Attribute; +import org.opensearch.autotagging.FeatureType; +import org.opensearch.autotagging.Rule; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.rule.attribute_extractor.AttributeExtractor; +import org.opensearch.rule.storage.DefaultAttributeValueStore; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class InMemoryRuleProcessingServiceTests extends OpenSearchTestCase { + InMemoryRuleProcessingService sut; + + public void setUp() throws Exception { + super.setUp(); + sut = new InMemoryRuleProcessingService(WLMFeatureType.WLM, DefaultAttributeValueStore::new); + } + + public void testAdd() { + sut.add(getRule(Set.of("test", "change"), "test_id")); + + List> extractors = getAttributeExtractors(List.of("test")); + Optional label = sut.evaluateLabel(extractors); + assertTrue(label.isPresent()); + assertEquals("test_id", label.get()); + } + + public void testRemove() { + Rule rule = getRule(Set.of("test", "change"), "test_id"); + sut.add(rule); + sut.remove(rule); + + List> extractors = getAttributeExtractors(List.of("test")); + Optional label = sut.evaluateLabel(extractors); + assertFalse(label.isPresent()); + } + + public void testEvaluateLabelForExactMatch() { + sut.add(getRule(Set.of("test1", "change"), "test_id")); + sut.add(getRule(Set.of("test", "double"), "test_id1")); + + List> extractors = getAttributeExtractors(List.of("test")); + Optional label = sut.evaluateLabel(extractors); + assertTrue(label.isPresent()); + assertEquals("test_id1", label.get()); + } + + public void testEvaluateLabelForMultiMatch() { + sut.add(getRule(Set.of("key1", "change"), "test_id")); + sut.add(getRule(Set.of("key2", "double"), "new_id")); + + List> extractors = getAttributeExtractors(List.of("key1", "key2")); + Optional label = sut.evaluateLabel(extractors); + assertFalse(label.isPresent()); + } + + public void testEvaluateLabelForNoMatch() { + sut.add(getRule(Set.of("test1", "change"), "test_id")); + sut.add(getRule(Set.of("test", "double"), "test_id")); + + List> extractors = getAttributeExtractors(List.of("dummy_test")); + Optional label = sut.evaluateLabel(extractors); + assertFalse(label.isPresent()); + } + + public void testEvaluateLabelForExactMatchWithLongestMatchingPrefixCase() { + sut.add(getRule(Set.of("test1", "change"), "test_id")); + sut.add(getRule(Set.of("test", "double"), "test_id1")); + + List> extractors = getAttributeExtractors(List.of("testing")); + Optional label = sut.evaluateLabel(extractors); + assertTrue(label.isPresent()); + assertEquals("test_id1", label.get()); + } + + public void testEvaluateLabelForNoMatchWithLongestMatchingPrefixCase() { + sut.add(getRule(Set.of("key1", "change"), "test_id")); + sut.add(getRule(Set.of("key12", "double"), "test_id1")); + + List> extractors = getAttributeExtractors(List.of("key")); + Optional label = sut.evaluateLabel(extractors); + assertFalse(label.isPresent()); + } + + private static Rule getRule(Set attributeValues, String label) { + return new Rule( + "test description", + Map.of(TestAttribute.TEST_ATTRIBUTE, attributeValues), + WLMFeatureType.WLM, + label, + "2025-02-24T07:42:10.123456Z" + ); + } + + private static List> getAttributeExtractors(List extractedAttributes) { + List> extractors = List.of(new AttributeExtractor() { + @Override + public Attribute getAttribute() { + return TestAttribute.TEST_ATTRIBUTE; + } + + @Override + public Iterable extract() { + return extractedAttributes; + } + }); + return extractors; + } + + public enum WLMFeatureType implements FeatureType { + WLM; + + @Override + public String getName() { + return ""; + } + + @Override + public Map getAllowedAttributesRegistry() { + return Map.of("test_attribute", TestAttribute.TEST_ATTRIBUTE); + } + + @Override + public void registerFeatureType() {} + } + + public enum TestAttribute implements Attribute { + TEST_ATTRIBUTE("test_attribute"), + INVALID_ATTRIBUTE("invalid_attribute"); + + private final String name; + + TestAttribute(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + @Override + public void validateAttribute() {} + + @Override + public void writeTo(StreamOutput out) throws IOException {} + } +} diff --git a/libs/autotagging-commons/src/test/java/org/opensearch/rule/storage/AttributeValueStoreFactoryTests.java b/libs/autotagging-commons/src/test/java/org/opensearch/rule/storage/AttributeValueStoreFactoryTests.java new file mode 100644 index 0000000000000..5cdc128c50620 --- /dev/null +++ b/libs/autotagging-commons/src/test/java/org/opensearch/rule/storage/AttributeValueStoreFactoryTests.java @@ -0,0 +1,38 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rule.storage; + +import org.opensearch.autotagging.Attribute; +import org.opensearch.rule.InMemoryRuleProcessingServiceTests.TestAttribute; +import org.opensearch.rule.InMemoryRuleProcessingServiceTests.WLMFeatureType; +import org.opensearch.test.OpenSearchTestCase; + +public class AttributeValueStoreFactoryTests extends OpenSearchTestCase { + AttributeValueStoreFactory sut; + + @Override + public void setUp() throws Exception { + super.setUp(); + sut = new AttributeValueStoreFactory(WLMFeatureType.WLM, DefaultAttributeValueStore::new); + } + + public void testFeatureLevelStoreInitialisation() { + for (Attribute attribute : WLMFeatureType.WLM.getAllowedAttributesRegistry().values()) { + assertTrue(sut.getAttributeValueStore(attribute) instanceof DefaultAttributeValueStore); + } + } + + public void testValidGetAttributeValueStore() { + assertTrue(sut.getAttributeValueStore(TestAttribute.TEST_ATTRIBUTE) instanceof DefaultAttributeValueStore); + } + + public void testInValidGetAttributeValueStore() { + assertThrows(IllegalArgumentException.class, () -> { sut.getAttributeValueStore(TestAttribute.INVALID_ATTRIBUTE); }); + } +} diff --git a/libs/autotagging-commons/src/test/java/org/opensearch/rule/storage/AttributeValueStoreTests.java b/libs/autotagging-commons/src/test/java/org/opensearch/rule/storage/AttributeValueStoreTests.java new file mode 100644 index 0000000000000..2340cc3327337 --- /dev/null +++ b/libs/autotagging-commons/src/test/java/org/opensearch/rule/storage/AttributeValueStoreTests.java @@ -0,0 +1,126 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.rule.storage; + +import org.apache.commons.collections4.trie.PatriciaTrie; +import org.opensearch.test.OpenSearchTestCase; + +import java.util.ArrayList; +import java.util.List; + +public class AttributeValueStoreTests extends OpenSearchTestCase { + + AttributeValueStore subjectUnderTest; + final static String ALPHA_NUMERIC = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; + + public void setUp() throws Exception { + super.setUp(); + subjectUnderTest = new DefaultAttributeValueStore<>(new PatriciaTrie<>()); + } + + public void testPut() { + subjectUnderTest.put("foo", "bar"); + assertEquals("bar", subjectUnderTest.get("foo").get()); + } + + public void testRemove() { + subjectUnderTest.put("foo", "bar"); + subjectUnderTest.remove("foo"); + assertEquals(0, subjectUnderTest.size()); + } + + public void tesGet() { + subjectUnderTest.put("foo", "bar"); + assertEquals("bar", subjectUnderTest.get("foo").get()); + } + + public void testGetWhenNoProperPrefixIsPresent() { + subjectUnderTest.put("foo", "bar"); + subjectUnderTest.put("foodip", "sing"); + assertTrue(subjectUnderTest.get("foxtail").isEmpty()); + subjectUnderTest.put("fox", "lucy"); + + assertFalse(subjectUnderTest.get("foxtail").isEmpty()); + } + + public void testClear() { + subjectUnderTest.put("foo", "bar"); + subjectUnderTest.clear(); + assertEquals(0, subjectUnderTest.size()); + } + + public void testConcurrentUpdatesAndReads() { + final List randomStrings = new ArrayList<>(); + for (int i = 0; i < 100; i++) { + randomStrings.add(generateRandom(20)); + } + List readerThreads = new ArrayList<>(); + List writerThreads = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + readerThreads.add(new AttributeValueStoreReader(subjectUnderTest, randomStrings)); + writerThreads.add(new AttributeValueStoreWriter(subjectUnderTest, randomStrings)); + } + + for (int ii = 0; ii < 10; ii++) { + readerThreads.get(ii).start(); + writerThreads.get(ii).start(); + } + } + + public static String generateRandom(int maxLength) { + int length = random().nextInt(maxLength) + 1; // +1 to avoid length 0 + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + sb.append(ALPHA_NUMERIC.charAt(random().nextInt(ALPHA_NUMERIC.length()))); + } + return sb.toString(); + } + + private static class AttributeValueStoreReader extends Thread { + private final AttributeValueStore subjectUnderTest; + private final List toReadKeys; + + public AttributeValueStoreReader(AttributeValueStore subjectUnderTest, List toReadKeys) { + super(); + this.subjectUnderTest = subjectUnderTest; + this.toReadKeys = toReadKeys; + } + + @Override + public void run() { + try { + Thread.sleep(random().nextInt(100)); + for (String key : toReadKeys) { + subjectUnderTest.get(key); + } + } catch (InterruptedException e) {} + } + } + + private static class AttributeValueStoreWriter extends Thread { + private final AttributeValueStore subjectUnderTest; + private final List toWriteKeys; + + public AttributeValueStoreWriter(AttributeValueStore subjectUnderTest, List toWriteKeys) { + super(); + this.subjectUnderTest = subjectUnderTest; + this.toWriteKeys = toWriteKeys; + } + + @Override + public void run() { + try { + Thread.sleep(random().nextInt(100)); + for (String key : toWriteKeys) { + subjectUnderTest.put(key, key); + } + } catch (InterruptedException e) {} + } + } +} diff --git a/plugins/workload-management/build.gradle b/plugins/workload-management/build.gradle index c73c63e84ed1f..5396a74361b77 100644 --- a/plugins/workload-management/build.gradle +++ b/plugins/workload-management/build.gradle @@ -19,9 +19,5 @@ opensearchplugin { } dependencies { - api 'org.apache.commons:commons-collections4:4.4' -} - -tasks.named("dependencyLicenses").configure { - mapping from: /commons-collections.*/, to: 'commons-collections' + api project(":libs:opensearch-autotagging-commons") } diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/attribute_extractor/IndicesExtractor.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/attribute_extractor/IndicesExtractor.java new file mode 100644 index 0000000000000..a3230ac919eb1 --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/attribute_extractor/IndicesExtractor.java @@ -0,0 +1,41 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.plugin.wlm.rule.attribute_extractor; + +import org.opensearch.action.IndicesRequest; +import org.opensearch.autotagging.Attribute; +import org.opensearch.rule.attribute_extractor.AttributeExtractor; + +import java.util.List; + +/** + * This class extracts the indices from a request + */ +public class IndicesExtractor implements AttributeExtractor { + private final IndicesRequest indicesRequest; + + /** + * Default constructor + * @param indicesRequest + */ + public IndicesExtractor(IndicesRequest indicesRequest) { + this.indicesRequest = indicesRequest; + } + + @Override + public Attribute getAttribute() { + // TODO: this will be replaced by WLM defined index_pattern attribute + return null; + } + + @Override + public Iterable extract() { + return List.of(indicesRequest.indices()); + } +} diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/attribute_extractor/package-info.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/attribute_extractor/package-info.java new file mode 100644 index 0000000000000..ca1cc902b6ca1 --- /dev/null +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/attribute_extractor/package-info.java @@ -0,0 +1,12 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/** + * This package contains feature attribute extractor interface and its implementations + */ +package org.opensearch.plugin.wlm.rule.attribute_extractor; diff --git a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/package-info.java b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/package-info.java index 85c0562dae5ee..d6a196cca4672 100644 --- a/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/package-info.java +++ b/plugins/workload-management/src/main/java/org/opensearch/plugin/wlm/rule/package-info.java @@ -6,8 +6,8 @@ * compatible open source license. */ -package org.opensearch.plugin.wlm.rule; /** * This package holds constructs for the Rule's in-memory storage, processing and syncing the in-memory view * with the index view */ +package org.opensearch.plugin.wlm.rule; diff --git a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rule/storage/AttributeValueStoreTests.java b/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rule/storage/AttributeValueStoreTests.java deleted file mode 100644 index 29c42e51efeb0..0000000000000 --- a/plugins/workload-management/src/test/java/org/opensearch/plugin/wlm/rule/storage/AttributeValueStoreTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.plugin.wlm.rule.storage; - -import org.apache.commons.collections4.trie.PatriciaTrie; -import org.opensearch.test.OpenSearchTestCase; - -public class AttributeValueStoreTests extends OpenSearchTestCase { - - AttributeValueStore subjectUnderTest; - - public void setUp() throws Exception { - super.setUp(); - subjectUnderTest = new DefaultAttributeValueStore<>(new PatriciaTrie<>()); - } - - public void testPut() { - subjectUnderTest.put("foo", "bar"); - assertEquals("bar", subjectUnderTest.get("foo").get()); - } - - public void testRemove() { - subjectUnderTest.put("foo", "bar"); - subjectUnderTest.remove("foo"); - assertEquals(0, subjectUnderTest.size()); - } - - public void tesGet() { - subjectUnderTest.put("foo", "bar"); - assertEquals("bar", subjectUnderTest.get("foo").get()); - } - - public void testGetWhenNoProperPrefixIsPresent() { - subjectUnderTest.put("foo", "bar"); - subjectUnderTest.put("foodip", "sing"); - assertTrue(subjectUnderTest.get("foxtail").isEmpty()); - subjectUnderTest.put("fox", "lucy"); - - assertFalse(subjectUnderTest.get("foxtail").isEmpty()); - } - - public void testClear() { - subjectUnderTest.put("foo", "bar"); - subjectUnderTest.clear(); - assertEquals(0, subjectUnderTest.size()); - } -}