[Extensions] REST API to initialize an extension and dynamically load it (opensearch-project#8029)

* Implemented REST API for initializing extension

Signed-off-by: Owais Kazi <[email protected]>

* Cleanup extensions.yml design

Signed-off-by: Owais Kazi <[email protected]>

* Added tests for RestInitializeExtensionAction

Signed-off-by: Owais Kazi <[email protected]>

* Pulled extensions REST request in extensions directory

Signed-off-by: Owais Kazi <[email protected]>

* Removed forbidden APIs from rest action and modified tests

Signed-off-by: Owais Kazi <[email protected]>

* Added entry in changelog

Signed-off-by: Owais Kazi <[email protected]>

* Added test for parse

Signed-off-by: Owais Kazi <[email protected]>

* Addressed PR comments

Signed-off-by: Owais Kazi <[email protected]>

* Addressed PR comments

Signed-off-by: Owais Kazi <[email protected]>

* Spotless Fixed

Signed-off-by: Owais Kazi <[email protected]>

* Handled exceptions

Signed-off-by: Owais Kazi <[email protected]>

* Handled test failure

Signed-off-by: Owais Kazi <[email protected]>

---------

Signed-off-by: Owais Kazi <[email protected]>
Signed-off-by: Shivansh Arora <[email protected]>

Author: owaiskazi19

Diff for: CHANGELOG.md

@@ -82,6 +82,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add descending order search optimization through reverse segment read. ([#7967](https://github.com/opensearch-project/OpenSearch/pull/7967))
 - Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020))
 - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038))
+- Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029)
 
 ### Dependencies
 - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814)

Diff for: server/src/main/java/org/opensearch/action/ActionModule.java

@@ -294,8 +294,10 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsFilter;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.extensions.ExtensionsManager;
 import org.opensearch.extensions.action.ExtensionProxyAction;
 import org.opensearch.extensions.action.ExtensionProxyTransportAction;
+import org.opensearch.extensions.rest.RestInitializeExtensionAction;
 import org.opensearch.index.seqno.RetentionLeaseActions;
 import org.opensearch.identity.IdentityService;
 import org.opensearch.indices.SystemIndices;
@@ -453,7 +455,7 @@
 import org.opensearch.rest.action.search.RestPutSearchPipelineAction;
 import org.opensearch.rest.action.search.RestSearchAction;
 import org.opensearch.rest.action.search.RestSearchScrollAction;
-import org.opensearch.rest.extensions.RestSendToExtensionAction;
+import org.opensearch.extensions.rest.RestSendToExtensionAction;
 import org.opensearch.tasks.Task;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.usage.UsageService;
@@ -508,6 +510,7 @@ public class ActionModule extends AbstractModule {
     private final RequestValidators<PutMappingRequest> mappingRequestValidators;
     private final RequestValidators<IndicesAliasesRequest> indicesAliasesRequestRequestValidators;
     private final ThreadPool threadPool;
+    private final ExtensionsManager extensionsManager;
 
     public ActionModule(
         Settings settings,
@@ -521,7 +524,8 @@ public ActionModule(
         CircuitBreakerService circuitBreakerService,
         UsageService usageService,
         SystemIndices systemIndices,
-        IdentityService identityService
+        IdentityService identityService,
+        ExtensionsManager extensionsManager
     ) {
         this.settings = settings;
         this.indexNameExpressionResolver = indexNameExpressionResolver;
@@ -530,6 +534,7 @@ public ActionModule(
         this.settingsFilter = settingsFilter;
         this.actionPlugins = actionPlugins;
         this.threadPool = threadPool;
+        this.extensionsManager = extensionsManager;
         actions = setupActions(actionPlugins);
         actionFilters = setupActionFilters(actionPlugins);
         dynamicActionRegistry = new DynamicActionRegistry();
@@ -947,6 +952,11 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
             registerHandler.accept(new RestDeleteSearchPipelineAction());
         }
 
+        // Extensions API
+        if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) {
+            registerHandler.accept(new RestInitializeExtensionAction(extensionsManager));
+        }
+
         for (ActionPlugin plugin : actionPlugins) {
             for (RestHandler handler : plugin.getRestHandlers(
                 settings,

Diff for: server/src/main/java/org/opensearch/bootstrap/Security.java

@@ -316,7 +316,6 @@ static void addFilePermissions(Permissions policy, Environment environment) thro
         addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libDir(), "read,readlink", false);
         addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesDir(), "read,readlink", false);
         addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsDir(), "read,readlink", false);
-        addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.extensionDir(), "read,readlink", false);
         addDirectoryPath(policy, "path.conf'", environment.configDir(), "read,readlink", false);
         // read-write dirs
         addDirectoryPath(policy, "java.io.tmpdir", environment.tmpDir(), "read,readlink,write,delete", false);

Diff for: server/src/main/java/org/opensearch/env/Environment.java

@@ -311,10 +311,6 @@ public Path pluginsDir() {
         return pluginsDir;
     }
 
-    public Path extensionDir() {
-        return extensionsDir;
-    }
-
     public Path binDir() {
         return binDir;
     }

Diff for: server/src/main/java/org/opensearch/extensions/ExtensionDependency.java

@@ -16,6 +16,10 @@
 import org.opensearch.common.io.stream.StreamInput;
 import org.opensearch.common.io.stream.StreamOutput;
 import org.opensearch.common.io.stream.Writeable;
+import org.opensearch.core.common.Strings;
+import org.opensearch.core.xcontent.XContentParser;
+
+import static org.opensearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
 
 /**
  * This class handles the dependent extensions information
@@ -25,6 +29,8 @@
 public class ExtensionDependency implements Writeable {
     private String uniqueId;
     private Version version;
+    private static final String UNIQUE_ID = "uniqueId";
+    private static final String VERSION = "version";
 
     public ExtensionDependency(String uniqueId, Version version) {
         this.uniqueId = uniqueId;
@@ -54,6 +60,39 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeVersion(version);
     }
 
+    public static ExtensionDependency parse(XContentParser parser) throws IOException {
+        String uniqueId = null;
+        Version version = null;
+        ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
+        while (parser.nextToken() != XContentParser.Token.END_OBJECT) {
+            String fieldName = parser.currentName();
+            parser.nextToken();
+
+            switch (fieldName) {
+                case UNIQUE_ID:
+                    uniqueId = parser.text();
+                    break;
+                case VERSION:
+                    try {
+                        version = Version.fromString(parser.text());
+                    } catch (IllegalArgumentException e) {
+                        throw e;
+                    }
+                    break;
+                default:
+                    parser.skipChildren();
+                    break;
+            }
+        }
+        if (Strings.isNullOrEmpty(uniqueId)) {
+            throw new IOException("Required field [uniqueId] is missing in the request for the dependent extension");
+        } else if (version == null) {
+            throw new IOException("Required field [version] is missing in the request for the dependent extension");
+        }
+        return new ExtensionDependency(uniqueId, version);
+
+    }
+
     /**
     * The uniqueId of the dependency extension
     *