From d98ff4f32001d9943be3106ff19c21e73f62ea96 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 7 Oct 2025 10:29:23 +0530 Subject: [PATCH 01/11] extensions: add sync functionality This change introduces support for synchronizing extension files across management servers in a clustered environment. - Adds new syncExtension API to trigger synchronization for a selected extension. - Implements service to create .tgz archive of the extension directory or selected files. - Generates signed share URL with HMAC signature and expiry. - Sends DownloadAndSyncExtensionFilesCommand to peer management servers. - Handles archive download, staging, and extraction on receiver side. - Adds Sync Extension action in the UI Extensions view. - Updates events, logging, and cleanup of temporary share files. Refactors all filesystem related code to extensions framework layer. Checksum for extensions is now calculated and compared for all files in the extension directory. Signed-off-by: Abhishek Kumar --- .../main/java/com/cloud/event/EventTypes.java | 2 + .../apache/cloudstack/api/ApiConstants.java | 2 + client/conf/server.properties.in | 16 + .../org/apache/cloudstack/ServerDaemon.java | 118 ++- .../cloudstack/ShareSignedUrlFilter.java | 89 ++ .../cloud/hypervisor/ExternalProvisioner.java | 12 - .../com/cloud/cluster/ClusterManagerImpl.java | 2 + .../cluster/dao/ManagementServerHostDao.java | 2 + .../dao/ManagementServerHostDaoImpl.java | 19 +- .../dao/ManagementServerHostDaoImplTest.java | 90 ++ .../extensions/api/SyncExtensionCmd.java | 139 +++ .../DownloadAndSyncExtensionFilesCommand.java | 62 ++ .../command/ExtensionBaseCommand.java | 6 + .../StartSyncExtensionFilesCommand.java | 42 + .../manager/ExtensionsFilesystemManager.java | 54 + .../ExtensionsFilesystemManagerImpl.java | 448 ++++++++ .../extensions/manager/ExtensionsManager.java | 3 + .../manager/ExtensionsManagerImpl.java | 236 ++++- .../manager/ExtensionsShareManager.java | 34 + .../manager/ExtensionsShareManagerImpl.java | 551 ++++++++++ .../framework/extensions/vo/ExtensionVO.java | 2 +- ...ring-framework-extensions-core-context.xml | 2 + .../ExtensionsFilesystemManagerImplTest.java | 468 +++++++++ .../manager/ExtensionsManagerImplTest.java | 464 +++++++- .../ExtensionsShareManagerImplTest.java | 760 ++++++++++++++ .../mom/webhook/WebhookDeliveryThread.java | 18 +- .../webhook/WebhookDeliveryThreadTest.java | 17 - .../ExternalPathPayloadProvisioner.java | 903 ---------------- .../ExternalPathPayloadProvisionerTest.java | 989 ------------------ ui/public/locales/en.json | 9 +- ui/src/config/section/extension.js | 10 + ui/src/views/extension/SyncExtension.vue | 180 ++++ .../utils/security/HMACSignUtil.java | 43 + .../utils/server/ServerPropertiesUtil.java | 105 ++ .../utils/security/HMACSignUtilTest.java | 60 ++ 35 files changed, 3928 insertions(+), 2029 deletions(-) create mode 100644 client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java create mode 100644 framework/cluster/src/test/java/com/cloud/cluster/dao/ManagementServerHostDaoImplTest.java create mode 100644 framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmd.java create mode 100644 framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadAndSyncExtensionFilesCommand.java create mode 100644 framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/StartSyncExtensionFilesCommand.java create mode 100644 framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java create mode 100644 framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java create mode 100644 framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManager.java create mode 100644 framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java create mode 100644 framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java create mode 100644 framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImplTest.java delete mode 100644 plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java delete mode 100644 plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java create mode 100644 ui/src/views/extension/SyncExtension.vue create mode 100644 utils/src/main/java/org/apache/cloudstack/utils/security/HMACSignUtil.java create mode 100644 utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java create mode 100644 utils/src/test/java/org/apache/cloudstack/utils/security/HMACSignUtilTest.java diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 38e601c790a7..52c9bf5f3571 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -844,6 +844,7 @@ public class EventTypes { public static final String EVENT_EXTENSION_CREATE = "EXTENSION.CREATE"; public static final String EVENT_EXTENSION_UPDATE = "EXTENSION.UPDATE"; public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE"; + public static final String EVENT_EXTENSION_SYNC = "EXTENSION.SYNC"; public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER"; public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER"; public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD"; @@ -1385,6 +1386,7 @@ public class EventTypes { entityEventDetails.put(EVENT_EXTENSION_CREATE, Extension.class); entityEventDetails.put(EVENT_EXTENSION_UPDATE, Extension.class); entityEventDetails.put(EVENT_EXTENSION_DELETE, Extension.class); + entityEventDetails.put(EVENT_EXTENSION_SYNC, Extension.class); entityEventDetails.put(EVENT_EXTENSION_RESOURCE_REGISTER, Extension.class); entityEventDetails.put(EVENT_EXTENSION_RESOURCE_UNREGISTER, Extension.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 475b845af9b1..238b17484916 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -547,6 +547,7 @@ public class ApiConstants { public static final String SOURCE_CIDR_LIST = "sourcecidrlist"; public static final String SOURCE_ZONE_ID = "sourcezoneid"; + public static final String SOURCE_MANAGEMENT_SERVER_ID = "sourcemanagementserverid"; public static final String SSL_VERIFICATION = "sslverification"; public static final String START_ASN = "startasn"; public static final String START_DATE = "startdate"; @@ -567,6 +568,7 @@ public class ApiConstants { public static final String SWAP_OWNER = "swapowner"; public static final String SYSTEM_VM_TYPE = "systemvmtype"; public static final String TAGS = "tags"; + public static final String TARGET_MANAGEMENT_SERVER_IDS = "targetmanagementserverids"; public static final String STORAGE_TAGS = "storagetags"; public static final String STORAGE_ACCESS_GROUPS = "storageaccessgroups"; public static final String STORAGE_ACCESS_GROUP = "storageaccessgroup"; diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in index 5958486b4dff..beb04ee8909c 100644 --- a/client/conf/server.properties.in +++ b/client/conf/server.properties.in @@ -62,3 +62,19 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@ # Thread pool configuration #threads.min=10 #threads.max=500 + +# These properties configure the share endpoint, which enables controlled file sharing through the management server. +# They allow administrators to enable or disable sharing, set the base directory for shared files, define cache +# behavior, restrict access to specific directories, and secure access with a secret key. This ensures flexible and +# secure file sharing for different modules such as extensions, etc. +# Enable or disable file sharing feature (true/false). Default is true +share.enabled=true +# The base directory from which files can be shared. Default is /share +# share.base.dir= +# The cache control header value to be used for shared files. Default is public,max-age=86400,immutable +# share.cache.control=public,max-age=86400,immutable +# Allow or disallow directory listing when accessing a directory. Default is false +# share.dir.allowed=false +# Secret key for securing links using HMAC signature. If not set then links will not be signed. Default is change-me +# It is recommended to change this value to a strong secret key in production +share.secret=change-me diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 196695e1fc6b..446f7728f758 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -24,15 +24,25 @@ import java.io.InputStream; import java.lang.management.ManagementFactory; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; import java.util.Properties; -import com.cloud.api.ApiServer; +import javax.servlet.DispatcherType; + +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.commons.daemon.Daemon; import org.apache.commons.daemon.DaemonContext; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.eclipse.jetty.jmx.MBeanContainer; import org.eclipse.jetty.server.ForwardedRequestCustomizer; +import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.RequestLog; @@ -46,14 +56,18 @@ import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.KeyStoreScanner; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; import org.eclipse.jetty.webapp.WebAppContext; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; +import com.cloud.api.ApiServer; import com.cloud.utils.Pair; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.server.ServerProperties; @@ -111,6 +125,12 @@ public class ServerDaemon implements Daemon { private int minThreads; private int maxThreads; + private boolean shareEnabled = false; + private String shareBaseDir; + private String shareCacheCtl; + private boolean shareDirList = false; + private String shareSecret; + ////////////////////////////////////////////////// /////////////// Public methods /////////////////// ////////////////////////////////////////////////// @@ -121,6 +141,14 @@ public static void main(final String... anArgs) throws Exception { daemon.start(); } + protected void initShareConfigFromProperties() { + setShareEnabled(ServerPropertiesUtil.getShareEnabled()); + setShareBaseDir(ServerPropertiesUtil.getShareBaseDirectory()); + setShareCacheCtl(ServerPropertiesUtil.getShareCacheControl()); + setShareDirList(ServerPropertiesUtil.getShareDirAllowed()); + setShareSecret(ServerPropertiesUtil.getShareSecret()); + } + @Override public void init(final DaemonContext context) { final File confFile = PropertiesUtil.findConfigFile("server.properties"); @@ -153,6 +181,7 @@ public void init(final DaemonContext context) { setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS)))); setMinThreads(Integer.valueOf(properties.getProperty(THREADS_MIN, "10"))); setMaxThreads(Integer.valueOf(properties.getProperty(THREADS_MAX, "500"))); + initShareConfigFromProperties(); } catch (final IOException e) { logger.warn("Failed to read configuration from server.properties file", e); } finally { @@ -288,6 +317,52 @@ private void createHttpsConnector(final HttpConfiguration httpConfig) { } } + /** + * Creates a Jetty context at /share to serve static files for modules (e.g. Extensions Framework). + * Controlled via server properties + * + * @return a configured Handler or null if disabled. + */ + private Handler createShareContextHandler() throws IOException { + if (!shareEnabled) { + logger.info("/{} context not mounted", ServerPropertiesUtil.SHARE_DIR); + return null; + } + + final Path base = Paths.get(shareBaseDir); + Files.createDirectories(base); + + final ServletContextHandler shareCtx = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + shareCtx.setContextPath("/" + ServerPropertiesUtil.SHARE_DIR); + shareCtx.setBaseResource(Resource.newResource(base.toAbsolutePath().toUri())); + + // Efficient static file serving + ServletHolder def = shareCtx.addServlet(DefaultServlet.class, "/*"); + def.setInitParameter("dirAllowed", Boolean.toString(shareDirList)); + def.setInitParameter("etags", "true"); + def.setInitParameter("cacheControl", shareCacheCtl); + def.setInitParameter("useFileMappedBuffer", "true"); + def.setInitParameter("acceptRanges", "true"); + + // Gzip using modern Jetty handler + org.eclipse.jetty.server.handler.gzip.GzipHandler gzipHandler = + new org.eclipse.jetty.server.handler.gzip.GzipHandler(); + gzipHandler.setMinGzipSize(1024); + gzipHandler.setIncludedMimeTypes( + "text/html", "text/plain", "text/css", "text/javascript", + "application/javascript", "application/json", "application/xml"); + gzipHandler.setHandler(shareCtx); + + // Optional signed-URL guard (path + "|" + exp => HMAC-SHA256, base64url) + if (StringUtils.isNotBlank(shareSecret)) { + shareCtx.addFilter(new FilterHolder(new ShareSignedUrlFilter(true, shareSecret)), + "/*", EnumSet.of(DispatcherType.REQUEST)); + } + + logger.info("Mounted /{} static context at baseDir={}", ServerPropertiesUtil.SHARE_DIR, base); + return shareCtx; + } + private Pair createHandlers() { final WebAppContext webApp = new WebAppContext(); webApp.setContextPath(contextPath); @@ -318,8 +393,23 @@ private Pair createHandlers() { rootRedirect.setNewContextURL(contextPath); rootRedirect.setPermanent(true); + // Optional /share handler (served by createShareContextHandler) + Handler shareHandler = null; + try { + shareHandler = createShareContextHandler(); + } catch (IOException e) { + logger.error("Failed to initialize /share context", e); + } + + List handlers = new java.util.ArrayList<>(); + handlers.add(log); + handlers.add(gzipHandler); + if (shareHandler != null) { + handlers.add(shareHandler); + } // Put rootRedirect at the end! - return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(log, gzipHandler, rootRedirect)); + handlers.add(rootRedirect); + return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(handlers.toArray(new Handler[0]))); } private RequestLog createRequestLog() { @@ -408,4 +498,24 @@ public void setMinThreads(int minThreads) { public void setMaxThreads(int maxThreads) { this.maxThreads = maxThreads; } + + public void setShareEnabled(boolean shareEnabled) { + this.shareEnabled = shareEnabled; + } + + public void setShareBaseDir(String shareBaseDir) { + this.shareBaseDir = shareBaseDir; + } + + public void setShareCacheCtl(String shareCacheCtl) { + this.shareCacheCtl = shareCacheCtl; + } + + public void setShareDirList(boolean shareDirList) { + this.shareDirList = shareDirList; + } + + public void setShareSecret(String shareSecret) { + this.shareSecret = shareSecret; + } } diff --git a/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java new file mode 100644 index 000000000000..75c177ae5a06 --- /dev/null +++ b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack; + +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.utils.security.HMACSignUtil; +import org.apache.commons.codec.DecoderException; + +/** + * Optional HMAC token check: /share/...?...&exp=1699999999&sig=BASE64URL(HMACSHA256(path|exp)) + */ +public class ShareSignedUrlFilter implements Filter { + private final boolean requireToken; + private final String secret; + + public ShareSignedUrlFilter(boolean requireToken, String secret) { + this.requireToken = requireToken; + this.secret = secret; + } + + @Override public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest r = (HttpServletRequest) req; + HttpServletResponse w = (HttpServletResponse) res; + + String expStr = r.getParameter("exp"); + String sig = r.getParameter("sig"); + + if (!requireToken && (expStr == null || sig == null)) { + chain.doFilter(req, res); + return; + } + if (expStr == null || sig == null) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); + return; + } + long exp; + try { + exp = Long.parseLong(expStr); + } catch (NumberFormatException e) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad exp"); + return; + } + if (Instant.now().getEpochSecond() > exp) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired"); + return; + } + String want = ""; + try { + String data = r.getRequestURI() + "|" + expStr; + want = HMACSignUtil.generateSignature(data, secret); + } catch (InvalidKeyException | NoSuchAlgorithmException | DecoderException e) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Auth error"); + return; + } + if (!want.equals(sig)) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad signature"); + return; + } + chain.doFilter(req, res); + } +} diff --git a/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java b/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java index c574a8be0175..e35403a8c7e5 100644 --- a/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java +++ b/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java @@ -35,18 +35,6 @@ public interface ExternalProvisioner extends Manager { - String getExtensionsPath(); - - String getExtensionPath(String relativePath); - - String getChecksumForExtensionPath(String extensionName, String relativePath); - - void prepareExtensionPath(String extensionName, boolean userDefined, String extensionRelativePath); - - void cleanupExtensionPath(String extensionName, String extensionRelativePath); - - void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory); - PrepareExternalProvisioningAnswer prepareExternalProvisioning(String hostGuid, String extensionName, String extensionRelativePath, PrepareExternalProvisioningCommand cmd); StartAnswer startInstance(String hostGuid, String extensionName, String extensionRelativePath, StartCommand cmd); diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java index 78924a10b32d..f2900af6a4a0 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java @@ -468,6 +468,8 @@ public String execute(final String strPeer, final long agentId, final String cmd return null; } + + @Override public ManagementServerHostVO getPeer(final String mgmtServerId) { return _mshostDao.findByMsid(Long.parseLong(mgmtServerId)); diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java index 6c8ffcac78b7..dd0019dd1181 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java @@ -61,4 +61,6 @@ public interface ManagementServerHostDao extends GenericDao listUpByIds(List ids); } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java index ec943a9c26be..96ac18cb9858 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java @@ -24,12 +24,11 @@ import java.util.List; import java.util.TimeZone; - +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.management.ManagementServerHost.State; import org.apache.commons.collections.CollectionUtils; import com.cloud.cluster.ClusterInvalidSessionException; -import org.apache.cloudstack.management.ManagementServerHost; -import org.apache.cloudstack.management.ManagementServerHost.State; import com.cloud.cluster.ManagementServerHostVO; import com.cloud.utils.DateUtil; import com.cloud.utils.db.DB; @@ -318,4 +317,18 @@ public ManagementServerHostVO findOneByLongestRuntime() { return CollectionUtils.isNotEmpty(msHosts) ? msHosts.get(0) : null; } + @Override + public List listUpByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("ids", sb.entity().getId(), SearchCriteria.Op.IN); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("ids", ids.toArray()); + sc.setParameters("state", ManagementServerHost.State.Up); + return listBy(sc); + } } diff --git a/framework/cluster/src/test/java/com/cloud/cluster/dao/ManagementServerHostDaoImplTest.java b/framework/cluster/src/test/java/com/cloud/cluster/dao/ManagementServerHostDaoImplTest.java new file mode 100644 index 000000000000..438e71b928cb --- /dev/null +++ b/framework/cluster/src/test/java/com/cloud/cluster/dao/ManagementServerHostDaoImplTest.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.List; + +import org.apache.cloudstack.management.ManagementServerHost; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class ManagementServerHostDaoImplTest { + + @Spy + @InjectMocks + private ManagementServerHostDaoImpl managementServerHostDao; + + @Test + public void listUpByIdsReturnsEmptyListWhenInputIsEmpty() { + List result = managementServerHostDao.listUpByIds(Collections.emptyList()); + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void listUpByIdsReturnsEmptyListWhenNoMatchingIds() {; + SearchBuilder mockSb = mock(SearchBuilder.class); + SearchCriteria mocSC = mock(SearchCriteria.class); + when(mockSb.entity()).thenReturn(mock(ManagementServerHostVO.class)); + when(mockSb.create()).thenReturn(mocSC); + doReturn(mockSb).when(managementServerHostDao).createSearchBuilder(); + doReturn(Collections.emptyList()).when(managementServerHostDao).listBy(any(SearchCriteria.class)); + List result = managementServerHostDao.listUpByIds(List.of(1L, 2L)); + assertNotNull(result); + assertTrue(result.isEmpty()); + verify(managementServerHostDao).createSearchBuilder(); + verify(mockSb).and(eq("ids"), anyLong(), eq(SearchCriteria.Op.IN)); + verify(mockSb).and(eq("state"), nullable(ManagementServerHost.State.class), eq(SearchCriteria.Op.EQ)); + verify(mockSb).done(); + verify(mocSC).setParameters(eq("ids"), anyLong(), anyLong()); + verify(mocSC).setParameters("state", ManagementServerHost.State.Up); + } + + @Test + public void listUpByIdsReturnsMatchingHostsWhenIdsAreValid() { + ManagementServerHostVO host1 = mock(ManagementServerHostVO.class); + ManagementServerHostVO host2 = mock(ManagementServerHostVO.class); + doReturn(List.of(host1, host2)).when(managementServerHostDao).listBy(any(SearchCriteria.class)); + List result = managementServerHostDao.listUpByIds(List.of(1L, 2L)); + assertNotNull(result); + assertEquals(2, result.size()); + assertTrue(result.contains(host1)); + assertTrue(result.contains(host2)); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmd.java new file mode 100644 index 000000000000..3ea1e98da95a --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmd.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.api; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ExtensionResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.user.Account; + +@APICommand(name = "syncExtension", + description = "To sync the extension files from one management server to other management server(s)", + responseObject = SuccessResponse.class, + responseHasSensitiveInfo = false, + entityType = {Extension.class}, + authorized = {RoleType.Admin}, + since = "4.23.0") +public class SyncExtensionCmd extends BaseAsyncCmd { + + @Inject + ExtensionsManager extensionsManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, required = true, + entityType = ExtensionResponse.class, description = "ID of the extension") + private Long id; + + @Parameter(name = ApiConstants.SOURCE_MANAGEMENT_SERVER_ID, type = BaseCmd.CommandType.UUID, required = true, + entityType = ManagementServerResponse.class, + description = "ID of the management server from which files are to be synced") + private Long sourceManagementServerId; + + @Parameter(name = ApiConstants.TARGET_MANAGEMENT_SERVER_IDS, type = BaseCmd.CommandType.LIST, + collectionType = BaseCmd.CommandType.UUID, entityType = ManagementServerResponse.class, + description="the IDs of the management servers to which the extension files are to be synced. " + + "If not specified, the files will be synced to all management servers") + private List targetManagementServerIds; + + @Parameter(name = ApiConstants.FILES, type = CommandType.LIST, collectionType = CommandType.STRING, + description = "List of files to sync. Specify absolute or relative paths. If not provided, all extension " + + "files will be synced.") + private List files; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Long getSourceManagementServerId() { + return sourceManagementServerId; + } + + public List getTargetManagementServerIds() { + return targetManagementServerIds; + } + + public List getFiles() { + return files; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + boolean result = extensionsManager.syncExtension(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to sync extension"); + } + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Extension; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_EXTENSION_SYNC; + } + + @Override + public String getEventDescription() { + return "Sync extension: " + getId(); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadAndSyncExtensionFilesCommand.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadAndSyncExtensionFilesCommand.java new file mode 100644 index 000000000000..538cf09b7b85 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadAndSyncExtensionFilesCommand.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.command; + +import org.apache.cloudstack.extension.Extension; + +public class DownloadAndSyncExtensionFilesCommand extends ExtensionServerActionBaseCommand { + + public enum SyncType { + Complete, + Partial + } + + private SyncType syncType = SyncType.Complete; + private final String downloadUrl; + private final String checksum; + private final long size; + private String path; + + public DownloadAndSyncExtensionFilesCommand(long msId, Extension extension, String downloadUrl, + long size, String checksum) { + super(msId, extension); + this.downloadUrl = downloadUrl; + this.size = size; + this.checksum = checksum; + } + + public SyncType getSyncType() { + return syncType; + } + + public void setSyncType(SyncType syncType) { + this.syncType = syncType; + } + + public String getDownloadUrl() { + return downloadUrl; + } + + public long getSize() { + return size; + } + + public String getChecksum() { + return checksum; + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java index ead3c2e4012e..4a340b98b469 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java @@ -25,6 +25,7 @@ public class ExtensionBaseCommand extends Command { private final long extensionId; private final String extensionName; private final boolean extensionUserDefined; + private final Extension.Type extensionType; private final String extensionRelativePath; private final Extension.State extensionState; @@ -32,6 +33,7 @@ protected ExtensionBaseCommand(Extension extension) { this.extensionId = extension.getId(); this.extensionName = extension.getName(); this.extensionUserDefined = extension.isUserDefined(); + this.extensionType = extension.getType(); this.extensionRelativePath = extension.getRelativePath(); this.extensionState = extension.getState(); } @@ -48,6 +50,10 @@ public boolean isExtensionUserDefined() { return extensionUserDefined; } + public Extension.Type getExtensionType() { + return extensionType; + } + public String getExtensionRelativePath() { return extensionRelativePath; } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/StartSyncExtensionFilesCommand.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/StartSyncExtensionFilesCommand.java new file mode 100644 index 000000000000..5e7f421988a1 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/StartSyncExtensionFilesCommand.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.command; + +import java.util.List; + +import org.apache.cloudstack.extension.Extension; + +public class StartSyncExtensionFilesCommand extends ExtensionServerActionBaseCommand { + private final List targetManagementServerIds; + private final List files; + + public StartSyncExtensionFilesCommand(long msId, Extension extension, List targetManagementServerIds, + List files) { + super(msId, extension); + this.targetManagementServerIds = targetManagementServerIds; + this.files = files; + } + + public List getTargetManagementServerIds() { + return targetManagementServerIds; + } + + public List getFiles() { + return files; + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java new file mode 100644 index 000000000000..341e7200ad63 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.extension.Extension; + +public interface ExtensionsFilesystemManager { + + String getExtensionsPath(); + + Path getExtensionRootPath(Extension extension); + + String getExtensionPath(String relativePath); + + String getExtensionCheckedPath(String extensionName, String extensionRelativePath); + + Map getChecksumMapForExtension(String extensionName, String relativePath); + + void prepareExtensionPath(String extensionName, boolean userDefined, Extension.Type type, String extensionRelativePath); + + void cleanupExtensionPath(String extensionName, String extensionRelativePath); + + void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory); + + Path getExtensionsStagingPath() throws IOException; + + String prepareExternalPayload(String extensionName, Map details) throws IOException; + + void deleteExtensionPayload(String extensionName, String payloadFileName); + + void validateExtensionFiles(Extension extension, List files); + + boolean packExtensionFilesAsTgz(Extension extension, Path extensionFilesPath, Path archivePath); +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java new file mode 100644 index 000000000000..966f8e4e6c9e --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java @@ -0,0 +1,448 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.FileUtil; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; + +public class ExtensionsFilesystemManagerImpl extends ManagerBase implements ExtensionsFilesystemManager { + + public static final Map BASE_EXTERNAL_SCRIPTS = + Map.of(Extension.Type.Orchestrator, "scripts/vm/hypervisor/external/provisioner/provisioner.sh"); + + private static final String EXTENSIONS = "extensions"; + private static final String EXTENSIONS_DEPLOYMENT_MODE_NAME = "extensions.deployment.mode"; + private static final String EXTENSIONS_DIRECTORY_PROD = "/usr/share/cloudstack-management/extensions"; + private static final String EXTENSIONS_DATA_DIRECTORY_PROD = System.getProperty("user.home") + File.separator + EXTENSIONS; + private static final String EXTENSIONS_DIRECTORY_DEV = EXTENSIONS; + private static final String EXTENSIONS_DATA_DIRECTORY_DEV = "client/target/extensions-data"; + + private String extensionsDirectory; + private String extensionsDataDirectory; + private ExecutorService payloadCleanupExecutor; + private ScheduledExecutorService payloadCleanupScheduler; + + private void initializeExtensionDirectories() { + String deploymentMode = ServerPropertiesUtil.getProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); + if ("developer".equals(deploymentMode)) { + extensionsDirectory = EXTENSIONS_DIRECTORY_DEV; + extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_DEV; + } else { + extensionsDirectory = EXTENSIONS_DIRECTORY_PROD; + extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_PROD; + } + } + + protected boolean checkExtensionsDirectory() { + File dir = new File(extensionsDirectory); + if (!dir.exists() || !dir.isDirectory() || !dir.canWrite()) { + logger.error("Extension directory [{}] is not properly set up. It must exist, be a directory, and be writeable", + dir.getAbsolutePath()); + return false; + } + if (!extensionsDirectory.equals(dir.getAbsolutePath())) { + extensionsDirectory = dir.getAbsolutePath(); + } + logger.info("Extensions directory path: {}", extensionsDirectory); + return true; + } + + protected void createOrCheckExtensionsDataDirectory() throws ConfigurationException { + File dir = new File(extensionsDataDirectory); + if (!dir.exists()) { + try { + Files.createDirectories(dir.toPath()); + } catch (IOException e) { + logger.error("Unable to create extensions data directory [{}]", dir.getAbsolutePath(), e); + throw new ConfigurationException("Unable to create extensions data directory path"); + } + } + if (!dir.isDirectory() || !dir.canWrite()) { + logger.error("Extensions data directory [{}] is not properly set up. It must exist, be a directory, and be writeable", + dir.getAbsolutePath()); + throw new ConfigurationException("Extensions data directory path is not accessible"); + } + extensionsDataDirectory = dir.getAbsolutePath(); + logger.info("Extensions data directory path: {}", extensionsDataDirectory); + } + + protected void scheduleExtensionPayloadDirectoryCleanup(String extensionName) { + try { + Future future = payloadCleanupExecutor.submit(() -> { + try { + cleanupExtensionData(extensionName, 1, false); + logger.trace("Cleaned up payload directory for extension: {}", extensionName); + } catch (Exception e) { + logger.warn("Exception during payload cleanup for extension: {} due to {}", extensionName, + e.getMessage()); + logger.trace(e); + } + }); + payloadCleanupScheduler.schedule(() -> { + try { + if (!future.isDone()) { + future.cancel(true); + logger.trace("Cancelled cleaning up payload directory for extension: {} as it " + + "running for more than 3 seconds", extensionName); + } + } catch (Exception e) { + logger.warn("Failed to cancel payload cleanup task for extension: {} due to {}", + extensionName, e.getMessage()); + logger.trace(e); + } + }, 3, TimeUnit.SECONDS); + } catch (RejectedExecutionException e) { + logger.warn("Payload cleanup task for extension: {} was rejected due to: {}", extensionName, + e.getMessage()); + logger.trace(e); + } + } + + protected static String getFileExtension(File file) { + String name = file.getName(); + int lastDot = name.lastIndexOf('.'); + return (lastDot == -1) ? "" : name.substring(lastDot + 1); + } + + protected Path getExtensionRootPath(String extensionName) { + final String normalizedName = Extension.getDirectoryName(extensionName); + final String extensionDir = extensionsDirectory + File.separator + normalizedName; + return Path.of(extensionDir); + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + initializeExtensionDirectories(); + checkExtensionsDirectory(); + createOrCheckExtensionsDataDirectory(); + return true; + } + + @Override + public boolean start() { + payloadCleanupExecutor = Executors.newSingleThreadExecutor(); + payloadCleanupScheduler = Executors.newSingleThreadScheduledExecutor(); + return true; + } + + @Override + public boolean stop() { + payloadCleanupExecutor.shutdown(); + payloadCleanupScheduler.shutdown(); + return true; + } + + @Override + public String getExtensionsPath() { + return extensionsDirectory; + } + + @Override + public Path getExtensionRootPath(Extension extension) { + return getExtensionRootPath(extension.getName()); + } + + @Override + public String getExtensionPath(String relativePath) { + return String.format("%s%s%s", extensionsDirectory, File.separator, relativePath); + } + + @Override + public String getExtensionCheckedPath(String extensionName, String extensionRelativePath) { + String path = getExtensionPath(extensionRelativePath); + File file = new File(path); + String errorSuffix = String.format("Entry point [%s] for extension: %s", path, extensionName); + if (!file.exists()) { + logger.error("{} does not exist", errorSuffix); + return null; + } + if (!file.isFile()) { + logger.error("{} is not a file", errorSuffix); + return null; + } + if (!file.canRead()) { + logger.error("{} is not readable", errorSuffix); + return null; + } + if (!file.canExecute()) { + logger.error("{} is not executable", errorSuffix); + return null; + } + return path; + } + + @Override + public Map getChecksumMapForExtension(String extensionName, String relativePath) { + String path = getExtensionCheckedPath(extensionName, relativePath); + if (StringUtils.isBlank(path)) { + return null; + } + try { + Path rootPath = getExtensionRootPath(extensionName); + Map fileChecksums = new TreeMap<>(); + java.util.List files = new java.util.ArrayList<>(); + try (Stream stream = Files.walk(rootPath)) { + stream.filter(Files::isRegularFile).forEach(files::add); + } + files.sort(Comparator.naturalOrder()); + for (Path filePath : files) { + String relative = rootPath.relativize(filePath).toString().replace(File.separatorChar, '/'); + String fileChecksum = DigestHelper.calculateChecksum(filePath.toFile()); + fileChecksums.put(relative, fileChecksum); + } + if (logger.isTraceEnabled()) { + String json = GsonHelper.getGson().toJson(fileChecksums); + logger.trace("Calculated individual file checksums for extension: {}: {}", extensionName, json); + } + return fileChecksums; + } catch (IOException | CloudRuntimeException e) { + return null; + } + } + + protected boolean packFilesAsTgz(Path sourcePath, Path archivePath) { + int result = Script.executeCommandForExitValue( + 60 * 1000, + Script.getExecutableAbsolutePath("tar"), + "-czpf", archivePath.toAbsolutePath().toString(), + "-C", sourcePath.toAbsolutePath().toString(), + "." + ); + return result == 0; + } + + @Override + public void prepareExtensionPath(String extensionName, boolean userDefined, Extension.Type type, String extensionRelativePath) { + logger.debug("Preparing entry point for Extension [name: {}, user-defined: {}]", extensionName, userDefined); + if (!userDefined) { + logger.debug("Skipping preparing entry point for inbuilt extension: {}", extensionName); + return; + } + CloudRuntimeException exception = + new CloudRuntimeException(String.format("Failed to prepare scripts for extension: %s", extensionName)); + String sourceScriptPath = Script.findScript("", BASE_EXTERNAL_SCRIPTS.get(type)); + if(sourceScriptPath == null) { + logger.debug("Base script is not available for preparing extension: {} of type: {}", + extensionName, type); + return; + } + String destinationPath = getExtensionPath(extensionRelativePath); + File destinationFile = new File(destinationPath); + File sourceFile = new File(sourceScriptPath); + if (!getFileExtension(sourceFile).equalsIgnoreCase(getFileExtension(destinationFile))) { + logger.error("Extension file type do not match with base file for extension: {} of type: {}", + extensionName, type); + return; + } + if (destinationFile.exists()) { + logger.info("File already exists at {} for extension: {}, skipping copy.", destinationPath, + extensionName); + return; + } + if (!checkExtensionsDirectory()) { + throw exception; + } + Path destinationPathObj = Paths.get(destinationPath); + Path destinationDirPath = destinationPathObj.getParent(); + if (destinationDirPath == null) { + logger.error("Failed to find parent directory for extension: {} script path {}", + extensionName, destinationPath); + throw exception; + } + try { + Files.createDirectories(destinationDirPath); + } catch (IOException e) { + logger.error("Failed to create directory: {} for extension: {}", destinationDirPath, + extensionName, e); + throw exception; + } + try { + Path sourcePath = Paths.get(sourceScriptPath); + Files.copy(sourcePath, destinationPathObj, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + logger.error("Failed to copy entry point file to [{}] for extension: {}", + destinationPath, extensionName, e); + throw exception; + } + logger.debug("Successfully prepared entry point [{}] for extension: {}", destinationPath, + extensionName); + } + + @Override + public void cleanupExtensionPath(String extensionName, String extensionRelativePath) { + String normalizedPath = extensionRelativePath; + if (normalizedPath.startsWith("/")) { + normalizedPath = normalizedPath.substring(1); + } + try { + Path rootPath = Paths.get(extensionsDirectory).toAbsolutePath().normalize(); + String extensionDirName = Extension.getDirectoryName(extensionName); + Path filePath = rootPath + .resolve(normalizedPath.startsWith(extensionDirName) ? extensionDirName : normalizedPath) + .normalize(); + if (!Files.exists(filePath)) { + return; + } + if (!Files.isDirectory(filePath) && !Files.isRegularFile(filePath)) { + throw new CloudRuntimeException( + String.format("Failed to cleanup path: %s for extension: %s as it either " + + "does not exist or is not a regular file/directory", + extensionName, extensionRelativePath)); + } + if (!FileUtil.deleteRecursively(filePath)) { + throw new CloudRuntimeException( + String.format("Failed to delete path: %s for extension: %s", + extensionName, filePath)); + } + } catch (IOException e) { + throw new CloudRuntimeException( + String.format("Failed to cleanup path: %s for extension: %s due to: %s", + extensionName, normalizedPath, e.getMessage()), e); + } + } + + @Override + public void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory) { + String extensionPayloadDirPath = extensionsDataDirectory + File.separator + extensionName; + Path dirPath = Paths.get(extensionPayloadDirPath); + if (!Files.exists(dirPath)) { + return; + } + try { + if (cleanupDirectory) { + try (Stream paths = Files.walk(dirPath)) { + paths.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + return; + } + long cutoffMillis = System.currentTimeMillis() - (olderThanDays * 24L * 60 * 60 * 1000); + long lastModified = Files.getLastModifiedTime(dirPath).toMillis(); + if (lastModified < cutoffMillis) { + return; + } + try (Stream paths = Files.walk(dirPath)) { + paths.filter(path -> !path.equals(dirPath)) + .filter(path -> { + try { + return Files.getLastModifiedTime(path).toMillis() < cutoffMillis; + } catch (IOException e) { + return false; + } + }) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } catch (IOException e) { + logger.warn("Failed to clean up extension payloads for {}: {}", extensionName, e.getMessage()); + } + } + + @Override + public Path getExtensionsStagingPath() throws IOException { + Path extensionsPath = Paths.get(extensionsDirectory).toAbsolutePath().normalize(); + Path stagingPath = extensionsPath.resolve(".staging"); + Files.createDirectories(stagingPath); + return stagingPath; + } + + @Override + public String prepareExternalPayload(String extensionName, Map details) throws IOException { + String json = GsonHelper.getGson().toJson(details); + String fileName = UUID.randomUUID() + ".json"; + String extensionPayloadDir = extensionsDataDirectory + File.separator + extensionName; + Path payloadDirPath = Paths.get(extensionPayloadDir); + if (!Files.exists(payloadDirPath)) { + Files.createDirectories(payloadDirPath); + } else { + scheduleExtensionPayloadDirectoryCleanup(extensionName); + } + Path payloadFile = payloadDirPath.resolve(fileName); + Files.writeString(payloadFile, json, StandardOpenOption.CREATE_NEW); + return payloadFile.toAbsolutePath().toString(); + } + + @Override + public void deleteExtensionPayload(String extensionName, String payloadFileName) { + logger.trace("Deleting payload file: {} for extension: {}", payloadFileName, extensionName); + FileUtil.deletePath(payloadFileName); + } + + @Override + public void validateExtensionFiles(Extension extension, List files) { + if (CollectionUtils.isEmpty(files)) { + return; + } + Path rootPath = getExtensionRootPath(extension); + File rootDir = rootPath.toFile(); + if (!rootDir.exists() || !rootDir.isDirectory()) { + throw new CloudRuntimeException("Extension directory does not exist: " + rootPath); + } + for (String filePath : files) { + File file = new File(filePath); + if (!file.isAbsolute()) { + file = new File(rootDir, filePath); + } + if (!file.exists()) { + throw new CloudRuntimeException("File does not exist: " + filePath); + } + } + } + + @Override + public boolean packExtensionFilesAsTgz(Extension extension, Path extensionFilesPath, Path archivePath) { + logger.debug("Packing files for {} from: {} to archive: {}", extension, + extensionFilesPath.toAbsolutePath(), archivePath.toAbsolutePath()); + return packFilesAsTgz(extensionFilesPath, archivePath); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java index 1b1a175c5975..14ccf56094cd 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java @@ -39,6 +39,7 @@ import org.apache.cloudstack.framework.extensions.api.ListExtensionsCmd; import org.apache.cloudstack.framework.extensions.api.RegisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.RunCustomActionCmd; +import org.apache.cloudstack.framework.extensions.api.SyncExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UnregisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateExtensionCmd; @@ -97,4 +98,6 @@ Pair extensionResourceMapDetailsNeedUpdate(final void updateExtensionResourceMapDetails(final long extensionResourceMapId, final Map details); Answer getInstanceConsole(VirtualMachine vm, Host host); + + boolean syncExtension(SyncExtensionCmd cmd); } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index f66c195399d1..53bfb677d4c0 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -72,14 +72,17 @@ import org.apache.cloudstack.framework.extensions.api.ListExtensionsCmd; import org.apache.cloudstack.framework.extensions.api.RegisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.RunCustomActionCmd; +import org.apache.cloudstack.framework.extensions.api.SyncExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UnregisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateExtensionCmd; import org.apache.cloudstack.framework.extensions.command.CleanupExtensionFilesCommand; +import org.apache.cloudstack.framework.extensions.command.DownloadAndSyncExtensionFilesCommand; import org.apache.cloudstack.framework.extensions.command.ExtensionRoutingUpdateCommand; import org.apache.cloudstack.framework.extensions.command.ExtensionServerActionBaseCommand; import org.apache.cloudstack.framework.extensions.command.GetExtensionPathChecksumCommand; import org.apache.cloudstack.framework.extensions.command.PrepareExtensionPathCommand; +import org.apache.cloudstack.framework.extensions.command.StartSyncExtensionFilesCommand; import org.apache.cloudstack.framework.extensions.dao.ExtensionCustomActionDao; import org.apache.cloudstack.framework.extensions.dao.ExtensionCustomActionDetailsDao; import org.apache.cloudstack.framework.extensions.dao.ExtensionDao; @@ -123,7 +126,6 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; -import com.cloud.hypervisor.ExternalProvisioner; import com.cloud.hypervisor.Hypervisor; import com.cloud.org.Cluster; import com.cloud.serializer.GsonHelper; @@ -179,9 +181,6 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana @Inject HostDetailsDao hostDetailsDao; - @Inject - ExternalProvisioner externalProvisioner; - @Inject ExtensionCustomActionDao extensionCustomActionDao; @@ -212,6 +211,12 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana @Inject RoleService roleService; + @Inject + ExtensionsFilesystemManager extensionsFilesystemManager; + + @Inject + ExtensionsShareManager extensionsShareManager; + private ScheduledExecutorService extensionPathStateCheckExecutor; protected String getDefaultExtensionRelativePath(String name) { @@ -272,9 +277,9 @@ protected boolean prepareExtensionPathOnMSPeer(Extension extension, ManagementSe } protected Pair prepareExtensionPathOnCurrentServer(String name, boolean userDefined, - String relativePath) { + Extension.Type type, String relativePath) { try { - externalProvisioner.prepareExtensionPath(name, userDefined, relativePath); + extensionsFilesystemManager.prepareExtensionPath(name, userDefined, type, relativePath); } catch (CloudRuntimeException e) { logger.error("Failed to prepare path for Extension [name: {}, userDefined: {}, relativePath: {}] on this server", name, userDefined, relativePath, e); @@ -294,8 +299,8 @@ protected boolean cleanupExtensionFilesOnMSPeer(Extension extension, ManagementS protected Pair cleanupExtensionFilesOnCurrentServer(String name, String relativePath) { try { - externalProvisioner.cleanupExtensionPath(name, relativePath); - externalProvisioner.cleanupExtensionData(name, 0, true); + extensionsFilesystemManager.cleanupExtensionPath(name, relativePath); + extensionsFilesystemManager.cleanupExtensionData(name, 0, true); } catch (CloudRuntimeException e) { logger.error("Failed to cleanup files for Extension [name: {}, relativePath: {}] on this server", name, relativePath, e); @@ -534,10 +539,40 @@ protected void checkOrchestratorTemplates(Long extensionId) { } } + protected void checkExtensionPathState(Extension extension) { + List msList = managementServerHostDao.listBy(ManagementServerHost.State.Up); + msList.removeIf(ms -> ms.getMsid() == ManagementServerNode.getManagementServerId()); + checkExtensionPathState(extension, msList); + } + + protected boolean compareChecksumMaps(Extension extension, long ms1Id, Map map1, + long ms2Id, Map map2) { + if (MapUtils.isEmpty(map1) || MapUtils.isEmpty(map2)) { + return false; + } + if (map1.size() != map2.size()) { + return false; + } + for (Map.Entry entry : map1.entrySet()) { + String map2Value = map2.get(entry.getKey()); + logger.debug("Comparing checksum for file {} of {} between [msid: {}, checksum: {}] and [msid: {}, checksum: {}]", + entry.getKey(), extension, ms1Id, entry.getValue(), ms2Id, map2Value); + if (!StringUtils.equals(entry.getValue(), map2Value)) { + logger.error("Checksum for file {} of {} is different [msid: {}, checksum: {}] and [msid: {}, checksum: {}]", + entry.getKey(), extension, ms1Id, entry.getValue(), ms2Id, + (StringUtils.isNotBlank(map2Value) ? map2Value : "unknown")); + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") protected void checkExtensionPathState(Extension extension, List msHosts) { - String checksum = externalProvisioner.getChecksumForExtensionPath(extension.getName(), + logger.debug("Checking path state for {} across management servers", extension); + Map checksumMap = extensionsFilesystemManager.getChecksumMapForExtension(extension.getName(), extension.getRelativePath()); - if (StringUtils.isBlank(checksum)) { + if (MapUtils.isEmpty(checksumMap)) { updateExtensionPathReady(extension, false); return; } @@ -548,10 +583,22 @@ protected void checkExtensionPathState(Extension extension, List msPeerChecksumResult = getChecksumForExtensionPathOnMSPeer(extension, msHost); - if (!msPeerChecksumResult.first() || !checksum.equals(msPeerChecksumResult.second())) { - logger.error("Path checksum for {} is different [msid: {}, checksum: {}] and [msid: {}, checksum: {}]", - extension, ManagementServerNode.getManagementServerId(), checksum, msHost.getMsid(), - (msPeerChecksumResult.first() ? msPeerChecksumResult.second() : "unknown")); + if (!msPeerChecksumResult.first() || StringUtils.isBlank(msPeerChecksumResult.second())) { + logger.error("{} returned failure or empty checksum map for {}", msHost, extension); + updateExtensionPathReady(extension, false); + return; + } + Map msPeerChecksumMap = null; + try { + msPeerChecksumMap = GsonHelper.getGson().fromJson(msPeerChecksumResult.second(), Map.class); + } catch (Exception e) { + logger.error("Failed to parse checksum map JSON from {} for {}: {}", + msHost, extension, e.getMessage(), e); + updateExtensionPathReady(extension, false); + return; + } + if (!compareChecksumMaps(extension, ManagementServerNode.getManagementServerId(), checksumMap, + msHost.getMsid(), msPeerChecksumMap)) { updateExtensionPathReady(extension, false); return; } @@ -561,7 +608,7 @@ protected void checkExtensionPathState(Extension extension, List msHosts = managementServerHostDao.listBy(ManagementServerHost.State.Up); for (ManagementServerHostVO msHost : msHosts) { if (msHost.getMsid() == ManagementServerNode.getManagementServerId()) { - prepared = prepared && prepareExtensionPathOnCurrentServer(extension.getName(), extension.isUserDefined(), + prepared = prepared && prepareExtensionPathOnCurrentServer(extension.getName(), + extension.isUserDefined(), + extension.getType(), extension.getRelativePath()).first(); continue; } @@ -917,7 +966,7 @@ public ExtensionResponse createExtensionResponse(Extension extension, ExtensionResponse response = new ExtensionResponse(extension.getUuid(), extension.getName(), extension.getDescription(), extension.getType().name()); response.setCreated(extension.getCreated()); - response.setPath(externalProvisioner.getExtensionPath(extension.getRelativePath())); + response.setPath(extensionsFilesystemManager.getExtensionPath(extension.getRelativePath())); response.setPathReady(extension.isPathReady()); response.setUserDefined(extension.isUserDefined()); response.setState(extension.getState().name()); @@ -1499,19 +1548,27 @@ public String handleExtensionServerCommands(ExtensionServerActionBaseCommand com Answer answer = new Answer(command, false, "Unsupported command"); if (command instanceof GetExtensionPathChecksumCommand) { final GetExtensionPathChecksumCommand cmd = (GetExtensionPathChecksumCommand)command; - String checksum = externalProvisioner.getChecksumForExtensionPath(extensionName, + Map checksumMap = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, extensionRelativePath); - answer = new Answer(cmd, StringUtils.isNotBlank(checksum), checksum); + answer = new Answer(cmd, MapUtils.isNotEmpty(checksumMap), GsonHelper.getGson().toJson(checksumMap)); } else if (command instanceof PrepareExtensionPathCommand) { final PrepareExtensionPathCommand cmd = (PrepareExtensionPathCommand)command; Pair result = prepareExtensionPathOnCurrentServer( - extensionName, cmd.isExtensionUserDefined(), extensionRelativePath); + extensionName, cmd.isExtensionUserDefined(), cmd.getExtensionType(), extensionRelativePath); answer = new Answer(cmd, result.first(), result.second()); } else if (command instanceof CleanupExtensionFilesCommand) { final CleanupExtensionFilesCommand cmd = (CleanupExtensionFilesCommand)command; Pair result = cleanupExtensionFilesOnCurrentServer(extensionName, extensionRelativePath); answer = new Answer(cmd, result.first(), result.second()); + } else if (command instanceof StartSyncExtensionFilesCommand) { + final StartSyncExtensionFilesCommand cmd = (StartSyncExtensionFilesCommand)command; + Pair result = startSyncExtensionFiles(cmd); + answer = new Answer(cmd, result.first(), result.second()); + } else if (command instanceof DownloadAndSyncExtensionFilesCommand) { + final DownloadAndSyncExtensionFilesCommand cmd = (DownloadAndSyncExtensionFilesCommand)command; + Pair result = downloadAndSyncExtensionFiles(cmd); + answer = new Answer(cmd, result.first(), result.second()); } final Answer[] answers = new Answer[1]; answers[0] = answer; @@ -1576,6 +1633,124 @@ public Answer getInstanceConsole(VirtualMachine vm, Host host) { return agentMgr.easySend(host.getId(), cmd); } + protected Pair syncExtensionUsingMSPeer(ExtensionVO extension, + ManagementServerHostVO sourceManagementServer, List targetManagementServers, + List files) { + logger.debug("Initiating sync for {} using {}", extension, sourceManagementServer); + final String msPeer = Long.toString(sourceManagementServer.getMsid()); + final Command[] cmds = new Command[1]; + cmds[0] = new StartSyncExtensionFilesCommand( + ManagementServerNode.getManagementServerId(), extension, + targetManagementServers.stream().map(ManagementServerHost::getUuid).collect(Collectors.toList()), + files); + String answersStr = clusterManager.execute(msPeer, 0L, GsonHelper.getGson().toJson(cmds), true); + return getResultFromAnswersString(answersStr, extension, sourceManagementServer, "sync"); + } + + protected Pair startSyncExtensionFiles(StartSyncExtensionFilesCommand cmd) { + final long extensionId = cmd.getExtensionId(); + final List targetManagementServerIds = cmd.getTargetManagementServerIds(); + final List files = cmd.getFiles(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + String msg = String.format("Unable to find extension with id: %d for starting sync", extensionId); + logger.error(msg); + return new Pair<>(false, msg); + } + if (CollectionUtils.isEmpty(targetManagementServerIds)) { + String msg = "No valid target management servers specified for starting sync"; + logger.error(msg); + return new Pair<>(false, msg); + } + List targetManagementServers = managementServerHostDao.listByUuids(targetManagementServerIds); + if (targetManagementServers.size() != targetManagementServerIds.size()) { + String msg = "Some of the specified target management servers are not found"; + logger.error(msg); + return new Pair<>(false, msg); + } + ManagementServerHost sourceManagementServer = managementServerHostDao.findById(ManagementServerNode.getManagementServerId()); + List targetManagementServerHosts = targetManagementServers.stream() + .map(msHostVO -> (ManagementServerHost) msHostVO) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(files)) { + try { + extensionsFilesystemManager.validateExtensionFiles(extension, files); + } catch (CloudRuntimeException cre) { + String msg = "Invalid file paths specified: " + cre.getMessage(); + logger.error(msg); + return new Pair<>(false, msg); + } + } + return extensionsShareManager.syncExtension(extension, sourceManagementServer, targetManagementServerHosts, + files); + } + + protected Pair downloadAndSyncExtensionFiles(DownloadAndSyncExtensionFilesCommand cmd) { + final long extensionId = cmd.getExtensionId(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + String msg = String.format("Unable to find extension with ID: %d for starting sync", extensionId); + logger.error(msg); + return new Pair<>(false, msg); + } + return extensionsShareManager.downloadAndApplyExtensionSync(extension, cmd); + } + + @Override + public boolean syncExtension(SyncExtensionCmd cmd) { + final long extensionId = cmd.getId(); + final long sourceManagementServerId = cmd.getSourceManagementServerId(); + List targetManagementServerIds = cmd.getTargetManagementServerIds(); + final List files = cmd.getFiles(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + throw new InvalidParameterValueException("Unable to find extension with the specified id"); + } + final ManagementServerHostVO sourceManagementServer = managementServerHostDao.findById(sourceManagementServerId); + if (sourceManagementServer == null || !ManagementServerHost.State.Up.equals(sourceManagementServer.getState())) { + throw new InvalidParameterValueException("Unable to find active source management server with the specified id"); + } + List targetManagementServers = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(targetManagementServerIds)) { + if (targetManagementServerIds.contains(sourceManagementServerId)) { + throw new InvalidParameterValueException("Source management server cannot be specified as target"); + } + List msList = managementServerHostDao.listUpByIds(targetManagementServerIds); + if (msList.size() != targetManagementServerIds.size()) { + throw new InvalidParameterValueException("Some of the specified target management servers are not found or not in Up state"); + } + targetManagementServers.addAll(msList); + } else { + List msList = managementServerHostDao.listBy(ManagementServerHost.State.Up); + msList.removeIf(ms -> ms.getId() == sourceManagementServerId); + targetManagementServers.addAll(msList); + } + if (CollectionUtils.isEmpty(targetManagementServers)) { + throw new InvalidParameterValueException("No valid target management servers found for syncing the extension"); + } + if (CollectionUtils.isNotEmpty(files)) { + try { + extensionsFilesystemManager.validateExtensionFiles(extension, files); + } catch (CloudRuntimeException cre) { + throw new InvalidParameterValueException("Invalid file paths specified: " + cre.getMessage()); + } + } + Pair result; + if (ManagementServerNode.getManagementServerId() != sourceManagementServer.getMsid()) { + result = syncExtensionUsingMSPeer(extension, sourceManagementServer, targetManagementServers, files); + } else { + result = extensionsShareManager.syncExtension(extension, sourceManagementServer, targetManagementServers, + files); + } + if (!result.first()) { + throw new CloudRuntimeException(String.format("Failed to sync extension '%s' via '%s': %s", + extension.getName(), sourceManagementServer.getName(), result.second())); + } + + checkExtensionPathState(extension); + return true; + } + @Override public Long getExtensionIdForCluster(long clusterId) { ExtensionResourceMapVO map = extensionResourceMapDao.findByResourceIdAndType(clusterId, @@ -1600,6 +1775,17 @@ public Extension getExtensionForCluster(long clusterId) { return extensionDao.findById(extensionId); } + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + try { + extensionPathStateCheckExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("Extension-Path-State-Check")); + } catch (final Exception e) { + throw new ConfigurationException("Unable to to configure ExtensionsManagerImpl"); + } + return true; + } + @Override public boolean start() { long pathStateCheckInterval = PathStateCheckInterval.value(); @@ -1612,12 +1798,9 @@ public boolean start() { } @Override - public boolean configure(String name, Map params) throws ConfigurationException { - try { - extensionPathStateCheckExecutor = Executors.newScheduledThreadPool(1, - new NamedThreadFactory("Extension-Path-State-Check")); - } catch (final Exception e) { - throw new ConfigurationException("Unable to to configure ExtensionsManagerImpl"); + public boolean stop() { + if (extensionPathStateCheckExecutor != null && !extensionPathStateCheckExecutor.isShutdown()) { + extensionPathStateCheckExecutor.shutdownNow(); } return true; } @@ -1637,6 +1820,7 @@ public List> getCommands() { cmds.add(UpdateExtensionCmd.class); cmds.add(RegisterExtensionCmd.class); cmds.add(UnregisterExtensionCmd.class); + cmds.add(SyncExtensionCmd.class); return cmds; } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManager.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManager.java new file mode 100644 index 000000000000..1623023ed56e --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManager.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + + +import java.util.List; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.framework.extensions.command.DownloadAndSyncExtensionFilesCommand; +import org.apache.cloudstack.management.ManagementServerHost; + +import com.cloud.utils.Pair; + +public interface ExtensionsShareManager { + Pair syncExtension(Extension extension, ManagementServerHost sourceManagementServer, + List targetManagementServers, List files); + + Pair downloadAndApplyExtensionSync(Extension extension, DownloadAndSyncExtensionFilesCommand cmd); +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java new file mode 100644 index 000000000000..020af902d338 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java @@ -0,0 +1,551 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.extensions.command.DownloadAndSyncExtensionFilesCommand; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.cloudstack.utils.security.HMACSignUtil; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.cluster.ClusterManager; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.FileUtil; +import com.cloud.utils.HttpUtils; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.script.Script; + +public class ExtensionsShareManagerImpl extends ManagerBase implements ExtensionsShareManager { + + protected static final String EXTENSIONS_SHARE_SUBDIR = "extensions"; + protected static final int DEFAULT_SHARE_LINK_VALIDITY_SECONDS = 3600; // 1 hour + + ConfigKey ShareLinkValidityInterval = new ConfigKey<>("Advanced", Integer.class, + "extension.share.link.validity.interval", String.valueOf(DEFAULT_SHARE_LINK_VALIDITY_SECONDS), + String.format("Interval (in seconds) for which the extension archive share link is valid. " + + "Default is %s seconds", DEFAULT_SHARE_LINK_VALIDITY_SECONDS), + false, ConfigKey.Scope.Global); + + @Inject + ExtensionsFilesystemManager extensionsFilesystemManager; + + @Inject + ClusterManager clusterManager; + + private ScheduledExecutorService extensionShareCleanupExecutor; + private int shareLinkValidityInterval; + + protected Path getExtensionsSharePath() { + String shareBaseDir = ServerPropertiesUtil.getShareBaseDirectory(); + shareBaseDir += File.separator + EXTENSIONS_SHARE_SUBDIR; + return Path.of(shareBaseDir); + } + + protected String getManagementServerBaseUrl(ManagementServerHost managementHost) { + boolean secure = Boolean.parseBoolean(ServerPropertiesUtil.getProperty("https.enable", + "false")); + final String scheme = secure ? "https" : "http"; + final String host = managementHost.getServiceIP(); + int port = secure + ? Integer.parseInt(ServerPropertiesUtil.getProperty("https.port", "8443")) + : Integer.parseInt(ServerPropertiesUtil.getProperty("http.port", "8080")); + + return String.format("%s://%s:%d", scheme, host, port); + } + + protected Pair getResultFromAnswersString(String answersStr, Extension extension, + ManagementServerHost msHost, String op) { + Answer[] answers = null; + try { + answers = GsonHelper.getGson().fromJson(answersStr, Answer[].class); + } catch (Exception e) { + logger.error("Failed to parse answer JSON during {} for {} on {}: {}", + op, extension, msHost, e.getMessage(), e); + return new Pair<>(false, e.getMessage()); + } + Answer answer = answers != null && answers.length > 0 ? answers[0] : null; + boolean result = false; + String details = "Unknown error"; + if (answer != null) { + result = answer.getResult(); + details = answer.getDetails(); + } + if (!result) { + logger.error("Failed to {} for {} on {} due to {}", op, extension, msHost, details); + return new Pair<>(false, details); + } + return new Pair<>(true, details); + } + + /** + * Creates a .tgz archive for the specified extension. + * If the files list is empty, the entire extension directory is archived. + * If the files list is not empty, only the specified relative files are archived; throws if any file is missing. + * + * @return ArchiveInfo containing the archive path, size, SHA-256 checksum, and sync type. + */ + protected ArchiveInfo createArchive(Extension extension, List files) throws IOException { + final boolean isPartial = CollectionUtils.isNotEmpty(files); + final DownloadAndSyncExtensionFilesCommand.SyncType syncType = + isPartial ? DownloadAndSyncExtensionFilesCommand.SyncType.Partial + : DownloadAndSyncExtensionFilesCommand.SyncType.Complete; + final Path extensionRootPath = extensionsFilesystemManager.getExtensionRootPath(extension); + final List toPack; + if (isPartial) { + toPack = new ArrayList<>(files.size()); + for (String rel : files) { + Path p = extensionRootPath.resolve(rel).normalize(); + if (!p.startsWith(extensionRootPath)) { + throw new SecurityException("Path escapes version dir: " + rel); + } + if (!Files.exists(p)) { + throw new NoSuchFileException("File not found: " + p.toAbsolutePath().toString()); + } + toPack.add(p); + } + } else { + toPack = List.of(extensionRootPath); + } + StringBuilder archiveName = new StringBuilder(Extension.getDirectoryName(extension.getName())) + .append("-").append(System.currentTimeMillis()).append(".tgz"); + if (isPartial) { + archiveName.insert(0, "partial-"); + } + Path archivePath = getExtensionsSharePath().resolve(archiveName.toString()); + + if (!packAsTgz(extension, extensionRootPath, toPack, archivePath)) { + throw new IOException("Failed to create archive " + archivePath); + } + + logger.info("Created archive {} from {} ({} files)", archivePath, extensionRootPath, toPack.size()); + + long size = Files.size(archivePath); + String checksum = DigestHelper.calculateChecksum(archivePath.toFile()); + + return new ArchiveInfo(archivePath, size, checksum, syncType); + } + + /** + * Generates a signed share URL for the given extension archive. + * The resulting URL format is: {baseUrl}/share/extensions/{archiveName}?exp={expiry}&sig={signature} + * + * @param managementServer the management server host generating the URL + * @param archivePath the path to the archive file + * @return the signed share URL for the archive + * @throws DecoderException if signature decoding fails + * @throws NoSuchAlgorithmException if the HMAC algorithm is not available + * @throws InvalidKeyException if the secret key is invalid + */ + protected String generateSignedArchiveUrl(ManagementServerHost managementServer, Path archivePath) + throws DecoderException, NoSuchAlgorithmException, InvalidKeyException { + final String baseUrl = getManagementServerBaseUrl(managementServer); + final long expiresAtEpochSec = System.currentTimeMillis() / 1000L + shareLinkValidityInterval; + final String secretKey = ServerPropertiesUtil.getShareSecret(); + String archiveName = archivePath.getFileName().toString(); + String uriPath = String.format("/%s/%s/%s", ServerPropertiesUtil.SHARE_DIR, EXTENSIONS_SHARE_SUBDIR, + archiveName); + String sig = ""; + if (StringUtils.isNotBlank(secretKey)) { + String payload = uriPath + "|" + expiresAtEpochSec; + sig = HMACSignUtil.generateSignature(payload, secretKey); + } + StringBuilder sb = new StringBuilder(); + sb.append(baseUrl).append(uriPath).append("?exp=").append(expiresAtEpochSec); + if (StringUtils.isNotBlank(sig)) { + sb.append("&sig=").append(URLEncoder.encode(sig, StandardCharsets.UTF_8)); + } + return sb.toString(); + } + + /** + * Build the DownloadAndSyncExtensionFilesCommand to send to a target MS. + */ + protected DownloadAndSyncExtensionFilesCommand buildCommand(long msId, Extension ext, ArchiveInfo archive, + String signedUrl) { + DownloadAndSyncExtensionFilesCommand cmd = + new DownloadAndSyncExtensionFilesCommand(msId, ext, signedUrl, archive.getSize(), archive.getChecksum()); + cmd.setSyncType(archive.getSyncType()); + return cmd; + } + + /** + * Packs the specified files or directories into a .tgz archive. + * If a single directory is provided, archives the entire directory. + * If multiple files are provided, copies them to a temporary directory preserving structure before archiving. + * + * @param extensionRootPath the root path of the extension + * @param toPack list of files or directories to include in the archive + * @param archivePath the destination path for the .tgz archive + * @return true if the archive was created successfully, false otherwise + * @throws IOException if an I/O error occurs during packing + */ + protected boolean packAsTgz(Extension extension, Path extensionRootPath, List toPack, Path archivePath) + throws IOException { + Files.createDirectories(archivePath.getParent()); + FileUtil.deletePath(archivePath.toAbsolutePath().toString()); + + Path sourceDir; + if (toPack.size() == 1 && Files.isDirectory(toPack.get(0))) { + sourceDir = toPack.get(0); + } else { + sourceDir = Files.createTempDirectory("pack-tmp-"); + for (Path p : toPack) { + Path rel = extensionRootPath.relativize(p); + Path dest = sourceDir.resolve(rel); + Files.createDirectories(dest.getParent()); + Files.copy(p, dest, StandardCopyOption.COPY_ATTRIBUTES); + } + } + + boolean result = extensionsFilesystemManager.packExtensionFilesAsTgz(extension, sourceDir, archivePath); + + if (!sourceDir.equals(extensionRootPath)) { + FileUtil.deleteRecursively(sourceDir); + } + + return result; + } + + protected static boolean extractTgz(Path archive, Path destRoot) throws IOException { + int result = Script.executeCommandForExitValue( + 60 * 1000, + Script.getExecutableAbsolutePath("tar"), + "-xpf", archive.toAbsolutePath().toString(), + "-C", destRoot.toAbsolutePath().toString() + ); + return result == 0; + } + + protected long downloadTo(String url, Path dest) throws IOException { + boolean result = HttpUtils.downloadFileWithProgress(url, dest.toString(), logger); + if (!result) { + throw new IOException("Download failed"); + } + if (!Files.exists(dest)) { + throw new IOException("Download failed: file not found"); + } + return Files.size(dest); + } + + /** + * Atomically replaces the target directory with the source directory. + * If the target exists, it is first moved to a backup location. + * Attempts an atomic move; falls back to a regular move if necessary. + * Cleans up the backup directory after a successful move. + * + * @param from the source directory to move + * @param to the target directory to replace + * @throws IOException if an I/O error occurs during the operation + */ + protected static void atomicReplaceDir(Path from, Path to) throws IOException { + Files.createDirectories(from); + Files.createDirectories(to.getParent()); + Path backup = to.getParent().resolve(to.getFileName().toString() + ".bak-" + System.currentTimeMillis()); + if (Files.exists(to)) { + try { + Files.move(to, backup, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + Files.move(to, backup); + } + } + try { + Files.move(from, to, StandardCopyOption.ATOMIC_MOVE); + FileUtil.deleteRecursively(backup); + } catch (IOException e) { + if (Files.exists(backup)) { + try { + Files.move(backup, to, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ignore) {} + } + throw e; + } + } + + /** + * Overlays files from the source directory into the target directory. + * For each file in `fromRoot`, copies it to the corresponding location in `targetRoot`, + * replacing existing files atomically when possible. + * Directory structure is preserved. + * + * @param fromRoot the source root directory containing files to overlay + * @param targetRoot the target root directory to overlay files into + * @throws IOException if an I/O error occurs during overlay + */ + protected static void overlayInto(Path fromRoot, Path targetRoot) throws IOException { + try (Stream stream = Files.walk(fromRoot)) { + stream.forEach(src -> { + try { + if (Files.isDirectory(src)) return; + Path rel = fromRoot.relativize(src); + Path dst = targetRoot.resolve(rel); + Files.createDirectories(dst.getParent()); + Path tmp = dst.getParent().resolve(dst.getFileName() + ".tmp-" + System.nanoTime()); + Files.copy(src, tmp, StandardCopyOption.REPLACE_EXISTING); + try { + Files.move(tmp, dst, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + Files.move(tmp, dst, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + }); + } + } + + /** + * Applies the extension synchronization by extracting the archive and updating the extension directory. + * For COMPLETE sync type, replaces the entire extension directory atomically. + * For PARTIAL sync type, overlays the extracted files into the existing extension directory. + * + * @param extension the extension to synchronize + * @param syncType the type of synchronization (COMPLETE or PARTIAL) + * @param tmpArchive the path to the temporary archive file + * @param extensionRootPath the root path of the extension directory + * @throws IOException if an I/O error occurs during extraction or application + */ + protected void applyExtensionSync(Extension extension, DownloadAndSyncExtensionFilesCommand.SyncType syncType, + Path tmpArchive, Path extensionRootPath) throws IOException { + logger.debug("Applying extension sync for {} with sync type {} from archive {}", extension, syncType, + tmpArchive); + Path stagingDir = extensionsFilesystemManager.getExtensionsStagingPath(); + Path applyRoot = Files.createTempDirectory(stagingDir, + Extension.getDirectoryName(extension.getName()) + "-"); + if (!extractTgz(tmpArchive, applyRoot)) { + throw new IOException("Failed to extract archive " + tmpArchive); + } + if (DownloadAndSyncExtensionFilesCommand.SyncType.Complete.equals(syncType)) { + atomicReplaceDir(applyRoot, extensionRootPath); + return; + } + overlayInto(applyRoot, extensionRootPath); + FileUtil.deleteRecursively(applyRoot); + } + + protected void cleanupExtensionsShareFiles(long cutoff) throws IOException { + Path sharePath = getExtensionsSharePath(); + if (!Files.exists(sharePath) || !Files.isDirectory(sharePath)) { + return; + } + try (Stream paths = Files.list(sharePath)) { + paths.filter(p -> p.getFileName().toString().endsWith(".tgz")) + .filter(p -> { + try { + return Files.getLastModifiedTime(p).toMillis() < cutoff; + } catch (IOException e) { + return false; + } + }) + .forEach(p -> { + try { + Files.delete(p); + logger.debug("Deleted expired extension archive {}", p); + } catch (IOException e) { + logger.warn("Failed to delete expired extension archive {}", p, e); + } + }); + } + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + try { + extensionShareCleanupExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("Extension-Share-Cleanup")); + } catch (final Exception e) { + throw new ConfigurationException("Unable to to configure ExtensionsManagerImpl"); + } + return true; + } + + @Override + public boolean start() { + int initialDelay = 120; + shareLinkValidityInterval = ShareLinkValidityInterval.value(); + logger.debug("Scheduling cleanup task for extension share archive with initial delay={}s and interval={}s", + initialDelay, shareLinkValidityInterval); + extensionShareCleanupExecutor.scheduleWithFixedDelay(new ShareCleanupWorker(), + initialDelay, shareLinkValidityInterval, TimeUnit.SECONDS); + return true; + } + + @Override + public boolean stop() { + if (extensionShareCleanupExecutor != null && !extensionShareCleanupExecutor.isShutdown()) { + extensionShareCleanupExecutor.shutdownNow(); + } + return true; + } + + @Override + public Pair syncExtension(Extension extension, ManagementServerHost sourceManagementServer, + List targetManagementServers, List files) { + ArchiveInfo archiveInfo = null; + try { + try { + archiveInfo = createArchive(extension, files); + } catch (IOException e) { + String msg = "Failed to create archive"; + logger.error("{} for {}", extension, msg, e); + return new Pair<>(false, msg); + } + String signedUrl; + try { + signedUrl = generateSignedArchiveUrl(sourceManagementServer, archiveInfo.getPath()); + } catch (DecoderException | NoSuchAlgorithmException | InvalidKeyException e) { + String msg = "Failed to generate signed URL"; + logger.error("{} for {} using {}", msg, extension, sourceManagementServer, e); + return new Pair<>(false, msg); + } + for (ManagementServerHost targetMs : targetManagementServers) { + String targetUrl = signedUrl + "&tgt=" + URLEncoder.encode(targetMs.getUuid(), StandardCharsets.UTF_8); + final String msPeer = Long.toString(targetMs.getMsid()); + final Command[] cmds = new Command[1]; + cmds[0] = buildCommand(sourceManagementServer.getMsid(), extension, archiveInfo, targetUrl); + String answersStr = clusterManager.execute(msPeer, 0L, GsonHelper.getGson().toJson(cmds), true); + Pair result = getResultFromAnswersString(answersStr, extension, targetMs, + "sync"); + if (!result.first()) { + String msg = "Sync failed"; + logger.error("{} for {} on {} due to: {}", msg, extension, targetMs, result.second()); + return new Pair<>(false, String.format("%s on management server: %s", msg, targetMs.getName())); + } + } + } finally { + if (archiveInfo != null) { + FileUtil.deletePath(archiveInfo.getPath().toAbsolutePath().toString()); + } + } + return new Pair<>(true, ""); + } + + @Override + public Pair downloadAndApplyExtensionSync(Extension extension, + DownloadAndSyncExtensionFilesCommand cmd) { + final Path extensionRootPath = extensionsFilesystemManager.getExtensionRootPath(extension); + Path tmpArchive = null; + try { + tmpArchive = Files.createTempFile("dl-", ".tgz"); + long contentLength = downloadTo(cmd.getDownloadUrl(), tmpArchive); + + if (cmd.getSize() > 0 && contentLength != -1 && cmd.getSize() != contentLength) { + return new Pair<>(false, String.format("Size mismatch: expected %d got %d", cmd.getSize(), + contentLength)); + } + + String got = DigestHelper.calculateChecksum(tmpArchive.toFile()); + if (!got.equalsIgnoreCase(cmd.getChecksum())) { + return new Pair<>(false, String.format("Checksum mismatch for archive. expected=%s got=%s", + cmd.getChecksum(), got)); + } + + applyExtensionSync(extension, cmd.getSyncType(), tmpArchive, extensionRootPath); + } catch (IOException e) { + String msg = String.format("Failed to download/apply sync for %s from %s: %s", extension, + cmd.getDownloadUrl(), e.getMessage()); + logger.error(msg, e); + return new Pair<>(false, msg); + } finally { + if (tmpArchive != null) { + FileUtil.deletePath(tmpArchive.toAbsolutePath().toString()); + } + } + return new Pair<>(true, ""); + } + + protected static class ArchiveInfo { + private final Path path; + private final long size; + private final String checksum; + private final DownloadAndSyncExtensionFilesCommand.SyncType syncType; + + public ArchiveInfo(Path path, long size, String checksum, + DownloadAndSyncExtensionFilesCommand.SyncType syncType) { + this.path = path; + this.size = size; + this.checksum = checksum; + this.syncType = syncType; + } + + public Path getPath() { return path; } + public long getSize() { return size; } + public String getChecksum() { return checksum; } + public DownloadAndSyncExtensionFilesCommand.SyncType getSyncType() { return syncType; } + } + + protected class ShareCleanupWorker extends ManagedContextRunnable { + protected void reallyRun() { + try { + long expiryMillis = shareLinkValidityInterval * 1100L; + long cutoff = System.currentTimeMillis() - expiryMillis; + cleanupExtensionsShareFiles(cutoff); + } catch (Exception e) { + logger.warn("Extensions share cleanup failed", e); + } + } + + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("ExtensionShareCleanup"); + try { + if (gcLock.lock(3)) { + try { + reallyRun(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java index 20423764c1c3..7dd0c7059d84 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java @@ -174,6 +174,6 @@ public void setRemoved(Date removed) { @Override public String toString() { - return String.format("Extension %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "name", "type")); + return String.format("Extension %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "name", "type", "userDefined")); } } diff --git a/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml b/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml index 9d44d8ff7f3d..150c4223452e 100644 --- a/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml +++ b/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml @@ -31,6 +31,8 @@ + + diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java new file mode 100644 index 000000000000..b7463a66b569 --- /dev/null +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java @@ -0,0 +1,468 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; +import java.util.stream.Stream; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.logging.log4j.Logger; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class ExtensionsFilesystemManagerImplTest { + + @Mock + private Logger logger; + + @Spy + @InjectMocks + private ExtensionsFilesystemManagerImpl extensionsFilesystemManager; + + private File tempDir; + private File tempDataDir; + private Properties testProperties; + private File testScript; + + @Before + public void setUp() throws IOException { + tempDir = Files.createTempDirectory("extensions-test").toFile(); + tempDataDir = Files.createTempDirectory("extensions-data-test").toFile(); + + testScript = new File(tempDir, "test-extension.sh"); + testScript.createNewFile(); + resetTestScript(); + + testProperties = new Properties(); + testProperties.setProperty("extensions.deployment.mode", "developer"); + + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDirectory", tempDir.getAbsolutePath()); + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", tempDataDir.getAbsolutePath()); + + try (MockedStatic propertiesUtilMock = Mockito.mockStatic(PropertiesUtil.class)) { + File mockPropsFile = mock(File.class); + propertiesUtilMock.when(() -> PropertiesUtil.findConfigFile(anyString())).thenReturn(mockPropsFile); + } + } + + private void resetTestScript() { + testScript.setExecutable(true); + testScript.setReadable(true); + testScript.setWritable(true); + } + + @Test + public void getExtensionRootPathReturnsCorrectPathForValidExtension() { + Extension extension = mock(Extension.class); + Mockito.when(extension.getName()).thenReturn("test-extension"); + String expectedPath = tempDir.getAbsolutePath() + File.separator + "test-extension"; + Path result = extensionsFilesystemManager.getExtensionRootPath(extension); + assertEquals(expectedPath, result.toString()); + } + + + @Test + public void testGetExtensionPath() { + String result = extensionsFilesystemManager.getExtensionPath("test-extension.sh"); + String expected = tempDir.getAbsolutePath() + File.separator + "test-extension.sh"; + assertEquals(expected, result); + } + + @Test + public void testGetExtensionCheckedPathValidFile() { + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); + + assertEquals(testScript.getAbsolutePath(), result); + } + + @Test + public void testGetExtensionCheckedPathFileNotExists() { + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "nonexistent.sh"); + + assertNull(result); + } + + @Test + public void testGetExtensionCheckedPathNoExecutePermissions() { + testScript.setExecutable(false); + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); + assertNull(result); + Mockito.verify(logger).error("{} is not executable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); + } + + @Test + public void testGetExtensionCheckedPathNoReadPermissions() { + testScript.setWritable(false); + testScript.setReadable(false); + Assume.assumeFalse("Skipping test as file can not be marked unreadable", testScript.canRead()); + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); + assertNull(result); + Mockito.verify(logger).error("{} is not readable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); + } + + @Test + public void testCheckExtensionsDirectoryValid() { + boolean result = extensionsFilesystemManager.checkExtensionsDirectory(); + assertTrue(result); + } + + @Test + public void testCheckExtensionsDirectoryInvalid() { + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDirectory", "/nonexistent/path"); + + boolean result = extensionsFilesystemManager.checkExtensionsDirectory(); + assertFalse(result); + } + + @Test + public void testCreateOrCheckExtensionsDataDirectory() throws ConfigurationException { + extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); + Mockito.verify(logger).info("Extensions data directory path: {}", tempDataDir.getAbsolutePath()); + } + + @Test(expected = ConfigurationException.class) + public void testCreateOrCheckExtensionsDataDirectoryCreateThrowsExceptionFail() throws ConfigurationException { + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", "/nonexistent/path"); + try(MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + filesMock.when(() -> Files.createDirectories(any())).thenThrow(new IOException("fail")); + extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); + } + } + + @Test(expected = ConfigurationException.class) + public void testCreateOrCheckExtensionsDataDirectoryNoCreateFail() throws ConfigurationException { + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", "/nonexistent/path"); + try(MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + filesMock.when(() -> Files.createDirectories(any())).thenReturn(mock(Path.class)); + extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); + } + } + + @Test + public void getChecksumMapForExtensionReturnsChecksumsForAllFiles() throws IOException { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + Path file1 = Files.createFile(rootPath.resolve("file1.txt")); + Path file2 = Files.createFile(rootPath.resolve("file2.txt")); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + + try (MockedStatic digestHelperMock = Mockito.mockStatic(DigestHelper.class)) { + digestHelperMock.when(() -> DigestHelper.calculateChecksum(file1.toFile())).thenReturn("checksum1"); + digestHelperMock.when(() -> DigestHelper.calculateChecksum(file2.toFile())).thenReturn("checksum2"); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNotNull(result); + for(Map.Entry entry : result.entrySet()) { + System.out.println(entry.getKey() + ": " + entry.getValue()); + } + assertTrue(result.size() > 2); + assertEquals("checksum1", result.get("file1.txt")); + assertEquals("checksum2", result.get("file2.txt")); + } + } + + @Test + public void getChecksumMapForExtensionReturnsNullForBlankPath() { + String extensionName = "test-extension"; + doReturn(null).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNull(result); + } + + @Test + public void getChecksumMapForExtensionHandlesIOExceptionDuringFileWalk() { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + try (MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + filesMock.when(() -> Files.walk(rootPath)).thenThrow(new IOException("File walk error")); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNull(result); + } + } + + @Test + public void getChecksumMapForExtensionHandlesChecksumCalculationFailure() throws IOException { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + Path file1 = Files.createFile(rootPath.resolve("file1.txt")); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + try (MockedStatic digestHelperMock = Mockito.mockStatic(DigestHelper.class)) { + digestHelperMock.when(() -> DigestHelper.calculateChecksum(file1.toFile())).thenThrow(new CloudRuntimeException("Checksum error")); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNull(result); + } + } + + @Test + public void getChecksumMapForExtensionReturnsEmptyMapWhenNoFilesExist() { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + try (MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + filesMock.when(() -> Files.walk(rootPath)).thenReturn(Stream.empty()); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNotNull(result); + assertTrue(result.isEmpty()); + } + } + + @Test + public void packFilesAsTgzReturnsTrue() { + Path sourcePath = tempDir.toPath(); + Path archivePath = Path.of("test-archive.tgz"); + try (MockedStatic + + diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/HMACSignUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/security/HMACSignUtil.java new file mode 100644 index 000000000000..13a1f68ecb2a --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/HMACSignUtil.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.security; + +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import javax.crypto.Mac; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Base64; + +public class HMACSignUtil { + final static String ALGORITHM = "HMACSHA256"; + + public static String generateSignature(String data, String key) + throws InvalidKeyException, NoSuchAlgorithmException, DecoderException { + Mac mac = Mac.getInstance(ALGORITHM); + SecretKey secretKey = new SecretKeySpec(key.getBytes(StandardCharsets.UTF_8), mac.getAlgorithm()); + mac.init(secretKey); + byte[] dataAsBytes = data.getBytes(StandardCharsets.UTF_8); + byte[] encodedText = mac.doFinal(dataAsBytes); + return new String(Base64.encodeBase64(encodedText)).trim(); + } +} diff --git a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java new file mode 100644 index 000000000000..8fd44e66e067 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.server; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloud.utils.PropertiesUtil; + +public class ServerPropertiesUtil { + public static final String SHARE_DIR = "share"; + private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); + private static final String PROPERTIES_FILE = "server.properties"; + + public static final String SHARE_ENABLED = "share.enabled"; + private static final String SHARE_BASE_DIR = "share.base.dir"; + public static final String SHARE_CACHE_CONTROL = "share.cache.control"; + public static final String SHARE_DIR_ALLOWED = "share.dir.allowed"; + private static final String SHARE_SECRET = "share.secret"; + + private static final AtomicReference propertiesRef = new AtomicReference<>(); + + public static String getProperty(String name) { + Properties props = propertiesRef.get(); + if (props != null) { + return props.getProperty(name); + } + File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); + if (propsFile == null) { + logger.error("{} file not found", PROPERTIES_FILE); + return null; + } + Properties tempProps = new Properties(); + try (FileInputStream is = new FileInputStream(propsFile)) { + tempProps.load(is); + } catch (IOException e) { + logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); + return null; + } + if (!propertiesRef.compareAndSet(null, tempProps)) { + tempProps = propertiesRef.get(); + } + return tempProps.getProperty(name); + } + + public static String getProperty(String name, String defaultValue) { + String value = getProperty(name); + if (value == null) { + value = defaultValue; + } + return value; + } + + public static boolean getShareEnabled() { + return Boolean.parseBoolean(getProperty(SHARE_ENABLED, "false")); + } + + public static String getShareBaseDirectory() { + String shareBaseDir = getProperty(SHARE_BASE_DIR); + if (StringUtils.isNotBlank(shareBaseDir)) { + return shareBaseDir; + } + boolean isMavenRun = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().contains("org.codehaus.plexus.classworlds.launcher.Launcher"); + if (isMavenRun) { + // when running from maven, use a share directory from client/target in the current working directory + return System.getProperty("user.dir") + File.separator + "client" + File.separator + "target" + File.separator + SHARE_DIR; + } + return System.getProperty("user.home") + File.separator + SHARE_DIR; + } + + public static String getShareCacheControl() { + return getProperty(SHARE_CACHE_CONTROL, "public,max-age=86400,immutable"); + } + + public static boolean getShareDirAllowed() { + return Boolean.parseBoolean(getProperty(SHARE_DIR_ALLOWED, "false")); + } + + public static String getShareSecret() { + return getProperty(SHARE_SECRET); + } +} diff --git a/utils/src/test/java/org/apache/cloudstack/utils/security/HMACSignUtilTest.java b/utils/src/test/java/org/apache/cloudstack/utils/security/HMACSignUtilTest.java new file mode 100644 index 000000000000..5cf653bbd31f --- /dev/null +++ b/utils/src/test/java/org/apache/cloudstack/utils/security/HMACSignUtilTest.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.security; + +import org.junit.Assert; +import org.junit.Test; + +public class HMACSignUtilTest { + + @Test + public void generateSignatureValidInputsReturnsExpectedSignature() throws Exception { + String data = "CloudStack works!"; + String key = "Pj4pnwSUBZ4wQFXw2zWdVY1k5Ku9bIy70wCNG1DmS8keO7QapCLw2Axtgc2nEPYzfFCfB38ATNLt6caDqU2dSw"; + String result = "HYLWSII5Ap23WeSaykNsIo6mOhmV3d18s5p2cq2ebCA="; + final String signature = HMACSignUtil.generateSignature(data, key); + Assert.assertNotNull(signature); + Assert.assertEquals(result, signature); + } + + @Test(expected = IllegalArgumentException.class) + public void generateSignatureInvalidKeyThrowsException() throws Exception { + final String data = "testData"; + final String key = ""; // Empty key + HMACSignUtil.generateSignature(data, key); + } + + @Test + public void generateSignatureDifferentInputsProduceDifferentSignatures() throws Exception { + final String data1 = "testData1"; + final String data2 = "testData2"; + final String key = "testKey"; + final String signature1 = HMACSignUtil.generateSignature(data1, key); + final String signature2 = HMACSignUtil.generateSignature(data2, key); + Assert.assertNotEquals(signature1, signature2); + } + + @Test + public void generateSignatureSameInputsProduceSameSignature() throws Exception { + final String data = "testData"; + final String key = "testKey"; + final String signature1 = HMACSignUtil.generateSignature(data, key); + final String signature2 = HMACSignUtil.generateSignature(data, key); + Assert.assertEquals(signature1, signature2); + } +} From 4a535bdacba66b72c59d0d97ddce687f07cf06af Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Oct 2025 12:01:24 +0530 Subject: [PATCH 02/11] more tests Signed-off-by: Abhishek Kumar --- .../cloudstack/ShareSignedUrlFilter.java | 3 +- .../cloudstack/ShareSignedUrlFilterTest.java | 136 ++++++++ .../extensions/api/SyncExtensionCmdTest.java | 124 ++++++++ .../ExtensionsFilesystemManagerImplTest.java | 298 ++++++++++++++++-- .../utils/server/ServerPropertiesUtil.java | 4 +- .../server/ServerPropertiesUtilTest.java | 96 ++++++ 6 files changed, 637 insertions(+), 24 deletions(-) create mode 100644 client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java create mode 100644 framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java create mode 100644 utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java diff --git a/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java index 75c177ae5a06..e9a2dc2ee12c 100644 --- a/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java +++ b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java @@ -45,7 +45,8 @@ public ShareSignedUrlFilter(boolean requireToken, String secret) { this.secret = secret; } - @Override public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) + @Override + public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) throws IOException, ServletException { HttpServletRequest r = (HttpServletRequest) req; HttpServletResponse w = (HttpServletResponse) res; diff --git a/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java new file mode 100644 index 000000000000..97c7d1655107 --- /dev/null +++ b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.time.Instant; + +import javax.servlet.FilterChain; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.utils.security.HMACSignUtil; +import org.junit.Test; + +public class ShareSignedUrlFilterTest { + + @Test + public void allowsRequestWhenTokenIsNotRequiredAndParametersAreMissing() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(false, "secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + when(mockRequest.getParameter("exp")).thenReturn(null); + when(mockRequest.getParameter("sig")).thenReturn(null); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockChain).doFilter(mockRequest, mockResponse); + } + + @Test + public void deniesRequestWhenTokenIsRequiredAndParametersAreMissing() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + when(mockRequest.getParameter("exp")).thenReturn(null); + when(mockRequest.getParameter("sig")).thenReturn(null); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); + verifyNoInteractions(mockChain); + } + + @Test + public void deniesRequestWhenExpirationIsInvalid() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + when(mockRequest.getParameter("exp")).thenReturn("invalid"); + when(mockRequest.getParameter("sig")).thenReturn("signature"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Bad exp"); + verifyNoInteractions(mockChain); + } + + @Test + public void deniesRequestWhenTokenIsExpired() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + when(mockRequest.getParameter("exp")).thenReturn(String.valueOf(Instant.now().getEpochSecond() - 10)); + when(mockRequest.getParameter("sig")).thenReturn("signature"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired"); + verifyNoInteractions(mockChain); + } + + @Test + public void deniesRequestWhenSignatureIsInvalid() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + when(mockRequest.getParameter("exp")).thenReturn(String.valueOf(Instant.now().getEpochSecond() + 1000)); + when(mockRequest.getParameter("sig")).thenReturn("invalidSignature"); + when(mockRequest.getRequestURI()).thenReturn("/share/resource"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Bad signature"); + verifyNoInteractions(mockChain); + } + + @Test + public void allowsRequestWhenSignatureIsValid() throws Exception { + String secret = "secret"; + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, secret); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + String exp = String.valueOf(Instant.now().getEpochSecond() + 1000); + String data = "/share/resource|" + exp; + String validSignature = HMACSignUtil.generateSignature(data, secret); + + when(mockRequest.getParameter("exp")).thenReturn(exp); + when(mockRequest.getParameter("sig")).thenReturn(validSignature); + when(mockRequest.getRequestURI()).thenReturn("/share/resource"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockChain).doFilter(mockRequest, mockResponse); + } +} diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java new file mode 100644 index 000000000000..7c43f48dee0a --- /dev/null +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.api; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.List; + +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; +import org.junit.Test; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.event.EventTypes; +import com.cloud.user.Account; + +public class SyncExtensionCmdTest { + + @Test + public void returnsIdWhenGetIdIsCalled() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + ReflectionTestUtils.setField(cmd, "id", 123L); + assertEquals(Long.valueOf(123), cmd.getId()); + } + + @Test + public void returnsSourceManagementServerIdWhenGetSourceManagementServerIdIsCalled() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + ReflectionTestUtils.setField(cmd, "sourceManagementServerId", 456L); + assertEquals(Long.valueOf(456), cmd.getSourceManagementServerId()); + } + + @Test + public void returnsTargetManagementServerIdsWhenGetTargetManagementServerIdsIsCalled() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + List targetIds = Arrays.asList(789L, 101L); + ReflectionTestUtils.setField(cmd, "targetManagementServerIds", targetIds); + assertEquals(targetIds, cmd.getTargetManagementServerIds()); + } + + @Test + public void returnsFilesWhenGetFilesIsCalled() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + List files = Arrays.asList("file1.txt", "file2.txt"); + ReflectionTestUtils.setField(cmd, "files", files); + assertEquals(files, cmd.getFiles()); + } + + @Test + public void executesSuccessfullyWhenSyncExtensionSucceeds() throws Exception { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + ExtensionsManager mockManager = mock(ExtensionsManager.class); + ReflectionTestUtils.setField(cmd, "extensionsManager", mockManager); + when(mockManager.syncExtension(cmd)).thenReturn(true); + + cmd.execute(); + + SuccessResponse response = (SuccessResponse) cmd.getResponseObject(); + assertTrue(response.getSuccess()); + } + + @Test(expected = ServerApiException.class) + public void throwsExceptionWhenSyncExtensionFails() throws Exception { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + ExtensionsManager mockManager = mock(ExtensionsManager.class); + ReflectionTestUtils.setField(cmd, "extensionsManager", mockManager); + when(mockManager.syncExtension(cmd)).thenReturn(false); + + cmd.execute(); + } + + @Test + public void returnsSystemAccountIdWhenGetEntityOwnerIdIsCalled() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + assertEquals(Account.ACCOUNT_ID_SYSTEM, cmd.getEntityOwnerId()); + } + + @Test + public void returnsExtensionResourceTypeWhenGetApiResourceTypeIsCalled() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + assertEquals(ApiCommandResourceType.Extension, cmd.getApiResourceType()); + } + + @Test + public void returnsIdWhenGetApiResourceIdIsCalled() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + ReflectionTestUtils.setField(cmd, "id", 123L); + assertEquals(Long.valueOf(123), cmd.getApiResourceId()); + } + + @Test + public void returnsCorrectEventType() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + assertEquals(EventTypes.EVENT_EXTENSION_SYNC, cmd.getEventType()); + } + + @Test + public void returnsCorrectEventDescription() { + SyncExtensionCmd cmd = new SyncExtensionCmd(); + ReflectionTestUtils.setField(cmd, "id", 123L); + assertEquals("Sync extension: 123", cmd.getEventDescription()); + } +} diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java index b7463a66b569..b65a74c52cc9 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java @@ -26,9 +26,12 @@ import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import java.io.File; import java.io.IOException; @@ -36,9 +39,17 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileTime; +import java.util.List; import java.util.Map; import java.util.Properties; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import javax.naming.ConfigurationException; @@ -58,6 +69,7 @@ import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +import com.cloud.utils.FileUtil; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.exception.CloudRuntimeException; import com.cloud.utils.script.Script; @@ -140,7 +152,7 @@ public void testGetExtensionCheckedPathNoExecutePermissions() { testScript.setExecutable(false); String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); assertNull(result); - Mockito.verify(logger).error("{} is not executable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); + verify(logger).error("{} is not executable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); } @Test @@ -150,7 +162,7 @@ public void testGetExtensionCheckedPathNoReadPermissions() { Assume.assumeFalse("Skipping test as file can not be marked unreadable", testScript.canRead()); String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); assertNull(result); - Mockito.verify(logger).error("{} is not readable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); + verify(logger).error("{} is not readable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); } @Test @@ -170,13 +182,13 @@ public void testCheckExtensionsDirectoryInvalid() { @Test public void testCreateOrCheckExtensionsDataDirectory() throws ConfigurationException { extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); - Mockito.verify(logger).info("Extensions data directory path: {}", tempDataDir.getAbsolutePath()); + verify(logger).info("Extensions data directory path: {}", tempDataDir.getAbsolutePath()); } @Test(expected = ConfigurationException.class) public void testCreateOrCheckExtensionsDataDirectoryCreateThrowsExceptionFail() throws ConfigurationException { ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", "/nonexistent/path"); - try(MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + try (MockedStatic filesMock = Mockito.mockStatic(Files.class)) { filesMock.when(() -> Files.createDirectories(any())).thenThrow(new IOException("fail")); extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); } @@ -185,12 +197,105 @@ public void testCreateOrCheckExtensionsDataDirectoryCreateThrowsExceptionFail() @Test(expected = ConfigurationException.class) public void testCreateOrCheckExtensionsDataDirectoryNoCreateFail() throws ConfigurationException { ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", "/nonexistent/path"); - try(MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + try (MockedStatic filesMock = Mockito.mockStatic(Files.class)) { filesMock.when(() -> Files.createDirectories(any())).thenReturn(mock(Path.class)); extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); } } + @Test + public void schedulesCleanupTaskSuccessfully() { + String extensionName = "test-extension"; + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(payloadCleanupExecutor).submit(any(Runnable.class)); + verify(payloadCleanupScheduler).schedule(any(Runnable.class), eq(3L), eq(TimeUnit.SECONDS)); + } + + @Test + public void handlesRejectedExecutionExceptionGracefully() { + String extensionName = "test-extension"; + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + doThrow(new RejectedExecutionException("Task rejected")) + .when(payloadCleanupExecutor).submit(any(Runnable.class)); + + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(logger).warn("Payload cleanup task for extension: {} was rejected due to: {}", extensionName, "Task rejected"); + } + + @Test + public void cancelsTaskIfNotCompletedWithinTimeout() { + String extensionName = "test-extension"; + Future mockFuture = mock(Future.class); + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(ScheduledFuture.class); + }).when(payloadCleanupScheduler).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + doReturn(mockFuture).when(payloadCleanupExecutor).submit(any(Runnable.class)); + Mockito.when(mockFuture.isDone()).thenReturn(false); + + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(mockFuture).cancel(true); + verify(logger).trace("Cancelled cleaning up payload directory for extension: {} as it running for more than 3 seconds", extensionName); + } + + @Test + public void logsExceptionDuringCleanupTask() { + String extensionName = "test-extension"; + doThrow(new RuntimeException("Cleanup error")) + .when(extensionsFilesystemManager).cleanupExtensionData(extensionName, 1, false); + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(Future.class); + }).when(payloadCleanupExecutor).submit(any(Runnable.class)); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(ScheduledFuture.class); + }).when(payloadCleanupScheduler).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(logger).warn("Exception during payload cleanup for extension: {} due to {}", extensionName, "Cleanup error"); + } + + @Test + public void logsExceptionWhenCancellingTaskFails() { + String extensionName = "test-extension"; + Future mockFuture = mock(Future.class); + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(ScheduledFuture.class); + }).when(payloadCleanupScheduler).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + doReturn(mockFuture).when(payloadCleanupExecutor).submit(any(Runnable.class)); + doThrow(new RuntimeException("Cancel error")).when(mockFuture).cancel(true); + + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(logger).warn("Failed to cancel payload cleanup task for extension: {} due to {}", extensionName, "Cancel error"); + } + @Test public void getChecksumMapForExtensionReturnsChecksumsForAllFiles() throws IOException { String extensionName = "test-extension"; @@ -205,7 +310,7 @@ public void getChecksumMapForExtensionReturnsChecksumsForAllFiles() throws IOExc digestHelperMock.when(() -> DigestHelper.calculateChecksum(file2.toFile())).thenReturn("checksum2"); Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); assertNotNull(result); - for(Map.Entry entry : result.entrySet()) { + for (Map.Entry entry : result.entrySet()) { System.out.println(entry.getKey() + ": " + entry.getValue()); } assertTrue(result.size() > 2); @@ -346,28 +451,132 @@ public void testPrepareExtensionPathFileAlreadyExists() { } @Test - public void testCleanupExtensionPath() throws IOException { - String extensionDirName = Extension.getDirectoryName("test-extension"); - File extensionDir = new File(tempDir, extensionDirName); - extensionDir.mkdirs(); - File testFile = new File(extensionDir, "test-file.txt"); - testFile.createNewFile(); + public void cleansUpFileWhenPathIsValid() throws IOException { + String extensionName = "test-extension"; + String extensionRelativePath = "test-file.txt"; + Path rootPath = Paths.get(tempDir.getAbsolutePath()); + Path filePath = rootPath.resolve(extensionRelativePath); + Files.createFile(filePath); + + extensionsFilesystemManager.cleanupExtensionPath(extensionName, extensionRelativePath); + + assertFalse(Files.exists(filePath)); + } + + @Test + public void doesNothingWhenPathDoesNotExist() { + String extensionName = "test-extension"; + String extensionRelativePath = "nonexistent-file.txt"; + + extensionsFilesystemManager.cleanupExtensionPath(extensionName, extensionRelativePath); + + Mockito.verifyNoInteractions(logger); + } + + @Test(expected = CloudRuntimeException.class) + public void throwsExceptionWhenPathIsNotFileOrDirectory() throws IOException { + String extensionName = "test-extension"; + String extensionRelativePath = "invalid-path"; + Path rootPath = Paths.get(tempDir.getAbsolutePath()); + Path invalidPath = rootPath.resolve(extensionRelativePath); + Files.createSymbolicLink(invalidPath, rootPath); + + extensionsFilesystemManager.cleanupExtensionPath(extensionName, extensionRelativePath); + } + + @Test(expected = CloudRuntimeException.class) + public void throwsExceptionWhenDeletionFails() throws IOException { + String extensionName = "test-extension"; + String extensionRelativePath = "undeletable-file.txt"; + Path rootPath = Paths.get(tempDir.getAbsolutePath()); + Path filePath = rootPath.resolve(extensionRelativePath); + Files.createFile(filePath); + try (MockedStatic fileUtilMock = Mockito.mockStatic(FileUtil.class)) { + fileUtilMock.when(() -> FileUtil.deleteRecursively(filePath)).thenReturn(false); + + extensionsFilesystemManager.cleanupExtensionPath(extensionName, extensionRelativePath); + } + } + + @Test + public void cleansUpDirectoryWhenPathIsValid() throws IOException { + String extensionName = "test-extension"; + String extensionRelativePath = "test-dir"; + Path rootPath = Paths.get(tempDir.getAbsolutePath()); + Path dirPath = rootPath.resolve(extensionRelativePath); + Files.createDirectories(dirPath); - extensionsFilesystemManager.cleanupExtensionPath("test-extension", extensionDirName + "/test-file.txt"); + extensionsFilesystemManager.cleanupExtensionPath(extensionName, extensionRelativePath); - assertFalse(testFile.exists()); + assertFalse(Files.exists(dirPath)); } @Test - public void testCleanupExtensionData() throws IOException { - File extensionDataDir = new File(tempDataDir, "test-extension"); - extensionDataDir.mkdirs(); - File testFile = new File(extensionDataDir, "test-file.txt"); - testFile.createNewFile(); + public void cleansUpEntireDirectoryWhenCleanupDirectoryIsTrue() throws IOException { + String extensionName = "test-extension"; + Path extensionDataDir = Paths.get(tempDataDir.getAbsolutePath(), extensionName); + Files.createDirectories(extensionDataDir); + Files.createFile(extensionDataDir.resolve("file1.txt")); + Files.createFile(extensionDataDir.resolve("file2.txt")); - extensionsFilesystemManager.cleanupExtensionData("test-extension", 1, true); + extensionsFilesystemManager.cleanupExtensionData(extensionName, 1, true); - assertFalse(extensionDataDir.exists()); + assertFalse(Files.exists(extensionDataDir)); + } + + @Test + public void cleansUpOldFilesWhenOlderThanDaysIsSpecified() throws IOException { + String extensionName = "test-extension"; + Path extensionDataDir = Paths.get(tempDataDir.getAbsolutePath(), extensionName); + Files.createDirectories(extensionDataDir); + Path oldFile = extensionDataDir.resolve("old-file.txt"); + Path newFile = extensionDataDir.resolve("new-file.txt"); + Files.createFile(oldFile); + Files.createFile(newFile); + Files.setLastModifiedTime(oldFile, FileTime.fromMillis(System.currentTimeMillis() - TimeUnit.DAYS.toMillis(2))); + Files.setLastModifiedTime(newFile, FileTime.fromMillis(System.currentTimeMillis())); + + extensionsFilesystemManager.cleanupExtensionData(extensionName, 1, false); + + assertFalse(Files.exists(oldFile)); + assertTrue(Files.exists(newFile)); + } + + @Test + public void doesNothingWhenDirectoryDoesNotExist() { + String extensionName = "nonexistent-extension"; + + extensionsFilesystemManager.cleanupExtensionData(extensionName, 1, true); + + Mockito.verifyNoInteractions(logger); + } + + @Test + public void handlesIOExceptionDuringFileWalkGracefully() throws IOException { + String extensionName = "test-extension"; + Path extensionDataDir = Paths.get(tempDataDir.getAbsolutePath(), extensionName); + Files.createDirectories(extensionDataDir); + try (MockedStatic filesMock = Mockito.mockStatic(Files.class)) { + filesMock.when(() -> Files.walk(extensionDataDir)).thenThrow(new IOException("File walk error")); + + extensionsFilesystemManager.cleanupExtensionData(extensionName, 1, false); + } + + assertTrue(Files.exists(extensionDataDir)); + } + + @Test + public void skipsFilesNotOlderThanSpecifiedDays() throws IOException { + String extensionName = "test-extension"; + Path extensionDataDir = Paths.get(tempDataDir.getAbsolutePath(), extensionName); + Files.createDirectories(extensionDataDir); + Path recentFile = extensionDataDir.resolve("recent-file.txt"); + Files.createFile(recentFile); + Files.setLastModifiedTime(recentFile, FileTime.fromMillis(System.currentTimeMillis())); + + extensionsFilesystemManager.cleanupExtensionData(extensionName, 1, false); + + assertTrue(Files.exists(recentFile)); } @Test @@ -446,6 +655,53 @@ public void getExtensionsStagingPathThrowsExceptionWhenDirectoryCannotBeCreated( extensionsFilesystemManager.getExtensionsStagingPath(); } + @Test + public void doesNothingWhenFileListIsEmpty() { + Extension extension = mock(Extension.class); + extensionsFilesystemManager.validateExtensionFiles(extension, null); + extensionsFilesystemManager.validateExtensionFiles(extension, List.of()); + } + + @Test(expected = CloudRuntimeException.class) + public void throwsExceptionWhenExtensionDirectoryDoesNotExist() { + Extension extension = mock(Extension.class); + doReturn(Paths.get("/nonexistent/path")).when(extensionsFilesystemManager).getExtensionRootPath(extension); + + extensionsFilesystemManager.validateExtensionFiles(extension, List.of("file1.txt")); + } + + @Test(expected = CloudRuntimeException.class) + public void throwsExceptionWhenFileDoesNotExist() { + Extension extension = mock(Extension.class); + Path rootPath = tempDir.toPath(); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extension); + + extensionsFilesystemManager.validateExtensionFiles(extension, List.of("nonexistent-file.txt")); + } + + @Test + public void validatesFilesWhenAllExist() throws IOException { + Extension extension = mock(Extension.class); + Path rootPath = tempDir.toPath(); + Path file1 = Files.createFile(rootPath.resolve("file1.txt")); + Path file2 = Files.createFile(rootPath.resolve("file2.txt")); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extension); + + extensionsFilesystemManager.validateExtensionFiles(extension, List.of("file1.txt", "file2.txt")); + + assertTrue(Files.exists(file1)); + assertTrue(Files.exists(file2)); + } + + @Test(expected = CloudRuntimeException.class) + public void throwsExceptionWhenRelativeFilePathIsInvalid() { + Extension extension = mock(Extension.class); + Path rootPath = tempDir.toPath(); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extension); + + extensionsFilesystemManager.validateExtensionFiles(extension, List.of("../invalid-path/file.txt")); + } + @Test public void packExtensionFilesAsTgzReturnsTrue() { Extension extension = mock(Extension.class); diff --git a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java index 8fd44e66e067..0ef622678651 100644 --- a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java +++ b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java @@ -33,7 +33,7 @@ public class ServerPropertiesUtil { public static final String SHARE_DIR = "share"; private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); - private static final String PROPERTIES_FILE = "server.properties"; + protected static final String PROPERTIES_FILE = "server.properties"; public static final String SHARE_ENABLED = "share.enabled"; private static final String SHARE_BASE_DIR = "share.base.dir"; @@ -41,7 +41,7 @@ public class ServerPropertiesUtil { public static final String SHARE_DIR_ALLOWED = "share.dir.allowed"; private static final String SHARE_SECRET = "share.secret"; - private static final AtomicReference propertiesRef = new AtomicReference<>(); + protected static final AtomicReference propertiesRef = new AtomicReference<>(); public static String getProperty(String name) { Properties props = propertiesRef.get(); diff --git a/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java new file mode 100644 index 000000000000..b4485dbfb9f0 --- /dev/null +++ b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.server; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockConstruction; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.util.Properties; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.PropertiesUtil; + +@RunWith(MockitoJUnitRunner.class) +public class ServerPropertiesUtilTest { + + @Test + public void returnsPropertyValueWhenPropertiesAreLoaded() { + Properties mockProperties = mock(Properties.class); + when(mockProperties.getProperty("key")).thenReturn("value"); + ServerPropertiesUtil.propertiesRef.set(mockProperties); + String result = ServerPropertiesUtil.getProperty("key"); + assertEquals("value", result); + } + + @Test + public void returnsNullWhenPropertyDoesNotExist() { + Properties mockProperties = mock(Properties.class); + ServerPropertiesUtil.propertiesRef.set(mockProperties); + assertNull(ServerPropertiesUtil.getProperty("nonexistentKey")); + } + + @Test + public void loadsPropertiesFromFileWhenNotCached() throws Exception { + File tempFile = Files.createTempFile("server", ".properties").toFile(); + tempFile.deleteOnExit(); + Files.writeString(tempFile.toPath(), "key=value\n"); + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(tempFile); + assertEquals("value", ServerPropertiesUtil.getProperty("key")); + } + } + + @Test + public void returnsNullWhenPropertiesFileNotFound() { + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(null); + assertNull(ServerPropertiesUtil.getProperty("key")); + } + } + + @Test + public void returnsNullWhenIOExceptionOccurs() { + File mockFile = mock(File.class); + try (MockedStatic mocked = mockStatic(PropertiesUtil.class); + MockedConstruction ignored = mockConstruction(FileInputStream.class); + MockedConstruction ignored1 = mockConstruction(Properties.class, (mock, context) -> { + doThrow(new IOException("Test IOException")).when(mock).load(any(FileInputStream.class)); + })) { + mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) + .thenReturn(mockFile); + assertNull(ServerPropertiesUtil.getProperty("key")); + } + } +} From 0337e34163f532f10d116aab5f6ed3f4067df3b5 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Oct 2025 12:07:50 +0530 Subject: [PATCH 03/11] missing file Signed-off-by: Abhishek Kumar --- .../ExternalPathPayloadProvisioner.java | 543 +++++++++++++ .../ExternalPathPayloadProvisionerTest.java | 741 ++++++++++++++++++ 2 files changed, 1284 insertions(+) create mode 100644 plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java create mode 100644 plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java diff --git a/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java new file mode 100644 index 000000000000..aaaecd982c6a --- /dev/null +++ b/plugins/hypervisors/external/src/main/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisioner.java @@ -0,0 +1,543 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.hypervisor.external.provisioner; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.inject.Inject; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsFilesystemManager; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.ObjectUtils; + +import com.cloud.agent.AgentManager; +import com.cloud.agent.api.GetExternalConsoleAnswer; +import com.cloud.agent.api.GetExternalConsoleCommand; +import com.cloud.agent.api.HostVmStateReportEntry; +import com.cloud.agent.api.PrepareExternalProvisioningAnswer; +import com.cloud.agent.api.PrepareExternalProvisioningCommand; +import com.cloud.agent.api.RebootAnswer; +import com.cloud.agent.api.RebootCommand; +import com.cloud.agent.api.RunCustomActionAnswer; +import com.cloud.agent.api.RunCustomActionCommand; +import com.cloud.agent.api.StartAnswer; +import com.cloud.agent.api.StartCommand; +import com.cloud.agent.api.StopAnswer; +import com.cloud.agent.api.StopCommand; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.ExternalProvisioner; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.HypervisorGuru; +import com.cloud.hypervisor.HypervisorGuruManager; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.json.JsonMergeUtil; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VirtualMachineProfileImpl; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.UserVmDao; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +public class ExternalPathPayloadProvisioner extends ManagerBase implements ExternalProvisioner { + + @Inject + UserVmDao _uservmDao; + + @Inject + HostDao hostDao; + + @Inject + HypervisorGuruManager hypervisorGuruManager; + + @Inject + ExtensionsManager extensionsManager; + + @Inject + ExtensionsFilesystemManager extensionsFilesystemManager; + + private static final List TRIVIAL_ACTIONS = Arrays.asList( + "status" + ); + + @Override + public String getName() { + return getClass().getSimpleName(); + } + + protected Map loadAccessDetails(Map> externalDetails, + VirtualMachineTO virtualMachineTO) { + Map modifiedDetails = new HashMap<>(); + if (MapUtils.isNotEmpty(externalDetails) && externalDetails.containsKey(ApiConstants.CALLER)) { + modifiedDetails.put(ApiConstants.CALLER, externalDetails.get(ApiConstants.CALLER)); + externalDetails.remove(ApiConstants.CALLER); + } + if (MapUtils.isNotEmpty(externalDetails)) { + modifiedDetails.put(ApiConstants.EXTERNAL_DETAILS, externalDetails); + } + if (virtualMachineTO != null) { + modifiedDetails.put(ApiConstants.VIRTUAL_MACHINE_ID, virtualMachineTO.getUuid()); + modifiedDetails.put(ApiConstants.VIRTUAL_MACHINE_NAME, virtualMachineTO.getName()); + modifiedDetails.put(VmDetailConstants.CLOUDSTACK_VM_DETAILS, virtualMachineTO); + } + return modifiedDetails; + } + + protected VirtualMachineTO getVirtualMachineTO(VirtualMachine vm) { + if (vm == null) { + return null; + } + final HypervisorGuru hvGuru = hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.External); + VirtualMachineProfile profile = new VirtualMachineProfileImpl(vm); + return hvGuru.implement(profile); + } + + protected String getSanitizedJsonStringForLog(String json) { + if (StringUtils.isBlank(json)) { + return json; + } + return json.replaceAll("(\"password\"\\s*:\\s*\")([^\"]*)(\")", "$1****$3"); + } + + protected String getExtensionConfigureError(String extensionName, String hostName) { + StringBuilder sb = new StringBuilder("Extension: ").append(extensionName).append(" not configured"); + if (StringUtils.isNotBlank(hostName)) { + sb.append(" for host: ").append(hostName); + } + return sb.toString(); + } + + @Override + public PrepareExternalProvisioningAnswer prepareExternalProvisioning(String hostName, + String extensionName, String extensionRelativePath, PrepareExternalProvisioningCommand cmd) { + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return new PrepareExternalProvisioningAnswer(cmd, false, getExtensionConfigureError(extensionName, hostName)); + } + VirtualMachineTO vmTO = cmd.getVirtualMachineTO(); + String vmUUID = vmTO.getUuid(); + logger.debug("Executing PrepareExternalProvisioningCommand in the external provisioner " + + "for the VM {} as part of VM deployment", vmUUID); + Map accessDetails = loadAccessDetails(cmd.getExternalDetails(), vmTO); + Pair result = prepareExternalProvisioningInternal(extensionName, extensionPath, + vmUUID, accessDetails, cmd.getWait()); + String output = result.second(); + if (!result.first()) { + return new PrepareExternalProvisioningAnswer(cmd, false, output); + } + if (StringUtils.isEmpty(output)) { + return new PrepareExternalProvisioningAnswer(cmd, result.first(), ""); + } + try { + String merged = JsonMergeUtil.mergeJsonPatch(GsonHelper.getGson().toJson(vmTO), result.second()); + VirtualMachineTO virtualMachineTO = GsonHelper.getGson().fromJson(merged, VirtualMachineTO.class); + return new PrepareExternalProvisioningAnswer(cmd, null, virtualMachineTO, null); + } catch (Exception e) { + logger.warn("Failed to parse the output from preparing external provisioning operation as " + + "part of VM deployment: {}", e.getMessage(), e); + return new PrepareExternalProvisioningAnswer(cmd, false, "Failed to parse VM"); + } + } + + @Override + public StartAnswer startInstance(String hostName, String extensionName, String extensionRelativePath, + StartCommand cmd) { + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return new StartAnswer(cmd, getExtensionConfigureError(extensionName, hostName)); + } + VirtualMachineTO virtualMachineTO = cmd.getVirtualMachine(); + Map accessDetails = loadAccessDetails(cmd.getExternalDetails(), virtualMachineTO); + String vmUUID = virtualMachineTO.getUuid(); + + logger.debug(String.format("Executing StartCommand in the external provisioner for VM %s", vmUUID)); + + Object deployvm = virtualMachineTO.getDetails().get("deployvm"); + boolean isDeploy = (deployvm != null && Boolean.parseBoolean((String)deployvm)); + String operation = isDeploy ? "Deploying" : "Starting"; + try { + Pair result = executeStartCommandOnExternalSystem(extensionName, isDeploy, + extensionPath, vmUUID, accessDetails, cmd.getWait()); + + if (!result.first()) { + String errMsg = String.format("%s VM %s on the external system failed: %s", operation, vmUUID, result.second()); + logger.debug(errMsg); + return new StartAnswer(cmd, result.second()); + } + logger.debug(String.format("%s VM %s on the external system", operation, vmUUID)); + return new StartAnswer(cmd); + + } catch (CloudRuntimeException e) { + String errMsg = String.format("%s VM %s on the external system failed: %s", operation, vmUUID, e.getMessage()); + logger.debug(errMsg); + return new StartAnswer(cmd, errMsg); + } + } + + private Pair executeStartCommandOnExternalSystem(String extensionName, boolean isDeploy, + String filename, String vmUUID, Map accessDetails, int wait) { + if (isDeploy) { + return deployInstanceOnExternalSystem(extensionName, filename, vmUUID, accessDetails, wait); + } else { + return startInstanceOnExternalSystem(extensionName, filename, vmUUID, accessDetails, wait); + } + } + + @Override + public StopAnswer stopInstance(String hostName, String extensionName, String extensionRelativePath, + StopCommand cmd) { + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return new StopAnswer(cmd, getExtensionConfigureError(extensionName, hostName), false); + } + logger.debug("Executing stop command on the external provisioner"); + VirtualMachineTO virtualMachineTO = cmd.getVirtualMachine(); + String vmUUID = cmd.getVirtualMachine().getUuid(); + logger.debug("Executing stop command in the external system for the VM {}", vmUUID); + Map accessDetails = loadAccessDetails(cmd.getExternalDetails(), virtualMachineTO); + Pair result = stopInstanceOnExternalSystem(extensionName, extensionPath, vmUUID, + accessDetails, cmd.getWait()); + if (result.first()) { + return new StopAnswer(cmd, null, true); + } else { + return new StopAnswer(cmd, result.second(), false); + } + } + + @Override + public RebootAnswer rebootInstance(String hostName, String extensionName, String extensionRelativePath, + RebootCommand cmd) { + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return new RebootAnswer(cmd, getExtensionConfigureError(extensionName, hostName), false); + } + VirtualMachineTO virtualMachineTO = cmd.getVirtualMachine(); + String vmUUID = virtualMachineTO.getUuid(); + logger.debug("Executing reboot command in the external system for the VM {}", vmUUID); + Map accessDetails = loadAccessDetails(cmd.getExternalDetails(), virtualMachineTO); + Pair result = rebootInstanceOnExternalSystem(extensionName, extensionPath, vmUUID, + accessDetails, cmd.getWait()); + if (result.first()) { + return new RebootAnswer(cmd, null, true); + } else { + return new RebootAnswer(cmd, result.second(), false); + } + } + + @Override + public StopAnswer expungeInstance(String hostName, String extensionName, String extensionRelativePath, + StopCommand cmd) { + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return new StopAnswer(cmd, getExtensionConfigureError(extensionName, hostName), false); + } + VirtualMachineTO virtualMachineTO = cmd.getVirtualMachine(); + String vmUUID = virtualMachineTO.getUuid(); + logger.debug("Executing stop command as part of expunge in the external system for the VM {}", vmUUID); + Map accessDetails = loadAccessDetails(cmd.getExternalDetails(), virtualMachineTO); + Pair result = deleteInstanceOnExternalSystem(extensionName, extensionPath, vmUUID, + accessDetails, cmd.getWait()); + if (result.first()) { + return new StopAnswer(cmd, null, true); + } else { + return new StopAnswer(cmd, result.second(), false); + } + } + + @Override + public Map getHostVmStateReport(long hostId, String extensionName, + String extensionRelativePath) { + final Map vmStates = new HashMap<>(); + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return vmStates; + } + HostVO host = hostDao.findById(hostId); + if (host == null) { + logger.error("Host with ID: {} not found", hostId); + return vmStates; + } + List allVms = _uservmDao.listByHostId(hostId); + allVms.addAll(_uservmDao.listByLastHostId(hostId)); + if (CollectionUtils.isEmpty(allVms)) { + logger.debug("No VMs found for the {}", host); + return vmStates; + } + Map> accessDetails = + extensionsManager.getExternalAccessDetails(host, null); + for (UserVmVO vm: allVms) { + VirtualMachine.PowerState powerState = getVmPowerState(vm, accessDetails, extensionName, extensionPath); + vmStates.put(vm.getInstanceName(), new HostVmStateReportEntry(powerState, "host-" + hostId)); + } + return vmStates; + } + + @Override + public GetExternalConsoleAnswer getInstanceConsole(String hostName, String extensionName, + String extensionRelativePath, GetExternalConsoleCommand cmd) { + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return new GetExternalConsoleAnswer(cmd, getExtensionConfigureError(extensionName, hostName)); + } + VirtualMachineTO virtualMachineTO = cmd.getVirtualMachine(); + String vmUUID = virtualMachineTO.getUuid(); + logger.debug("Executing getconsole command in the external system for the VM {}", vmUUID); + Map accessDetails = loadAccessDetails(cmd.getExternalDetails(), virtualMachineTO); + Pair result = getInstanceConsoleOnExternalSystem(extensionName, extensionPath, vmUUID, + accessDetails, cmd.getWait()); + if (result == null) { + return new GetExternalConsoleAnswer(cmd, "No response from external system"); + } + String output = result.second(); + if (!result.first()) { + return new GetExternalConsoleAnswer(cmd, output); + } + logger.debug("Received console details from the external system: {}", + getSanitizedJsonStringForLog(output)); + try { + JsonObject jsonObj = JsonParser.parseString(output).getAsJsonObject(); + JsonObject consoleObj = jsonObj.has("console") ? jsonObj.getAsJsonObject("console") : null; + if (consoleObj == null) { + logger.error("Missing console object in external console output: {}", + getSanitizedJsonStringForLog(output)); + return new GetExternalConsoleAnswer(cmd, "Missing console object in output"); + } + String url = consoleObj.has("url") ? consoleObj.get("url").getAsString() : null; + String host = consoleObj.has("host") ? consoleObj.get("host").getAsString() : null; + Integer port = consoleObj.has("port") ? Integer.valueOf(consoleObj.get("port").getAsString()) : null; + String password = consoleObj.has("password") ? consoleObj.get("password").getAsString() : null; + boolean passwordOneTimeUseOnly = consoleObj.has("passwordonetimeuseonly") && + consoleObj.get("passwordonetimeuseonly").getAsBoolean(); + String protocol = consoleObj.has("protocol") ? consoleObj.get("protocol").getAsString() : null; + if (url == null && ObjectUtils.anyNull(host, port)) { + logger.error("Missing required fields in external console output: {}", + getSanitizedJsonStringForLog(output)); + return new GetExternalConsoleAnswer(cmd, "Missing required fields in output"); + } + return new GetExternalConsoleAnswer(cmd, url, host, port, password, passwordOneTimeUseOnly, protocol); + } catch (RuntimeException e) { + logger.error("Failed to parse output for getInstanceConsole: {}", e.getMessage(), e); + return new GetExternalConsoleAnswer(cmd, "Failed to parse output"); + } + } + + @Override + public RunCustomActionAnswer runCustomAction(String hostName, String extensionName, + String extensionRelativePath, RunCustomActionCommand cmd) { + String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extensionName, extensionRelativePath); + if (StringUtils.isEmpty(extensionPath)) { + return new RunCustomActionAnswer(cmd, false, getExtensionConfigureError(extensionName, hostName)); + } + final String actionName = cmd.getActionName(); + final Map parameters = cmd.getParameters(); + logger.debug("Executing custom action '{}' in the external system", actionName); + Map accessDetails = loadAccessDetails(cmd.getExternalDetails(), cmd.getVmTO()); + accessDetails.put(ApiConstants.ACTION, actionName); + if (MapUtils.isNotEmpty(parameters)) { + accessDetails.put(ApiConstants.PARAMETERS, parameters); + } + Pair result = runCustomActionOnExternalSystem(extensionName, extensionPath, + actionName, accessDetails, cmd.getWait()); + return new RunCustomActionAnswer(cmd, result.first(), result.second()); + } + + protected Pair runCustomActionOnExternalSystem(String extensionName, String filename, + String actionName, Map accessDetails, int wait) { + return executeExternalCommand(extensionName, actionName, accessDetails, wait, + String.format("Failed to execute custom action '%s' on external system", actionName), filename); + } + + protected VirtualMachine.PowerState getPowerStateFromString(String powerStateStr) { + if (StringUtils.isBlank(powerStateStr)) { + return VirtualMachine.PowerState.PowerUnknown; + } + if (powerStateStr.equalsIgnoreCase(VirtualMachine.PowerState.PowerOn.toString())) { + return VirtualMachine.PowerState.PowerOn; + } else if (powerStateStr.equalsIgnoreCase(VirtualMachine.PowerState.PowerOff.toString())) { + return VirtualMachine.PowerState.PowerOff; + } + return VirtualMachine.PowerState.PowerUnknown; + } + + protected VirtualMachine.PowerState parsePowerStateFromResponse(UserVmVO userVmVO, String response) { + logger.debug("Power status response from the external system for {} : {}", userVmVO, response); + if (StringUtils.isBlank(response)) { + logger.warn("Empty response while trying to fetch the power status of the {}", userVmVO); + return VirtualMachine.PowerState.PowerUnknown; + } + if (!response.trim().startsWith("{")) { + return getPowerStateFromString(response); + } + try { + JsonObject jsonObj = new JsonParser().parse(response).getAsJsonObject(); + String powerState = jsonObj.has("power_state") ? jsonObj.get("power_state").getAsString() : null; + return getPowerStateFromString(powerState); + } catch (Exception e) { + logger.warn("Failed to parse power status response: {} for {} as JSON: {}", + response, userVmVO, e.getMessage()); + return VirtualMachine.PowerState.PowerUnknown; + } + } + + private VirtualMachine.PowerState getVmPowerState(UserVmVO userVmVO, Map> accessDetails, + String extensionName, String extensionPath) { + VirtualMachineTO virtualMachineTO = getVirtualMachineTO(userVmVO); + accessDetails.put(ApiConstants.VIRTUAL_MACHINE, virtualMachineTO.getExternalDetails()); + Map modifiedDetails = loadAccessDetails(accessDetails, virtualMachineTO); + String vmUUID = userVmVO.getUuid(); + logger.debug("Trying to get VM power status from the external system for {}", userVmVO); + Pair result = getInstanceStatusOnExternalSystem(extensionName, extensionPath, vmUUID, + modifiedDetails, AgentManager.Wait.value()); + if (!result.first()) { + logger.warn("Failure response received while trying to fetch the power status of the {} : {}", + userVmVO, result.second()); + return VirtualMachine.PowerState.PowerUnknown; + } + return parsePowerStateFromResponse(userVmVO, result.second()); + } + public Pair prepareExternalProvisioningInternal(String extensionName, String filename, + String vmUUID, Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "prepare", accessDetails, wait, + String.format("Failed to prepare external provisioner for deploying VM %s on external system", vmUUID), + filename); + } + + public Pair deployInstanceOnExternalSystem(String extensionName, String filename, String vmUUID, + Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "create", accessDetails, wait, + String.format("Failed to create the instance %s on external system", vmUUID), filename); + } + + public Pair startInstanceOnExternalSystem(String extensionName, String filename, String vmUUID, + Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "start", accessDetails, wait, + String.format("Failed to start the instance %s on external system", vmUUID), filename); + } + + public Pair stopInstanceOnExternalSystem(String extensionName, String filename, String vmUUID, + Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "stop", accessDetails, wait, + String.format("Failed to stop the instance %s on external system", vmUUID), filename); + } + + public Pair rebootInstanceOnExternalSystem(String extensionName, String filename, String vmUUID, + Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "reboot", accessDetails, wait, + String.format("Failed to reboot the instance %s on external system", vmUUID), filename); + } + + public Pair deleteInstanceOnExternalSystem(String extensionName, String filename, String vmUUID, + Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "delete", accessDetails, wait, + String.format("Failed to delete the instance %s on external system", vmUUID), filename); + } + + public Pair getInstanceStatusOnExternalSystem(String extensionName, String filename, String vmUUID, + Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "status", accessDetails, wait, + String.format("Failed to get the instance power status %s on external system", vmUUID), filename); + } + + public Pair getInstanceConsoleOnExternalSystem(String extensionName, String filename, + String vmUUID, Map accessDetails, int wait) { + return executeExternalCommand(extensionName, "getconsole", accessDetails, wait, + String.format("Failed to get the instance console %s on external system", vmUUID), filename); + } + + public Pair executeExternalCommand(String extensionName, String action, + Map accessDetails, int wait, String errorLogPrefix, String file) { + try { + Path executablePath = Paths.get(file).toAbsolutePath().normalize(); + if (!Files.isExecutable(executablePath)) { + logger.error("{}: File is not executable: {}", errorLogPrefix, executablePath); + return new Pair<>(false, "File is not executable"); + } + if (wait == 0) { + wait = AgentManager.Wait.value(); + } + List command = new ArrayList<>(); + command.add(executablePath.toString()); + command.add(action); + String dataFile = extensionsFilesystemManager.prepareExternalPayload(extensionName, accessDetails); + command.add(dataFile); + command.add(Integer.toString(wait)); + ProcessBuilder builder = new ProcessBuilder(command); + builder.redirectErrorStream(true); + + logger.debug("Executing {} for command: {} with wait: {} and data file: {}", executablePath, + action, wait, dataFile); + + Process process = builder.start(); + boolean finished = process.waitFor(wait, TimeUnit.SECONDS); + if (!finished) { + process.destroyForcibly(); + logger.error("{}: External API execution timed out after {} seconds", errorLogPrefix, wait); + return new Pair<>(false, "Timeout"); + } + StringBuilder output = new StringBuilder(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line; + while ((line = reader.readLine()) != null) { + output.append(line).append(System.lineSeparator()); + } + } + int exitCode = process.exitValue(); + if (exitCode != 0) { + logger.warn("{}: External API execution failed with exit code {}", errorLogPrefix, exitCode); + return new Pair<>(false, "Exit code: " + exitCode + ", Output: " + output.toString().trim()); + } + deleteExtensionPayloadFile(extensionName, action, dataFile); + return new Pair<>(true, output.toString().trim()); + + } catch (IOException | InterruptedException e) { + logger.error("{}: External operation failed", errorLogPrefix, e); + throw new CloudRuntimeException(String.format("%s: External operation failed", errorLogPrefix), e); + } + } + + protected void deleteExtensionPayloadFile(String extensionName, String action, String payloadFileName) { + if (!TRIVIAL_ACTIONS.contains(action)) { + logger.trace("Skipping deletion of payload file: {} for extension: {}, action: {}", + payloadFileName, extensionName, action); + return; + } + extensionsFilesystemManager.deleteExtensionPayload(extensionName, payloadFileName); + } +} diff --git a/plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java b/plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java new file mode 100644 index 000000000000..2a562fa8f484 --- /dev/null +++ b/plugins/hypervisors/external/src/test/java/org/apache/cloudstack/hypervisor/external/provisioner/ExternalPathPayloadProvisionerTest.java @@ -0,0 +1,741 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// +package org.apache.cloudstack.hypervisor.external.provisioner; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyMap; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsFilesystemManager; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; +import org.apache.logging.log4j.Logger; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.GetExternalConsoleAnswer; +import com.cloud.agent.api.GetExternalConsoleCommand; +import com.cloud.agent.api.HostVmStateReportEntry; +import com.cloud.agent.api.PrepareExternalProvisioningAnswer; +import com.cloud.agent.api.PrepareExternalProvisioningCommand; +import com.cloud.agent.api.RebootAnswer; +import com.cloud.agent.api.RebootCommand; +import com.cloud.agent.api.RunCustomActionAnswer; +import com.cloud.agent.api.RunCustomActionCommand; +import com.cloud.agent.api.StartAnswer; +import com.cloud.agent.api.StartCommand; +import com.cloud.agent.api.StopAnswer; +import com.cloud.agent.api.StopCommand; +import com.cloud.agent.api.to.VirtualMachineTO; +import com.cloud.host.HostVO; +import com.cloud.host.dao.HostDao; +import com.cloud.hypervisor.Hypervisor; +import com.cloud.hypervisor.HypervisorGuru; +import com.cloud.hypervisor.HypervisorGuruManager; +import com.cloud.serializer.GsonHelper; +import com.cloud.template.VirtualMachineTemplate; +import com.cloud.utils.Pair; +import com.cloud.vm.UserVmVO; +import com.cloud.vm.VirtualMachine; +import com.cloud.vm.VirtualMachineProfile; +import com.cloud.vm.VmDetailConstants; +import com.cloud.vm.dao.UserVmDao; + +@RunWith(MockitoJUnitRunner.class) +public class ExternalPathPayloadProvisionerTest { + + @Spy + @InjectMocks + private ExternalPathPayloadProvisioner provisioner; + + @Mock + private UserVmDao userVmDao; + + @Mock + private HostDao hostDao; + + @Mock + private HypervisorGuruManager hypervisorGuruManager; + + @Mock + private HypervisorGuru hypervisorGuru; + + @Mock + private Logger logger; + + @Mock + private ExtensionsManager extensionsManager; + + @Mock + private ExtensionsFilesystemManager extensionsFilesystemManager; + + @Test + public void testLoadAccessDetails() { + Map> externalDetails = new HashMap<>(); + externalDetails.put(ApiConstants.EXTENSION, Map.of("key1", "value1")); + + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + when(vmTO.getName()).thenReturn("test-vm"); + + Map result = provisioner.loadAccessDetails(externalDetails, vmTO); + + assertNotNull(result); + assertEquals(externalDetails, result.get(ApiConstants.EXTERNAL_DETAILS)); + assertEquals("test-uuid", result.get(ApiConstants.VIRTUAL_MACHINE_ID)); + assertEquals("test-vm", result.get(ApiConstants.VIRTUAL_MACHINE_NAME)); + assertEquals(vmTO, result.get("cloudstack.vm.details")); + } + + @Test + public void testLoadAccessDetailsWithNullExternalDetails() { + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + when(vmTO.getName()).thenReturn("test-vm"); + + Map result = provisioner.loadAccessDetails(null, vmTO); + + assertNotNull(result); + assertNull(result.get(ApiConstants.EXTERNAL_DETAILS)); + assertEquals("test-uuid", result.get(ApiConstants.VIRTUAL_MACHINE_ID)); + assertEquals("test-vm", result.get(ApiConstants.VIRTUAL_MACHINE_NAME)); + } + + @Test + public void testLoadAccessDetails_WithCaller() { + Map> externalDetails = new HashMap<>(); + externalDetails.put(ApiConstants.EXTENSION, Map.of("key1", "value1")); + externalDetails.put(ApiConstants.CALLER, Map.of("key2", "value2")); + Map result = provisioner.loadAccessDetails(externalDetails, null); + + assertNotNull(result); + assertNotNull(result.get(ApiConstants.EXTERNAL_DETAILS)); + assertNotNull(((Map) result.get(ApiConstants.EXTERNAL_DETAILS)).get(ApiConstants.EXTENSION)); + assertNotNull(result.get(ApiConstants.CALLER)); + assertNull(result.get(VmDetailConstants.CLOUDSTACK_VM_DETAILS)); + } + + private String setupCheckedExtensionPath() { + String path = "test-extension.sh"; + when(extensionsFilesystemManager.getExtensionCheckedPath(anyString(), anyString())).thenReturn(path); + return path; + } + + @Test + public void testPrepareExternalProvisioning() { + String path = setupCheckedExtensionPath(); + PrepareExternalProvisioningCommand cmd = mock(PrepareExternalProvisioningCommand.class); + VirtualMachineTO vmTO = new VirtualMachineTO(1, "test-vm", VirtualMachine.Type.User, 1, 1000, 256, + 512, VirtualMachineTemplate.BootloaderType.HVM, "OS", false, false, "Pass"); + vmTO.setUuid("test-uuid"); + + when(cmd.getVirtualMachineTO()).thenReturn(vmTO); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(true, "{\"nics\":[{\"uuid\":\"test-net-uuid\",\"mac\":\"00:00:00:01:02:03\"}]}")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + PrepareExternalProvisioningAnswer answer = provisioner.prepareExternalProvisioning( + "host-name", "test-extension", path, cmd); + + assertTrue(answer.getResult()); + assertEquals("test-net-uuid", answer.getVirtualMachineTO().getNics()[0].getNetworkUuid()); + assertEquals("00:00:00:01:02:03", answer.getVirtualMachineTO().getNics()[0].getMac()); + } + + @Test + public void testPrepareExternalProvisioning_ExtensionNotConfigured() { + PrepareExternalProvisioningCommand cmd = mock(PrepareExternalProvisioningCommand.class); + + String extensionName = "test-extension"; + String hostName = "host-name"; + PrepareExternalProvisioningAnswer answer = provisioner.prepareExternalProvisioning( + hostName, extensionName, "nonexistent.sh", cmd); + + assertFalse(answer.getResult()); + assertNotNull(answer); + assertEquals(String.format("Extension: %s not configured for host: %s", extensionName, hostName), answer.getDetails()); + } + + @Test + public void testStartInstance() { + String path = setupCheckedExtensionPath(); + StartCommand cmd = mock(StartCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(true, "{\"status\": \"success\", \"message\": \"Instance started\"}")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + StartAnswer answer = provisioner.startInstance("host-name", "test-extension", path, cmd); + + assertTrue(answer.getResult()); + verify(logger).debug("Starting VM test-uuid on the external system"); + } + + @Test + public void testStartInstanceDeploy() { + String path = setupCheckedExtensionPath(); + StartCommand cmd = mock(StartCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + Map details = new HashMap<>(); + details.put("deployvm", "true"); + when(vmTO.getDetails()).thenReturn(details); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(true, "{\"status\": \"success\", \"message\": \"Instance started\"}")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + StartAnswer answer = provisioner.startInstance("host-name", "test-extension", path, cmd); + + assertTrue(answer.getResult()); + verify(logger).debug("Deploying VM test-uuid on the external system"); + } + + @Test + public void testStartInstanceError() { + String path = setupCheckedExtensionPath(); + StartCommand cmd = mock(StartCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(false, "{\"error\": \"Instance failed to start\"}")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + StartAnswer answer = provisioner.startInstance("host-name", "test-extension", path, cmd); + + assertFalse(answer.getResult()); + assertEquals("{\"error\": \"Instance failed to start\"}", answer.getDetails()); + verify(logger).debug("Starting VM test-uuid on the external system failed: {\"error\": \"Instance failed to start\"}"); + } + + @Test + public void testStopInstance() { + String path = setupCheckedExtensionPath(); + StopCommand cmd = mock(StopCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(true, "success")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + StopAnswer answer = provisioner.stopInstance("host-name", "test-extension", path, cmd); + + assertTrue(answer.getResult()); + } + + @Test + public void testRebootInstance() { + String path = setupCheckedExtensionPath(); + RebootCommand cmd = mock(RebootCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(true, "success")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + RebootAnswer answer = provisioner.rebootInstance("host-name", "test-extension", path, cmd); + + assertTrue(answer.getResult()); + } + + @Test + public void testExpungeInstance() { + String path = setupCheckedExtensionPath(); + StopCommand cmd = mock(StopCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(vmTO.getUuid()).thenReturn("test-uuid"); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(true, "success")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + StopAnswer answer = provisioner.expungeInstance("host-name", "test-extension", path, cmd); + + assertTrue(answer.getResult()); + } + + @Test + public void testGetHostVmStateReport() { + String path = setupCheckedExtensionPath(); + HostVO host = mock(HostVO.class); + when(hostDao.findById(anyLong())).thenReturn(host); + + UserVmVO vm = mock(UserVmVO.class); + when(vm.getUuid()).thenReturn("test-uuid"); + when(vm.getInstanceName()).thenReturn("test-instance"); + + List vms = new ArrayList<>(); + vms.add(vm); + when(userVmDao.listByHostId(anyLong())).thenReturn(vms); + when(userVmDao.listByLastHostId(anyLong())).thenReturn(new ArrayList<>()); + + when(hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.External)).thenReturn(hypervisorGuru); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(hypervisorGuru.implement(any(VirtualMachineProfile.class))).thenReturn(vmTO); + + doReturn(new Pair<>(true, "PowerOn")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + Map result = provisioner.getHostVmStateReport(1L, "test-extension", path); + + assertNotNull(result); + assertEquals(1, result.size()); + assertTrue(result.containsKey("test-instance")); + } + + @Test + public void testGetHostVmStateReportHostNotFound() { + String path = setupCheckedExtensionPath(); + Map result = provisioner.getHostVmStateReport(1L, "test-extension", path); + + assertNotNull(result); + assertTrue(result.isEmpty()); + verify(logger).error("Host with ID: {} not found", 1L); + } + + @Test + public void testRunCustomAction() { + String path = setupCheckedExtensionPath(); + RunCustomActionCommand cmd = mock(RunCustomActionCommand.class); + when(cmd.getActionName()).thenReturn("test-action"); + when(cmd.getParameters()).thenReturn(new HashMap<>()); + when(cmd.getExternalDetails()).thenReturn(new HashMap<>()); + when(cmd.getWait()).thenReturn(30); + + doReturn(new Pair<>(true, "success")).when(provisioner) + .executeExternalCommand(anyString(), anyString(), anyMap(), anyInt(), anyString(), anyString()); + + RunCustomActionAnswer answer = provisioner.runCustomAction("host-name", "test-extension", path, cmd); + + assertTrue(answer.getResult()); + verify(logger).debug("Executing custom action '{}' in the external system", "test-action"); + } + + @Test + public void testExecuteExternalCommand() throws IOException { + when(extensionsFilesystemManager.prepareExternalPayload(anyString(), anyMap())).thenAnswer( + invocation -> { + String extensionName = invocation.getArgument(0, String.class); + Map details = invocation.getArgument(1, Map.class); + File tempFile = File.createTempFile(extensionName, ".json"); + String json = GsonHelper.getGson().toJson(details); + Files.writeString(tempFile.toPath(), json); + tempFile.deleteOnExit(); + return tempFile.getAbsolutePath(); + } + ); + String scriptContent = "#!/bin/bash\nif grep -q '{\"test\":\"value\"}' \"$2\"; then\necho 'success'\nexit 0\nelse\necho 'fail'\nexit 1\nfi"; + File testScript = File.createTempFile("test-extension", ".sh"); + Files.writeString(testScript.toPath(), scriptContent); + testScript.setExecutable(true); + testScript.deleteOnExit(); + + Map accessDetails = new HashMap<>(); + accessDetails.put("test", "value"); + + Pair result = provisioner.executeExternalCommand( + "test-extension", "test-action", accessDetails, 30, "Test error", testScript.getAbsolutePath()); + + assertTrue(result.first()); + assertTrue(result.second().contains("success")); + } + + @Test + public void testExecuteExternalCommandFileNotExecutable() { + Map accessDetails = new HashMap<>(); + + Pair result = provisioner.executeExternalCommand( + "test-extension", "test-action", accessDetails, 30, "Test error", "/nonexistent/path"); + + assertFalse(result.first()); + assertEquals("File is not executable", result.second()); + } + + @Test + public void deleteExtensionPayloadFile_DeletesFile_WhenActionIsTrivial() { + String extensionName = "test-extension"; + String action = "status"; // in TRIVIAL_ACTIONS + String payloadFileName = "/tmp/test-payload.json"; + provisioner.deleteExtensionPayloadFile(extensionName, action, payloadFileName); + verify(extensionsFilesystemManager).deleteExtensionPayload(extensionName, payloadFileName); + } + + @Test + public void deleteExtensionPayloadFile_DoesNothing_WhenActionIsNotTrivial() { + String extensionName = "test-extension"; + String action = "start"; + String payloadFileName = "/tmp/test-payload.json"; + provisioner.deleteExtensionPayloadFile(extensionName, action, payloadFileName); + verify(logger).trace( + "Skipping deletion of payload file: {} for extension: {}, action: {}", + payloadFileName, extensionName, action); + } + + @Test + public void getPowerStateFromStringReturnsPowerOnForValidInput() { + VirtualMachine.PowerState result = provisioner.getPowerStateFromString("PowerOn"); + assertEquals(VirtualMachine.PowerState.PowerOn, result); + } + + @Test + public void getPowerStateFromStringReturnsPowerOffForValidInput() { + VirtualMachine.PowerState result = provisioner.getPowerStateFromString("PowerOff"); + assertEquals(VirtualMachine.PowerState.PowerOff, result); + } + + @Test + public void getPowerStateFromStringReturnsPowerUnknownForInvalidInput() { + VirtualMachine.PowerState result = provisioner.getPowerStateFromString("InvalidState"); + assertEquals(VirtualMachine.PowerState.PowerUnknown, result); + } + + @Test + public void getPowerStateFromStringReturnsPowerUnknownForBlankInput() { + VirtualMachine.PowerState result = provisioner.getPowerStateFromString(""); + assertEquals(VirtualMachine.PowerState.PowerUnknown, result); + } + + @Test + public void parsePowerStateFromResponseReturnsPowerOnForValidJson() { + String response = "{\"power_state\":\"PowerOn\"}"; + UserVmVO vm = mock(UserVmVO.class); + VirtualMachine.PowerState result = provisioner.parsePowerStateFromResponse(vm, response); + assertEquals(VirtualMachine.PowerState.PowerOn, result); + } + + @Test + public void parsePowerStateFromResponseReturnsPowerOffForValidJson() { + String response = "{\"power_state\":\"PowerOff\"}"; + UserVmVO vm = mock(UserVmVO.class); + VirtualMachine.PowerState result = provisioner.parsePowerStateFromResponse(vm, response); + assertEquals(VirtualMachine.PowerState.PowerOff, result); + } + + @Test + public void parsePowerStateFromResponseReturnsPowerUnknownForInvalidJson() { + String response = "{\"invalid_key\":\"value\"}"; + UserVmVO vm = mock(UserVmVO.class); + VirtualMachine.PowerState result = provisioner.parsePowerStateFromResponse(vm, response); + assertEquals(VirtualMachine.PowerState.PowerUnknown, result); + } + + @Test + public void parsePowerStateFromResponseReturnsPowerUnknownForMalformedJson() { + String response = "{power_state:PowerOn"; + UserVmVO vm = mock(UserVmVO.class); + VirtualMachine.PowerState result = provisioner.parsePowerStateFromResponse(vm, response); + assertEquals(VirtualMachine.PowerState.PowerUnknown, result); + } + + @Test + public void parsePowerStateFromResponseReturnsPowerUnknownForBlankResponse() { + String response = ""; + UserVmVO vm = mock(UserVmVO.class); + VirtualMachine.PowerState result = provisioner.parsePowerStateFromResponse(vm, response); + assertEquals(VirtualMachine.PowerState.PowerUnknown, result); + } + + @Test + public void parsePowerStateFromResponseReturnsPowerStateForPlainTextResponse() { + String response = "PowerOn"; + UserVmVO vm = mock(UserVmVO.class); + VirtualMachine.PowerState result = provisioner.parsePowerStateFromResponse(vm, response); + assertEquals(VirtualMachine.PowerState.PowerOn, result); + } + + @Test + public void getVirtualMachineTOReturnsNullWhenVmIsNull() { + VirtualMachineTO result = provisioner.getVirtualMachineTO(null); + assertNull(result); + } + + @Test + public void getVirtualMachineTOReturnsValidTOWhenVmIsNotNull() { + VirtualMachine vm = mock(VirtualMachine.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(hypervisorGuruManager.getGuru(Hypervisor.HypervisorType.External)).thenReturn(hypervisorGuru); + when(hypervisorGuru.implement(any(VirtualMachineProfile.class))).thenReturn(vmTO); + VirtualMachineTO result = provisioner.getVirtualMachineTO(vm); + assertNotNull(result); + assertEquals(vmTO, result); + verify(hypervisorGuruManager).getGuru(Hypervisor.HypervisorType.External); + verify(hypervisorGuru).implement(any(VirtualMachineProfile.class)); + } + + @Test + public void getInstanceConsoleReturnsAnswerWhenConsoleDetailsAreValid() { + String path = setupCheckedExtensionPath(); + GetExternalConsoleCommand cmd = mock(GetExternalConsoleCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(vmTO.getUuid()).thenReturn("test-uuid"); + + Map accessDetails = new HashMap<>(); + when(provisioner.loadAccessDetails(any(), eq(vmTO))).thenReturn(accessDetails); + + String validOutput = "{\"console\":{\"host\":\"127.0.0.1\",\"port\":5900,\"password\":\"pass\",\"protocol\":\"vnc\"}}"; + doReturn(new Pair<>(true, validOutput)).when(provisioner) + .getInstanceConsoleOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + GetExternalConsoleAnswer result = provisioner.getInstanceConsole("host-name", "test-extension", path, cmd); + + assertNotNull(result); + assertEquals("127.0.0.1", result.getHost()); + Integer port = 5900; + assertEquals(port, result.getPort()); + assertEquals("pass", result.getPassword()); + assertEquals("vnc", result.getProtocol()); + } + + @Test + public void getInstanceConsoleReturnsErrorWhenExtensionNotConfigured() { + GetExternalConsoleCommand cmd = mock(GetExternalConsoleCommand.class); + when(extensionsFilesystemManager.getExtensionCheckedPath(anyString(), anyString())).thenReturn(null); + + String extensionName = "test-extension"; + String hostName = "host-name"; + GetExternalConsoleAnswer result = provisioner.getInstanceConsole(hostName, + extensionName, "test-extension.sh", cmd); + + assertNotNull(result); + assertEquals(String.format("Extension: %s not configured for host: %s", extensionName, hostName), result.getDetails()); + } + + @Test + public void getInstanceConsoleReturnsErrorWhenExternalSystemFails() { + String path = setupCheckedExtensionPath(); + GetExternalConsoleCommand cmd = mock(GetExternalConsoleCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(vmTO.getUuid()).thenReturn("test-uuid"); + + doReturn(new Pair<>(false, "External system error")).when(provisioner) + .getInstanceConsoleOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + GetExternalConsoleAnswer result = provisioner.getInstanceConsole("host-name", "test-extension", path, cmd); + + assertNotNull(result); + assertEquals("External system error", result.getDetails()); + } + + @Test + public void getInstanceConsoleReturnsErrorWhenConsoleObjectIsMissing() { + String path = setupCheckedExtensionPath(); + GetExternalConsoleCommand cmd = mock(GetExternalConsoleCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(vmTO.getUuid()).thenReturn("test-uuid"); + + String invalidOutput = "{\"invalid_key\":\"value\"}"; + doReturn(new Pair<>(true, invalidOutput)).when(provisioner) + .getInstanceConsoleOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + GetExternalConsoleAnswer result = provisioner.getInstanceConsole("host-name", "test-extension", path, cmd); + + assertNotNull(result); + assertEquals("Missing console object in output", result.getDetails()); + } + + @Test + public void getInstanceConsoleReturnsErrorWhenRequiredFieldsAreMissing() { + String path = setupCheckedExtensionPath(); + GetExternalConsoleCommand cmd = mock(GetExternalConsoleCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(vmTO.getUuid()).thenReturn("test-uuid"); + + String incompleteOutput = "{\"console\":{\"host\":\"127.0.0.1\"}}"; + doReturn(new Pair<>(true, incompleteOutput)).when(provisioner) + .getInstanceConsoleOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + GetExternalConsoleAnswer result = provisioner.getInstanceConsole("host-name", "test-extension", path, cmd); + + assertNotNull(result); + assertEquals("Missing required fields in output", result.getDetails()); + } + + @Test + public void getInstanceConsoleReturnsErrorWhenOutputParsingFails() { + String path = setupCheckedExtensionPath(); + GetExternalConsoleCommand cmd = mock(GetExternalConsoleCommand.class); + VirtualMachineTO vmTO = mock(VirtualMachineTO.class); + when(cmd.getVirtualMachine()).thenReturn(vmTO); + when(vmTO.getUuid()).thenReturn("test-uuid"); + + String malformedOutput = "{console:invalid}"; + doReturn(new Pair<>(true, malformedOutput)).when(provisioner) + .getInstanceConsoleOnExternalSystem(anyString(), anyString(), anyString(), anyMap(), anyInt()); + + GetExternalConsoleAnswer result = provisioner.getInstanceConsole("host-name", "test-extension", path, cmd); + + assertNotNull(result); + assertEquals("Failed to parse output", result.getDetails()); + } + + @Test + public void getInstanceConsoleOnExternalSystemReturnsSuccessWhenCommandExecutesSuccessfully() { + String extensionName = "test-extension"; + String filename = "test-script.sh"; + String vmUUID = "test-vm-uuid"; + Map accessDetails = new HashMap<>(); + int wait = 30; + + doReturn(new Pair<>(true, "Console details")).when(provisioner) + .executeExternalCommand(eq(extensionName), eq("getconsole"), eq(accessDetails), eq(wait), anyString(), eq(filename)); + + Pair result = provisioner.getInstanceConsoleOnExternalSystem(extensionName, filename, vmUUID, accessDetails, wait); + + assertTrue(result.first()); + assertEquals("Console details", result.second()); + } + + @Test + public void getInstanceConsoleOnExternalSystemReturnsFailureWhenCommandFails() { + String extensionName = "test-extension"; + String filename = "test-script.sh"; + String vmUUID = "test-vm-uuid"; + Map accessDetails = new HashMap<>(); + int wait = 30; + + doReturn(new Pair<>(false, "Failed to get console")).when(provisioner) + .executeExternalCommand(eq(extensionName), eq("getconsole"), eq(accessDetails), eq(wait), anyString(), eq(filename)); + + Pair result = provisioner.getInstanceConsoleOnExternalSystem(extensionName, filename, vmUUID, accessDetails, wait); + + assertFalse(result.first()); + assertEquals("Failed to get console", result.second()); + } + + @Test + public void getInstanceConsoleOnExternalSystemHandlesNullResponseGracefully() { + String extensionName = "test-extension"; + String filename = "test-script.sh"; + String vmUUID = "test-vm-uuid"; + Map accessDetails = new HashMap<>(); + int wait = 30; + + doReturn(null).when(provisioner) + .executeExternalCommand(eq(extensionName), eq("getconsole"), eq(accessDetails), eq(wait), anyString(), eq(filename)); + + Pair result = provisioner.getInstanceConsoleOnExternalSystem(extensionName, filename, vmUUID, accessDetails, wait); + + assertNull(result); + } + + @Test + public void getSanitizedJsonStringForLogReturnsNullWhenInputIsNull() { + String result = provisioner.getSanitizedJsonStringForLog(null); + assertNull(result); + } + + @Test + public void getSanitizedJsonStringForLogReturnsEmptyWhenInputIsEmpty() { + String result = provisioner.getSanitizedJsonStringForLog(""); + assertEquals("", result); + } + + @Test + public void getSanitizedJsonStringForLogReturnsSameStringWhenNoPasswordField() { + String json = "{\"key\":\"value\"}"; + String result = provisioner.getSanitizedJsonStringForLog(json); + assertEquals(json, result); + } + + @Test + public void getSanitizedJsonStringForLogMasksPasswordField() { + String json = "{\"password\":\"secret\"}"; + String result = provisioner.getSanitizedJsonStringForLog(json); + assertEquals("{\"password\":\"****\"}", result); + } + + @Test + public void getSanitizedJsonStringForLogHandlesMultiplePasswordFields() { + String json = "{\"password\":\"secret\",\"nested\":{\"password\":\"anotherSecret\"}}"; + String result = provisioner.getSanitizedJsonStringForLog(json); + assertEquals("{\"password\":\"****\",\"nested\":{\"password\":\"****\"}}", result); + } + + @Test + public void getSanitizedJsonStringForLogHandlesMalformedJsonGracefully() { + String json = "{password:\"secret\""; + String result = provisioner.getSanitizedJsonStringForLog(json); + assertEquals("{password:\"secret\"", result); + } + + @Test + public void getExtensionConfigureErrorReturnsMessageWhenHostNameIsNotBlank() { + String result = provisioner.getExtensionConfigureError("test-extension", "test-host"); + assertEquals("Extension: test-extension not configured for host: test-host", result); + } + + @Test + public void getExtensionConfigureErrorReturnsMessageWhenHostNameIsBlank() { + String result = provisioner.getExtensionConfigureError("test-extension", ""); + assertEquals("Extension: test-extension not configured", result); + } + + @Test + public void getExtensionConfigureErrorReturnsMessageWhenHostNameIsNull() { + String result = provisioner.getExtensionConfigureError("test-extension", null); + assertEquals("Extension: test-extension not configured", result); + } +} From c91a9706dc423224713dad0236cc0ba8f5dcd47c Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Oct 2025 12:13:13 +0530 Subject: [PATCH 04/11] fix line endings Signed-off-by: Abhishek Kumar --- .../cloudstack/ShareSignedUrlFilterTest.java | 48 +++++++++---------- .../extensions/api/SyncExtensionCmdTest.java | 26 +++++----- .../server/ServerPropertiesUtilTest.java | 8 ++-- 3 files changed, 41 insertions(+), 41 deletions(-) diff --git a/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java index 97c7d1655107..5eb9f87309c1 100644 --- a/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java +++ b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java @@ -39,80 +39,80 @@ public void allowsRequestWhenTokenIsNotRequiredAndParametersAreMissing() throws HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - + when(mockRequest.getParameter("exp")).thenReturn(null); when(mockRequest.getParameter("sig")).thenReturn(null); - + filter.doFilter(mockRequest, mockResponse, mockChain); - + verify(mockChain).doFilter(mockRequest, mockResponse); } - + @Test public void deniesRequestWhenTokenIsRequiredAndParametersAreMissing() throws Exception { ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - + when(mockRequest.getParameter("exp")).thenReturn(null); when(mockRequest.getParameter("sig")).thenReturn(null); - + filter.doFilter(mockRequest, mockResponse, mockChain); - + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); verifyNoInteractions(mockChain); } - + @Test public void deniesRequestWhenExpirationIsInvalid() throws Exception { ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - + when(mockRequest.getParameter("exp")).thenReturn("invalid"); when(mockRequest.getParameter("sig")).thenReturn("signature"); - + filter.doFilter(mockRequest, mockResponse, mockChain); - + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Bad exp"); verifyNoInteractions(mockChain); } - + @Test public void deniesRequestWhenTokenIsExpired() throws Exception { ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - + when(mockRequest.getParameter("exp")).thenReturn(String.valueOf(Instant.now().getEpochSecond() - 10)); when(mockRequest.getParameter("sig")).thenReturn("signature"); - + filter.doFilter(mockRequest, mockResponse, mockChain); - + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired"); verifyNoInteractions(mockChain); } - + @Test public void deniesRequestWhenSignatureIsInvalid() throws Exception { ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - + when(mockRequest.getParameter("exp")).thenReturn(String.valueOf(Instant.now().getEpochSecond() + 1000)); when(mockRequest.getParameter("sig")).thenReturn("invalidSignature"); when(mockRequest.getRequestURI()).thenReturn("/share/resource"); - + filter.doFilter(mockRequest, mockResponse, mockChain); - + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Bad signature"); verifyNoInteractions(mockChain); } - + @Test public void allowsRequestWhenSignatureIsValid() throws Exception { String secret = "secret"; @@ -120,17 +120,17 @@ public void allowsRequestWhenSignatureIsValid() throws Exception { HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - + String exp = String.valueOf(Instant.now().getEpochSecond() + 1000); String data = "/share/resource|" + exp; String validSignature = HMACSignUtil.generateSignature(data, secret); - + when(mockRequest.getParameter("exp")).thenReturn(exp); when(mockRequest.getParameter("sig")).thenReturn(validSignature); when(mockRequest.getRequestURI()).thenReturn("/share/resource"); - + filter.doFilter(mockRequest, mockResponse, mockChain); - + verify(mockChain).doFilter(mockRequest, mockResponse); } } diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java index 7c43f48dee0a..63eb5c48258d 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java @@ -43,14 +43,14 @@ public void returnsIdWhenGetIdIsCalled() { ReflectionTestUtils.setField(cmd, "id", 123L); assertEquals(Long.valueOf(123), cmd.getId()); } - + @Test public void returnsSourceManagementServerIdWhenGetSourceManagementServerIdIsCalled() { SyncExtensionCmd cmd = new SyncExtensionCmd(); ReflectionTestUtils.setField(cmd, "sourceManagementServerId", 456L); assertEquals(Long.valueOf(456), cmd.getSourceManagementServerId()); } - + @Test public void returnsTargetManagementServerIdsWhenGetTargetManagementServerIdsIsCalled() { SyncExtensionCmd cmd = new SyncExtensionCmd(); @@ -58,7 +58,7 @@ public void returnsTargetManagementServerIdsWhenGetTargetManagementServerIdsIsCa ReflectionTestUtils.setField(cmd, "targetManagementServerIds", targetIds); assertEquals(targetIds, cmd.getTargetManagementServerIds()); } - + @Test public void returnsFilesWhenGetFilesIsCalled() { SyncExtensionCmd cmd = new SyncExtensionCmd(); @@ -66,55 +66,55 @@ public void returnsFilesWhenGetFilesIsCalled() { ReflectionTestUtils.setField(cmd, "files", files); assertEquals(files, cmd.getFiles()); } - + @Test public void executesSuccessfullyWhenSyncExtensionSucceeds() throws Exception { SyncExtensionCmd cmd = new SyncExtensionCmd(); ExtensionsManager mockManager = mock(ExtensionsManager.class); ReflectionTestUtils.setField(cmd, "extensionsManager", mockManager); when(mockManager.syncExtension(cmd)).thenReturn(true); - + cmd.execute(); - + SuccessResponse response = (SuccessResponse) cmd.getResponseObject(); assertTrue(response.getSuccess()); } - + @Test(expected = ServerApiException.class) public void throwsExceptionWhenSyncExtensionFails() throws Exception { SyncExtensionCmd cmd = new SyncExtensionCmd(); ExtensionsManager mockManager = mock(ExtensionsManager.class); ReflectionTestUtils.setField(cmd, "extensionsManager", mockManager); when(mockManager.syncExtension(cmd)).thenReturn(false); - + cmd.execute(); } - + @Test public void returnsSystemAccountIdWhenGetEntityOwnerIdIsCalled() { SyncExtensionCmd cmd = new SyncExtensionCmd(); assertEquals(Account.ACCOUNT_ID_SYSTEM, cmd.getEntityOwnerId()); } - + @Test public void returnsExtensionResourceTypeWhenGetApiResourceTypeIsCalled() { SyncExtensionCmd cmd = new SyncExtensionCmd(); assertEquals(ApiCommandResourceType.Extension, cmd.getApiResourceType()); } - + @Test public void returnsIdWhenGetApiResourceIdIsCalled() { SyncExtensionCmd cmd = new SyncExtensionCmd(); ReflectionTestUtils.setField(cmd, "id", 123L); assertEquals(Long.valueOf(123), cmd.getApiResourceId()); } - + @Test public void returnsCorrectEventType() { SyncExtensionCmd cmd = new SyncExtensionCmd(); assertEquals(EventTypes.EVENT_EXTENSION_SYNC, cmd.getEventType()); } - + @Test public void returnsCorrectEventDescription() { SyncExtensionCmd cmd = new SyncExtensionCmd(); diff --git a/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java index b4485dbfb9f0..21e0b63ddc58 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java @@ -51,14 +51,14 @@ public void returnsPropertyValueWhenPropertiesAreLoaded() { String result = ServerPropertiesUtil.getProperty("key"); assertEquals("value", result); } - + @Test public void returnsNullWhenPropertyDoesNotExist() { Properties mockProperties = mock(Properties.class); ServerPropertiesUtil.propertiesRef.set(mockProperties); assertNull(ServerPropertiesUtil.getProperty("nonexistentKey")); } - + @Test public void loadsPropertiesFromFileWhenNotCached() throws Exception { File tempFile = Files.createTempFile("server", ".properties").toFile(); @@ -70,7 +70,7 @@ public void loadsPropertiesFromFileWhenNotCached() throws Exception { assertEquals("value", ServerPropertiesUtil.getProperty("key")); } } - + @Test public void returnsNullWhenPropertiesFileNotFound() { try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { @@ -79,7 +79,7 @@ public void returnsNullWhenPropertiesFileNotFound() { assertNull(ServerPropertiesUtil.getProperty("key")); } } - + @Test public void returnsNullWhenIOExceptionOccurs() { File mockFile = mock(File.class); From ecbdcd66e7ecc37dfa23d6e9c8bb09d04aa82e80 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 10 Oct 2025 16:17:13 +0530 Subject: [PATCH 05/11] do not use mockConstruction Signed-off-by: Abhishek Kumar --- .../server/ServerPropertiesUtilTest.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java index 21e0b63ddc58..2eece43e47b1 100644 --- a/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java +++ b/utils/src/test/java/org/apache/cloudstack/utils/server/ServerPropertiesUtilTest.java @@ -19,22 +19,18 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockConstruction; import static org.mockito.Mockito.mockStatic; import static org.mockito.Mockito.when; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.nio.file.Files; import java.util.Properties; +import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; -import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.junit.MockitoJUnitRunner; @@ -43,6 +39,11 @@ @RunWith(MockitoJUnitRunner.class) public class ServerPropertiesUtilTest { + @After + public void clearCache() { + ServerPropertiesUtil.propertiesRef.set(null); + } + @Test public void returnsPropertyValueWhenPropertiesAreLoaded() { Properties mockProperties = mock(Properties.class); @@ -81,15 +82,13 @@ public void returnsNullWhenPropertiesFileNotFound() { } @Test - public void returnsNullWhenIOExceptionOccurs() { - File mockFile = mock(File.class); - try (MockedStatic mocked = mockStatic(PropertiesUtil.class); - MockedConstruction ignored = mockConstruction(FileInputStream.class); - MockedConstruction ignored1 = mockConstruction(Properties.class, (mock, context) -> { - doThrow(new IOException("Test IOException")).when(mock).load(any(FileInputStream.class)); - })) { + public void returnsNullWhenIOExceptionOccurs() throws IOException { + File tempFile = Files.createTempFile("bad", ".properties").toFile(); + tempFile.deleteOnExit(); + Files.writeString(tempFile.toPath(), "\u0000\u0000\u0000"); + try (MockedStatic mocked = mockStatic(PropertiesUtil.class)) { mocked.when(() -> PropertiesUtil.findConfigFile(ServerPropertiesUtil.PROPERTIES_FILE)) - .thenReturn(mockFile); + .thenReturn(tempFile); assertNull(ServerPropertiesUtil.getProperty("key")); } } From 2c70e034d937a516a572968c9b121d2cf090cf08 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Sat, 11 Oct 2025 14:24:10 +0530 Subject: [PATCH 06/11] changes Signed-off-by: Abhishek Kumar --- .../org/apache/cloudstack/ServerDaemon.java | 2 +- .../cloudstack/ShareSignedUrlFilter.java | 9 +- .../cloudstack/ShareSignedUrlFilterTest.java | 81 ++----- .../manager/ExtensionsFilesystemManager.java | 2 - .../ExtensionsFilesystemManagerImpl.java | 18 -- .../manager/ExtensionsShareManagerImpl.java | 28 +-- .../ExtensionsFilesystemManagerImplTest.java | 64 ----- .../ExtensionsShareManagerImplTest.java | 227 ++++++++---------- 8 files changed, 138 insertions(+), 293 deletions(-) diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 446f7728f758..1b1b851d0d60 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -355,7 +355,7 @@ private Handler createShareContextHandler() throws IOException { // Optional signed-URL guard (path + "|" + exp => HMAC-SHA256, base64url) if (StringUtils.isNotBlank(shareSecret)) { - shareCtx.addFilter(new FilterHolder(new ShareSignedUrlFilter(true, shareSecret)), + shareCtx.addFilter(new FilterHolder(new ShareSignedUrlFilter(shareSecret)), "/*", EnumSet.of(DispatcherType.REQUEST)); } diff --git a/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java index e9a2dc2ee12c..704c867eec9c 100644 --- a/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java +++ b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java @@ -32,16 +32,15 @@ import org.apache.cloudstack.utils.security.HMACSignUtil; import org.apache.commons.codec.DecoderException; +import org.apache.commons.lang3.StringUtils; /** * Optional HMAC token check: /share/...?...&exp=1699999999&sig=BASE64URL(HMACSHA256(path|exp)) */ public class ShareSignedUrlFilter implements Filter { - private final boolean requireToken; private final String secret; - public ShareSignedUrlFilter(boolean requireToken, String secret) { - this.requireToken = requireToken; + public ShareSignedUrlFilter(String secret) { this.secret = secret; } @@ -54,10 +53,6 @@ public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) String expStr = r.getParameter("exp"); String sig = r.getParameter("sig"); - if (!requireToken && (expStr == null || sig == null)) { - chain.doFilter(req, res); - return; - } if (expStr == null || sig == null) { w.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); return; diff --git a/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java index 5eb9f87309c1..7377d8d5b19d 100644 --- a/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java +++ b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java @@ -34,39 +34,8 @@ public class ShareSignedUrlFilterTest { @Test - public void allowsRequestWhenTokenIsNotRequiredAndParametersAreMissing() throws Exception { - ShareSignedUrlFilter filter = new ShareSignedUrlFilter(false, "secret"); - HttpServletRequest mockRequest = mock(HttpServletRequest.class); - HttpServletResponse mockResponse = mock(HttpServletResponse.class); - FilterChain mockChain = mock(FilterChain.class); - - when(mockRequest.getParameter("exp")).thenReturn(null); - when(mockRequest.getParameter("sig")).thenReturn(null); - - filter.doFilter(mockRequest, mockResponse, mockChain); - - verify(mockChain).doFilter(mockRequest, mockResponse); - } - - @Test - public void deniesRequestWhenTokenIsRequiredAndParametersAreMissing() throws Exception { - ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); - HttpServletRequest mockRequest = mock(HttpServletRequest.class); - HttpServletResponse mockResponse = mock(HttpServletResponse.class); - FilterChain mockChain = mock(FilterChain.class); - - when(mockRequest.getParameter("exp")).thenReturn(null); - when(mockRequest.getParameter("sig")).thenReturn(null); - - filter.doFilter(mockRequest, mockResponse, mockChain); - - verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); - verifyNoInteractions(mockChain); - } - - @Test - public void deniesRequestWhenExpirationIsInvalid() throws Exception { - ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); + public void deniesRequestWhenExpParameterIsWithinDeltaButInvalid() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter("secret"); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); @@ -81,56 +50,56 @@ public void deniesRequestWhenExpirationIsInvalid() throws Exception { } @Test - public void deniesRequestWhenTokenIsExpired() throws Exception { - ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); + public void allowsRequestWhenExpParameterIsValidAndWithinDelta() throws Exception { + String secret = "secret"; + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(secret); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - when(mockRequest.getParameter("exp")).thenReturn(String.valueOf(Instant.now().getEpochSecond() - 10)); - when(mockRequest.getParameter("sig")).thenReturn("signature"); + String exp = String.valueOf(Instant.now().getEpochSecond() + 50); // Within delta + String data = "/share/resource|" + exp; + String validSignature = HMACSignUtil.generateSignature(data, secret); + + when(mockRequest.getParameter("exp")).thenReturn(exp); + when(mockRequest.getParameter("sig")).thenReturn(validSignature); + when(mockRequest.getRequestURI()).thenReturn("/share/resource"); filter.doFilter(mockRequest, mockResponse, mockChain); - verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired"); - verifyNoInteractions(mockChain); + verify(mockChain).doFilter(mockRequest, mockResponse); } @Test - public void deniesRequestWhenSignatureIsInvalid() throws Exception { - ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, "secret"); + public void deniesRequestWhenExpParameterIsValidButOutsideDelta() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter("secret"); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - when(mockRequest.getParameter("exp")).thenReturn(String.valueOf(Instant.now().getEpochSecond() + 1000)); - when(mockRequest.getParameter("sig")).thenReturn("invalidSignature"); - when(mockRequest.getRequestURI()).thenReturn("/share/resource"); + String exp = String.valueOf(Instant.now().getEpochSecond() - 200); // Outside delta + when(mockRequest.getParameter("exp")).thenReturn(exp); + when(mockRequest.getParameter("sig")).thenReturn("signature"); filter.doFilter(mockRequest, mockResponse, mockChain); - verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Bad signature"); + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired"); verifyNoInteractions(mockChain); } @Test - public void allowsRequestWhenSignatureIsValid() throws Exception { - String secret = "secret"; - ShareSignedUrlFilter filter = new ShareSignedUrlFilter(true, secret); + public void deniesRequestWhenSignatureIsValidButExpParameterIsMissing() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter("secret"); HttpServletRequest mockRequest = mock(HttpServletRequest.class); HttpServletResponse mockResponse = mock(HttpServletResponse.class); FilterChain mockChain = mock(FilterChain.class); - String exp = String.valueOf(Instant.now().getEpochSecond() + 1000); - String data = "/share/resource|" + exp; - String validSignature = HMACSignUtil.generateSignature(data, secret); - - when(mockRequest.getParameter("exp")).thenReturn(exp); - when(mockRequest.getParameter("sig")).thenReturn(validSignature); - when(mockRequest.getRequestURI()).thenReturn("/share/resource"); + when(mockRequest.getParameter("exp")).thenReturn(null); + when(mockRequest.getParameter("sig")).thenReturn("validSignature"); filter.doFilter(mockRequest, mockResponse, mockChain); - verify(mockChain).doFilter(mockRequest, mockResponse); + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); + verifyNoInteractions(mockChain); } } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java index 341e7200ad63..b9f0252dad2c 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java @@ -49,6 +49,4 @@ public interface ExtensionsFilesystemManager { void deleteExtensionPayload(String extensionName, String payloadFileName); void validateExtensionFiles(Extension extension, List files); - - boolean packExtensionFilesAsTgz(Extension extension, Path extensionFilesPath, Path archivePath); } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java index 966f8e4e6c9e..03bff7249564 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java @@ -248,17 +248,6 @@ public Map getChecksumMapForExtension(String extensionName, Stri } } - protected boolean packFilesAsTgz(Path sourcePath, Path archivePath) { - int result = Script.executeCommandForExitValue( - 60 * 1000, - Script.getExecutableAbsolutePath("tar"), - "-czpf", archivePath.toAbsolutePath().toString(), - "-C", sourcePath.toAbsolutePath().toString(), - "." - ); - return result == 0; - } - @Override public void prepareExtensionPath(String extensionName, boolean userDefined, Extension.Type type, String extensionRelativePath) { logger.debug("Preparing entry point for Extension [name: {}, user-defined: {}]", extensionName, userDefined); @@ -438,11 +427,4 @@ public void validateExtensionFiles(Extension extension, List files) { } } } - - @Override - public boolean packExtensionFilesAsTgz(Extension extension, Path extensionFilesPath, Path archivePath) { - logger.debug("Packing files for {} from: {} to archive: {}", extension, - extensionFilesPath.toAbsolutePath(), archivePath.toAbsolutePath()); - return packFilesAsTgz(extensionFilesPath, archivePath); - } } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java index 020af902d338..0d4dceafadf2 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java @@ -44,6 +44,7 @@ import org.apache.cloudstack.framework.extensions.command.DownloadAndSyncExtensionFilesCommand; import org.apache.cloudstack.managed.context.ManagedContextRunnable; import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.utils.filesystem.ArchiveUtil; import org.apache.cloudstack.utils.security.DigestHelper; import org.apache.cloudstack.utils.security.HMACSignUtil; import org.apache.cloudstack.utils.server.ServerPropertiesUtil; @@ -126,13 +127,13 @@ protected Pair getResultFromAnswersString(String answersStr, Ex } /** - * Creates a .tgz archive for the specified extension. + * Creates an archive for the specified extension. * If the files list is empty, the entire extension directory is archived. * If the files list is not empty, only the specified relative files are archived; throws if any file is missing. * * @return ArchiveInfo containing the archive path, size, SHA-256 checksum, and sync type. */ - protected ArchiveInfo createArchive(Extension extension, List files) throws IOException { + protected ArchiveInfo createArchiveForSync(Extension extension, List files) throws IOException { final boolean isPartial = CollectionUtils.isNotEmpty(files); final DownloadAndSyncExtensionFilesCommand.SyncType syncType = isPartial ? DownloadAndSyncExtensionFilesCommand.SyncType.Partial @@ -161,7 +162,7 @@ protected ArchiveInfo createArchive(Extension extension, List files) thr } Path archivePath = getExtensionsSharePath().resolve(archiveName.toString()); - if (!packAsTgz(extension, extensionRootPath, toPack, archivePath)) { + if (!packArchiveForSync(extension, extensionRootPath, toPack, archivePath)) { throw new IOException("Failed to create archive " + archivePath); } @@ -227,7 +228,7 @@ protected DownloadAndSyncExtensionFilesCommand buildCommand(long msId, Extension * @return true if the archive was created successfully, false otherwise * @throws IOException if an I/O error occurs during packing */ - protected boolean packAsTgz(Extension extension, Path extensionRootPath, List toPack, Path archivePath) + protected boolean packArchiveForSync(Extension extension, Path extensionRootPath, List toPack, Path archivePath) throws IOException { Files.createDirectories(archivePath.getParent()); FileUtil.deletePath(archivePath.toAbsolutePath().toString()); @@ -244,8 +245,9 @@ protected boolean packAsTgz(Extension extension, Path extensionRootPath, List syncExtension(Extension extension, ManagementServer ArchiveInfo archiveInfo = null; try { try { - archiveInfo = createArchive(extension, files); + archiveInfo = createArchiveForSync(extension, files); } catch (IOException e) { String msg = "Failed to create archive"; logger.error("{} for {}", extension, msg, e); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java index b65a74c52cc9..b9cc063491c3 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java @@ -43,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; @@ -368,49 +367,6 @@ public void getChecksumMapForExtensionReturnsEmptyMapWhenNoFilesExist() { } } - @Test - public void packFilesAsTgzReturnsTrue() { - Path sourcePath = tempDir.toPath(); - Path archivePath = Path.of("test-archive.tgz"); - try (MockedStatic