diff --git a/api/src/main/java/com/cloud/event/EventTypes.java b/api/src/main/java/com/cloud/event/EventTypes.java index 38e601c790a7..2b3210280dc3 100644 --- a/api/src/main/java/com/cloud/event/EventTypes.java +++ b/api/src/main/java/com/cloud/event/EventTypes.java @@ -844,6 +844,8 @@ public class EventTypes { public static final String EVENT_EXTENSION_CREATE = "EXTENSION.CREATE"; public static final String EVENT_EXTENSION_UPDATE = "EXTENSION.UPDATE"; public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE"; + public static final String EVENT_EXTENSION_SYNC = "EXTENSION.SYNC"; + public static final String EVENT_EXTENSION_DOWNLOAD = "EXTENSION.DOWNLOAD"; public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER"; public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER"; public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD"; @@ -1385,6 +1387,7 @@ public class EventTypes { entityEventDetails.put(EVENT_EXTENSION_CREATE, Extension.class); entityEventDetails.put(EVENT_EXTENSION_UPDATE, Extension.class); entityEventDetails.put(EVENT_EXTENSION_DELETE, Extension.class); + entityEventDetails.put(EVENT_EXTENSION_SYNC, Extension.class); entityEventDetails.put(EVENT_EXTENSION_RESOURCE_REGISTER, Extension.class); entityEventDetails.put(EVENT_EXTENSION_RESOURCE_UNREGISTER, Extension.class); entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class); diff --git a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java index 475b845af9b1..238b17484916 100644 --- a/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java +++ b/api/src/main/java/org/apache/cloudstack/api/ApiConstants.java @@ -547,6 +547,7 @@ public class ApiConstants { public static final String SOURCE_CIDR_LIST = "sourcecidrlist"; public static final String SOURCE_ZONE_ID = "sourcezoneid"; + public static final String SOURCE_MANAGEMENT_SERVER_ID = "sourcemanagementserverid"; public static final String SSL_VERIFICATION = "sslverification"; public static final String START_ASN = "startasn"; public static final String START_DATE = "startdate"; @@ -567,6 +568,7 @@ public class ApiConstants { public static final String SWAP_OWNER = "swapowner"; public static final String SYSTEM_VM_TYPE = "systemvmtype"; public static final String TAGS = "tags"; + public static final String TARGET_MANAGEMENT_SERVER_IDS = "targetmanagementserverids"; public static final String STORAGE_TAGS = "storagetags"; public static final String STORAGE_ACCESS_GROUPS = "storageaccessgroups"; public static final String STORAGE_ACCESS_GROUP = "storageaccessgroup"; diff --git a/api/src/main/java/org/apache/cloudstack/api/response/ExtractResponse.java b/api/src/main/java/org/apache/cloudstack/api/response/ExtractResponse.java index 3d22dfe092ce..e3b9f76ccb57 100644 --- a/api/src/main/java/org/apache/cloudstack/api/response/ExtractResponse.java +++ b/api/src/main/java/org/apache/cloudstack/api/response/ExtractResponse.java @@ -18,21 +18,20 @@ import java.util.Date; -import com.google.gson.annotations.SerializedName; - import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.api.BaseResponse; import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; public class ExtractResponse extends BaseResponse { @SerializedName(ApiConstants.ID) @Param(description = "the id of extracted object") - private String id; + protected String id; @SerializedName(ApiConstants.NAME) @Param(description = "the name of the extracted object") - private String name; + protected String name; @SerializedName("extractId") @Param(description = "the upload id of extracted object") @@ -80,7 +79,7 @@ public class ExtractResponse extends BaseResponse { @SerializedName(ApiConstants.URL) @Param(description = "if mode = upload then url of the uploaded entity. if mode = download the url from which the entity can be downloaded") - private String url; + protected String url; public ExtractResponse() { } diff --git a/client/conf/server.properties.in b/client/conf/server.properties.in index 5958486b4dff..beb04ee8909c 100644 --- a/client/conf/server.properties.in +++ b/client/conf/server.properties.in @@ -62,3 +62,19 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@ # Thread pool configuration #threads.min=10 #threads.max=500 + +# These properties configure the share endpoint, which enables controlled file sharing through the management server. +# They allow administrators to enable or disable sharing, set the base directory for shared files, define cache +# behavior, restrict access to specific directories, and secure access with a secret key. This ensures flexible and +# secure file sharing for different modules such as extensions, etc. +# Enable or disable file sharing feature (true/false). Default is true +share.enabled=true +# The base directory from which files can be shared. Default is /share +# share.base.dir= +# The cache control header value to be used for shared files. Default is public,max-age=86400,immutable +# share.cache.control=public,max-age=86400,immutable +# Allow or disallow directory listing when accessing a directory. Default is false +# share.dir.allowed=false +# Secret key for securing links using HMAC signature. If not set then links will not be signed. Default is change-me +# It is recommended to change this value to a strong secret key in production +share.secret=change-me diff --git a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java index 196695e1fc6b..7ab1e71fdd15 100644 --- a/client/src/main/java/org/apache/cloudstack/ServerDaemon.java +++ b/client/src/main/java/org/apache/cloudstack/ServerDaemon.java @@ -24,15 +24,25 @@ import java.io.InputStream; import java.lang.management.ManagementFactory; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; import java.util.Properties; -import com.cloud.api.ApiServer; +import javax.servlet.DispatcherType; + +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; import org.apache.commons.daemon.Daemon; import org.apache.commons.daemon.DaemonContext; import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.eclipse.jetty.jmx.MBeanContainer; import org.eclipse.jetty.server.ForwardedRequestCustomizer; +import org.eclipse.jetty.server.Handler; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; import org.eclipse.jetty.server.RequestLog; @@ -46,14 +56,18 @@ import org.eclipse.jetty.server.handler.RequestLogHandler; import org.eclipse.jetty.server.handler.gzip.GzipHandler; import org.eclipse.jetty.server.session.SessionHandler; +import org.eclipse.jetty.servlet.DefaultServlet; +import org.eclipse.jetty.servlet.FilterHolder; +import org.eclipse.jetty.servlet.ServletContextHandler; +import org.eclipse.jetty.servlet.ServletHolder; +import org.eclipse.jetty.util.resource.Resource; import org.eclipse.jetty.util.ssl.KeyStoreScanner; import org.eclipse.jetty.util.ssl.SslContextFactory; import org.eclipse.jetty.util.thread.QueuedThreadPool; import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler; import org.eclipse.jetty.webapp.WebAppContext; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.LogManager; +import com.cloud.api.ApiServer; import com.cloud.utils.Pair; import com.cloud.utils.PropertiesUtil; import com.cloud.utils.server.ServerProperties; @@ -111,6 +125,12 @@ public class ServerDaemon implements Daemon { private int minThreads; private int maxThreads; + private boolean shareEnabled = false; + private String shareBaseDir; + private String shareCacheCtl; + private boolean shareDirList = false; + private String shareSecret; + ////////////////////////////////////////////////// /////////////// Public methods /////////////////// ////////////////////////////////////////////////// @@ -121,6 +141,22 @@ public static void main(final String... anArgs) throws Exception { daemon.start(); } + protected void initShareConfigFromProperties() { + setShareEnabled(ServerPropertiesUtil.getShareEnabled()); + setShareBaseDir(ServerPropertiesUtil.getShareBaseDirectory()); + setShareCacheCtl(ServerPropertiesUtil.getShareCacheControl()); + setShareDirList(ServerPropertiesUtil.getShareDirAllowed()); + setShareSecret(ServerPropertiesUtil.getShareSecret()); + + logger.info(String.format("/%s static context enabled=%s, baseDir=%s, dirList=%s, cacheCtl=%s, secret=%s", + ServerPropertiesUtil.SHARE_DIR, + shareEnabled, + shareBaseDir, + shareDirList, + shareCacheCtl, + (StringUtils.isNotBlank(shareSecret) ? "configured" : "not configured"))); + } + @Override public void init(final DaemonContext context) { final File confFile = PropertiesUtil.findConfigFile("server.properties"); @@ -153,6 +189,7 @@ public void init(final DaemonContext context) { setMaxFormKeys(Integer.valueOf(properties.getProperty(REQUEST_MAX_FORM_KEYS_KEY, String.valueOf(DEFAULT_REQUEST_MAX_FORM_KEYS)))); setMinThreads(Integer.valueOf(properties.getProperty(THREADS_MIN, "10"))); setMaxThreads(Integer.valueOf(properties.getProperty(THREADS_MAX, "500"))); + initShareConfigFromProperties(); } catch (final IOException e) { logger.warn("Failed to read configuration from server.properties file", e); } finally { @@ -288,6 +325,52 @@ private void createHttpsConnector(final HttpConfiguration httpConfig) { } } + /** + * Creates a Jetty context at /share to serve static files for modules (e.g. Extensions Framework). + * Controlled via server properties + * + * @return a configured Handler or null if disabled. + */ + private Handler createShareContextHandler() throws IOException { + if (!shareEnabled) { + logger.info("/{} context not mounted", ServerPropertiesUtil.SHARE_DIR); + return null; + } + + final Path base = Paths.get(shareBaseDir); + Files.createDirectories(base); + + final ServletContextHandler shareCtx = new ServletContextHandler(ServletContextHandler.NO_SESSIONS); + shareCtx.setContextPath("/" + ServerPropertiesUtil.SHARE_DIR); + shareCtx.setBaseResource(Resource.newResource(base.toAbsolutePath().toUri())); + + // Efficient static file serving + ServletHolder def = shareCtx.addServlet(DefaultServlet.class, "/*"); + def.setInitParameter("dirAllowed", Boolean.toString(shareDirList)); + def.setInitParameter("etags", "true"); + def.setInitParameter("cacheControl", shareCacheCtl); + def.setInitParameter("useFileMappedBuffer", "true"); + def.setInitParameter("acceptRanges", "true"); + + // Gzip using modern Jetty handler + org.eclipse.jetty.server.handler.gzip.GzipHandler gzipHandler = + new org.eclipse.jetty.server.handler.gzip.GzipHandler(); + gzipHandler.setMinGzipSize(1024); + gzipHandler.setIncludedMimeTypes( + "text/html", "text/plain", "text/css", "text/javascript", + "application/javascript", "application/json", "application/xml"); + gzipHandler.setHandler(shareCtx); + + // Optional signed-URL guard (path + "|" + exp => HMAC-SHA256, base64url) + if (StringUtils.isNotBlank(shareSecret)) { + shareCtx.addFilter(new FilterHolder(new ShareSignedUrlFilter(shareSecret)), + "/*", EnumSet.of(DispatcherType.REQUEST)); + } + + logger.info("Mounted /{} static context at baseDir={}", ServerPropertiesUtil.SHARE_DIR, base); + return shareCtx; + } + private Pair createHandlers() { final WebAppContext webApp = new WebAppContext(); webApp.setContextPath(contextPath); @@ -318,8 +401,23 @@ private Pair createHandlers() { rootRedirect.setNewContextURL(contextPath); rootRedirect.setPermanent(true); + // Optional /share handler (served by createShareContextHandler) + Handler shareHandler = null; + try { + shareHandler = createShareContextHandler(); + } catch (IOException e) { + logger.error("Failed to initialize /share context", e); + } + + List handlers = new java.util.ArrayList<>(); + handlers.add(log); + handlers.add(gzipHandler); + if (shareHandler != null) { + handlers.add(shareHandler); + } // Put rootRedirect at the end! - return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(log, gzipHandler, rootRedirect)); + handlers.add(rootRedirect); + return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(handlers.toArray(new Handler[0]))); } private RequestLog createRequestLog() { @@ -408,4 +506,24 @@ public void setMinThreads(int minThreads) { public void setMaxThreads(int maxThreads) { this.maxThreads = maxThreads; } + + public void setShareEnabled(boolean shareEnabled) { + this.shareEnabled = shareEnabled; + } + + public void setShareBaseDir(String shareBaseDir) { + this.shareBaseDir = shareBaseDir; + } + + public void setShareCacheCtl(String shareCacheCtl) { + this.shareCacheCtl = shareCacheCtl; + } + + public void setShareDirList(boolean shareDirList) { + this.shareDirList = shareDirList; + } + + public void setShareSecret(String shareSecret) { + this.shareSecret = shareSecret; + } } diff --git a/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java new file mode 100644 index 000000000000..eb6b0320dfec --- /dev/null +++ b/client/src/main/java/org/apache/cloudstack/ShareSignedUrlFilter.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack; + +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.utils.security.HMACSignUtil; +import org.apache.commons.codec.DecoderException; + +/** + * HMAC token check: /share/...?...&exp=1699999999&sig=BASE64URL(HMACSHA256(path|exp)) + */ +public class ShareSignedUrlFilter implements Filter { + private final String secret; + + public ShareSignedUrlFilter(String secret) { + this.secret = secret; + } + + @Override + public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) + throws IOException, ServletException { + HttpServletRequest r = (HttpServletRequest) req; + HttpServletResponse w = (HttpServletResponse) res; + + String expStr = r.getParameter("exp"); + String sig = r.getParameter("sig"); + + if (expStr == null || sig == null) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); + return; + } + long exp; + try { + exp = Long.parseLong(expStr); + } catch (NumberFormatException e) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad exp"); + return; + } + if (Instant.now().getEpochSecond() > exp) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired"); + return; + } + String want = ""; + try { + String data = r.getRequestURI() + "|" + expStr; + want = HMACSignUtil.generateSignature(data, secret); + } catch (InvalidKeyException | NoSuchAlgorithmException | DecoderException e) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Auth error"); + return; + } + if (!want.equals(sig)) { + w.sendError(HttpServletResponse.SC_FORBIDDEN, "Bad signature"); + return; + } + chain.doFilter(req, res); + } +} diff --git a/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java new file mode 100644 index 000000000000..6ca2f4a97d04 --- /dev/null +++ b/client/src/test/java/org/apache/cloudstack/ShareSignedUrlFilterTest.java @@ -0,0 +1,113 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import java.time.Instant; + +import javax.servlet.FilterChain; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.cloudstack.utils.security.HMACSignUtil; +import org.junit.Test; + +/** + * Unit tests for {@link ShareSignedUrlFilter}. + * + * Tests cover: + * - invalid or missing `exp` parameter + * - expired tokens outside the allowed time delta + * - valid signatures when `exp` is within the allowed delta + */ +public class ShareSignedUrlFilterTest { + + @Test + public void deniesRequestWhenExpParameterIsWithinDeltaButInvalid() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter("secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + when(mockRequest.getParameter("exp")).thenReturn("invalid"); + when(mockRequest.getParameter("sig")).thenReturn("signature"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Bad exp"); + verifyNoInteractions(mockChain); + } + + @Test + public void allowsRequestWhenExpParameterIsValidAndWithinDelta() throws Exception { + String secret = "secret"; + ShareSignedUrlFilter filter = new ShareSignedUrlFilter(secret); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + String exp = String.valueOf(Instant.now().getEpochSecond() + 50); // Within delta + String data = "/share/resource|" + exp; + String validSignature = HMACSignUtil.generateSignature(data, secret); + + when(mockRequest.getParameter("exp")).thenReturn(exp); + when(mockRequest.getParameter("sig")).thenReturn(validSignature); + when(mockRequest.getRequestURI()).thenReturn("/share/resource"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockChain).doFilter(mockRequest, mockResponse); + } + + @Test + public void deniesRequestWhenExpParameterIsValidButOutsideDelta() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter("secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + String exp = String.valueOf(Instant.now().getEpochSecond() - 200); // Outside delta + when(mockRequest.getParameter("exp")).thenReturn(exp); + when(mockRequest.getParameter("sig")).thenReturn("signature"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Token expired"); + verifyNoInteractions(mockChain); + } + + @Test + public void deniesRequestWhenSignatureIsValidButExpParameterIsMissing() throws Exception { + ShareSignedUrlFilter filter = new ShareSignedUrlFilter("secret"); + HttpServletRequest mockRequest = mock(HttpServletRequest.class); + HttpServletResponse mockResponse = mock(HttpServletResponse.class); + FilterChain mockChain = mock(FilterChain.class); + + when(mockRequest.getParameter("exp")).thenReturn(null); + when(mockRequest.getParameter("sig")).thenReturn("validSignature"); + + filter.doFilter(mockRequest, mockResponse, mockChain); + + verify(mockResponse).sendError(HttpServletResponse.SC_FORBIDDEN, "Missing token"); + verifyNoInteractions(mockChain); + } +} diff --git a/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java b/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java index c574a8be0175..e35403a8c7e5 100644 --- a/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java +++ b/engine/components-api/src/main/java/com/cloud/hypervisor/ExternalProvisioner.java @@ -35,18 +35,6 @@ public interface ExternalProvisioner extends Manager { - String getExtensionsPath(); - - String getExtensionPath(String relativePath); - - String getChecksumForExtensionPath(String extensionName, String relativePath); - - void prepareExtensionPath(String extensionName, boolean userDefined, String extensionRelativePath); - - void cleanupExtensionPath(String extensionName, String extensionRelativePath); - - void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory); - PrepareExternalProvisioningAnswer prepareExternalProvisioning(String hostGuid, String extensionName, String extensionRelativePath, PrepareExternalProvisioningCommand cmd); StartAnswer startInstance(String hostGuid, String extensionName, String extensionRelativePath, StartCommand cmd); diff --git a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java index 78924a10b32d..f2900af6a4a0 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/ClusterManagerImpl.java @@ -468,6 +468,8 @@ public String execute(final String strPeer, final long agentId, final String cmd return null; } + + @Override public ManagementServerHostVO getPeer(final String mgmtServerId) { return _mshostDao.findByMsid(Long.parseLong(mgmtServerId)); diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java index 6c8ffcac78b7..dd0019dd1181 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDao.java @@ -61,4 +61,6 @@ public interface ManagementServerHostDao extends GenericDao listUpByIds(List ids); } diff --git a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java index ec943a9c26be..96ac18cb9858 100644 --- a/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java +++ b/framework/cluster/src/main/java/com/cloud/cluster/dao/ManagementServerHostDaoImpl.java @@ -24,12 +24,11 @@ import java.util.List; import java.util.TimeZone; - +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.management.ManagementServerHost.State; import org.apache.commons.collections.CollectionUtils; import com.cloud.cluster.ClusterInvalidSessionException; -import org.apache.cloudstack.management.ManagementServerHost; -import org.apache.cloudstack.management.ManagementServerHost.State; import com.cloud.cluster.ManagementServerHostVO; import com.cloud.utils.DateUtil; import com.cloud.utils.db.DB; @@ -318,4 +317,18 @@ public ManagementServerHostVO findOneByLongestRuntime() { return CollectionUtils.isNotEmpty(msHosts) ? msHosts.get(0) : null; } + @Override + public List listUpByIds(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return new ArrayList<>(); + } + SearchBuilder sb = createSearchBuilder(); + sb.and("ids", sb.entity().getId(), SearchCriteria.Op.IN); + sb.and("state", sb.entity().getState(), SearchCriteria.Op.EQ); + sb.done(); + SearchCriteria sc = sb.create(); + sc.setParameters("ids", ids.toArray()); + sc.setParameters("state", ManagementServerHost.State.Up); + return listBy(sc); + } } diff --git a/framework/cluster/src/test/java/com/cloud/cluster/dao/ManagementServerHostDaoImplTest.java b/framework/cluster/src/test/java/com/cloud/cluster/dao/ManagementServerHostDaoImplTest.java new file mode 100644 index 000000000000..438e71b928cb --- /dev/null +++ b/framework/cluster/src/test/java/com/cloud/cluster/dao/ManagementServerHostDaoImplTest.java @@ -0,0 +1,90 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package com.cloud.cluster.dao; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collections; +import java.util.List; + +import org.apache.cloudstack.management.ManagementServerHost; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.cluster.ManagementServerHostVO; +import com.cloud.utils.db.SearchBuilder; +import com.cloud.utils.db.SearchCriteria; + +@RunWith(MockitoJUnitRunner.class) +public class ManagementServerHostDaoImplTest { + + @Spy + @InjectMocks + private ManagementServerHostDaoImpl managementServerHostDao; + + @Test + public void listUpByIdsReturnsEmptyListWhenInputIsEmpty() { + List result = managementServerHostDao.listUpByIds(Collections.emptyList()); + assertNotNull(result); + assertTrue(result.isEmpty()); + } + + @Test + public void listUpByIdsReturnsEmptyListWhenNoMatchingIds() {; + SearchBuilder mockSb = mock(SearchBuilder.class); + SearchCriteria mocSC = mock(SearchCriteria.class); + when(mockSb.entity()).thenReturn(mock(ManagementServerHostVO.class)); + when(mockSb.create()).thenReturn(mocSC); + doReturn(mockSb).when(managementServerHostDao).createSearchBuilder(); + doReturn(Collections.emptyList()).when(managementServerHostDao).listBy(any(SearchCriteria.class)); + List result = managementServerHostDao.listUpByIds(List.of(1L, 2L)); + assertNotNull(result); + assertTrue(result.isEmpty()); + verify(managementServerHostDao).createSearchBuilder(); + verify(mockSb).and(eq("ids"), anyLong(), eq(SearchCriteria.Op.IN)); + verify(mockSb).and(eq("state"), nullable(ManagementServerHost.State.class), eq(SearchCriteria.Op.EQ)); + verify(mockSb).done(); + verify(mocSC).setParameters(eq("ids"), anyLong(), anyLong()); + verify(mocSC).setParameters("state", ManagementServerHost.State.Up); + } + + @Test + public void listUpByIdsReturnsMatchingHostsWhenIdsAreValid() { + ManagementServerHostVO host1 = mock(ManagementServerHostVO.class); + ManagementServerHostVO host2 = mock(ManagementServerHostVO.class); + doReturn(List.of(host1, host2)).when(managementServerHostDao).listBy(any(SearchCriteria.class)); + List result = managementServerHostDao.listUpByIds(List.of(1L, 2L)); + assertNotNull(result); + assertEquals(2, result.size()); + assertTrue(result.contains(host1)); + assertTrue(result.contains(host2)); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/DownloadExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/DownloadExtensionCmd.java new file mode 100644 index 000000000000..5eb13d547ca0 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/DownloadExtensionCmd.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.api; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ExtensionResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.framework.extensions.api.response.DownloadExtensionResponse; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.user.Account; + +@APICommand(name = "downloadExtension", + description = "To download the extension files as an archive", + responseObject = SuccessResponse.class, + responseHasSensitiveInfo = false, + entityType = {Extension.class}, + authorized = {RoleType.Admin}, + since = "4.23.0") +public class DownloadExtensionCmd extends BaseAsyncCmd { + + @Inject + ExtensionsManager extensionsManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = CommandType.UUID, required = true, + entityType = ExtensionResponse.class, description = "ID of the extension") + private Long id; + + @Parameter(name = ApiConstants.MANAGEMENT_SERVER_ID, type = CommandType.UUID, + entityType = ManagementServerResponse.class, + description = "ID of the management server from which files are to be downloaded") + private Long managementServerId; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Long getManagementServerId() { + return managementServerId; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + DownloadExtensionResponse response = extensionsManager.downloadExtension(this); + if (response != null) { + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to download extension"); + } + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Extension; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_EXTENSION_DOWNLOAD; + } + + @Override + public String getEventDescription() { + return "Download extension: " + getId(); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmd.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmd.java new file mode 100644 index 000000000000..3ea1e98da95a --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmd.java @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.api; + +import java.util.List; + +import javax.inject.Inject; + +import org.apache.cloudstack.acl.RoleType; +import org.apache.cloudstack.api.APICommand; +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.BaseAsyncCmd; +import org.apache.cloudstack.api.BaseCmd; +import org.apache.cloudstack.api.Parameter; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.ExtensionResponse; +import org.apache.cloudstack.api.response.ManagementServerResponse; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; + +import com.cloud.event.EventTypes; +import com.cloud.exception.ConcurrentOperationException; +import com.cloud.user.Account; + +@APICommand(name = "syncExtension", + description = "To sync the extension files from one management server to other management server(s)", + responseObject = SuccessResponse.class, + responseHasSensitiveInfo = false, + entityType = {Extension.class}, + authorized = {RoleType.Admin}, + since = "4.23.0") +public class SyncExtensionCmd extends BaseAsyncCmd { + + @Inject + ExtensionsManager extensionsManager; + + ///////////////////////////////////////////////////// + //////////////// API parameters ///////////////////// + ///////////////////////////////////////////////////// + + @Parameter(name = ApiConstants.ID, type = BaseCmd.CommandType.UUID, required = true, + entityType = ExtensionResponse.class, description = "ID of the extension") + private Long id; + + @Parameter(name = ApiConstants.SOURCE_MANAGEMENT_SERVER_ID, type = BaseCmd.CommandType.UUID, required = true, + entityType = ManagementServerResponse.class, + description = "ID of the management server from which files are to be synced") + private Long sourceManagementServerId; + + @Parameter(name = ApiConstants.TARGET_MANAGEMENT_SERVER_IDS, type = BaseCmd.CommandType.LIST, + collectionType = BaseCmd.CommandType.UUID, entityType = ManagementServerResponse.class, + description="the IDs of the management servers to which the extension files are to be synced. " + + "If not specified, the files will be synced to all management servers") + private List targetManagementServerIds; + + @Parameter(name = ApiConstants.FILES, type = CommandType.LIST, collectionType = CommandType.STRING, + description = "List of files to sync. Specify absolute or relative paths. If not provided, all extension " + + "files will be synced.") + private List files; + + ///////////////////////////////////////////////////// + /////////////////// Accessors /////////////////////// + ///////////////////////////////////////////////////// + + public Long getId() { + return id; + } + + public Long getSourceManagementServerId() { + return sourceManagementServerId; + } + + public List getTargetManagementServerIds() { + return targetManagementServerIds; + } + + public List getFiles() { + return files; + } + + ///////////////////////////////////////////////////// + /////////////// API Implementation/////////////////// + ///////////////////////////////////////////////////// + + @Override + public void execute() throws ServerApiException, ConcurrentOperationException { + boolean result = extensionsManager.syncExtension(this); + if (result) { + SuccessResponse response = new SuccessResponse(getCommandName()); + response.setSuccess(result); + setResponseObject(response); + } else { + throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to sync extension"); + } + } + + @Override + public long getEntityOwnerId() { + return Account.ACCOUNT_ID_SYSTEM; + } + + @Override + public ApiCommandResourceType getApiResourceType() { + return ApiCommandResourceType.Extension; + } + + @Override + public Long getApiResourceId() { + return getId(); + } + + @Override + public String getEventType() { + return EventTypes.EVENT_EXTENSION_SYNC; + } + + @Override + public String getEventDescription() { + return "Sync extension: " + getId(); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/response/DownloadExtensionResponse.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/response/DownloadExtensionResponse.java new file mode 100644 index 000000000000..bccbca365768 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/api/response/DownloadExtensionResponse.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.api.response; + +import org.apache.cloudstack.api.ApiConstants; +import org.apache.cloudstack.api.response.ExtractResponse; + +import com.cloud.serializer.Param; +import com.google.gson.annotations.SerializedName; + +public class DownloadExtensionResponse extends ExtractResponse { + + @SerializedName(ApiConstants.MANAGEMENT_SERVER_ID) + @Param(description = "the management server ID of the host") + private String managementServerId; + + @SerializedName(ApiConstants.MANAGEMENT_SERVER_NAME) + @Param(description = "the management server name of the host") + private String managementServerName; + + public String getManagementServerId() { + return managementServerId; + } + + public void setManagementServerId(String managementServerId) { + this.managementServerId = managementServerId; + } + + public String getManagementServerName() { + return managementServerName; + } + + public void setManagementServerName(String managementServerName) { + this.managementServerName = managementServerName; + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadAndSyncExtensionFilesCommand.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadAndSyncExtensionFilesCommand.java new file mode 100644 index 000000000000..538cf09b7b85 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadAndSyncExtensionFilesCommand.java @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.command; + +import org.apache.cloudstack.extension.Extension; + +public class DownloadAndSyncExtensionFilesCommand extends ExtensionServerActionBaseCommand { + + public enum SyncType { + Complete, + Partial + } + + private SyncType syncType = SyncType.Complete; + private final String downloadUrl; + private final String checksum; + private final long size; + private String path; + + public DownloadAndSyncExtensionFilesCommand(long msId, Extension extension, String downloadUrl, + long size, String checksum) { + super(msId, extension); + this.downloadUrl = downloadUrl; + this.size = size; + this.checksum = checksum; + } + + public SyncType getSyncType() { + return syncType; + } + + public void setSyncType(SyncType syncType) { + this.syncType = syncType; + } + + public String getDownloadUrl() { + return downloadUrl; + } + + public long getSize() { + return size; + } + + public String getChecksum() { + return checksum; + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadExtensionFilesCommand.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadExtensionFilesCommand.java new file mode 100644 index 000000000000..a313ecd20a29 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/DownloadExtensionFilesCommand.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.command; + +import org.apache.cloudstack.extension.Extension; + +public class DownloadExtensionFilesCommand extends ExtensionServerActionBaseCommand { + + public DownloadExtensionFilesCommand(long msId, Extension extension) { + super(msId, extension); + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java index ead3c2e4012e..4a340b98b469 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/ExtensionBaseCommand.java @@ -25,6 +25,7 @@ public class ExtensionBaseCommand extends Command { private final long extensionId; private final String extensionName; private final boolean extensionUserDefined; + private final Extension.Type extensionType; private final String extensionRelativePath; private final Extension.State extensionState; @@ -32,6 +33,7 @@ protected ExtensionBaseCommand(Extension extension) { this.extensionId = extension.getId(); this.extensionName = extension.getName(); this.extensionUserDefined = extension.isUserDefined(); + this.extensionType = extension.getType(); this.extensionRelativePath = extension.getRelativePath(); this.extensionState = extension.getState(); } @@ -48,6 +50,10 @@ public boolean isExtensionUserDefined() { return extensionUserDefined; } + public Extension.Type getExtensionType() { + return extensionType; + } + public String getExtensionRelativePath() { return extensionRelativePath; } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/StartSyncExtensionFilesCommand.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/StartSyncExtensionFilesCommand.java new file mode 100644 index 000000000000..5e7f421988a1 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/command/StartSyncExtensionFilesCommand.java @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.command; + +import java.util.List; + +import org.apache.cloudstack.extension.Extension; + +public class StartSyncExtensionFilesCommand extends ExtensionServerActionBaseCommand { + private final List targetManagementServerIds; + private final List files; + + public StartSyncExtensionFilesCommand(long msId, Extension extension, List targetManagementServerIds, + List files) { + super(msId, extension); + this.targetManagementServerIds = targetManagementServerIds; + this.files = files; + } + + public List getTargetManagementServerIds() { + return targetManagementServerIds; + } + + public List getFiles() { + return files; + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java new file mode 100644 index 000000000000..b9f0252dad2c --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManager.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.apache.cloudstack.extension.Extension; + +public interface ExtensionsFilesystemManager { + + String getExtensionsPath(); + + Path getExtensionRootPath(Extension extension); + + String getExtensionPath(String relativePath); + + String getExtensionCheckedPath(String extensionName, String extensionRelativePath); + + Map getChecksumMapForExtension(String extensionName, String relativePath); + + void prepareExtensionPath(String extensionName, boolean userDefined, Extension.Type type, String extensionRelativePath); + + void cleanupExtensionPath(String extensionName, String extensionRelativePath); + + void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory); + + Path getExtensionsStagingPath() throws IOException; + + String prepareExternalPayload(String extensionName, Map details) throws IOException; + + void deleteExtensionPayload(String extensionName, String payloadFileName); + + void validateExtensionFiles(Extension extension, List files); +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java new file mode 100644 index 000000000000..d1038f64b7ef --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImpl.java @@ -0,0 +1,430 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.FileUtil; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; + +public class ExtensionsFilesystemManagerImpl extends ManagerBase implements ExtensionsFilesystemManager { + + public static final Map BASE_EXTERNAL_SCRIPTS = + Map.of(Extension.Type.Orchestrator, "scripts/vm/hypervisor/external/provisioner/provisioner.sh"); + + private static final String EXTENSIONS = "extensions"; + private static final String EXTENSIONS_DEPLOYMENT_MODE_NAME = "extensions.deployment.mode"; + private static final String EXTENSIONS_DIRECTORY_PROD = "/usr/share/cloudstack-management/extensions"; + private static final String EXTENSIONS_DATA_DIRECTORY_PROD = System.getProperty("user.home") + File.separator + EXTENSIONS; + private static final String EXTENSIONS_DIRECTORY_DEV = EXTENSIONS; + private static final String EXTENSIONS_DATA_DIRECTORY_DEV = "client/target/extensions-data"; + + private String extensionsDirectory; + private String extensionsDataDirectory; + private ExecutorService payloadCleanupExecutor; + private ScheduledExecutorService payloadCleanupScheduler; + + private void initializeExtensionDirectories() { + String deploymentMode = ServerPropertiesUtil.getProperty(EXTENSIONS_DEPLOYMENT_MODE_NAME); + if ("developer".equals(deploymentMode)) { + extensionsDirectory = EXTENSIONS_DIRECTORY_DEV; + extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_DEV; + } else { + extensionsDirectory = EXTENSIONS_DIRECTORY_PROD; + extensionsDataDirectory = EXTENSIONS_DATA_DIRECTORY_PROD; + } + } + + protected boolean checkExtensionsDirectory() { + File dir = new File(extensionsDirectory); + if (!dir.exists() || !dir.isDirectory() || !dir.canWrite()) { + logger.error("Extension directory [{}] is not properly set up. It must exist, be a directory, and be writeable", + dir.getAbsolutePath()); + return false; + } + if (!extensionsDirectory.equals(dir.getAbsolutePath())) { + extensionsDirectory = dir.getAbsolutePath(); + } + logger.info("Extensions directory path: {}", extensionsDirectory); + return true; + } + + protected void createOrCheckExtensionsDataDirectory() throws ConfigurationException { + File dir = new File(extensionsDataDirectory); + if (!dir.exists()) { + try { + Files.createDirectories(dir.toPath()); + } catch (IOException e) { + logger.error("Unable to create extensions data directory [{}]", dir.getAbsolutePath(), e); + throw new ConfigurationException("Unable to create extensions data directory path"); + } + } + if (!dir.isDirectory() || !dir.canWrite()) { + logger.error("Extensions data directory [{}] is not properly set up. It must exist, be a directory, and be writeable", + dir.getAbsolutePath()); + throw new ConfigurationException("Extensions data directory path is not accessible"); + } + extensionsDataDirectory = dir.getAbsolutePath(); + logger.info("Extensions data directory path: {}", extensionsDataDirectory); + } + + protected void scheduleExtensionPayloadDirectoryCleanup(String extensionName) { + try { + Future future = payloadCleanupExecutor.submit(() -> { + try { + cleanupExtensionData(extensionName, 1, false); + logger.trace("Cleaned up payload directory for extension: {}", extensionName); + } catch (Exception e) { + logger.warn("Exception during payload cleanup for extension: {} due to {}", extensionName, + e.getMessage()); + logger.trace(e); + } + }); + payloadCleanupScheduler.schedule(() -> { + try { + if (!future.isDone()) { + future.cancel(true); + logger.trace("Cancelled cleaning up payload directory for extension: {} as it " + + "running for more than 3 seconds", extensionName); + } + } catch (Exception e) { + logger.warn("Failed to cancel payload cleanup task for extension: {} due to {}", + extensionName, e.getMessage()); + logger.trace(e); + } + }, 3, TimeUnit.SECONDS); + } catch (RejectedExecutionException e) { + logger.warn("Payload cleanup task for extension: {} was rejected due to: {}", extensionName, + e.getMessage()); + logger.trace(e); + } + } + + protected static String getFileExtension(File file) { + String name = file.getName(); + int lastDot = name.lastIndexOf('.'); + return (lastDot == -1) ? "" : name.substring(lastDot + 1); + } + + protected Path getExtensionRootPath(String extensionName) { + final String normalizedName = Extension.getDirectoryName(extensionName); + final String extensionDir = extensionsDirectory + File.separator + normalizedName; + return Path.of(extensionDir); + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + super.configure(name, params); + + initializeExtensionDirectories(); + checkExtensionsDirectory(); + createOrCheckExtensionsDataDirectory(); + return true; + } + + @Override + public boolean start() { + payloadCleanupExecutor = Executors.newSingleThreadExecutor(); + payloadCleanupScheduler = Executors.newSingleThreadScheduledExecutor(); + return true; + } + + @Override + public boolean stop() { + payloadCleanupExecutor.shutdown(); + payloadCleanupScheduler.shutdown(); + return true; + } + + @Override + public String getExtensionsPath() { + return extensionsDirectory; + } + + @Override + public Path getExtensionRootPath(Extension extension) { + return getExtensionRootPath(extension.getName()); + } + + @Override + public String getExtensionPath(String relativePath) { + return String.format("%s%s%s", extensionsDirectory, File.separator, relativePath); + } + + @Override + public String getExtensionCheckedPath(String extensionName, String extensionRelativePath) { + String path = getExtensionPath(extensionRelativePath); + File file = new File(path); + String errorSuffix = String.format("Entry point [%s] for extension: %s", path, extensionName); + if (!file.exists()) { + logger.error("{} does not exist", errorSuffix); + return null; + } + if (!file.isFile()) { + logger.error("{} is not a file", errorSuffix); + return null; + } + if (!file.canRead()) { + logger.error("{} is not readable", errorSuffix); + return null; + } + if (!file.canExecute()) { + logger.error("{} is not executable", errorSuffix); + return null; + } + return path; + } + + @Override + public Map getChecksumMapForExtension(String extensionName, String relativePath) { + String path = getExtensionCheckedPath(extensionName, relativePath); + if (StringUtils.isBlank(path)) { + return null; + } + try { + Path rootPath = getExtensionRootPath(extensionName); + Map fileChecksums = new TreeMap<>(); + java.util.List files = new java.util.ArrayList<>(); + try (Stream stream = Files.walk(rootPath)) { + stream.filter(Files::isRegularFile).forEach(files::add); + } + files.sort(Comparator.naturalOrder()); + for (Path filePath : files) { + String relative = rootPath.relativize(filePath).toString().replace(File.separatorChar, '/'); + String fileChecksum = DigestHelper.calculateChecksum(filePath.toFile()); + fileChecksums.put(relative, fileChecksum); + } + if (logger.isTraceEnabled()) { + String json = GsonHelper.getGson().toJson(fileChecksums); + logger.trace("Calculated individual file checksums for extension: {}: {}", extensionName, json); + } + return fileChecksums; + } catch (IOException | CloudRuntimeException e) { + return null; + } + } + + @Override + public void prepareExtensionPath(String extensionName, boolean userDefined, Extension.Type type, String extensionRelativePath) { + logger.debug("Preparing entry point for Extension [name: {}, user-defined: {}]", extensionName, userDefined); + if (!userDefined) { + logger.debug("Skipping preparing entry point for inbuilt extension: {}", extensionName); + return; + } + CloudRuntimeException exception = + new CloudRuntimeException(String.format("Failed to prepare scripts for extension: %s", extensionName)); + String sourceScriptPath = Script.findScript("", BASE_EXTERNAL_SCRIPTS.get(type)); + if(sourceScriptPath == null) { + logger.debug("Base script is not available for preparing extension: {} of type: {}", + extensionName, type); + return; + } + String destinationPath = getExtensionPath(extensionRelativePath); + File destinationFile = new File(destinationPath); + File sourceFile = new File(sourceScriptPath); + if (!getFileExtension(sourceFile).equalsIgnoreCase(getFileExtension(destinationFile))) { + logger.error("Extension file type do not match with base file for extension: {} of type: {}", + extensionName, type); + return; + } + if (destinationFile.exists()) { + logger.info("File already exists at {} for extension: {}, skipping copy.", destinationPath, + extensionName); + return; + } + if (!checkExtensionsDirectory()) { + throw exception; + } + Path destinationPathObj = Paths.get(destinationPath); + Path destinationDirPath = destinationPathObj.getParent(); + if (destinationDirPath == null) { + logger.error("Failed to find parent directory for extension: {} script path {}", + extensionName, destinationPath); + throw exception; + } + try { + Files.createDirectories(destinationDirPath); + } catch (IOException e) { + logger.error("Failed to create directory: {} for extension: {}", destinationDirPath, + extensionName, e); + throw exception; + } + try { + Path sourcePath = Paths.get(sourceScriptPath); + Files.copy(sourcePath, destinationPathObj, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + logger.error("Failed to copy entry point file to [{}] for extension: {}", + destinationPath, extensionName, e); + throw exception; + } + logger.debug("Successfully prepared entry point [{}] for extension: {}", destinationPath, + extensionName); + } + + @Override + public void cleanupExtensionPath(String extensionName, String extensionRelativePath) { + String normalizedPath = extensionRelativePath; + if (normalizedPath.startsWith("/")) { + normalizedPath = normalizedPath.substring(1); + } + try { + Path rootPath = Paths.get(extensionsDirectory).toAbsolutePath().normalize(); + String extensionDirName = Extension.getDirectoryName(extensionName); + Path filePath = rootPath + .resolve(normalizedPath.startsWith(extensionDirName) ? extensionDirName : normalizedPath) + .normalize(); + if (!Files.exists(filePath)) { + return; + } + if (!Files.isDirectory(filePath) && !Files.isRegularFile(filePath)) { + throw new CloudRuntimeException( + String.format("Failed to cleanup path: %s for extension: %s as it either " + + "does not exist or is not a regular file/directory", + extensionName, extensionRelativePath)); + } + if (!FileUtil.deleteRecursively(filePath)) { + throw new CloudRuntimeException( + String.format("Failed to delete path: %s for extension: %s", + extensionName, filePath)); + } + } catch (IOException e) { + throw new CloudRuntimeException( + String.format("Failed to cleanup path: %s for extension: %s due to: %s", + extensionName, normalizedPath, e.getMessage()), e); + } + } + + @Override + public void cleanupExtensionData(String extensionName, int olderThanDays, boolean cleanupDirectory) { + String extensionPayloadDirPath = extensionsDataDirectory + File.separator + extensionName; + Path dirPath = Paths.get(extensionPayloadDirPath); + if (!Files.exists(dirPath)) { + return; + } + try { + if (cleanupDirectory) { + try (Stream paths = Files.walk(dirPath)) { + paths.sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + return; + } + long cutoffMillis = System.currentTimeMillis() - (olderThanDays * 24L * 60 * 60 * 1000); + long lastModified = Files.getLastModifiedTime(dirPath).toMillis(); + if (lastModified < cutoffMillis) { + return; + } + try (Stream paths = Files.walk(dirPath)) { + paths.filter(path -> !path.equals(dirPath)) + .filter(path -> { + try { + return Files.getLastModifiedTime(path).toMillis() < cutoffMillis; + } catch (IOException e) { + return false; + } + }) + .sorted(Comparator.reverseOrder()) + .map(Path::toFile) + .forEach(File::delete); + } + } catch (IOException e) { + logger.warn("Failed to clean up extension payloads for {}: {}", extensionName, e.getMessage()); + } + } + + @Override + public Path getExtensionsStagingPath() throws IOException { + Path extensionsPath = Paths.get(extensionsDirectory).toAbsolutePath().normalize(); + Path stagingPath = extensionsPath.resolve(".staging"); + Files.createDirectories(stagingPath); + return stagingPath; + } + + @Override + public String prepareExternalPayload(String extensionName, Map details) throws IOException { + String json = GsonHelper.getGson().toJson(details); + String fileName = UUID.randomUUID() + ".json"; + String extensionPayloadDir = extensionsDataDirectory + File.separator + extensionName; + Path payloadDirPath = Paths.get(extensionPayloadDir); + if (!Files.exists(payloadDirPath)) { + Files.createDirectories(payloadDirPath); + } else { + scheduleExtensionPayloadDirectoryCleanup(extensionName); + } + Path payloadFile = payloadDirPath.resolve(fileName); + Files.writeString(payloadFile, json, StandardOpenOption.CREATE_NEW); + return payloadFile.toAbsolutePath().toString(); + } + + @Override + public void deleteExtensionPayload(String extensionName, String payloadFilePath) { + logger.trace("Deleting payload file: {} for extension: {}", payloadFilePath, extensionName); + FileUtil.deletePath(payloadFilePath); + } + + @Override + public void validateExtensionFiles(Extension extension, List files) { + if (CollectionUtils.isEmpty(files)) { + return; + } + Path rootPath = getExtensionRootPath(extension); + File rootDir = rootPath.toFile(); + if (!rootDir.exists() || !rootDir.isDirectory()) { + throw new CloudRuntimeException("Extension directory does not exist: " + rootPath); + } + for (String filePath : files) { + File file = new File(filePath); + if (!file.isAbsolute()) { + file = new File(rootDir, filePath); + } + if (!file.exists()) { + throw new CloudRuntimeException("File does not exist: " + filePath); + } + } + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java index 1b1a175c5975..e7090c1bdd9d 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManager.java @@ -35,13 +35,16 @@ import org.apache.cloudstack.framework.extensions.api.CreateExtensionCmd; import org.apache.cloudstack.framework.extensions.api.DeleteCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.DeleteExtensionCmd; +import org.apache.cloudstack.framework.extensions.api.DownloadExtensionCmd; import org.apache.cloudstack.framework.extensions.api.ListCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.ListExtensionsCmd; import org.apache.cloudstack.framework.extensions.api.RegisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.RunCustomActionCmd; +import org.apache.cloudstack.framework.extensions.api.SyncExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UnregisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateExtensionCmd; +import org.apache.cloudstack.framework.extensions.api.response.DownloadExtensionResponse; import org.apache.cloudstack.framework.extensions.command.ExtensionServerActionBaseCommand; import com.cloud.agent.api.Answer; @@ -97,4 +100,8 @@ Pair extensionResourceMapDetailsNeedUpdate(final void updateExtensionResourceMapDetails(final long extensionResourceMapId, final Map details); Answer getInstanceConsole(VirtualMachine vm, Host host); + + boolean syncExtension(SyncExtensionCmd cmd); + + DownloadExtensionResponse downloadExtension(DownloadExtensionCmd cmd); } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java index f66c195399d1..0b3066f78e3d 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsManagerImpl.java @@ -68,18 +68,24 @@ import org.apache.cloudstack.framework.extensions.api.CreateExtensionCmd; import org.apache.cloudstack.framework.extensions.api.DeleteCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.DeleteExtensionCmd; +import org.apache.cloudstack.framework.extensions.api.DownloadExtensionCmd; import org.apache.cloudstack.framework.extensions.api.ListCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.ListExtensionsCmd; import org.apache.cloudstack.framework.extensions.api.RegisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.RunCustomActionCmd; +import org.apache.cloudstack.framework.extensions.api.SyncExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UnregisterExtensionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateCustomActionCmd; import org.apache.cloudstack.framework.extensions.api.UpdateExtensionCmd; +import org.apache.cloudstack.framework.extensions.api.response.DownloadExtensionResponse; import org.apache.cloudstack.framework.extensions.command.CleanupExtensionFilesCommand; +import org.apache.cloudstack.framework.extensions.command.DownloadAndSyncExtensionFilesCommand; +import org.apache.cloudstack.framework.extensions.command.DownloadExtensionFilesCommand; import org.apache.cloudstack.framework.extensions.command.ExtensionRoutingUpdateCommand; import org.apache.cloudstack.framework.extensions.command.ExtensionServerActionBaseCommand; import org.apache.cloudstack.framework.extensions.command.GetExtensionPathChecksumCommand; import org.apache.cloudstack.framework.extensions.command.PrepareExtensionPathCommand; +import org.apache.cloudstack.framework.extensions.command.StartSyncExtensionFilesCommand; import org.apache.cloudstack.framework.extensions.dao.ExtensionCustomActionDao; import org.apache.cloudstack.framework.extensions.dao.ExtensionCustomActionDetailsDao; import org.apache.cloudstack.framework.extensions.dao.ExtensionDao; @@ -123,7 +129,6 @@ import com.cloud.host.HostVO; import com.cloud.host.dao.HostDao; import com.cloud.host.dao.HostDetailsDao; -import com.cloud.hypervisor.ExternalProvisioner; import com.cloud.hypervisor.Hypervisor; import com.cloud.org.Cluster; import com.cloud.serializer.GsonHelper; @@ -179,9 +184,6 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana @Inject HostDetailsDao hostDetailsDao; - @Inject - ExternalProvisioner externalProvisioner; - @Inject ExtensionCustomActionDao extensionCustomActionDao; @@ -212,6 +214,12 @@ public class ExtensionsManagerImpl extends ManagerBase implements ExtensionsMana @Inject RoleService roleService; + @Inject + ExtensionsFilesystemManager extensionsFilesystemManager; + + @Inject + ExtensionsShareManager extensionsShareManager; + private ScheduledExecutorService extensionPathStateCheckExecutor; protected String getDefaultExtensionRelativePath(String name) { @@ -272,9 +280,9 @@ protected boolean prepareExtensionPathOnMSPeer(Extension extension, ManagementSe } protected Pair prepareExtensionPathOnCurrentServer(String name, boolean userDefined, - String relativePath) { + Extension.Type type, String relativePath) { try { - externalProvisioner.prepareExtensionPath(name, userDefined, relativePath); + extensionsFilesystemManager.prepareExtensionPath(name, userDefined, type, relativePath); } catch (CloudRuntimeException e) { logger.error("Failed to prepare path for Extension [name: {}, userDefined: {}, relativePath: {}] on this server", name, userDefined, relativePath, e); @@ -294,8 +302,8 @@ protected boolean cleanupExtensionFilesOnMSPeer(Extension extension, ManagementS protected Pair cleanupExtensionFilesOnCurrentServer(String name, String relativePath) { try { - externalProvisioner.cleanupExtensionPath(name, relativePath); - externalProvisioner.cleanupExtensionData(name, 0, true); + extensionsFilesystemManager.cleanupExtensionPath(name, relativePath); + extensionsFilesystemManager.cleanupExtensionData(name, 0, true); } catch (CloudRuntimeException e) { logger.error("Failed to cleanup files for Extension [name: {}, relativePath: {}] on this server", name, relativePath, e); @@ -534,10 +542,40 @@ protected void checkOrchestratorTemplates(Long extensionId) { } } + protected void checkExtensionPathState(Extension extension) { + List msList = managementServerHostDao.listBy(ManagementServerHost.State.Up); + msList.removeIf(ms -> ms.getMsid() == ManagementServerNode.getManagementServerId()); + checkExtensionPathState(extension, msList); + } + + protected boolean compareChecksumMaps(Extension extension, long ms1Id, Map map1, + long ms2Id, Map map2) { + if (MapUtils.isEmpty(map1) || MapUtils.isEmpty(map2)) { + return false; + } + if (map1.size() != map2.size()) { + return false; + } + for (Map.Entry entry : map1.entrySet()) { + String map2Value = map2.get(entry.getKey()); + logger.debug("Comparing checksum for file {} of {} between [msid: {}, checksum: {}] and [msid: {}, checksum: {}]", + entry.getKey(), extension, ms1Id, entry.getValue(), ms2Id, map2Value); + if (!StringUtils.equals(entry.getValue(), map2Value)) { + logger.error("Checksum for file {} of {} is different [msid: {}, checksum: {}] and [msid: {}, checksum: {}]", + entry.getKey(), extension, ms1Id, entry.getValue(), ms2Id, + (StringUtils.isNotBlank(map2Value) ? map2Value : "unknown")); + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") protected void checkExtensionPathState(Extension extension, List msHosts) { - String checksum = externalProvisioner.getChecksumForExtensionPath(extension.getName(), + logger.debug("Checking path state for {} across management servers", extension); + Map checksumMap = extensionsFilesystemManager.getChecksumMapForExtension(extension.getName(), extension.getRelativePath()); - if (StringUtils.isBlank(checksum)) { + if (MapUtils.isEmpty(checksumMap)) { updateExtensionPathReady(extension, false); return; } @@ -548,10 +586,22 @@ protected void checkExtensionPathState(Extension extension, List msPeerChecksumResult = getChecksumForExtensionPathOnMSPeer(extension, msHost); - if (!msPeerChecksumResult.first() || !checksum.equals(msPeerChecksumResult.second())) { - logger.error("Path checksum for {} is different [msid: {}, checksum: {}] and [msid: {}, checksum: {}]", - extension, ManagementServerNode.getManagementServerId(), checksum, msHost.getMsid(), - (msPeerChecksumResult.first() ? msPeerChecksumResult.second() : "unknown")); + if (!msPeerChecksumResult.first() || StringUtils.isBlank(msPeerChecksumResult.second())) { + logger.error("{} returned failure or empty checksum map for {}", msHost, extension); + updateExtensionPathReady(extension, false); + return; + } + Map msPeerChecksumMap = null; + try { + msPeerChecksumMap = GsonHelper.getGson().fromJson(msPeerChecksumResult.second(), Map.class); + } catch (Exception e) { + logger.error("Failed to parse checksum map JSON from {} for {}: {}", + msHost, extension, e.getMessage(), e); + updateExtensionPathReady(extension, false); + return; + } + if (!compareChecksumMaps(extension, ManagementServerNode.getManagementServerId(), checksumMap, + msHost.getMsid(), msPeerChecksumMap)) { updateExtensionPathReady(extension, false); return; } @@ -561,7 +611,7 @@ protected void checkExtensionPathState(Extension extension, List msHosts = managementServerHostDao.listBy(ManagementServerHost.State.Up); for (ManagementServerHostVO msHost : msHosts) { if (msHost.getMsid() == ManagementServerNode.getManagementServerId()) { - prepared = prepared && prepareExtensionPathOnCurrentServer(extension.getName(), extension.isUserDefined(), + prepared = prepared && prepareExtensionPathOnCurrentServer(extension.getName(), + extension.isUserDefined(), + extension.getType(), extension.getRelativePath()).first(); continue; } @@ -917,7 +969,7 @@ public ExtensionResponse createExtensionResponse(Extension extension, ExtensionResponse response = new ExtensionResponse(extension.getUuid(), extension.getName(), extension.getDescription(), extension.getType().name()); response.setCreated(extension.getCreated()); - response.setPath(externalProvisioner.getExtensionPath(extension.getRelativePath())); + response.setPath(extensionsFilesystemManager.getExtensionPath(extension.getRelativePath())); response.setPathReady(extension.isPathReady()); response.setUserDefined(extension.isUserDefined()); response.setState(extension.getState().name()); @@ -1499,19 +1551,31 @@ public String handleExtensionServerCommands(ExtensionServerActionBaseCommand com Answer answer = new Answer(command, false, "Unsupported command"); if (command instanceof GetExtensionPathChecksumCommand) { final GetExtensionPathChecksumCommand cmd = (GetExtensionPathChecksumCommand)command; - String checksum = externalProvisioner.getChecksumForExtensionPath(extensionName, + Map checksumMap = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, extensionRelativePath); - answer = new Answer(cmd, StringUtils.isNotBlank(checksum), checksum); + answer = new Answer(cmd, MapUtils.isNotEmpty(checksumMap), GsonHelper.getGson().toJson(checksumMap)); } else if (command instanceof PrepareExtensionPathCommand) { final PrepareExtensionPathCommand cmd = (PrepareExtensionPathCommand)command; Pair result = prepareExtensionPathOnCurrentServer( - extensionName, cmd.isExtensionUserDefined(), extensionRelativePath); + extensionName, cmd.isExtensionUserDefined(), cmd.getExtensionType(), extensionRelativePath); answer = new Answer(cmd, result.first(), result.second()); } else if (command instanceof CleanupExtensionFilesCommand) { final CleanupExtensionFilesCommand cmd = (CleanupExtensionFilesCommand)command; Pair result = cleanupExtensionFilesOnCurrentServer(extensionName, extensionRelativePath); answer = new Answer(cmd, result.first(), result.second()); + } else if (command instanceof StartSyncExtensionFilesCommand) { + final StartSyncExtensionFilesCommand cmd = (StartSyncExtensionFilesCommand)command; + Pair result = startSyncExtensionFiles(cmd); + answer = new Answer(cmd, result.first(), result.second()); + } else if (command instanceof DownloadAndSyncExtensionFilesCommand) { + final DownloadAndSyncExtensionFilesCommand cmd = (DownloadAndSyncExtensionFilesCommand)command; + Pair result = downloadAndSyncExtensionFiles(cmd); + answer = new Answer(cmd, result.first(), result.second()); + } else if (command instanceof DownloadExtensionFilesCommand) { + final DownloadExtensionFilesCommand cmd = (DownloadExtensionFilesCommand)command; + Pair result = downloadExtensionFiles(cmd); + answer = new Answer(cmd, result.first(), result.second()); } final Answer[] answers = new Answer[1]; answers[0] = answer; @@ -1576,6 +1640,183 @@ public Answer getInstanceConsole(VirtualMachine vm, Host host) { return agentMgr.easySend(host.getId(), cmd); } + protected Pair syncExtensionUsingMSPeer(ExtensionVO extension, + ManagementServerHostVO sourceManagementServer, List targetManagementServers, + List files) { + logger.debug("Initiating sync for {} using {}", extension, sourceManagementServer); + final String msPeer = Long.toString(sourceManagementServer.getMsid()); + final Command[] cmds = new Command[1]; + cmds[0] = new StartSyncExtensionFilesCommand( + ManagementServerNode.getManagementServerId(), extension, + targetManagementServers.stream().map(ManagementServerHost::getUuid).collect(Collectors.toList()), + files); + String answersStr = clusterManager.execute(msPeer, 0L, GsonHelper.getGson().toJson(cmds), true); + return getResultFromAnswersString(answersStr, extension, sourceManagementServer, "sync"); + } + + protected Pair startSyncExtensionFiles(StartSyncExtensionFilesCommand cmd) { + final long extensionId = cmd.getExtensionId(); + final List targetManagementServerIds = cmd.getTargetManagementServerIds(); + final List files = cmd.getFiles(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + String msg = String.format("Unable to find extension with id: %d for starting sync", extensionId); + logger.error(msg); + return new Pair<>(false, msg); + } + if (CollectionUtils.isEmpty(targetManagementServerIds)) { + String msg = "No valid target management servers specified for starting sync"; + logger.error(msg); + return new Pair<>(false, msg); + } + List targetManagementServers = managementServerHostDao.listByUuids(targetManagementServerIds); + if (targetManagementServers.size() != targetManagementServerIds.size()) { + String msg = "Some of the specified target management servers are not found"; + logger.error(msg); + return new Pair<>(false, msg); + } + ManagementServerHost sourceManagementServer = managementServerHostDao.findById(ManagementServerNode.getManagementServerId()); + List targetManagementServerHosts = targetManagementServers.stream() + .map(msHostVO -> (ManagementServerHost) msHostVO) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(files)) { + try { + extensionsFilesystemManager.validateExtensionFiles(extension, files); + } catch (CloudRuntimeException cre) { + String msg = "Invalid file paths specified: " + cre.getMessage(); + logger.error(msg); + return new Pair<>(false, msg); + } + } + return extensionsShareManager.syncExtension(extension, sourceManagementServer, targetManagementServerHosts, + files); + } + + protected Pair downloadAndSyncExtensionFiles(DownloadAndSyncExtensionFilesCommand cmd) { + final long extensionId = cmd.getExtensionId(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + String msg = String.format("Unable to find extension with ID: %d for starting sync", extensionId); + logger.error(msg); + return new Pair<>(false, msg); + } + return extensionsShareManager.downloadAndApplyExtensionSync(extension, cmd); + } + + @Override + public boolean syncExtension(SyncExtensionCmd cmd) { + final long extensionId = cmd.getId(); + final long sourceManagementServerId = cmd.getSourceManagementServerId(); + List targetManagementServerIds = cmd.getTargetManagementServerIds(); + final List files = cmd.getFiles(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + throw new InvalidParameterValueException("Unable to find extension with the specified id"); + } + final ManagementServerHostVO sourceManagementServer = managementServerHostDao.findById(sourceManagementServerId); + if (sourceManagementServer == null || !ManagementServerHost.State.Up.equals(sourceManagementServer.getState())) { + throw new InvalidParameterValueException("Unable to find active source management server with the specified id"); + } + List targetManagementServers = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(targetManagementServerIds)) { + if (targetManagementServerIds.contains(sourceManagementServerId)) { + throw new InvalidParameterValueException("Source management server cannot be specified as target"); + } + List msList = managementServerHostDao.listUpByIds(targetManagementServerIds); + if (msList.size() != targetManagementServerIds.size()) { + throw new InvalidParameterValueException("Some of the specified target management servers are not found or not in Up state"); + } + targetManagementServers.addAll(msList); + } else { + List msList = managementServerHostDao.listBy(ManagementServerHost.State.Up); + msList.removeIf(ms -> ms.getId() == sourceManagementServerId); + targetManagementServers.addAll(msList); + } + if (CollectionUtils.isEmpty(targetManagementServers)) { + throw new InvalidParameterValueException("No valid target management servers found for syncing the extension"); + } + if (CollectionUtils.isNotEmpty(files)) { + try { + extensionsFilesystemManager.validateExtensionFiles(extension, files); + } catch (CloudRuntimeException cre) { + throw new InvalidParameterValueException("Invalid file paths specified: " + cre.getMessage()); + } + } + Pair result; + if (ManagementServerNode.getManagementServerId() != sourceManagementServer.getMsid()) { + result = syncExtensionUsingMSPeer(extension, sourceManagementServer, targetManagementServers, files); + } else { + result = extensionsShareManager.syncExtension(extension, sourceManagementServer, targetManagementServers, + files); + } + if (result == null || !result.first()) { + String msg = result == null ? "Null result received for sync operation" : result.second(); + throw new CloudRuntimeException(String.format("Failed to sync extension '%s' via '%s': %s", + extension.getName(), sourceManagementServer.getName(), msg)); + } + + checkExtensionPathState(extension); + return true; + } + + protected Pair downloadExtensionUsingMSPeer(ExtensionVO extension, + ManagementServerHostVO managementServer) { + logger.debug("Initiating download for {} using {}", extension, managementServer); + final String msPeer = Long.toString(managementServer.getMsid()); + final Command[] cmds = new Command[1]; + cmds[0] = new DownloadExtensionFilesCommand( + ManagementServerNode.getManagementServerId(), extension); + String answersStr = clusterManager.execute(msPeer, 0L, GsonHelper.getGson().toJson(cmds), true); + return getResultFromAnswersString(answersStr, extension, managementServer, "download"); + } + + protected Pair downloadExtensionFiles(DownloadExtensionFilesCommand cmd) { + final long extensionId = cmd.getExtensionId(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + String msg = String.format("Unable to find extension with id: %d for starting sync", extensionId); + logger.error(msg); + return new Pair<>(false, msg); + } + return extensionsShareManager.downloadExtension(extension, + managementServerHostDao.findByMsid(ManagementServerNode.getManagementServerId())); + } + + @Override + public DownloadExtensionResponse downloadExtension(DownloadExtensionCmd cmd) { + final long extensionId = cmd.getId(); + final Long managementServerId = cmd.getManagementServerId(); + final ExtensionVO extension = extensionDao.findById(extensionId); + if (extension == null) { + throw new InvalidParameterValueException("Unable to find extension with the specified id"); + } + final ManagementServerHostVO managementServer = managementServerId == null ? + managementServerHostDao.findByMsid(ManagementServerNode.getManagementServerId()) : + managementServerHostDao.findById(managementServerId); + if (managementServer == null || !ManagementServerHost.State.Up.equals(managementServer.getState())) { + throw new InvalidParameterValueException("Unable to find active source management server with the specified id"); + } + Pair result; + if (ManagementServerNode.getManagementServerId() != managementServer.getMsid()) { + result = downloadExtensionUsingMSPeer(extension, managementServer); + } else { + result = extensionsShareManager.downloadExtension(extension, managementServer); + } + if (result == null || !result.first()) { + String msg = result == null ? "Null result received for download operation" : result.second(); + throw new CloudRuntimeException(String.format("Failed to download extension '%s' via '%s': %s", + extension.getName(), managementServer.getName(), msg)); + } + DownloadExtensionResponse response = new DownloadExtensionResponse(); + response.setId(extension.getUuid()); + response.setName(extension.getName()); + response.setManagementServerId(managementServer.getUuid()); + response.setManagementServerName(managementServer.getName()); + response.setUrl(result.second()); + response.setObjectName("extension"); + return response; + } + @Override public Long getExtensionIdForCluster(long clusterId) { ExtensionResourceMapVO map = extensionResourceMapDao.findByResourceIdAndType(clusterId, @@ -1600,6 +1841,17 @@ public Extension getExtensionForCluster(long clusterId) { return extensionDao.findById(extensionId); } + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + try { + extensionPathStateCheckExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("Extension-Path-State-Check")); + } catch (final Exception e) { + throw new ConfigurationException("Unable to to configure ExtensionsManagerImpl"); + } + return true; + } + @Override public boolean start() { long pathStateCheckInterval = PathStateCheckInterval.value(); @@ -1612,12 +1864,9 @@ public boolean start() { } @Override - public boolean configure(String name, Map params) throws ConfigurationException { - try { - extensionPathStateCheckExecutor = Executors.newScheduledThreadPool(1, - new NamedThreadFactory("Extension-Path-State-Check")); - } catch (final Exception e) { - throw new ConfigurationException("Unable to to configure ExtensionsManagerImpl"); + public boolean stop() { + if (extensionPathStateCheckExecutor != null && !extensionPathStateCheckExecutor.isShutdown()) { + extensionPathStateCheckExecutor.shutdownNow(); } return true; } @@ -1637,6 +1886,8 @@ public List> getCommands() { cmds.add(UpdateExtensionCmd.class); cmds.add(RegisterExtensionCmd.class); cmds.add(UnregisterExtensionCmd.class); + cmds.add(SyncExtensionCmd.class); + cmds.add(DownloadExtensionCmd.class); return cmds; } diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManager.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManager.java new file mode 100644 index 000000000000..03caa3b86cf4 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManager.java @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + + +import java.util.List; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.framework.extensions.command.DownloadAndSyncExtensionFilesCommand; +import org.apache.cloudstack.management.ManagementServerHost; + +import com.cloud.utils.Pair; + +public interface ExtensionsShareManager { + Pair syncExtension(Extension extension, ManagementServerHost sourceManagementServer, + List targetManagementServers, List files); + + Pair downloadAndApplyExtensionSync(Extension extension, DownloadAndSyncExtensionFilesCommand cmd); + + Pair downloadExtension(Extension extension, ManagementServerHost managementServer); +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java new file mode 100644 index 000000000000..af0f9c1a5087 --- /dev/null +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsShareManagerImpl.java @@ -0,0 +1,605 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import javax.inject.Inject; +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.framework.config.ConfigKey; +import org.apache.cloudstack.framework.extensions.command.DownloadAndSyncExtensionFilesCommand; +import org.apache.cloudstack.managed.context.ManagedContextRunnable; +import org.apache.cloudstack.management.ManagementServerHost; +import org.apache.cloudstack.utils.filesystem.ArchiveUtil; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.cloudstack.utils.security.HMACSignUtil; +import org.apache.cloudstack.utils.server.ServerPropertiesUtil; +import org.apache.commons.codec.DecoderException; +import org.apache.commons.collections.CollectionUtils; + +import com.cloud.agent.api.Answer; +import com.cloud.agent.api.Command; +import com.cloud.cluster.ClusterManager; +import com.cloud.serializer.GsonHelper; +import com.cloud.utils.FileUtil; +import com.cloud.utils.HttpUtils; +import com.cloud.utils.Pair; +import com.cloud.utils.StringUtils; +import com.cloud.utils.component.ManagerBase; +import com.cloud.utils.concurrency.NamedThreadFactory; +import com.cloud.utils.db.GlobalLock; +import com.cloud.utils.exception.CloudRuntimeException; + +public class ExtensionsShareManagerImpl extends ManagerBase implements ExtensionsShareManager { + + protected static final String EXTENSIONS_SHARE_SUBDIR = "extensions"; + protected static final int DEFAULT_SHARE_LINK_VALIDITY_SECONDS = 3600; // 1 hour + + ConfigKey ShareLinkValidityInterval = new ConfigKey<>("Advanced", Integer.class, + "extension.share.link.validity.interval", String.valueOf(DEFAULT_SHARE_LINK_VALIDITY_SECONDS), + String.format("Interval (in seconds) for which the extension archive share link is valid. " + + "Default is %s seconds", DEFAULT_SHARE_LINK_VALIDITY_SECONDS), + false, ConfigKey.Scope.Global); + + @Inject + ExtensionsFilesystemManager extensionsFilesystemManager; + + @Inject + ClusterManager clusterManager; + + private ScheduledExecutorService extensionShareCleanupExecutor; + private int shareLinkValidityInterval; + private boolean serverShareEnabled = true; + + protected Path getExtensionsSharePath() { + String shareBaseDir = ServerPropertiesUtil.getShareBaseDirectory(); + shareBaseDir += File.separator + EXTENSIONS_SHARE_SUBDIR; + return Path.of(shareBaseDir); + } + + protected String getManagementServerBaseUrl(ManagementServerHost managementHost) { + boolean secure = Boolean.parseBoolean(ServerPropertiesUtil.getProperty("https.enable", + "false")); + final String scheme = secure ? "https" : "http"; + final String host = managementHost.getServiceIP(); + int port = secure + ? Integer.parseInt(ServerPropertiesUtil.getProperty("https.port", "8443")) + : Integer.parseInt(ServerPropertiesUtil.getProperty("http.port", "8080")); + + return String.format("%s://%s:%d", scheme, host, port); + } + + protected Pair getResultFromAnswersString(String answersStr, Extension extension, + ManagementServerHost msHost, String op) { + Answer[] answers = null; + try { + answers = GsonHelper.getGson().fromJson(answersStr, Answer[].class); + } catch (Exception e) { + logger.error("Failed to parse answer JSON during {} for {} on {}: {}", + op, extension, msHost, e.getMessage(), e); + return new Pair<>(false, e.getMessage()); + } + Answer answer = answers != null && answers.length > 0 ? answers[0] : null; + boolean result = false; + String details = "Unknown error"; + if (answer != null) { + result = answer.getResult(); + details = answer.getDetails(); + } + if (!result) { + logger.error("Failed to {} for {} on {} due to {}", op, extension, msHost, details); + return new Pair<>(false, details); + } + return new Pair<>(true, details); + } + + /** + * Creates an archive for the specified extension. + * If the files list is empty, the entire extension directory is archived. + * If the files list is not empty, only the specified relative files are archived; throws if any file is missing. + * + * @return ArchiveInfo containing the archive path, size, SHA-256 checksum, and sync type. + */ + protected ArchiveInfo createArchiveForSync(Extension extension, List files) throws IOException { + final String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extension.getName(), + extension.getRelativePath()); + if (extensionPath == null) { + throw new IOException(String.format("Path not found %s", extension.getRelativePath())); + } + final boolean isPartial = CollectionUtils.isNotEmpty(files); + final DownloadAndSyncExtensionFilesCommand.SyncType syncType = + isPartial ? DownloadAndSyncExtensionFilesCommand.SyncType.Partial + : DownloadAndSyncExtensionFilesCommand.SyncType.Complete; + final Path extensionRootPath = extensionsFilesystemManager.getExtensionRootPath(extension); + final List toPack; + if (isPartial) { + toPack = new ArrayList<>(files.size()); + for (String rel : files) { + Path p = extensionRootPath.resolve(rel).normalize(); + if (!p.startsWith(extensionRootPath)) { + throw new SecurityException("File path escapes extension directory: " + rel); + } + if (!Files.exists(p)) { + throw new NoSuchFileException("File not found: " + p.toAbsolutePath().toString()); + } + toPack.add(p); + } + } else { + toPack = List.of(extensionRootPath); + } + StringBuilder archiveName = new StringBuilder(Extension.getDirectoryName(extension.getName())) + .append("-").append(System.currentTimeMillis()).append(".tgz"); + if (isPartial) { + archiveName.insert(0, "partial-"); + } + Path archivePath = getExtensionsSharePath().resolve(archiveName.toString()); + + if (!packArchiveForSync(extension, extensionRootPath, toPack, archivePath)) { + throw new IOException("Failed to create archive " + archivePath); + } + + logger.info("Created archive {} from {} ({} files)", archivePath, extensionRootPath, toPack.size()); + + long size = Files.size(archivePath); + String checksum = DigestHelper.calculateChecksum(archivePath.toFile()); + + return new ArchiveInfo(archivePath, size, checksum, syncType); + } + + protected ArchiveInfo createArchiveForDownload(Extension extension) throws IOException { + final String extensionPath = extensionsFilesystemManager.getExtensionCheckedPath(extension.getName(), + extension.getRelativePath()); + if (extensionPath == null) { + throw new IOException(String.format("Path not found %s", extension.getRelativePath())); + } + final Path extensionRootPath = extensionsFilesystemManager.getExtensionRootPath(extension); + String archiveName = Extension.getDirectoryName(extension.getName()) + + "-" + System.currentTimeMillis() + ".zip"; + Path archivePath = getExtensionsSharePath().resolve(archiveName); + + if (!packArchiveForDownload(extension, extensionRootPath, archivePath)) { + throw new IOException("Failed to create archive " + archivePath); + } + + logger.info("Created archive {} from {}", archivePath, extensionRootPath); + + long size = Files.size(archivePath); + String checksum = DigestHelper.calculateChecksum(archivePath.toFile()); + + return new ArchiveInfo(archivePath, size, checksum, DownloadAndSyncExtensionFilesCommand.SyncType.Complete); + } + + /** + * Generates a signed share URL for the given extension archive. + * The resulting URL format is: {baseUrl}/share/extensions/{archiveName}?exp={expiry}&sig={signature} + * + * @param managementServer the management server host generating the URL + * @param archivePath the path to the archive file + * @return the signed share URL for the archive + * @throws DecoderException if signature decoding fails + * @throws NoSuchAlgorithmException if the HMAC algorithm is not available + * @throws InvalidKeyException if the secret key is invalid + */ + protected String generateSignedArchiveUrl(ManagementServerHost managementServer, Path archivePath) + throws DecoderException, NoSuchAlgorithmException, InvalidKeyException, CloudRuntimeException { + if (!serverShareEnabled) { + throw new CloudRuntimeException("Share context is disabled on this management server in server.properties"); + } + final String baseUrl = getManagementServerBaseUrl(managementServer); + final long expiresAtEpochSec = System.currentTimeMillis() / 1000L + shareLinkValidityInterval; + final String secretKey = ServerPropertiesUtil.getShareSecret(); + String archiveName = archivePath.getFileName().toString(); + String uriPath = String.format("/%s/%s/%s", ServerPropertiesUtil.SHARE_DIR, EXTENSIONS_SHARE_SUBDIR, + archiveName); + String sig = ""; + if (StringUtils.isNotBlank(secretKey)) { + String payload = uriPath + "|" + expiresAtEpochSec; + sig = HMACSignUtil.generateSignature(payload, secretKey); + } + StringBuilder sb = new StringBuilder(); + sb.append(baseUrl).append(uriPath).append("?exp=").append(expiresAtEpochSec); + if (StringUtils.isNotBlank(sig)) { + sb.append("&sig=").append(URLEncoder.encode(sig, StandardCharsets.UTF_8)); + } + return sb.toString(); + } + + /** + * Build the DownloadAndSyncExtensionFilesCommand to send to a target MS. + */ + protected DownloadAndSyncExtensionFilesCommand buildCommand(long msId, Extension ext, ArchiveInfo archive, + String signedUrl) { + DownloadAndSyncExtensionFilesCommand cmd = + new DownloadAndSyncExtensionFilesCommand(msId, ext, signedUrl, archive.getSize(), archive.getChecksum()); + cmd.setSyncType(archive.getSyncType()); + return cmd; + } + + /** + * Packs the specified files or directories into a .tgz archive. + * If a single directory is provided, archives the entire directory. + * If multiple files are provided, copies them to a temporary directory preserving structure before archiving. + * + * @param extensionRootPath the root path of the extension + * @param toPack list of files or directories to include in the archive + * @param archivePath the destination path for the .tgz archive + * @return true if the archive was created successfully, false otherwise + * @throws IOException if an I/O error occurs during packing + */ + protected boolean packArchiveForSync(Extension extension, Path extensionRootPath, List toPack, Path archivePath) + throws IOException { + Files.createDirectories(archivePath.getParent()); + FileUtil.deletePath(archivePath.toAbsolutePath().toString()); + + Path sourceDir; + if (toPack.size() == 1 && Files.isDirectory(toPack.get(0))) { + sourceDir = toPack.get(0); + } else { + sourceDir = Files.createTempDirectory("pack-tmp-"); + for (Path p : toPack) { + Path rel = extensionRootPath.relativize(p); + Path dest = sourceDir.resolve(rel); + Files.createDirectories(dest.getParent()); + Files.copy(p, dest, StandardCopyOption.COPY_ATTRIBUTES); + } + } + logger.debug("Packing files for sync for {} from: {} to archive: {}", extension, sourceDir, + archivePath.toAbsolutePath()); + boolean result = ArchiveUtil.packPath(ArchiveUtil.ArchiveFormat.TGZ, sourceDir, archivePath, 60); + + if (!sourceDir.equals(extensionRootPath)) { + FileUtil.deleteRecursively(sourceDir); + } + + return result; + } + protected boolean packArchiveForDownload(Extension extension, Path extensionRootPath, Path archivePath) + throws IOException { + Files.createDirectories(archivePath.getParent()); + FileUtil.deletePath(archivePath.toAbsolutePath().toString()); + logger.debug("Packing files for download for {} from: {} to archive: {}", extension, extensionRootPath, + archivePath.toAbsolutePath()); + + return ArchiveUtil.packPath(ArchiveUtil.ArchiveFormat.ZIP, extensionRootPath, archivePath, 60); + } + + protected long downloadTo(String url, Path dest) throws IOException { + boolean result = HttpUtils.downloadFileWithProgress(url, dest.toString(), logger); + if (!result) { + throw new IOException("Download failed"); + } + if (!Files.exists(dest)) { + throw new IOException("Download failed: file not found"); + } + return Files.size(dest); + } + + /** + * Atomically replaces the target directory with the source directory. + * If the target exists, it is first moved to a backup location. + * Attempts an atomic move; falls back to a regular move if necessary. + * Cleans up the backup directory after a successful move. + * + * @param from the source directory to move + * @param to the target directory to replace + * @throws IOException if an I/O error occurs during the operation + */ + protected static void atomicReplaceDir(Path from, Path to) throws IOException { + Files.createDirectories(from); + Files.createDirectories(to.getParent()); + Path backup = to.getParent().resolve(to.getFileName().toString() + ".bak-" + System.currentTimeMillis()); + if (Files.exists(to)) { + try { + Files.move(to, backup, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException e) { + Files.move(to, backup); + } + } + try { + Files.move(from, to, StandardCopyOption.ATOMIC_MOVE); + FileUtil.deleteRecursively(backup); + } catch (IOException e) { + if (Files.exists(backup)) { + try { + Files.move(backup, to, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ignore) {} + } + throw e; + } + } + + /** + * Overlays files from the source directory into the target directory. + * For each file in `fromRoot`, copies it to the corresponding location in `targetRoot`, + * replacing existing files atomically when possible. + * Directory structure is preserved. + * + * @param fromRoot the source root directory containing files to overlay + * @param targetRoot the target root directory to overlay files into + * @throws IOException if an I/O error occurs during overlay + */ + protected static void overlayInto(Path fromRoot, Path targetRoot) throws IOException { + try (Stream stream = Files.walk(fromRoot)) { + stream.forEach(src -> { + try { + if (Files.isDirectory(src)) return; + Path rel = fromRoot.relativize(src); + Path dst = targetRoot.resolve(rel); + Files.createDirectories(dst.getParent()); + Path tmp = dst.getParent().resolve(dst.getFileName() + ".tmp-" + System.nanoTime()); + Files.copy(src, tmp, StandardCopyOption.REPLACE_EXISTING); + try { + Files.move(tmp, dst, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + Files.move(tmp, dst, StandardCopyOption.REPLACE_EXISTING); + } + } catch (IOException ex) { + throw new UncheckedIOException(ex); + } + }); + } + } + + /** + * Applies the extension synchronization by extracting the archive and updating the extension directory. + * For COMPLETE sync type, replaces the entire extension directory atomically. + * For PARTIAL sync type, overlays the extracted files into the existing extension directory. + * + * @param extension the extension to synchronize + * @param syncType the type of synchronization (COMPLETE or PARTIAL) + * @param tmpArchive the path to the temporary archive file + * @param extensionRootPath the root path of the extension directory + * @throws IOException if an I/O error occurs during extraction or application + */ + protected void applyExtensionSync(Extension extension, DownloadAndSyncExtensionFilesCommand.SyncType syncType, + Path tmpArchive, Path extensionRootPath) throws IOException { + logger.debug("Applying extension sync for {} with sync type {} from archive {}", extension, syncType, + tmpArchive); + Path stagingDir = extensionsFilesystemManager.getExtensionsStagingPath(); + Path applyRoot = Files.createTempDirectory(stagingDir, + Extension.getDirectoryName(extension.getName()) + "-"); + if (!ArchiveUtil.extractToPath(ArchiveUtil.ArchiveFormat.TGZ, tmpArchive, applyRoot, 60)) { + throw new IOException("Failed to extract archive " + tmpArchive); + } + if (DownloadAndSyncExtensionFilesCommand.SyncType.Complete.equals(syncType)) { + atomicReplaceDir(applyRoot, extensionRootPath); + return; + } + overlayInto(applyRoot, extensionRootPath); + FileUtil.deleteRecursively(applyRoot); + } + + protected void cleanupExtensionsShareFiles(long cutoff) throws IOException { + Path sharePath = getExtensionsSharePath(); + if (!Files.exists(sharePath) || !Files.isDirectory(sharePath)) { + return; + } + try (Stream paths = Files.list(sharePath)) { + paths.filter(p -> p.getFileName().toString().endsWith(".tgz")) + .filter(p -> { + try { + return Files.getLastModifiedTime(p).toMillis() < cutoff; + } catch (IOException e) { + return false; + } + }) + .forEach(p -> { + try { + Files.delete(p); + logger.debug("Deleted expired extension archive {}", p); + } catch (IOException e) { + logger.warn("Failed to delete expired extension archive {}", p, e); + } + }); + } + } + + @Override + public boolean configure(String name, Map params) throws ConfigurationException { + try { + extensionShareCleanupExecutor = Executors.newScheduledThreadPool(1, + new NamedThreadFactory("Extension-Share-Cleanup")); + } catch (final Exception e) { + throw new ConfigurationException("Unable to to configure ExtensionsManagerImpl"); + } + return true; + } + + @Override + public boolean start() { + int initialDelay = 120; + shareLinkValidityInterval = ShareLinkValidityInterval.value(); + logger.debug("Scheduling cleanup task for extension share archive with initial delay={}s and interval={}s", + initialDelay, shareLinkValidityInterval); + extensionShareCleanupExecutor.scheduleWithFixedDelay(new ShareCleanupWorker(), + initialDelay, shareLinkValidityInterval, TimeUnit.SECONDS); + serverShareEnabled = ServerPropertiesUtil.getShareEnabled(); + return true; + } + + @Override + public boolean stop() { + if (extensionShareCleanupExecutor != null && !extensionShareCleanupExecutor.isShutdown()) { + extensionShareCleanupExecutor.shutdownNow(); + } + return true; + } + + @Override + public Pair syncExtension(Extension extension, ManagementServerHost sourceManagementServer, + List targetManagementServers, List files) { + ArchiveInfo archiveInfo = null; + try { + try { + archiveInfo = createArchiveForSync(extension, files); + } catch (IOException e) { + String msg = "Failed to create archive"; + logger.error("{} for {}", extension, msg, e); + return new Pair<>(false, msg); + } + String signedUrl; + try { + signedUrl = generateSignedArchiveUrl(sourceManagementServer, archiveInfo.getPath()); + } catch (DecoderException | NoSuchAlgorithmException | InvalidKeyException | CloudRuntimeException e) { + String msg = "Failed to generate signed URL"; + logger.error("{} for {} using {}", msg, extension, sourceManagementServer, e); + return new Pair<>(false, msg); + } + for (ManagementServerHost targetMs : targetManagementServers) { + String targetUrl = signedUrl + "&tgt=" + URLEncoder.encode(targetMs.getUuid(), StandardCharsets.UTF_8); + final String msPeer = Long.toString(targetMs.getMsid()); + final Command[] cmds = new Command[1]; + cmds[0] = buildCommand(sourceManagementServer.getMsid(), extension, archiveInfo, targetUrl); + String answersStr = clusterManager.execute(msPeer, 0L, GsonHelper.getGson().toJson(cmds), true); + Pair result = getResultFromAnswersString(answersStr, extension, targetMs, + "sync"); + if (!result.first()) { + String msg = "Sync failed"; + logger.error("{} for {} on {} due to: {}", msg, extension, targetMs, result.second()); + return new Pair<>(false, String.format("%s on management server: %s", msg, targetMs.getName())); + } + } + } finally { + if (archiveInfo != null) { + FileUtil.deletePath(archiveInfo.getPath().toAbsolutePath().toString()); + } + } + return new Pair<>(true, ""); + } + + @Override + public Pair downloadAndApplyExtensionSync(Extension extension, + DownloadAndSyncExtensionFilesCommand cmd) { + final Path extensionRootPath = extensionsFilesystemManager.getExtensionRootPath(extension); + Path tmpArchive = null; + try { + tmpArchive = Files.createTempFile("dl-", ".tgz"); + long contentLength = downloadTo(cmd.getDownloadUrl(), tmpArchive); + + if (cmd.getSize() > 0 && contentLength != -1 && cmd.getSize() != contentLength) { + return new Pair<>(false, String.format("Size mismatch: expected %d got %d", cmd.getSize(), + contentLength)); + } + + String got = DigestHelper.calculateChecksum(tmpArchive.toFile()); + if (!got.equalsIgnoreCase(cmd.getChecksum())) { + return new Pair<>(false, String.format("Checksum mismatch for archive. expected=%s got=%s", + cmd.getChecksum(), got)); + } + + applyExtensionSync(extension, cmd.getSyncType(), tmpArchive, extensionRootPath); + } catch (IOException e) { + String msg = String.format("Failed to download/apply sync for %s from %s: %s", extension, + cmd.getDownloadUrl(), e.getMessage()); + logger.error(msg, e); + return new Pair<>(false, msg); + } finally { + if (tmpArchive != null) { + FileUtil.deletePath(tmpArchive.toAbsolutePath().toString()); + } + } + return new Pair<>(true, ""); + } + + @Override + public Pair downloadExtension(Extension extension, ManagementServerHost managementServer) { + ArchiveInfo archiveInfo; + try { + archiveInfo = createArchiveForDownload(extension); + } catch (IOException e) { + String msg = "Failed to create archive"; + logger.error("{} for {}", extension, msg, e); + return new Pair<>(false, msg); + } + try { + String signedUrl = generateSignedArchiveUrl(managementServer, archiveInfo.getPath()); + return new Pair<>(true, signedUrl); + } catch (DecoderException | NoSuchAlgorithmException | InvalidKeyException e) { + String msg = "Failed to generate signed URL"; + logger.error("{} for {} using {}", msg, extension, managementServer, e); + return new Pair<>(false, msg); + } + } + + protected static class ArchiveInfo { + private final Path path; + private final long size; + private final String checksum; + private final DownloadAndSyncExtensionFilesCommand.SyncType syncType; + + public ArchiveInfo(Path path, long size, String checksum, + DownloadAndSyncExtensionFilesCommand.SyncType syncType) { + this.path = path; + this.size = size; + this.checksum = checksum; + this.syncType = syncType; + } + + public Path getPath() { return path; } + public long getSize() { return size; } + public String getChecksum() { return checksum; } + public DownloadAndSyncExtensionFilesCommand.SyncType getSyncType() { return syncType; } + } + + protected class ShareCleanupWorker extends ManagedContextRunnable { + protected void reallyRun() { + try { + long expiryMillis = shareLinkValidityInterval * 1100L; + long cutoff = System.currentTimeMillis() - expiryMillis; + cleanupExtensionsShareFiles(cutoff); + } catch (Exception e) { + logger.warn("Extensions share cleanup failed", e); + } + } + + @Override + protected void runInContext() { + GlobalLock gcLock = GlobalLock.getInternLock("ExtensionShareCleanup"); + try { + if (gcLock.lock(3)) { + try { + reallyRun(); + } finally { + gcLock.unlock(); + } + } + } finally { + gcLock.releaseRef(); + } + } + } +} diff --git a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java index 20423764c1c3..7dd0c7059d84 100644 --- a/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java +++ b/framework/extensions/src/main/java/org/apache/cloudstack/framework/extensions/vo/ExtensionVO.java @@ -174,6 +174,6 @@ public void setRemoved(Date removed) { @Override public String toString() { - return String.format("Extension %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "name", "type")); + return String.format("Extension %s", ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "id", "uuid", "name", "type", "userDefined")); } } diff --git a/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml b/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml index 9d44d8ff7f3d..150c4223452e 100644 --- a/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml +++ b/framework/extensions/src/main/resources/META-INF/cloudstack/core/spring-framework-extensions-core-context.xml @@ -31,6 +31,8 @@ + + diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/AddCustomActionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/AddCustomActionCmdTest.java index b25de85a69d5..ff3c7d9bdcb3 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/AddCustomActionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/AddCustomActionCmdTest.java @@ -40,9 +40,10 @@ import org.apache.cloudstack.extension.ExtensionCustomAction; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; import org.apache.commons.collections.MapUtils; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; @@ -51,15 +52,11 @@ @RunWith(MockitoJUnitRunner.class) public class AddCustomActionCmdTest { - private AddCustomActionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() throws Exception { - cmd = new AddCustomActionCmd(); - extensionsManager = mock(ExtensionsManager.class); - cmd.extensionsManager = extensionsManager; - } + @InjectMocks + private AddCustomActionCmd cmd = new AddCustomActionCmd(); private void setField(String fieldName, Object value) { ReflectionTestUtils.setField(cmd, fieldName, value); diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java index 2edb6ea48e3f..c9600e9de803 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/CreateExtensionCmdTest.java @@ -28,8 +28,13 @@ import org.apache.commons.collections.MapUtils; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.junit.MockitoJUnitRunner; +@RunWith(MockitoJUnitRunner.class) public class CreateExtensionCmdTest { + @InjectMocks CreateExtensionCmd cmd = new CreateExtensionCmd(); @Test diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteCustomActionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteCustomActionCmdTest.java index 3a217d009fa7..f46c7b03dc20 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteCustomActionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteCustomActionCmdTest.java @@ -24,23 +24,23 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +@RunWith(MockitoJUnitRunner.class) public class DeleteCustomActionCmdTest { - private DeleteCustomActionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() throws Exception { - cmd = Mockito.spy(new DeleteCustomActionCmd()); - extensionsManager = Mockito.mock(ExtensionsManager.class); - java.lang.reflect.Field field = DeleteCustomActionCmd.class.getDeclaredField("extensionsManager"); - field.setAccessible(true); - field.set(cmd, extensionsManager); - } + @Spy + @InjectMocks + private DeleteCustomActionCmd cmd; @Test public void getIdReturnsNullWhenUnset() throws Exception { diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteExtensionCmdTest.java index 9078a05dfda7..3c09b39c1c7c 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DeleteExtensionCmdTest.java @@ -26,22 +26,24 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.api.response.SuccessResponse; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class DeleteExtensionCmdTest { - private DeleteExtensionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() { - cmd = Mockito.spy(new DeleteExtensionCmd()); - extensionsManager = Mockito.mock(ExtensionsManager.class); - cmd.extensionsManager = extensionsManager; - } + @Spy + @InjectMocks + private DeleteExtensionCmd cmd; @Test public void getIdReturnsNullWhenUnset() { diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DownloadExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DownloadExtensionCmdTest.java new file mode 100644 index 000000000000..94f98ebd17c1 --- /dev/null +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/DownloadExtensionCmdTest.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.api; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ApiErrorCode; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.framework.extensions.api.response.DownloadExtensionResponse; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.event.EventTypes; +import com.cloud.user.Account; + +@RunWith(MockitoJUnitRunner.class) +public class DownloadExtensionCmdTest { + + @Mock + private ExtensionsManager extensionsManager; + + @Spy + @InjectMocks + private DownloadExtensionCmd cmd; + + @Test + public void getIdReturnsExpected() { + ReflectionTestUtils.setField(cmd, "id", 123L); + assertEquals(Long.valueOf(123L), cmd.getId()); + } + + @Test + public void getManagementServerIdReturnsExpected() { + ReflectionTestUtils.setField(cmd, "managementServerId", 123L); + assertEquals(Long.valueOf(123L), cmd.getManagementServerId()); + } + + @Test + public void executeSetsResponseObjectWhenDownloadSucceeds() { + DownloadExtensionResponse response = mock(DownloadExtensionResponse.class); + when(extensionsManager.downloadExtension(any(DownloadExtensionCmd.class))).thenReturn(response); + cmd.execute(); + verify(cmd).setResponseObject(response); + } + + @Test + public void executeThrowsExceptionWhenDownloadFails() { + when(extensionsManager.downloadExtension(cmd)).thenReturn(null); + ServerApiException exception = assertThrows(ServerApiException.class, cmd::execute); + assertEquals(ApiErrorCode.INTERNAL_ERROR, exception.getErrorCode()); + assertEquals("Failed to download extension", exception.getDescription()); + } + + @Test + public void getEntityOwnerIdReturnsSystemAccount() { + assertEquals(Account.ACCOUNT_ID_SYSTEM, cmd.getEntityOwnerId()); + } + + @Test + public void getApiResourceTypeReturnsExtension() { + assertEquals(ApiCommandResourceType.Extension, cmd.getApiResourceType()); + } + + @Test + public void getEventTypeReturnsCorrectEventType() { + assertEquals(EventTypes.EVENT_EXTENSION_DOWNLOAD, cmd.getEventType()); + } + + @Test + public void getEventDescriptionReturnsCorrectDescription() { + Long id = 123L; + ReflectionTestUtils.setField(cmd, "id", id); + assertEquals("Download extension: 123", cmd.getEventDescription()); + } + +} diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListCustomActionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListCustomActionCmdTest.java index 6648b840c1cf..ac3c8e240b8a 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListCustomActionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListCustomActionCmdTest.java @@ -33,22 +33,23 @@ import org.apache.cloudstack.api.response.ExtensionCustomActionResponse; import org.apache.cloudstack.api.response.ListResponse; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class ListCustomActionCmdTest { - private ListCustomActionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() { - cmd = new ListCustomActionCmd(); - extensionsManager = mock(ExtensionsManager.class); - ReflectionTestUtils.setField(cmd, "extensionsManager", extensionsManager); - } + @Spy + @InjectMocks + private ListCustomActionCmd cmd; private void setField(String fieldName, Object value) { ReflectionTestUtils.setField(cmd, fieldName, value); @@ -140,14 +141,10 @@ public void isEnabledReturnsFalseWhenSetFalse() { @Test public void executeSetsListResponse() { List responses = Arrays.asList(mock(ExtensionCustomActionResponse.class)); - when(extensionsManager.listCustomActions(cmd)).thenReturn(responses); - - ListCustomActionCmd spyCmd = Mockito.spy(cmd); - doNothing().when(spyCmd).setResponseObject(any()); - - spyCmd.execute(); - - verify(extensionsManager).listCustomActions(spyCmd); - verify(spyCmd).setResponseObject(any(ListResponse.class)); + when(extensionsManager.listCustomActions(any(ListCustomActionCmd.class))).thenReturn(responses); + doNothing().when(cmd).setResponseObject(any()); + cmd.execute(); + verify(extensionsManager).listCustomActions(cmd); + verify(cmd).setResponseObject(any(ListResponse.class)); } } diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListExtensionsCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListExtensionsCmdTest.java index 1ca601293a37..0baa068de12d 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListExtensionsCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/ListExtensionsCmdTest.java @@ -19,6 +19,11 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import java.util.Arrays; import java.util.Collections; @@ -26,20 +31,29 @@ import java.util.List; import org.apache.cloudstack.api.ApiConstants; -import org.junit.Before; +import org.apache.cloudstack.api.response.ExtensionResponse; +import org.apache.cloudstack.api.response.ListResponse; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; import com.cloud.exception.InvalidParameterValueException; +@RunWith(MockitoJUnitRunner.class) public class ListExtensionsCmdTest { - private ListExtensionsCmd cmd; - @Before - public void setUp() { - cmd = new ListExtensionsCmd(); - } + @Mock + private ExtensionsManager extensionsManager; + + @Spy + @InjectMocks + private ListExtensionsCmd cmd; private void setPrivateField(String fieldName, Object value) { ReflectionTestUtils.setField(cmd, fieldName, value); @@ -89,4 +103,14 @@ public void testGetDetailsWithInvalidValueThrowsException() { setPrivateField("details", detailsList); cmd.getDetails(); } + + @Test + public void executeSetsListResponse() { + List responses = Arrays.asList(mock(ExtensionResponse.class)); + when(extensionsManager.listExtensions(any(ListExtensionsCmd.class))).thenReturn(responses); + doNothing().when(cmd).setResponseObject(any()); + cmd.execute(); + verify(extensionsManager).listExtensions(cmd); + verify(cmd).setResponseObject(any(ListResponse.class)); + } } diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RegisterExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RegisterExtensionCmdTest.java index b5281342d4c7..5eb45c12aedc 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RegisterExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RegisterExtensionCmdTest.java @@ -37,22 +37,23 @@ import org.apache.cloudstack.api.response.ExtensionResponse; import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class RegisterExtensionCmdTest { - private RegisterExtensionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() { - cmd = Mockito.spy(new RegisterExtensionCmd()); - extensionsManager = mock(ExtensionsManager.class); - ReflectionTestUtils.setField(cmd, "extensionsManager", extensionsManager); - } + @Spy + @InjectMocks + private RegisterExtensionCmd cmd; @Test public void extensionIdReturnsNullWhenUnset() { diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RunCustomActionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RunCustomActionCmdTest.java index 0fb4f628d275..29f96690693f 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RunCustomActionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/RunCustomActionCmdTest.java @@ -34,22 +34,23 @@ import org.apache.cloudstack.api.ServerApiException; import org.apache.cloudstack.extension.CustomActionResultResponse; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class RunCustomActionCmdTest { - private RunCustomActionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() { - cmd = Mockito.spy(new RunCustomActionCmd()); - extensionsManager = mock(ExtensionsManager.class); - ReflectionTestUtils.setField(cmd, "extensionsManager", extensionsManager); - } + @Spy + @InjectMocks + private RunCustomActionCmd cmd; @Test public void customActionIdReturnsNullWhenUnset() { diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java new file mode 100644 index 000000000000..7b0ff7ab4eee --- /dev/null +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/SyncExtensionCmdTest.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.api; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Arrays; +import java.util.List; + +import org.apache.cloudstack.api.ApiCommandResourceType; +import org.apache.cloudstack.api.ServerApiException; +import org.apache.cloudstack.api.response.SuccessResponse; +import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.event.EventTypes; +import com.cloud.user.Account; + +@RunWith(MockitoJUnitRunner.class) +public class SyncExtensionCmdTest { + + @Mock + private ExtensionsManager extensionsManager; + + @Spy + @InjectMocks + private SyncExtensionCmd cmd; + + @Test + public void returnsIdWhenGetIdIsCalled() { + ReflectionTestUtils.setField(cmd, "id", 123L); + assertEquals(Long.valueOf(123), cmd.getId()); + } + + @Test + public void returnsSourceManagementServerIdWhenGetSourceManagementServerIdIsCalled() { + ReflectionTestUtils.setField(cmd, "sourceManagementServerId", 456L); + assertEquals(Long.valueOf(456), cmd.getSourceManagementServerId()); + } + + @Test + public void returnsTargetManagementServerIdsWhenGetTargetManagementServerIdsIsCalled() { + List targetIds = Arrays.asList(789L, 101L); + ReflectionTestUtils.setField(cmd, "targetManagementServerIds", targetIds); + assertEquals(targetIds, cmd.getTargetManagementServerIds()); + } + + @Test + public void returnsFilesWhenGetFilesIsCalled() { + List files = Arrays.asList("file1.txt", "file2.txt"); + ReflectionTestUtils.setField(cmd, "files", files); + assertEquals(files, cmd.getFiles()); + } + + @Test + public void executesSuccessfullyWhenSyncExtensionSucceeds() { + when(extensionsManager.syncExtension(cmd)).thenReturn(true); + cmd.execute(); + SuccessResponse response = (SuccessResponse) cmd.getResponseObject(); + assertTrue(response.getSuccess()); + } + + @Test(expected = ServerApiException.class) + public void throwsExceptionWhenSyncExtensionFails() { + ExtensionsManager mockManager = mock(ExtensionsManager.class); + ReflectionTestUtils.setField(cmd, "extensionsManager", mockManager); + when(mockManager.syncExtension(cmd)).thenReturn(false); + + cmd.execute(); + } + + @Test + public void returnsSystemAccountIdWhenGetEntityOwnerIdIsCalled() { + assertEquals(Account.ACCOUNT_ID_SYSTEM, cmd.getEntityOwnerId()); + } + + @Test + public void returnsExtensionResourceTypeWhenGetApiResourceTypeIsCalled() { + assertEquals(ApiCommandResourceType.Extension, cmd.getApiResourceType()); + } + + @Test + public void returnsIdWhenGetApiResourceIdIsCalled() { + ReflectionTestUtils.setField(cmd, "id", 123L); + assertEquals(Long.valueOf(123), cmd.getApiResourceId()); + } + + @Test + public void returnsCorrectEventType() { + assertEquals(EventTypes.EVENT_EXTENSION_SYNC, cmd.getEventType()); + } + + @Test + public void returnsCorrectEventDescription() { + ReflectionTestUtils.setField(cmd, "id", 123L); + assertEquals("Sync extension: 123", cmd.getEventDescription()); + } +} diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UnregisterExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UnregisterExtensionCmdTest.java index f3c26f71f706..b9639f8c7e49 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UnregisterExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UnregisterExtensionCmdTest.java @@ -34,22 +34,23 @@ import org.apache.cloudstack.api.response.ExtensionResponse; import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; -import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class UnregisterExtensionCmdTest { - private UnregisterExtensionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() { - cmd = Mockito.spy(new UnregisterExtensionCmd()); - extensionsManager = mock(ExtensionsManager.class); - ReflectionTestUtils.setField(cmd, "extensionsManager", extensionsManager); - } + @Spy + @InjectMocks + private UnregisterExtensionCmd cmd; @Test public void extensionIdReturnsNullWhenUnset() { diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateCustomActionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateCustomActionCmdTest.java index 5ba17111c1ce..7c6b9f7dc2db 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateCustomActionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateCustomActionCmdTest.java @@ -39,22 +39,23 @@ import org.apache.cloudstack.extension.ExtensionCustomAction; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; import org.apache.commons.collections.MapUtils; -import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class UpdateCustomActionCmdTest { - private UpdateCustomActionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() { - cmd = Mockito.spy(new UpdateCustomActionCmd()); - extensionsManager = mock(ExtensionsManager.class); - ReflectionTestUtils.setField(cmd, "extensionsManager", extensionsManager); - } + @Spy + @InjectMocks + private UpdateCustomActionCmd cmd; @Test public void idReturnsValueWhenSet() { diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java index f0a3a6fcf219..adb12d9e8bcc 100644 --- a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/api/UpdateExtensionCmdTest.java @@ -38,22 +38,23 @@ import org.apache.cloudstack.extension.Extension; import org.apache.cloudstack.framework.extensions.manager.ExtensionsManager; import org.apache.commons.collections.MapUtils; -import org.junit.Before; import org.junit.Test; -import org.mockito.Mockito; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; import org.springframework.test.util.ReflectionTestUtils; +@RunWith(MockitoJUnitRunner.class) public class UpdateExtensionCmdTest { - private UpdateExtensionCmd cmd; + @Mock private ExtensionsManager extensionsManager; - @Before - public void setUp() { - cmd = Mockito.spy(new UpdateExtensionCmd()); - extensionsManager = mock(ExtensionsManager.class); - ReflectionTestUtils.setField(cmd, "extensionsManager", extensionsManager); - } + @Spy + @InjectMocks + private UpdateExtensionCmd cmd; @Test public void idReturnsNullWhenUnset() { diff --git a/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java new file mode 100644 index 000000000000..19e0f962be18 --- /dev/null +++ b/framework/extensions/src/test/java/org/apache/cloudstack/framework/extensions/manager/ExtensionsFilesystemManagerImplTest.java @@ -0,0 +1,670 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.framework.extensions.manager; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.doNothing; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.FileTime; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import javax.naming.ConfigurationException; + +import org.apache.cloudstack.extension.Extension; +import org.apache.cloudstack.utils.security.DigestHelper; +import org.apache.logging.log4j.Logger; +import org.junit.Assume; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.Spy; +import org.mockito.junit.MockitoJUnitRunner; +import org.springframework.test.util.ReflectionTestUtils; + +import com.cloud.utils.FileUtil; +import com.cloud.utils.PropertiesUtil; +import com.cloud.utils.exception.CloudRuntimeException; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class ExtensionsFilesystemManagerImplTest { + + @Mock + private Logger logger; + + @Spy + @InjectMocks + private ExtensionsFilesystemManagerImpl extensionsFilesystemManager; + + private File tempDir; + private File tempDataDir; + private Properties testProperties; + private File testScript; + + @Before + public void setUp() throws IOException { + tempDir = Files.createTempDirectory("extensions-test").toFile(); + tempDataDir = Files.createTempDirectory("extensions-data-test").toFile(); + + testScript = new File(tempDir, "test-extension.sh"); + testScript.createNewFile(); + resetTestScript(); + + testProperties = new Properties(); + testProperties.setProperty("extensions.deployment.mode", "developer"); + + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDirectory", tempDir.getAbsolutePath()); + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", tempDataDir.getAbsolutePath()); + + try (MockedStatic propertiesUtilMock = mockStatic(PropertiesUtil.class)) { + File mockPropsFile = mock(File.class); + propertiesUtilMock.when(() -> PropertiesUtil.findConfigFile(anyString())).thenReturn(mockPropsFile); + } + } + + private void resetTestScript() { + testScript.setExecutable(true); + testScript.setReadable(true); + testScript.setWritable(true); + } + + @Test + public void getExtensionRootPathReturnsCorrectPathForValidExtension() { + Extension extension = mock(Extension.class); + Mockito.when(extension.getName()).thenReturn("test-extension"); + String expectedPath = tempDir.getAbsolutePath() + File.separator + "test-extension"; + Path result = extensionsFilesystemManager.getExtensionRootPath(extension); + assertEquals(expectedPath, result.toString()); + } + + + @Test + public void testGetExtensionPath() { + String result = extensionsFilesystemManager.getExtensionPath("test-extension.sh"); + String expected = tempDir.getAbsolutePath() + File.separator + "test-extension.sh"; + assertEquals(expected, result); + } + + @Test + public void testGetExtensionCheckedPathValidFile() { + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); + + assertEquals(testScript.getAbsolutePath(), result); + } + + @Test + public void testGetExtensionCheckedPathFileNotExists() { + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "nonexistent.sh"); + + assertNull(result); + } + + @Test + public void testGetExtensionCheckedPathNoExecutePermissions() { + testScript.setExecutable(false); + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); + assertNull(result); + verify(logger).error("{} is not executable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); + } + + @Test + public void testGetExtensionCheckedPathNoReadPermissions() { + testScript.setWritable(false); + testScript.setReadable(false); + Assume.assumeFalse("Skipping test as file can not be marked unreadable", testScript.canRead()); + String result = extensionsFilesystemManager.getExtensionCheckedPath("test-extension", "test-extension.sh"); + assertNull(result); + verify(logger).error("{} is not readable", "Entry point [" + testScript.getAbsolutePath() + "] for extension: test-extension"); + } + + @Test + public void testCheckExtensionsDirectoryValid() { + boolean result = extensionsFilesystemManager.checkExtensionsDirectory(); + assertTrue(result); + } + + @Test + public void testCheckExtensionsDirectoryInvalid() { + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDirectory", "/nonexistent/path"); + + boolean result = extensionsFilesystemManager.checkExtensionsDirectory(); + assertFalse(result); + } + + @Test + public void testCreateOrCheckExtensionsDataDirectory() throws ConfigurationException { + extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); + verify(logger).info("Extensions data directory path: {}", tempDataDir.getAbsolutePath()); + } + + @Test(expected = ConfigurationException.class) + public void testCreateOrCheckExtensionsDataDirectoryCreateThrowsExceptionFail() throws ConfigurationException { + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", "/nonexistent/path"); + try (MockedStatic filesMock = mockStatic(Files.class)) { + filesMock.when(() -> Files.createDirectories(any())).thenThrow(new IOException("fail")); + extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); + } + } + + @Test(expected = ConfigurationException.class) + public void testCreateOrCheckExtensionsDataDirectoryNoCreateFail() throws ConfigurationException { + ReflectionTestUtils.setField(extensionsFilesystemManager, "extensionsDataDirectory", "/nonexistent/path"); + try (MockedStatic filesMock = mockStatic(Files.class)) { + filesMock.when(() -> Files.createDirectories(any())).thenReturn(mock(Path.class)); + extensionsFilesystemManager.createOrCheckExtensionsDataDirectory(); + } + } + + @Test + public void schedulesCleanupTaskSuccessfully() { + String extensionName = "test-extension"; + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(payloadCleanupExecutor).submit(any(Runnable.class)); + verify(payloadCleanupScheduler).schedule(any(Runnable.class), eq(3L), eq(TimeUnit.SECONDS)); + } + + @Test + public void handlesRejectedExecutionExceptionGracefully() { + String extensionName = "test-extension"; + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + doThrow(new RejectedExecutionException("Task rejected")) + .when(payloadCleanupExecutor).submit(any(Runnable.class)); + + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(logger).warn("Payload cleanup task for extension: {} was rejected due to: {}", extensionName, "Task rejected"); + } + + @Test + public void cancelsTaskIfNotCompletedWithinTimeout() { + String extensionName = "test-extension"; + Future mockFuture = mock(Future.class); + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(ScheduledFuture.class); + }).when(payloadCleanupScheduler).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + doReturn(mockFuture).when(payloadCleanupExecutor).submit(any(Runnable.class)); + Mockito.when(mockFuture.isDone()).thenReturn(false); + + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(mockFuture).cancel(true); + verify(logger).trace("Cancelled cleaning up payload directory for extension: {} as it running for more than 3 seconds", extensionName); + } + + @Test + public void logsExceptionDuringCleanupTask() { + String extensionName = "test-extension"; + doThrow(new RuntimeException("Cleanup error")) + .when(extensionsFilesystemManager).cleanupExtensionData(extensionName, 1, false); + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(Future.class); + }).when(payloadCleanupExecutor).submit(any(Runnable.class)); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(ScheduledFuture.class); + }).when(payloadCleanupScheduler).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(logger).warn("Exception during payload cleanup for extension: {} due to {}", extensionName, "Cleanup error"); + } + + @Test + public void logsExceptionWhenCancellingTaskFails() { + String extensionName = "test-extension"; + Future mockFuture = mock(Future.class); + ExecutorService payloadCleanupExecutor = mock(ExecutorService.class); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupExecutor", payloadCleanupExecutor); + ScheduledExecutorService payloadCleanupScheduler = mock(ScheduledExecutorService.class); + doAnswer(invocation -> { + Runnable runnable = invocation.getArgument(0); + runnable.run(); + return mock(ScheduledFuture.class); + }).when(payloadCleanupScheduler).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class)); + ReflectionTestUtils.setField(extensionsFilesystemManager, "payloadCleanupScheduler", payloadCleanupScheduler); + doReturn(mockFuture).when(payloadCleanupExecutor).submit(any(Runnable.class)); + doThrow(new RuntimeException("Cancel error")).when(mockFuture).cancel(true); + + extensionsFilesystemManager.scheduleExtensionPayloadDirectoryCleanup(extensionName); + + verify(logger).warn("Failed to cancel payload cleanup task for extension: {} due to {}", extensionName, "Cancel error"); + } + + @Test + public void getChecksumMapForExtensionReturnsChecksumsForAllFiles() throws IOException { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + Path file1 = Files.createFile(rootPath.resolve("file1.txt")); + Path file2 = Files.createFile(rootPath.resolve("file2.txt")); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + + try (MockedStatic digestHelperMock = mockStatic(DigestHelper.class)) { + digestHelperMock.when(() -> DigestHelper.calculateChecksum(file1.toFile())).thenReturn("checksum1"); + digestHelperMock.when(() -> DigestHelper.calculateChecksum(file2.toFile())).thenReturn("checksum2"); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNotNull(result); + for (Map.Entry entry : result.entrySet()) { + System.out.println(entry.getKey() + ": " + entry.getValue()); + } + assertTrue(result.size() > 2); + assertEquals("checksum1", result.get("file1.txt")); + assertEquals("checksum2", result.get("file2.txt")); + } + } + + @Test + public void getChecksumMapForExtensionReturnsNullForBlankPath() { + String extensionName = "test-extension"; + doReturn(null).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNull(result); + } + + @Test + public void getChecksumMapForExtensionHandlesIOExceptionDuringFileWalk() { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + try (MockedStatic filesMock = mockStatic(Files.class)) { + filesMock.when(() -> Files.walk(rootPath)).thenThrow(new IOException("File walk error")); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNull(result); + } + } + + @Test + public void getChecksumMapForExtensionHandlesChecksumCalculationFailure() throws IOException { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + Path file1 = Files.createFile(rootPath.resolve("file1.txt")); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + try (MockedStatic digestHelperMock = mockStatic(DigestHelper.class)) { + digestHelperMock.when(() -> DigestHelper.calculateChecksum(file1.toFile())).thenThrow(new CloudRuntimeException("Checksum error")); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNull(result); + } + } + + @Test + public void getChecksumMapForExtensionReturnsEmptyMapWhenNoFilesExist() { + String extensionName = "test-extension"; + Path rootPath = tempDir.toPath(); + doReturn(rootPath).when(extensionsFilesystemManager).getExtensionRootPath(extensionName); + doReturn(rootPath.toString()).when(extensionsFilesystemManager).getExtensionCheckedPath(extensionName, ""); + try (MockedStatic filesMock = mockStatic(Files.class)) { + filesMock.when(() -> Files.walk(rootPath)).thenReturn(Stream.empty()); + Map result = extensionsFilesystemManager.getChecksumMapForExtension(extensionName, ""); + assertNotNull(result); + assertTrue(result.isEmpty()); + } + } + + @Test + public void testPrepareExtensionPath() throws IOException { + try (MockedStatic + + diff --git a/utils/src/main/java/org/apache/cloudstack/utils/filesystem/ArchiveUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/filesystem/ArchiveUtil.java new file mode 100644 index 000000000000..52bd4fe2434c --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/filesystem/ArchiveUtil.java @@ -0,0 +1,151 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.filesystem; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import com.cloud.utils.script.Script; + +public class ArchiveUtil { + + public enum ArchiveFormat { + TGZ("tar", "tar"), + ZIP("zip", "unzip"); + + private final String packToolName; + private final String extractToolName; + + ArchiveFormat(String packToolName, String extractToolName) { + this.packToolName = packToolName; + this.extractToolName = extractToolName; + } + + public String getPackToolName() { + return packToolName; + } + + public String getExtractToolName() { + return extractToolName; + } + } + + private static String[] getPackCommandParams(ArchiveFormat format, Path sourcePath, Path archivePath) { + String toolPath = Script.getExecutableAbsolutePath(format.getPackToolName()); + if (format == ArchiveFormat.ZIP) { + return new String[]{ + toolPath, + "-r", + archivePath.toAbsolutePath().toString(), + sourcePath.toAbsolutePath().toString() + }; + } + return new String[]{ + toolPath, + "-czpf", archivePath.toAbsolutePath().toString(), + "-C", sourcePath.toAbsolutePath().toString(), + "." + }; + } + + private static String[] getExtractCommandParams(ArchiveFormat format, Path archive, Path destinationPath) { + String toolPath = Script.getExecutableAbsolutePath(format.getExtractToolName()); + if (format == ArchiveFormat.ZIP) { + return new String[]{ + toolPath, + archive.toAbsolutePath().toString(), + "-d", destinationPath.toAbsolutePath().toString() + }; + } + return new String[]{ + toolPath, + "-xpf", archive.toAbsolutePath().toString(), + "-C", destinationPath.toAbsolutePath().toString() + }; + } + + protected static boolean packDirectoryPathUsingJavaZip(Path sourcePath, Path archivePath) { + final AtomicBoolean failed = new AtomicBoolean(false); + if (!Files.exists(sourcePath)) { + return false; + } + + try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(archivePath))) { + if (Files.isDirectory(sourcePath)) { + try (Stream stream = Files.walk(sourcePath)) { + stream.filter(path -> !Files.isDirectory(path)) + .forEach(path -> { + if (failed.get()) { + return; + } + try { + Path relativePath = sourcePath.relativize(path); + String entryName = relativePath.toString().replace(File.separatorChar, '/'); + ZipEntry zipEntry = new ZipEntry(entryName); + zs.putNextEntry(zipEntry); + Files.copy(path, zs); + zs.closeEntry(); + } catch (IOException e) { + failed.set(true); + } + }); + } + } else if (Files.isRegularFile(sourcePath)) { + try { + String entryName = sourcePath.getFileName().toString().replace(File.separatorChar, '/'); + ZipEntry zipEntry = new ZipEntry(entryName); + zs.putNextEntry(zipEntry); + Files.copy(sourcePath, zs); + zs.closeEntry(); + } catch (IOException e) { + return false; + } + } else { + return false; + } + } catch (IOException e) { + return false; + } + return !failed.get(); + } + + public static boolean packPath(ArchiveFormat format, Path sourcePath, Path archivePath, int timeoutSeconds) { + if (format == ArchiveFormat.ZIP) { + return packDirectoryPathUsingJavaZip(sourcePath, archivePath); + } + int result = Script.executeCommandForExitValue( + timeoutSeconds * 1000L, + getPackCommandParams(format, sourcePath, archivePath) + ); + return result == 0; + } + + public static boolean extractToPath(ArchiveFormat format, Path archivePath, Path destinationPath, int timeoutSeconds) { + int result = Script.executeCommandForExitValue( + timeoutSeconds * 1000L, + getExtractCommandParams(format, archivePath, destinationPath) + ); + return result == 0; + } +} diff --git a/utils/src/main/java/org/apache/cloudstack/utils/security/HMACSignUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/security/HMACSignUtil.java new file mode 100644 index 000000000000..13a1f68ecb2a --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/security/HMACSignUtil.java @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.security; + +import java.nio.charset.StandardCharsets; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +import javax.crypto.Mac; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Base64; + +public class HMACSignUtil { + final static String ALGORITHM = "HMACSHA256"; + + public static String generateSignature(String data, String key) + throws InvalidKeyException, NoSuchAlgorithmException, DecoderException { + Mac mac = Mac.getInstance(ALGORITHM); + SecretKey secretKey = new SecretKeySpec(key.getBytes(StandardCharsets.UTF_8), mac.getAlgorithm()); + mac.init(secretKey); + byte[] dataAsBytes = data.getBytes(StandardCharsets.UTF_8); + byte[] encodedText = mac.doFinal(dataAsBytes); + return new String(Base64.encodeBase64(encodedText)).trim(); + } +} diff --git a/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java new file mode 100644 index 000000000000..09c02d7799d9 --- /dev/null +++ b/utils/src/main/java/org/apache/cloudstack/utils/server/ServerPropertiesUtil.java @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.server; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cloud.utils.PropertiesUtil; + +public class ServerPropertiesUtil { + public static final String SHARE_DIR = "share"; + private static final Logger logger = LoggerFactory.getLogger(ServerPropertiesUtil.class); + protected static final String PROPERTIES_FILE = "server.properties"; + + public static final String SHARE_ENABLED = "share.enabled"; + private static final String SHARE_BASE_DIR = "share.base.dir"; + public static final String SHARE_CACHE_CONTROL = "share.cache.control"; + public static final String SHARE_DIR_ALLOWED = "share.dir.allowed"; + private static final String SHARE_SECRET = "share.secret"; + + protected static final AtomicReference propertiesRef = new AtomicReference<>(); + + public static String getProperty(String name) { + Properties props = propertiesRef.get(); + if (props != null) { + return props.getProperty(name); + } + File propsFile = PropertiesUtil.findConfigFile(PROPERTIES_FILE); + if (propsFile == null) { + logger.error("{} file not found", PROPERTIES_FILE); + return null; + } + Properties tempProps = new Properties(); + try (FileInputStream is = new FileInputStream(propsFile)) { + tempProps.load(is); + } catch (IOException e) { + logger.error("Error loading {}: {}", PROPERTIES_FILE, e.getMessage(), e); + return null; + } + if (!propertiesRef.compareAndSet(null, tempProps)) { + tempProps = propertiesRef.get(); + } + return tempProps.getProperty(name); + } + + public static String getProperty(String name, String defaultValue) { + String value = getProperty(name); + if (value == null) { + value = defaultValue; + } + return value; + } + + public static boolean getShareEnabled() { + return Boolean.parseBoolean(getProperty(SHARE_ENABLED, "true")); + } + + public static String getShareBaseDirectory() { + String shareBaseDir = getProperty(SHARE_BASE_DIR); + if (StringUtils.isNotBlank(shareBaseDir)) { + return shareBaseDir; + } + boolean isMavenRun = ManagementFactory.getRuntimeMXBean().getInputArguments().toString().contains("org.codehaus.plexus.classworlds.launcher.Launcher"); + if (isMavenRun) { + // when running from maven, use a share directory from client/target in the current working directory + return System.getProperty("user.dir") + File.separator + "client" + File.separator + "target" + File.separator + SHARE_DIR; + } + return System.getProperty("user.home") + File.separator + SHARE_DIR; + } + + public static String getShareCacheControl() { + return getProperty(SHARE_CACHE_CONTROL, "public,max-age=86400,immutable"); + } + + public static boolean getShareDirAllowed() { + return Boolean.parseBoolean(getProperty(SHARE_DIR_ALLOWED, "false")); + } + + public static String getShareSecret() { + return getProperty(SHARE_SECRET); + } +} diff --git a/utils/src/test/java/org/apache/cloudstack/utils/filesystem/ArchiveUtilTest.java b/utils/src/test/java/org/apache/cloudstack/utils/filesystem/ArchiveUtilTest.java new file mode 100644 index 000000000000..11aa4f433824 --- /dev/null +++ b/utils/src/test/java/org/apache/cloudstack/utils/filesystem/ArchiveUtilTest.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.cloudstack.utils.filesystem; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.ArgumentMatchers.anyLong; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.utils.FileUtil; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class ArchiveUtilTest { + + private final List cleanupPaths = new ArrayList<>(); + + @Before + public void setup() { + cleanupPaths.clear(); + } + + @After + public void cleanup() { + for (int i = cleanupPaths.size() - 1; i >= 0; i--) { + Path path = cleanupPaths.get(i); + try { + if (Files.exists(path)) { + if (Files.isDirectory(path)) { + FileUtil.deleteRecursively(path); + } else { + Files.delete(path); + } + } + } catch (IOException e) { + // ignore + } + } + } + + private void runPackPathTest(ArchiveUtil.ArchiveFormat archiveFormat, boolean shouldSucceed) { + Path sourcePath = Path.of("source-path"); + Path archivePath = Path.of("test-archive." + archiveFormat.name().toLowerCase()); + try (MockedStatic