extensions: add sync, download functionalities

api/src/main/java/com/cloud/event/EventTypes.java

@@ -846,6 +846,8 @@ public class EventTypes {
     public static final String EVENT_EXTENSION_CREATE = "EXTENSION.CREATE";
     public static final String EVENT_EXTENSION_UPDATE = "EXTENSION.UPDATE";
     public static final String EVENT_EXTENSION_DELETE = "EXTENSION.DELETE";
+    public static final String EVENT_EXTENSION_SYNC = "EXTENSION.SYNC";
+    public static final String EVENT_EXTENSION_DOWNLOAD = "EXTENSION.DOWNLOAD";
     public static final String EVENT_EXTENSION_RESOURCE_REGISTER = "EXTENSION.RESOURCE.REGISTER";
     public static final String EVENT_EXTENSION_RESOURCE_UNREGISTER = "EXTENSION.RESOURCE.UNREGISTER";
     public static final String EVENT_EXTENSION_CUSTOM_ACTION_ADD = "EXTENSION.CUSTOM.ACTION.ADD";
@@ -1388,6 +1390,7 @@ public class EventTypes {
         entityEventDetails.put(EVENT_EXTENSION_CREATE, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_UPDATE, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_DELETE, Extension.class);
+        entityEventDetails.put(EVENT_EXTENSION_SYNC, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_RESOURCE_REGISTER, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_RESOURCE_UNREGISTER, Extension.class);
         entityEventDetails.put(EVENT_EXTENSION_CUSTOM_ACTION_ADD, ExtensionCustomAction.class);

api/src/main/java/com/cloud/storage/Upload.java

@@ -40,7 +40,7 @@ public static enum Status {
     }
 
     public static enum Type {
-        VOLUME, SNAPSHOT, TEMPLATE, ISO
+        VOLUME, SNAPSHOT, TEMPLATE, ISO, ARCHIVE
     }
 
     public static enum Mode {

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -556,6 +556,7 @@ public class ApiConstants {
 
     public static final String SOURCE_CIDR_LIST = "sourcecidrlist";
     public static final String SOURCE_ZONE_ID = "sourcezoneid";
+    public static final String SOURCE_MANAGEMENT_SERVER_ID = "sourcemanagementserverid";
     public static final String SSL_VERIFICATION = "sslverification";
     public static final String START_ASN = "startasn";
     public static final String START_DATE = "startdate";
@@ -576,6 +577,7 @@ public class ApiConstants {
     public static final String SWAP_OWNER = "swapowner";
     public static final String SYSTEM_VM_TYPE = "systemvmtype";
     public static final String TAGS = "tags";
+    public static final String TARGET_MANAGEMENT_SERVER_IDS = "targetmanagementserverids";
     public static final String STORAGE_TAGS = "storagetags";
     public static final String STORAGE_ACCESS_GROUPS = "storageaccessgroups";
     public static final String STORAGE_ACCESS_GROUP = "storageaccessgroup";

api/src/main/java/org/apache/cloudstack/api/response/ExtractResponse.java

@@ -18,12 +18,11 @@
 
 import java.util.Date;
 
-import com.google.gson.annotations.SerializedName;
-
 import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
 import com.cloud.serializer.Param;
+import com.google.gson.annotations.SerializedName;
 
 public class ExtractResponse extends BaseResponse {
     @SerializedName(ApiConstants.ID)

client/conf/server.properties.in

@@ -67,3 +67,17 @@ extensions.deployment.mode=@EXTENSIONSDEPLOYMENTMODE@
 # `downloadrepository` key value from the metadata file in template URLs. If not specified, the original template URL
 # will be used for download.
 # system.vm.templates.download.repository=http://download.cloudstack.org/systemvm/
+
+# These properties configure the share endpoint, which enables controlled file sharing through the management server.
+# They allow administrators to enable or disable sharing, set the base directory for shared files, define cache
+# behavior, restrict access to specific directories, and secure access with a secret key. This ensures flexible and
+# secure file sharing for different modules such as extensions, etc.
+# Enable or disable file sharing feature (true/false). Default is true
+share.enabled=true
+# The base directory from which files can be shared. Default is <HOME_DIRECTORY_OF_CLOUD_USER>/share
+# share.base.dir=
+# The cache control header value to be used for shared files. Default is public,max-age=86400,immutable
+# share.cache.control=public,max-age=86400,immutable
+# Secret key for securing links using HMAC signature. If not set then links will not be signed. Default is change-me
+# It is recommended to change this value to a strong secret key in production
+share.secret=change-me

client/src/main/java/org/apache/cloudstack/ServerDaemon.java

@@ -24,12 +24,24 @@
 import java.io.InputStream;
 import java.lang.management.ManagementFactory;
 import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.EnumSet;
+import java.util.List;
 import java.util.Properties;
 
+import javax.servlet.DispatcherType;
+
+import org.apache.cloudstack.servlet.ShareSignedUrlFilter;
+import org.apache.cloudstack.utils.server.ServerPropertiesUtil;
 import org.apache.commons.daemon.Daemon;
 import org.apache.commons.daemon.DaemonContext;
 import org.apache.commons.lang3.StringUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.eclipse.jetty.jmx.MBeanContainer;
+import org.eclipse.jetty.server.Handler;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.RequestLog;
@@ -43,13 +55,16 @@
 import org.eclipse.jetty.server.handler.RequestLogHandler;
 import org.eclipse.jetty.server.handler.gzip.GzipHandler;
 import org.eclipse.jetty.server.session.SessionHandler;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
+import org.eclipse.jetty.servlet.ServletHolder;
+import org.eclipse.jetty.util.resource.Resource;
 import org.eclipse.jetty.util.ssl.KeyStoreScanner;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.QueuedThreadPool;
 import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
 import org.eclipse.jetty.webapp.WebAppContext;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.LogManager;
 
 import com.cloud.utils.Pair;
 import com.cloud.utils.PropertiesUtil;
@@ -156,6 +171,13 @@ public void init(final DaemonContext context) {
         }
         logger.info(String.format("Initializing server daemon on %s, with http.enable=%s, http.port=%s, https.enable=%s, https.port=%s, context.path=%s",
                 bindInterface, httpEnable, httpPort, httpsEnable, httpsPort, contextPath));
+
+        if (ServerPropertiesUtil.getShareEnabled()) {
+            logger.info("/{} static context for file-sharing is enabled, baseDir={}, cacheCtl={}, secret={}",
+                    ServerPropertiesUtil.SHARE_DIR, ServerPropertiesUtil.getShareBaseDirectory(),
+                    ServerPropertiesUtil.getShareCacheControl(),
+                    (StringUtils.isNotBlank(ServerPropertiesUtil.getShareSecret()) ? "configured" : "not configured"));
+        }
     }
 
     @Override
@@ -264,6 +286,48 @@ private void createHttpsConnector(final HttpConfiguration httpConfig) {
         }
     }
 
+    /**
+     * Creates a Jetty context at /share to serve static files for modules (e.g. Extensions Framework).
+     * Controlled via server properties
+     *
+     * @return a configured Handler or null if disabled.
+     */
+    private Handler createShareContextHandler() throws IOException {
+        if (!ServerPropertiesUtil.getShareEnabled()) {
+            logger.info("/{} context not mounted", ServerPropertiesUtil.SHARE_DIR);
+            return null;
+        }
+
+        final Path base = Paths.get(ServerPropertiesUtil.getShareBaseDirectory());
+        Files.createDirectories(base);
+
+        final ServletContextHandler shareCtx = new ServletContextHandler(ServletContextHandler.NO_SESSIONS);
+        shareCtx.setContextPath("/" + ServerPropertiesUtil.SHARE_DIR);
+        shareCtx.setBaseResource(Resource.newResource(base.toAbsolutePath().toUri()));
+
+        // Efficient static file serving
+        ServletHolder def = shareCtx.addServlet(DefaultServlet.class, "/*");
+        def.setInitParameter("dirAllowed", "false");
+        def.setInitParameter("etags", "true");
+        def.setInitParameter("cacheControl", ServerPropertiesUtil.getShareCacheControl());
+        def.setInitParameter("useFileMappedBuffer", "true");
+        def.setInitParameter("acceptRanges", "true");
+
+        // Gzip using modern Jetty handler
+        org.eclipse.jetty.server.handler.gzip.GzipHandler gzipHandler =
+                new org.eclipse.jetty.server.handler.gzip.GzipHandler();
+        gzipHandler.setMinGzipSize(1024);
+        gzipHandler.setIncludedMimeTypes(
+                "text/html", "text/plain", "text/css", "text/javascript",
+                "application/javascript", "application/json", "application/xml");
+        gzipHandler.setHandler(shareCtx);
+        shareCtx.addFilter(new FilterHolder(new ShareSignedUrlFilter()), "/*",
+                EnumSet.of(DispatcherType.REQUEST));
+
+        logger.info("Mounted /{} static context at baseDir={}", ServerPropertiesUtil.SHARE_DIR, base);
+        return shareCtx;
+    }
+
     private Pair<SessionHandler,HandlerCollection> createHandlers() {
         final WebAppContext webApp = new WebAppContext();
         webApp.setContextPath(contextPath);
@@ -294,8 +358,23 @@ private Pair<SessionHandler,HandlerCollection> createHandlers() {
         rootRedirect.setNewContextURL(contextPath);
         rootRedirect.setPermanent(true);
 
+        // Optional /share handler (served by createShareContextHandler)
+        Handler shareHandler = null;
+        try {
+            shareHandler = createShareContextHandler();
+        } catch (IOException e) {
+            logger.error("Failed to initialize /share context", e);
+        }
+
+        List<Handler> handlers = new java.util.ArrayList<>();
+        handlers.add(log);
+        handlers.add(gzipHandler);
+        if (shareHandler != null) {
+            handlers.add(shareHandler);
+        }
         // Put rootRedirect at the end!
-        return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(log, gzipHandler, rootRedirect));
+        handlers.add(rootRedirect);
+        return new Pair<>(webApp.getSessionHandler(), new HandlerCollection(handlers.toArray(new Handler[0])));
     }
 
     private RequestLog createRequestLog() {

client/src/main/webapp/WEB-INF/web.xml

@@ -54,6 +54,12 @@
         <load-on-startup>6</load-on-startup>
     </servlet>
 
+    <servlet>
+        <servlet-name>shareServlet</servlet-name>
+        <servlet-class>org.apache.cloudstack.servlet.ShareServlet</servlet-class>
+        <load-on-startup>7</load-on-startup>
+    </servlet>
+
     <servlet-mapping>
          <servlet-name>apiServlet</servlet-name>
          <url-pattern>/api/*</url-pattern>
@@ -64,9 +70,24 @@
          <url-pattern>/console</url-pattern>
     </servlet-mapping>
 
+    <servlet-mapping>
+        <servlet-name>shareServlet</servlet-name>
+        <url-pattern>/share/*</url-pattern>
+    </servlet-mapping>
+
     <error-page>
         <exception-type>java.lang.Exception</exception-type>
         <location>/error.html</location>
     </error-page>
 
+    <filter>
+        <filter-name>share-signed-url</filter-name>
+        <filter-class>org.apache.cloudstack.servlet.ShareSignedUrlFilter</filter-class>
+    </filter>
+    <filter-mapping>
+        <filter-name>share-signed-url</filter-name>
+        <url-pattern>/share/*</url-pattern>
+        <dispatcher>REQUEST</dispatcher>
+    </filter-mapping>
+
 </web-app>

core/src/main/java/com/cloud/storage/template/HttpTemplateDownloader.java

@@ -259,7 +259,7 @@ private boolean copyBytes(File file, InputStream in, RandomAccessFile out) throw
                 break;
             }
             offset = writeBlock(bytesRead, out, buffer, offset);
-            if (!ResourceType.SNAPSHOT.equals(resourceType)
+            if (resourceType.shouldVerifyFormat()
                     && !verifyFormat.isVerifiedFormat()
                     && (offset >= MIN_FORMAT_VERIFICATION_SIZE || offset >= remoteSize)) {
                 verifyFormat.invoke();

core/src/main/java/org/apache/cloudstack/storage/command/DownloadCommand.java

@@ -20,6 +20,7 @@
 package org.apache.cloudstack.storage.command;
 
 import org.apache.cloudstack.api.InternalIdentity;
+import org.apache.cloudstack.storage.to.DownloadableArchiveObjectTO;
 import org.apache.cloudstack.storage.to.SnapshotObjectTO;
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
@@ -34,7 +35,26 @@
 public class DownloadCommand extends AbstractDownloadCommand implements InternalIdentity {
 
     public static enum ResourceType {
-        VOLUME, TEMPLATE, SNAPSHOT
+        VOLUME(true, true),
+        TEMPLATE(true, true),
+        SNAPSHOT(false, false),
+        ARCHIVE(false, false);
+
+        private final boolean requiresPostDownloadProcessing;
+        private final boolean verifyFormat;
+
+        ResourceType(boolean requiresPostDownloadProcessing, boolean verifyFormat) {
+            this.requiresPostDownloadProcessing = requiresPostDownloadProcessing;
+            this.verifyFormat = verifyFormat;
+        }
+
+        public boolean doesRequirePostDownloadProcessing() {
+            return requiresPostDownloadProcessing;
+        }
+
+        public boolean shouldVerifyFormat() {
+            return verifyFormat;
+        }
     }
 
     private boolean hvm;
@@ -114,6 +134,19 @@ public DownloadCommand(SnapshotObjectTO snapshot, Long maxDownloadSizeInBytes, S
         this.resourceType = ResourceType.SNAPSHOT;
     }
 
+    public DownloadCommand(DownloadableArchiveObjectTO archive, Long maxDownloadSizeInBytes, String url) {
+        super(archive.getName(), url, archive.getFormat(), archive.getAccountId());
+        _store = archive.getDataStore();
+        installPath = archive.getPath();
+        id = archive.getId();
+        checksum = archive.getChecksum();
+        if (_store instanceof NfsTO) {
+            setSecUrl(((NfsTO)_store).getUrl());
+        }
+        this.maxDownloadSizeInBytes = maxDownloadSizeInBytes;
+        this.resourceType = ResourceType.ARCHIVE;
+    }
+
     @Override
     public long getId() {
         return id;