Introduce new native backup provider (KNIB)

agent/conf/agent.properties

@@ -457,3 +457,7 @@ iscsi.session.cleanup.enabled=false
 
 # Instance conversion VIRT_V2V_TMPDIR env var
 #convert.instance.env.virtv2v.tmpdir=
+
+# Timeout (in seconds) for QCOW2 delta merge operations, mainly used for classic volume snapshots, disk-only VM snapshots on file-based storage, and the KNIB plugin.
+# If a value of 0 or less is informed, the default will be used.
+# qcow2.delta.merge.timeout=259200

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -156,7 +156,8 @@ public class AgentProperties{
     public static final Property<Integer> CMDS_TIMEOUT = new Property<>("cmds.timeout", 7200);
 
     /**
-     * The timeout (in seconds) for the snapshot merge operation, mainly used for classic volume snapshots and disk-only VM snapshots on file-based storage.<br>
+     * The timeout (in seconds) for QCOW2 delta merge operations, mainly used for classic volume snapshots, disk-only VM snapshots on file-based storage, and the KNIB plugin.
+     * If a value of 0 or less is informed, the default will be used.<br>
      * This configuration is only considered if libvirt.events.enabled is also true. <br>
      * Data type: Integer.<br>
      * Default value: <code>259200</code>

api/src/main/java/com/cloud/agent/api/to/DataObjectType.java

@@ -19,5 +19,5 @@
 package com.cloud.agent.api.to;
 
 public enum DataObjectType {
-    VOLUME, SNAPSHOT, TEMPLATE, ARCHIVE
+    VOLUME, SNAPSHOT, TEMPLATE, ARCHIVE, BACKUP
 }

api/src/main/java/com/cloud/hypervisor/HypervisorGuru.java

@@ -20,6 +20,7 @@
 import java.util.Map;
 
 import org.apache.cloudstack.backup.Backup;
+import org.apache.cloudstack.backup.BackupProvider;
 import org.apache.cloudstack.framework.config.ConfigKey;
 
 import com.cloud.agent.api.Command;
@@ -94,10 +95,10 @@ public interface HypervisorGuru extends Adapter {
     Map<String, String> getClusterSettings(long vmId);
 
     VirtualMachine importVirtualMachineFromBackup(long zoneId, long domainId, long accountId, long userId,
-                                                  String vmInternalName, Backup backup) throws Exception;
+                                                  String vmInternalName, Backup backup, BackupProvider backupProvider) throws Exception;
 
     boolean attachRestoredVolumeToVirtualMachine(long zoneId, String location, Backup.VolumeInfo volumeInfo,
-                                                 VirtualMachine vm, long poolId, Backup backup) throws Exception;
+                                                 VirtualMachine vm, long poolId, Backup backup, BackupProvider backupProvider) throws Exception;
     /**
      * Will generate commands to migrate a vm to a pool. For now this will only work for stopped VMs on Vmware.
      *

api/src/main/java/com/cloud/storage/Volume.java

@@ -60,7 +60,9 @@ enum State {
         UploadError(false, "Volume upload encountered some error"),
         UploadAbandoned(false, "Volume upload is abandoned since the upload was never initiated within a specified time"),
         Attaching(true, "The volume is attaching to a VM from Ready state."),
-        Restoring(true, "The volume is being restored from backup.");
+        Restoring(true, "The volume is being restored from backup."),
+        Consolidating(true, "The volume is being flattened."),
+        RestoreError(false, "The volume restore encountered an error.");
 
         boolean _transitional;
 
@@ -153,6 +155,10 @@ public String getDescription() {
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Destroy, Event.RestoreRequested, Restoring, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Restoring, Event.RestoreSucceeded, Ready, null));
             s_fsm.addTransition(new StateMachine2.Transition<State, Event>(Restoring, Event.RestoreFailed, Ready, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Ready, Event.ConsolidationRequested, Consolidating, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Consolidating, Event.OperationSucceeded, Ready, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(Consolidating, Event.OperationFailed, RestoreError, null));
+            s_fsm.addTransition(new StateMachine2.Transition<>(RestoreError, Event.RestoreFailed, RestoreError, null));
         }
     }
 
@@ -179,7 +185,8 @@ enum Event {
         OperationTimeout,
         RestoreRequested,
         RestoreSucceeded,
-        RestoreFailed;
+        RestoreFailed,
+        ConsolidationRequested
     }
 
     /**

api/src/main/java/com/cloud/storage/VolumeApiService.java

@@ -107,7 +107,7 @@ public interface VolumeApiService {
 
     Volume attachVolumeToVM(AttachVolumeCmd command);
 
-    Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS);
+    Volume attachVolumeToVM(Long vmId, Long volumeId, Long deviceId, Boolean allowAttachForSharedFS, boolean allowAttachOnRestoring);
 
     Volume detachVolumeViaDestroyVM(long vmId, long volumeId);
 

api/src/main/java/com/cloud/vm/VirtualMachine.java

@@ -58,7 +58,10 @@ public enum State {
         Error(false, "VM is in error"),
         Unknown(false, "VM state is unknown."),
         Shutdown(false, "VM state is shutdown from inside"),
-        Restoring(true, "VM is being restored from backup");
+        Restoring(true, "VM is being restored from backup"),
+        BackingUp(true, "VM is being backed up"),
+        BackupError(false, "VM backup is in a inconsistent state. Operator should analyse the logs and restore the VM"),
+        RestoreError(false, "VM restore left the VM in a inconsistent state. Operator should analyse the logs and restore the VM");
 
         private final boolean _transitional;
         String _description;
@@ -131,6 +134,14 @@ public static StateMachine2<State, VirtualMachine.Event, VirtualMachine> getStat
             s_fsm.addTransition(new Transition<State, Event>(State.Destroyed, Event.RestoringRequested, State.Restoring, null));
             s_fsm.addTransition(new Transition<State, Event>(State.Restoring, Event.RestoringSuccess, State.Stopped, null));
             s_fsm.addTransition(new Transition<State, Event>(State.Restoring, Event.RestoringFailed, State.Stopped, null));
+            s_fsm.addTransition(new Transition<>(State.Running, Event.BackupRequested, State.BackingUp, null));
+            s_fsm.addTransition(new Transition<>(State.Stopped, Event.BackupRequested, State.BackingUp, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.BackupSucceededRunning, State.Running, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.BackupSucceededStopped, State.Stopped, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.OperationFailedToError, State.BackupError, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.OperationFailedToRunning, State.Running, null));
+            s_fsm.addTransition(new Transition<>(State.BackingUp, Event.OperationFailedToStopped, State.Stopped, null));
+            s_fsm.addTransition(new Transition<State, Event>(State.RestoreError, Event.RestoringFailed, State.RestoreError, null));
 
             s_fsm.addTransition(new Transition<State, Event>(State.Starting, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running, Arrays.asList(new Impact[]{Impact.USAGE})));
             s_fsm.addTransition(new Transition<State, Event>(State.Stopping, VirtualMachine.Event.FollowAgentPowerOnReport, State.Running, null));
@@ -209,6 +220,8 @@ public enum Event {
         ExpungeOperation,
         OperationSucceeded,
         OperationFailed,
+        OperationFailedToRunning,
+        OperationFailedToStopped,
         OperationFailedToError,
         OperationRetry,
         AgentReportShutdowned,
@@ -218,6 +231,9 @@ public enum Event {
         RestoringRequested,
         RestoringFailed,
         RestoringSuccess,
+        BackupRequested,
+        BackupSucceededStopped,
+        BackupSucceededRunning,
 
         // added for new VMSync logic
         FollowAgentPowerOnReport,

api/src/main/java/com/cloud/vm/VmDetailConstants.java

@@ -129,4 +129,7 @@ public interface VmDetailConstants {
     String EXTERNAL_DETAIL_PREFIX = "External:";
     String CLOUDSTACK_VM_DETAILS = "cloudstack.vm.details";
     String CLOUDSTACK_VLAN = "cloudstack.vlan";
+
+    // KNIB specific
+    String LINKED_VOLUMES_SECONDARY_STORAGE_UUIDS = "linkedVolumesSecondaryStorageUuids";
 }

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -518,6 +518,7 @@ public class ApiConstants {
     public static final String QUALIFIERS = "qualifiers";
     public static final String QUERY_FILTER = "queryfilter";
     public static final String QUIESCE_VM = "quiescevm";
+    public static final String QUICK_RESTORE = "quickrestore";
     public static final String SCHEDULE = "schedule";
     public static final String SCHEDULE_ID = "scheduleid";
     public static final String SCOPE = "scope";
@@ -566,6 +567,7 @@ public class ApiConstants {
     public static final String STATE = "state";
     public static final String STATS = "stats";
     public static final String STATUS = "status";
+    public static final String COMPRESSION_STATUS = "compressionstatus";
     public static final String STORAGE_TYPE = "storagetype";
     public static final String STORAGE_POLICY = "storagepolicy";
     public static final String STORAGE_MOTION_ENABLED = "storagemotionenabled";
@@ -657,6 +659,7 @@ public class ApiConstants {
     public static final String ETCD_SERVICE_OFFERING_NAME = "etcdofferingname";
     public static final String REMOVE_VLAN = "removevlan";
     public static final String VLAN_ID = "vlanid";
+    public static final String ISOLATED = "isolated";
     public static final String ISOLATED_PVLAN = "isolatedpvlan";
     public static final String ISOLATED_PVLAN_TYPE = "isolatedpvlantype";
     public static final String ISOLATION_URI = "isolationuri";
@@ -1171,6 +1174,7 @@ public class ApiConstants {
     public static final String CLEAN_UP_EXTRA_CONFIG = "cleanupextraconfig";
     public static final String CLEAN_UP_PARAMETERS = "cleanupparameters";
     public static final String VIRTUAL_SIZE = "virtualsize";
+    public static final String UNCOMPRESSED_SIZE = "uncompressedsize";
     public static final String NETSCALER_CONTROLCENTER_ID = "netscalercontrolcenterid";
     public static final String NETSCALER_SERVICEPACKAGE_ID = "netscalerservicepackageid";
     public static final String FETCH_ROUTER_HEALTH_CHECK_RESULTS = "fetchhealthcheckresults";
@@ -1353,6 +1357,10 @@ public class ApiConstants {
 
     public static final String VMWARE_DC = "vmwaredc";
 
+    public static final String PARAMETER_DESCRIPTION_ISOLATED_BACKUPS = "Whether the backup will be isolated, defaults to false. " +
+            "Isolated backups are always created as full backups in independent chains. Therefore, they will never depend on any existing backup chain " +
+            "and no backup chain will depend on them. Currently only supported for the KNIB provider.";
+
     public static final String CSS = "css";
 
     public static final String JSON_CONFIGURATION = "jsonconfiguration";
@@ -1373,6 +1381,24 @@ public class ApiConstants {
     public static final String OBSOLETE_PARAMETERS = "obsoleteparameters";
     public static final String EXCLUDED_PARAMETERS = "excludedparameters";
 
+    public static final String COMPRESS = "compress";
+
+    public static final String VALIDATE = "validate";
+
+    public static final String ALLOW_QUICK_RESTORE = "allowquickrestore";
+
+    public static final String ALLOW_EXTRACT_FILE = "allowextractfile";
+
+    public static final String BACKUP_CHAIN_SIZE = "backupchainsize";
+
+    public static final String COMPRESSION_LIBRARY = "compressionlibrary";
+    public static final String ATTEMPTS = "attempts";
+
+    public static final String EXECUTING = "executing";
+
+    public static final String SCHEDULED = "scheduled";
+    public static final String SCHEDULED_DATE = "scheduleddate";
+
     /**
      * This enum specifies IO Drivers, each option controls specific policies on I/O.
      * Qemu guests support "threads" and "native" options Since 0.8.8 ; "io_uring" is supported Since 6.3.0 (QEMU 5.0).

api/src/main/java/org/apache/cloudstack/api/ResponseGenerator.java

@@ -22,6 +22,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.cloudstack.api.response.NativeBackupOfferingResponse;
+import org.apache.cloudstack.backup.NativeBackupOffering;
 import org.apache.cloudstack.api.response.ConsoleSessionResponse;
 import org.apache.cloudstack.consoleproxy.ConsoleSession;
 import org.apache.cloudstack.affinity.AffinityGroup;
@@ -583,4 +585,6 @@ List<TemplateResponse> createTemplateResponses(ResponseView view, VirtualMachine
     GuiThemeResponse createGuiThemeResponse(GuiThemeJoin guiThemeJoin);
 
     ConsoleSessionResponse createConsoleSessionResponse(ConsoleSession consoleSession, ResponseView responseView);
+
+    NativeBackupOfferingResponse createNativeBackupOfferingResponse(NativeBackupOffering offering);
 }

api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupCmd.java

@@ -81,6 +81,12 @@ public class CreateBackupCmd extends BaseAsyncCreateCmd {
             since = "4.21.0")
     private Boolean quiesceVM;
 
+    @Parameter(name = ApiConstants.ISOLATED,
+            type = CommandType.BOOLEAN,
+            description = ApiConstants.PARAMETER_DESCRIPTION_ISOLATED_BACKUPS,
+            since = "4.23.0")
+    private boolean isolated;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -101,6 +107,10 @@ public Boolean getQuiesceVM() {
         return quiesceVM;
     }
 
+    public boolean isIsolated() {
+        return isolated;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/backup/CreateBackupScheduleCmd.java

@@ -87,6 +87,12 @@ public class CreateBackupScheduleCmd extends BaseCmd {
             since = "4.21.0")
     private Boolean quiesceVM;
 
+    @Parameter(name = ApiConstants.ISOLATED,
+            type = CommandType.BOOLEAN,
+            description = ApiConstants.PARAMETER_DESCRIPTION_ISOLATED_BACKUPS,
+            since = "4.23.0")
+    private boolean isolated;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -115,6 +121,10 @@ public Boolean getQuiesceVM() {
         return quiesceVM;
     }
 
+    public boolean isIsolated() {
+        return isolated;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////

api/src/main/java/org/apache/cloudstack/api/command/user/backup/ListBackupCompressionJobsCmd.java

@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.backup;
+
+import com.cloud.exception.ConcurrentOperationException;
+import com.cloud.exception.InsufficientCapacityException;
+import com.cloud.exception.NetworkRuleConflictException;
+import com.cloud.exception.ResourceAllocationException;
+import com.cloud.exception.ResourceUnavailableException;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ApiConstants;
+import org.apache.cloudstack.api.BaseListCmd;
+import org.apache.cloudstack.api.Parameter;
+import org.apache.cloudstack.api.ServerApiException;
+import org.apache.cloudstack.api.response.BackupCompressionJobResponse;
+import org.apache.cloudstack.api.response.BackupResponse;
+import org.apache.cloudstack.api.response.HostResponse;
+import org.apache.cloudstack.api.response.ListResponse;
+import org.apache.cloudstack.api.response.ZoneResponse;
+
+@APICommand(name = "listBackupCompressionJobs", description = "List backup compression jobs", responseObject = BackupCompressionJobResponse.class,
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false, authorized = {RoleType.Admin}, since = "4.23.0")
+public class ListBackupCompressionJobsCmd extends BaseListCmd {
+
+    @Parameter(name = ApiConstants.ID, type = CommandType.LONG, entityType = BackupCompressionJobResponse.class, description = "List only job with given ID.")
+    private Long id;
+
+    @Parameter(name = ApiConstants.BACKUP_ID, type = CommandType.UUID, entityType = BackupResponse.class, description = "List jobs for the given backup.")
+    private Long backupId;
+
+    @Parameter(name = ApiConstants.HOST_ID, type = CommandType.UUID, entityType = HostResponse.class, description = "List jobs in the given host, implies executing.")
+    private Long hostId;
+
+    @Parameter(name = ApiConstants.ZONE_ID, type = CommandType.UUID, entityType = ZoneResponse.class, description = "List jobs in the given zone.")
+    private Long zoneId;
+
+    @Parameter(name = ApiConstants.TYPE, type = CommandType.STRING, description = "List jobs with the given type. Accepted values are Start and Finalize.")
+    private String type;
+
+    @Parameter(name = ApiConstants.EXECUTING, type = CommandType.BOOLEAN, description = "List executing jobs.")
+    private Boolean executing;
+
+    @Parameter(name = ApiConstants.SCHEDULED, type = CommandType.BOOLEAN, description = "List scheduled jobs.")
+    private Boolean scheduled;
+
+    /////////////////////////////////////////////////////
+    /////////////////// Accessors ///////////////////////
+    /////////////////////////////////////////////////////
+
+    public Long getId() {
+        return id;
+    }
+
+    public Long getBackupId() {
+        return backupId;
+    }
+
+    public Long getHostId() {
+        return hostId;
+    }
+
+    public Long getZoneId() {
+        return zoneId;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public boolean getExecuting() {
+        return Boolean.TRUE.equals(executing);
+    }
+
+    public boolean getScheduled() {
+        return Boolean.TRUE.equals(scheduled);
+    }
+
+    /////////////////////////////////////////////////////
+    /////////////// API Implementation///////////////////
+    /////////////////////////////////////////////////////
+
+    @Override
+    public void execute() throws ResourceUnavailableException, InsufficientCapacityException, ServerApiException, ConcurrentOperationException, ResourceAllocationException,
+            NetworkRuleConflictException {
+        ListResponse<BackupCompressionJobResponse> response = _queryService.listBackupCompressionJobs(this);
+        response.setResponseName(getCommandName());
+        this.setResponseObject(response);
+    }
+}