diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java index 030d9747d6cd..39a346db2dc3 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java @@ -58,6 +58,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.cloud.utils.script.OutputInterpreter; import org.apache.cloudstack.agent.directdownload.DirectDownloadAnswer; import org.apache.cloudstack.agent.directdownload.DirectDownloadCommand; import org.apache.cloudstack.direct.download.DirectDownloadHelper; @@ -1115,7 +1116,14 @@ public Answer backupSnapshot(final CopyCommand cmd) { } } else { final Script command = new Script(_manageSnapshotPath, cmd.getWaitInMillSeconds(), logger); - command.add("-b", isCreatedFromVmSnapshot ? snapshotDisk.getPath() : snapshot.getPath()); + String backupPath; + if (primaryPool.getType() == StoragePoolType.CLVM) { + backupPath = snapshotDisk.getPath(); + logger.debug("Using snapshotDisk path for CLVM backup: " + backupPath); + } else { + backupPath = isCreatedFromVmSnapshot ? snapshotDisk.getPath() : snapshot.getPath(); + } + command.add("-b", backupPath); command.add(NAME_OPTION, snapshotName); command.add("-p", snapshotDestPath); @@ -1172,7 +1180,11 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject if ((backupSnapshotAfterTakingSnapshot == null || BooleanUtils.toBoolean(backupSnapshotAfterTakingSnapshot)) && deleteSnapshotOnPrimary) { try { - Files.deleteIfExists(Paths.get(snapshotPath)); + if (primaryPool.getType() == StoragePoolType.CLVM) { + deleteClvmSnapshot(snapshotPath); + } else { + Files.deleteIfExists(Paths.get(snapshotPath)); + } } catch (IOException ex) { logger.error("Failed to delete snapshot [{}] on primary storage [{}].", snapshot.getId(), snapshot.getName(), ex); } @@ -1181,6 +1193,324 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject } } + /** + * Delete a CLVM snapshot using comprehensive cleanup. + * For CLVM, the snapshot path stored in DB is: /dev/vgname/volumeuuid/snapshotuuid + * This method handles: + * 1. Checking if snapshot artifacts still exist + * 2. Device-mapper snapshot entry removal + * 3. COW volume removal + * 4. -real device restoration if this is the last snapshot + * + * @param snapshotPath The snapshot path from database + * @param checkExistence If true, checks if snapshot exists before cleanup (for explicit deletion) + * If false, always performs cleanup (for post-backup cleanup) + * @return true if cleanup was performed, false if snapshot didn't exist (when checkExistence=true) + */ + private boolean deleteClvmSnapshot(String snapshotPath, boolean checkExistence) { + logger.info("Starting CLVM snapshot deletion for path: {}, checkExistence: {}", snapshotPath, checkExistence); + + try { + // Parse the snapshot path: /dev/acsvg/volume-uuid/snapshot-uuid + String[] pathParts = snapshotPath.split("/"); + if (pathParts.length < 5) { + logger.warn("Invalid CLVM snapshot path format: {}, expected format: /dev/vgname/volume-uuid/snapshot-uuid", snapshotPath); + return false; + } + + String vgName = pathParts[2]; + String volumeUuid = pathParts[3]; + String snapshotUuid = pathParts[4]; + + logger.info("Parsed snapshot path - VG: {}, Volume: {}, Snapshot: {}", vgName, volumeUuid, snapshotUuid); + + // Compute MD5 hash of snapshot UUID (same as managesnapshot.sh does) + String md5Hash = computeMd5Hash(snapshotUuid); + logger.debug("Computed MD5 hash for snapshot UUID {}: {}", snapshotUuid, md5Hash); + + // Check if snapshot exists (if requested) + if (checkExistence) { + String cowLvPath = "/dev/" + vgName + "/" + md5Hash + "-cow"; + Script checkCow = new Script("/usr/sbin/lvs", 5000, logger); + checkCow.add("--noheadings"); + checkCow.add(cowLvPath); + String checkResult = checkCow.execute(); + + if (checkResult != null) { + // COW volume doesn't exist - snapshot was already cleaned up + logger.info("CLVM snapshot {} was already deleted, no cleanup needed", snapshotUuid); + return false; + } + logger.info("CLVM snapshot artifacts still exist for {}, performing cleanup", snapshotUuid); + } + + // Check if this is the last snapshot for the volume + boolean isLastSnapshot = isLastSnapshotForVolume(vgName, volumeUuid); + logger.info("Is last snapshot for volume {}: {}", volumeUuid, isLastSnapshot); + + // Perform clean-up + cleanupClvmSnapshotArtifacts(vgName, volumeUuid, md5Hash, isLastSnapshot); + + logger.info("Successfully deleted CLVM snapshot: {}", snapshotPath); + return true; + + } catch (Exception ex) { + logger.error("Exception while deleting CLVM snapshot {}", snapshotPath, ex); + return false; + } + } + + /** + * Delete a CLVM snapshot after backup (always performs cleanup without checking existence). + * Convenience method for backward compatibility. + */ + private void deleteClvmSnapshot(String snapshotPath) { + deleteClvmSnapshot(snapshotPath, false); + } + + /** + * Check if this is the last snapshot for a given volume in the VG. + * + * @param vgName The volume group name + * @param volumeUuid The origin volume UUID + * @return true if this is the last (or only) snapshot for the volume + */ + private boolean isLastSnapshotForVolume(String vgName, String volumeUuid) { + try { + Script listSnapshots = new Script("/usr/sbin/lvs", 10000, logger); + listSnapshots.add("--noheadings"); + listSnapshots.add("-o"); + listSnapshots.add("lv_name,origin"); + listSnapshots.add(vgName); + + logger.debug("Checking snapshot count for volume {} in VG {}", volumeUuid, vgName); + + final OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = listSnapshots.execute(parser); + + if (result == null) { + String output = parser.getLines(); + if (output != null && !output.isEmpty()) { + int snapshotCount = 0; + String[] lines = output.split("\n"); + String escapedUuid = volumeUuid.replace("-", "--"); + + for (String line : lines) { + String trimmedLine = line.trim(); + if (!trimmedLine.isEmpty()) { + String[] parts = trimmedLine.split("\\s+"); + if (parts.length >= 2) { + String origin = parts[1]; + if (origin.equals(volumeUuid) || origin.equals(escapedUuid)) { + snapshotCount++; + } + } + } + } + + logger.debug("Found {} snapshot(s) for volume {}", snapshotCount, volumeUuid); + return snapshotCount <= 1; + } + } + logger.debug("Could not determine snapshot count, assuming not last snapshot"); + return false; + + } catch (Exception e) { + logger.warn("Exception while checking if last snapshot: {}", e.getMessage()); + return false; + } + } + + /** + * Clean up CLVM snapshot artifacts including device-mapper entries, COW volumes, + * and potentially restore the -real device if this is the last snapshot. + * + * @param vgName The volume group name + * @param originVolumeUuid The UUID of the origin volume + * @param snapshotMd5Hash The MD5 hash of the snapshot UUID + * @param isLastSnapshot Whether this is the last snapshot of the origin volume + */ + private void cleanupClvmSnapshotArtifacts(String vgName, String originVolumeUuid, String snapshotMd5Hash, boolean isLastSnapshot) { + logger.info("Cleaning up CLVM snapshot artifacts: VG={}, Origin={}, SnapshotHash={}, IsLastSnapshot={}", + vgName, originVolumeUuid, snapshotMd5Hash, isLastSnapshot); + + try { + String vgNameEscaped = vgName.replace("-", "--"); + String originEscaped = originVolumeUuid.replace("-", "--"); + + String snapshotDevice = vgNameEscaped + "-" + snapshotMd5Hash; + + removeSnapshotDeviceMapperEntry(snapshotDevice); + + removeCowVolume(vgName, snapshotMd5Hash); + + if (isLastSnapshot) { + logger.info("Step 3: This is the last snapshot, restoring origin volume {} from snapshot-origin state", originVolumeUuid); + restoreOriginVolumeFromSnapshotState(vgName, originVolumeUuid, vgNameEscaped, originEscaped); + } else { + logger.info("Step 3: Skipped - other snapshots still exist for volume {}", originVolumeUuid); + } + + logger.info("Successfully cleaned up CLVM snapshot artifacts"); + + } catch (Exception ex) {kvmstoragep + logger.error("Exception during CLVM snapshot artifact cleanup: {}", ex.getMessage(), ex); + } + } + + private void removeSnapshotDeviceMapperEntry(String snapshotDevice) { + logger.info("Step 1: Removing snapshot device-mapper entry: {}", snapshotDevice); + + Script dmRemoveSnapshot = new Script("/usr/sbin/dmsetup", 10000, logger); + dmRemoveSnapshot.add("remove"); + dmRemoveSnapshot.add(snapshotDevice); + + logger.debug("Executing: dmsetup remove {}", snapshotDevice); + String dmResult = dmRemoveSnapshot.execute(); + if (dmResult == null) { + logger.info("Successfully removed device-mapper entry: {}", snapshotDevice); + } else { + logger.debug("dmsetup remove returned: {} (may already be removed)", dmResult); + } + } + + private void removeCowVolume(String vgName, String snapshotMd5Hash) { + String cowLvName = snapshotMd5Hash + "-cow"; + String cowLvPath = "/dev/" + vgName + "/" + cowLvName; + logger.info("Step 2: Removing COW volume: {}", cowLvPath); + + Script removeCow = new Script("/usr/sbin/lvremove", 10000, logger); + removeCow.add("-f"); + removeCow.add(cowLvPath); + + logger.debug("Executing: lvremove -f {}", cowLvPath); + String cowResult = removeCow.execute(); + if (cowResult == null) { + logger.info("Successfully removed COW volume: {}", cowLvPath); + } else { + logger.warn("Failed to remove COW volume {}: {}", cowLvPath, cowResult); + } + } + + /** + * Restore an origin volume from snapshot-origin state back to normal state. + * This removes the -real device and reconfigures the volume device-mapper entry. + * Should only be called when deleting the last snapshot of a volume. + * + * @param vgName The volume group name + * @param volumeUuid The volume UUID + * @param vgNameEscaped The VG name with hyphens doubled for device-mapper + * @param volumeEscaped The volume UUID with hyphens doubled for device-mapper + */ + private void restoreOriginVolumeFromSnapshotState(String vgName, String volumeUuid, String vgNameEscaped, String volumeEscaped) { + try { + String originDevice = vgNameEscaped + "-" + volumeEscaped; + String realDevice = originDevice + "-real"; + + logger.info("Restoring volume {} from snapshot-origin state", volumeUuid); + + // Check if -real device exists + Script checkReal = new Script("/usr/sbin/dmsetup", 5000, logger); + checkReal.add("info"); + checkReal.add(realDevice); + + logger.debug("Checking if -real device exists: dmsetup info {}", realDevice); + String checkResult = checkReal.execute(); + if (checkResult != null) { + logger.debug("No -real device found for {}, volume may already be in normal state", volumeUuid); + return; + } + + logger.info("Found -real device, proceeding with restoration"); + + suspendOriginDevice(originDevice); + + logger.debug("Getting device-mapper table from -real device"); + Script getTable = new Script("/usr/sbin/dmsetup", 5000, logger); + getTable.add("table"); + getTable.add(realDevice); + + OutputInterpreter.AllLinesParser tableParser = new OutputInterpreter.AllLinesParser(); + String tableResult = getTable.execute(tableParser); + String realTable = tableParser.getLines(); + + resumeAndRemoveRealDevice(originDevice, realDevice, tableResult, realTable, volumeUuid); + + } catch (Exception ex) { + logger.error("Exception during volume restoration from snapshot-origin state: {}", ex.getMessage(), ex); + } + } + + private void suspendOriginDevice(String originDevice) { + logger.debug("Suspending origin device: {}", originDevice); + Script suspendOrigin = new Script("/usr/sbin/dmsetup", 5000, logger); + suspendOrigin.add("suspend"); + suspendOrigin.add(originDevice); + String suspendResult = suspendOrigin.execute(); + if (suspendResult != null) { + logger.warn("Failed to suspend origin device {}: {}", originDevice, suspendResult); + } + } + + private void resumeAndRemoveRealDevice(String originDevice, String realDevice, String tableResult, String realTable, String volumeUuid) { + if (tableResult == null && realTable != null && !realTable.isEmpty()) { + logger.debug("Restoring original table to origin device: {}", realTable); + + Script loadTable = new Script("/bin/bash", 10000, logger); + loadTable.add("-c"); + loadTable.add("echo '" + realTable + "' | /usr/sbin/dmsetup load " + originDevice); + + String loadResult = loadTable.execute(); + if (loadResult != null) { + logger.warn("Failed to load table to origin device: {}", loadResult); + } + + logger.debug("Resuming origin device"); + Script resumeOrigin = new Script("/usr/sbin/dmsetup", 5000, logger); + resumeOrigin.add("resume"); + resumeOrigin.add(originDevice); + String resumeResult = resumeOrigin.execute(); + if (resumeResult != null) { + logger.warn("Failed to resume origin device: {}", resumeResult); + } + + logger.debug("Removing -real device"); + Script removeReal = new Script("/usr/sbin/dmsetup", 5000, logger); + removeReal.add("remove"); + removeReal.add(realDevice); + String removeResult = removeReal.execute(); + if (removeResult == null) { + logger.info("Successfully removed -real device and restored origin volume {}", volumeUuid); + } else { + logger.warn("Failed to remove -real device: {}", removeResult); + } + } else { + logger.warn("Failed to get table from -real device, aborting restoration"); + Script resumeOrigin = new Script("/usr/sbin/dmsetup", 5000, logger); + resumeOrigin.add("resume"); + resumeOrigin.add(originDevice); + resumeOrigin.execute(); + } + } + /** + * Compute MD5 hash of a string, matching what managesnapshot.sh does: + * echo "${snapshot}" | md5sum -t | awk '{ print $1 }' + */ + private String computeMd5Hash(String input) { + try { + java.security.MessageDigest md = java.security.MessageDigest.getInstance("MD5"); + byte[] array = md.digest((input + "\n").getBytes("UTF-8")); + StringBuilder sb = new StringBuilder(); + for (byte b : array) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (Exception e) { + logger.error("Failed to compute MD5 hash for: {}", input, e); + return input; + } + } + protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map params, DataStoreTO store) throws LibvirtException, InternalErrorException { DiskDef iso = new DiskDef(); @@ -1842,8 +2172,14 @@ public Answer createSnapshot(final CreateObjectCommand cmd) { } } - if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state) && volume.requiresEncryption()) { - throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported"); + if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state)) { + if (volume.requiresEncryption()) { + throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported"); + } + + if (StoragePoolType.CLVM == primaryStore.getPoolType()) { + throw new CloudRuntimeException("VM is running, live snapshots aren't supported with CLVM primary storage"); + } } KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid()); @@ -2880,6 +3216,25 @@ public Answer deleteSnapshot(final DeleteCommand cmd) { if (snapshotTO.isKvmIncrementalSnapshot()) { deleteCheckpoint(snapshotTO); } + } else if (primaryPool.getType() == StoragePoolType.CLVM) { + // For CLVM, snapshots are typically already deleted from primary storage during backup + // via deleteSnapshotOnPrimary in the backupSnapshot finally block. + // This is called when the user explicitly deletes the snapshot via UI/API. + // We check if the snapshot still exists and clean it up if needed. + logger.info("Processing CLVM snapshot deletion (id={}, name={}, path={}) on primary storage", + snapshotTO.getId(), snapshotTO.getName(), snapshotTO.getPath()); + + String snapshotPath = snapshotTO.getPath(); + if (snapshotPath != null && !snapshotPath.isEmpty()) { + boolean wasDeleted = deleteClvmSnapshot(snapshotPath, true); + if (wasDeleted) { + logger.info("Successfully cleaned up CLVM snapshot {} from primary storage", snapshotName); + } else { + logger.info("CLVM snapshot {} was already deleted from primary storage during backup, no cleanup needed", snapshotName); + } + } else { + logger.debug("CLVM snapshot path is null or empty, assuming already cleaned up"); + } } else { logger.warn("Operation not implemented for storage pool type of " + primaryPool.getType().toString()); throw new InternalErrorException("Operation not implemented for storage pool type of " + primaryPool.getType().toString()); diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java index a03daeb197bf..f0658fee62af 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java @@ -34,6 +34,7 @@ import com.cloud.agent.properties.AgentProperties; import com.cloud.agent.properties.AgentPropertiesFileHandler; +import com.cloud.utils.script.OutputInterpreter; import org.apache.cloudstack.api.ApiConstants; import org.apache.cloudstack.utils.cryptsetup.KeyFile; import org.apache.cloudstack.utils.qemu.QemuImageOptions; @@ -254,9 +255,12 @@ public StorageVol getVolume(StoragePool pool, String volName) { try { vol = pool.storageVolLookupByName(volName); - logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool"); + if (vol != null) { + logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool"); + } } catch (LibvirtException e) { - throw new CloudRuntimeException("Could not find volume " + volName + ": " + e.getMessage()); + logger.debug("Volume " + volName + " still not found after pool refresh: " + e.getMessage()); + return null; } } @@ -663,6 +667,17 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { try { StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid); + + // Check if volume was found - if null, treat as not found and trigger fallback for CLVM + if (vol == null) { + logger.debug("Volume " + volumeUuid + " not found in libvirt, will check for CLVM fallback"); + if (pool.getType() == StoragePoolType.CLVM) { + return getPhysicalDisk(volumeUuid, pool, libvirtPool); + } + + throw new CloudRuntimeException("Volume " + volumeUuid + " not found in libvirt pool"); + } + KVMPhysicalDisk disk; LibvirtStorageVolumeDef voldef = getStorageVolumeDef(libvirtPool.getPool().getConnect(), vol); disk = new KVMPhysicalDisk(vol.getPath(), vol.getName(), pool); @@ -693,11 +708,156 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) { } return disk; } catch (LibvirtException e) { - logger.debug("Failed to get physical disk:", e); + logger.debug("Failed to get volume from libvirt: " + e.getMessage()); + // For CLVM, try direct block device access as fallback + if (pool.getType() == StoragePoolType.CLVM) { + return getPhysicalDisk(volumeUuid, pool, libvirtPool); + } + throw new CloudRuntimeException(e.toString()); } } + private KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool, LibvirtStoragePool libvirtPool) { + logger.info("CLVM volume not visible to libvirt, attempting direct block device access for volume: {}", volumeUuid); + + try { + logger.debug("Refreshing libvirt storage pool: {}", pool.getUuid()); + libvirtPool.getPool().refresh(0); + + StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid); + if (vol != null) { + logger.info("Volume found after pool refresh: {}", volumeUuid); + KVMPhysicalDisk disk; + LibvirtStorageVolumeDef voldef = getStorageVolumeDef(libvirtPool.getPool().getConnect(), vol); + disk = new KVMPhysicalDisk(vol.getPath(), vol.getName(), pool); + disk.setSize(vol.getInfo().allocation); + disk.setVirtualSize(vol.getInfo().capacity); + disk.setFormat(voldef.getFormat() == LibvirtStorageVolumeDef.VolumeFormat.QCOW2 ? + PhysicalDiskFormat.QCOW2 : PhysicalDiskFormat.RAW); + return disk; + } + } catch (LibvirtException refreshEx) { + logger.debug("Pool refresh failed or volume still not found: {}", refreshEx.getMessage()); + } + + // Still not found after refresh, try direct block device access + return getPhysicalDiskViaDirectBlockDevice(volumeUuid, pool); + } + + private String getVgName(KVMStoragePool pool, String sourceDir) { + String vgName = sourceDir; + if (vgName.startsWith("/")) { + String[] parts = vgName.split("/"); + List tokens = Arrays.stream(parts) + .filter(s -> !s.isEmpty()).collect(Collectors.toList()); + + vgName = tokens.size() > 1 ? tokens.get(1) + : tokens.size() == 1 ? tokens.get(0) + : ""; + } + return vgName; + } + + /** + * For CLVM volumes that exist in LVM but are not visible to libvirt, + * access them directly via block device path. + */ + private KVMPhysicalDisk getPhysicalDiskViaDirectBlockDevice(String volumeUuid, KVMStoragePool pool) { + try { + // For CLVM, pool sourceDir contains the VG path (e.g., "/dev/acsvg") + // Extract the VG name + String sourceDir = pool.getLocalPath(); + if (sourceDir == null || sourceDir.isEmpty()) { + throw new CloudRuntimeException("CLVM pool sourceDir is not set, cannot determine VG name"); + } + String vgName = getVgName(pool, sourceDir); + logger.debug("Using VG name: {} (from sourceDir: {}) ", vgName, sourceDir); + + // Check if the LV exists in LVM using lvs command + logger.debug("Checking if volume {} exsits in VG {}", volumeUuid, vgName); + Script checkLvCmd = new Script("/usr/sbin/lvs", 5000, logger); + checkLvCmd.add("--noheadings"); + checkLvCmd.add("--unbuffered"); + checkLvCmd.add(vgName + "/" + volumeUuid); + + String checkResult = checkLvCmd.execute(); + if (checkResult != null) { + logger.debug("Volume {} does not exist in VG {}: {}", volumeUuid, vgName, checkResult); + throw new CloudRuntimeException(String.format("Storage volume not found: no storage vol with matching name '%s'", volumeUuid)); + } + + logger.info("Volume {} exists in LVM but not visible to libvirt, accessing directly", volumeUuid); + + // Try standard device path first + String lvPath = "/dev/" + vgName + "/" + volumeUuid; + File lvDevice = new File(lvPath); + + if (!lvDevice.exists()) { + // Try device-mapper path with escaped hyphens + String vgNameEscaped = vgName.replace("-", "--"); + String volumeUuidEscaped = volumeUuid.replace("-", "--"); + lvPath = "/dev/mapper/" + vgNameEscaped + "-" + volumeUuidEscaped; + lvDevice = new File(lvPath); + + if (!lvDevice.exists()) { + logger.warn("Volume exists in LVM but device node not found: {}", volumeUuid); + throw new CloudRuntimeException(String.format("Could not find volume %s " + + "in VG %s - volume exists in LVM but device node not accessible", volumeUuid, vgName)); + } + } + + long size = 0; + try { + Script lvsCmd = new Script("/usr/sbin/lvs", 5000, logger); + lvsCmd.add("--noheadings"); + lvsCmd.add("--units"); + lvsCmd.add("b"); + lvsCmd.add("-o"); + lvsCmd.add("lv_size"); + lvsCmd.add(lvPath); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String result = lvsCmd.execute(parser); + + String output = null; + if (result == null) { + output = parser.getLines(); + } else { + output = result; + } + + if (output != null && !output.isEmpty()) { + String sizeStr = output.trim().replaceAll("[^0-9]", ""); + if (!sizeStr.isEmpty()) { + size = Long.parseLong(sizeStr); + } + } + } catch (Exception sizeEx) { + logger.warn("Failed to get size for CLVM volume via lvs: {}", sizeEx.getMessage()); + if (lvDevice.isFile()) { + size = lvDevice.length(); + } + } + + KVMPhysicalDisk disk = new KVMPhysicalDisk(lvPath, volumeUuid, pool); + disk.setFormat(PhysicalDiskFormat.RAW); + disk.setSize(size); + disk.setVirtualSize(size); + + logger.info("Successfully accessed CLVM volume via direct block device: {} " + + "with size: {} bytes",lvPath, size); + + return disk; + + } catch (CloudRuntimeException ex) { + throw ex; + } catch (Exception ex) { + logger.error("Failed to access CLVM volume via direct block device: {}",volumeUuid, ex); + throw new CloudRuntimeException(String.format("Could not find volume %s: %s ",volumeUuid, ex.getMessage())); + } + } + /** * adjust refcount */ @@ -1227,7 +1387,16 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool; try { StorageVol vol = getVolume(libvirtPool.getPool(), uuid); - logger.debug("Instructing libvirt to remove volume " + uuid + " from pool " + pool.getUuid()); + if (vol == null) { + logger.warn("Volume {} not found in libvirt pool {}, it may have been already deleted", uuid, pool.getUuid()); + + // For CLVM, attempt direct LVM cleanup in case the volume exists but libvirt can't see it + if (pool.getType() == StoragePoolType.CLVM) { + return cleanupCLVMVolume(uuid, pool); + } + return true; + } + logger.debug("Instructing libvirt to remove volume {} from pool {}", uuid, pool.getUuid()); if(Storage.ImageFormat.DIR.equals(format)){ deleteDirVol(libvirtPool, vol); } else { @@ -1236,10 +1405,83 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag vol.free(); return true; } catch (LibvirtException e) { + // For CLVM, if libvirt fails, try direct LVM cleanup + if (pool.getType() == StoragePoolType.CLVM) { + logger.warn("Libvirt failed to delete CLVM volume {}, attempting direct LVM cleanup: {}", uuid, e.getMessage()); + return cleanupCLVMVolume(uuid, pool); + } throw new CloudRuntimeException(e.toString()); } } + /** + * Clean up CLVM volume and its snapshots directly using LVM commands. + * This is used as a fallback when libvirt cannot find or delete the volume. + */ + private boolean cleanupCLVMVolume(String uuid, KVMStoragePool pool) { + logger.info("Starting direct LVM cleanup for CLVM volume: {} in pool: {}", uuid, pool.getUuid()); + + try { + String sourceDir = pool.getLocalPath(); + if (sourceDir == null || sourceDir.isEmpty()) { + logger.debug("Source directory is null or empty, cannot determine VG name for CLVM pool {}, skipping direct cleanup", pool.getUuid()); + return true; + } + String vgName = getVgName(pool, sourceDir); + logger.info("Determined VG name: {} for pool: {}", vgName, pool.getUuid()); + + if (vgName == null || vgName.isEmpty()) { + logger.warn("Cannot determine VG name for CLVM pool {}, skipping direct cleanup", pool.getUuid()); + return true; + } + + String lvPath = "/dev/" + vgName + "/" + uuid; + logger.debug("Volume path: {}", lvPath); + + // Check if the LV exists + Script checkLvs = new Script("lvs", 5000, logger); + checkLvs.add("--noheadings"); + checkLvs.add("--unbuffered"); + checkLvs.add(lvPath); + + logger.info("Checking if volume exists: lvs --noheadings --unbuffered {}", lvPath); + String checkResult = checkLvs.execute(); + + if (checkResult != null) { + logger.info("CLVM volume {} does not exist in LVM (check returned: {}), considering it as already deleted", uuid, checkResult); + return true; + } + + logger.info("Volume {} exists, proceeding with cleanup", uuid); + + logger.info("Step 1: Zero-filling volume {} for security", uuid); + secureZeroFillVolume(lvPath, uuid); + + logger.info("Step 2: Removing volume {}", uuid); + Script removeLv = new Script("lvremove", 10000, logger); + removeLv.add("-f"); + removeLv.add(lvPath); + + logger.info("Executing command: lvremove -f {}", lvPath); + String removeResult = removeLv.execute(); + + if (removeResult == null) { + logger.info("Successfully removed CLVM volume {} using direct LVM cleanup", uuid); + return true; + } else { + logger.warn("Command 'lvremove -f {}' returned error: {}", lvPath, removeResult); + if (removeResult.contains("not found") || removeResult.contains("Failed to find")) { + logger.info("CLVM volume {} not found during cleanup, considering it as already deleted", uuid); + return true; + } + return false; + } + } catch (Exception ex) { + logger.error("Exception during CLVM volume cleanup for {}: {}", uuid, ex.getMessage(), ex); + return true; + } + } + /** * This function copies a physical disk from Secondary Storage to Primary Storage * or from Primary to Primary Storage @@ -1737,6 +1979,80 @@ private void deleteVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtEx vol.delete(0); } + /** + * Securely zero-fill a volume before deletion to prevent data leakage. + * Uses blkdiscard (fast TRIM) as primary method, with dd zero-fill as fallback. + * + * @param lvPath The full path to the logical volume (e.g., /dev/vgname/lvname) + * @param volumeUuid The UUID of the volume for logging purposes + */ + private void secureZeroFillVolume(String lvPath, String volumeUuid) { + logger.info("Starting secure zero-fill for CLVM volume: {} at path: {}", volumeUuid, lvPath); + + boolean blkdiscardSuccess = false; + + // Try blkdiscard first (fast - sends TRIM commands) + try { + Script blkdiscard = new Script("blkdiscard", 300000, logger); // 5 minute timeout + blkdiscard.add("-f"); // Force flag to suppress confirmation prompts + blkdiscard.add(lvPath); + + String result = blkdiscard.execute(); + if (result == null) { + logger.info("Successfully zero-filled CLVM volume {} using blkdiscard (TRIM)", volumeUuid); + blkdiscardSuccess = true; + } else { + // Check if the error is "Operation not supported" - common with thick LVM without TRIM support + if (result.contains("Operation not supported") || result.contains("BLKDISCARD ioctl failed")) { + logger.info("blkdiscard not supported for volume {} (device doesn't support TRIM/DISCARD), using dd fallback", volumeUuid); + } else { + logger.warn("blkdiscard failed for volume {}: {}, will try dd fallback", volumeUuid, result); + } + } + } catch (Exception e) { + logger.warn("Exception during blkdiscard for volume {}: {}, will try dd fallback", volumeUuid, e.getMessage()); + } + + // Fallback to dd zero-fill (slow but thorough) + if (!blkdiscardSuccess) { + logger.info("Attempting zero-fill using dd for CLVM volume: {}", volumeUuid); + try { + // Use bash to chain commands with proper error handling + // nice -n 19: lowest CPU priority + // ionice -c 2 -n 7: best-effort I/O scheduling with lowest priority + // oflag=direct: bypass cache for more predictable performance + // || true at the end ensures the command doesn't fail even if dd returns error (which it does when disk is full - expected) + String command = String.format( + "nice -n 19 ionice -c 2 -n 7 dd if=/dev/zero of=%s bs=1M oflag=direct 2>&1 || true", + lvPath + ); + + Script ddZeroFill = new Script("/bin/bash", 3600000, logger); // 60 minute timeout for large volumes + ddZeroFill.add("-c"); + ddZeroFill.add(command); + + OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser(); + String ddResult = ddZeroFill.execute(parser); + String output = parser.getLines(); + + // dd writes to stderr even on success, check for completion indicators + if (output != null && (output.contains("copied") || output.contains("records in") || + output.contains("No space left on device"))) { + logger.info("Successfully zero-filled CLVM volume {} using dd", volumeUuid); + } else if (ddResult == null) { + logger.info("Zero-fill completed for CLVM volume {}", volumeUuid); + } else { + logger.warn("dd zero-fill for volume {} completed with output: {}", volumeUuid, + output != null ? output : ddResult); + } + } catch (Exception e) { + // Log warning but don't fail the deletion - zero-fill is a best-effort security measure + logger.warn("Failed to zero-fill CLVM volume {} before deletion: {}. Proceeding with deletion anyway.", + volumeUuid, e.getMessage()); + } + } + } + private void deleteDirVol(LibvirtStoragePool pool, StorageVol vol) throws LibvirtException { Script.runSimpleBashScript("rm -r --interactive=never " + vol.getPath()); } diff --git a/scripts/storage/qcow2/managesnapshot.sh b/scripts/storage/qcow2/managesnapshot.sh index 3650bdd9b6f6..46e194cd3a85 100755 --- a/scripts/storage/qcow2/managesnapshot.sh +++ b/scripts/storage/qcow2/managesnapshot.sh @@ -212,12 +212,12 @@ backup_snapshot() { return 1 fi - qemuimg_ret=$($qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}") + qemuimg_ret=$($qemu_img convert $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}" 2>&1) ret_code=$? - if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]] + if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]]) then forceShareFlag="" - $qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}" + $qemu_img convert $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}" ret_code=$? fi if [ $ret_code -gt 0 ] @@ -240,9 +240,9 @@ backup_snapshot() { # Backup VM snapshot qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk 2>&1) ret_code=$? - if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]]; then + if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]]); then forceShareFlag="" - qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk) + qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk 2>&1) ret_code=$? fi @@ -251,11 +251,11 @@ backup_snapshot() { return 1 fi - qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null) + qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1) ret_code=$? - if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"convert: invalid option -- 'U'"* ]]; then + if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]]); then forceShareFlag="" - qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null) + qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1) ret_code=$? fi