Skip to content

CLVM enhancements and fixes #12617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
Pearl1594 wants to merge 1 commit into
base: main
Choose a base branch
from
+269 −16
Draft

CLVM enhancements and fixes #12617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 96 additions & 4 deletions ...s/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/KVMStorageProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1115,7 +1115,14 @@ public Answer backupSnapshot(final CopyCommand cmd) {
}
} else {
final Script command = new Script(_manageSnapshotPath, cmd.getWaitInMillSeconds(), logger);
command.add("-b", isCreatedFromVmSnapshot ? snapshotDisk.getPath() : snapshot.getPath());
String backupPath;
if (primaryPool.getType() == StoragePoolType.CLVM) {
backupPath = snapshotDisk.getPath();
logger.debug("Using snapshotDisk path for CLVM backup: " + backupPath);
} else {
backupPath = isCreatedFromVmSnapshot ? snapshotDisk.getPath() : snapshot.getPath();
}
command.add("-b", backupPath);
command.add(NAME_OPTION, snapshotName);
command.add("-p", snapshotDestPath);

Expand Down Expand Up @@ -1172,7 +1179,11 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject

if ((backupSnapshotAfterTakingSnapshot == null || BooleanUtils.toBoolean(backupSnapshotAfterTakingSnapshot)) && deleteSnapshotOnPrimary) {
try {
Files.deleteIfExists(Paths.get(snapshotPath));
if (primaryPool.getType() == StoragePoolType.CLVM) {
deleteClvmSnapshot(snapshotPath);
} else {
Files.deleteIfExists(Paths.get(snapshotPath));
}
} catch (IOException ex) {
logger.error("Failed to delete snapshot [{}] on primary storage [{}].", snapshot.getId(), snapshot.getName(), ex);
}
Expand All @@ -1181,6 +1192,81 @@ private void deleteSnapshotOnPrimary(final CopyCommand cmd, final SnapshotObject
}
}

/**
* Delete a CLVM snapshot using lvremove command.
* For CLVM, the snapshot path stored in DB is: /dev/vgname/volumeuuid/snapshotuuid
* However, managesnapshot.sh creates the actual snapshot using MD5 hash of the snapshot UUID.
* The actual device is at: /dev/mapper/vgname-MD5(snapshotuuid)
* We need to compute the MD5 hash and remove both the snapshot LV and its COW volume.
*/
private void deleteClvmSnapshot(String snapshotPath) {
try {
// Parse the snapshot path: /dev/acsvg/volume-uuid/snapshot-uuid
// Extract VG name and snapshot UUID
String[] pathParts = snapshotPath.split("/");
if (pathParts.length < 5) {
logger.warn("Invalid CLVM snapshot path format: " + snapshotPath + ", skipping deletion");
return;
}

String vgName = pathParts[2];
String snapshotUuid = pathParts[4];

// Compute MD5 hash of snapshot UUID (same as managesnapshot.sh does)
String md5Hash = computeMd5Hash(snapshotUuid);

logger.debug("Deleting CLVM snapshot for UUID: " + snapshotUuid + " (MD5: " + md5Hash + ")");

// Remove the snapshot device mapper entry
// The snapshot device is at: /dev/mapper/vgname-md5hash
String vgNameEscaped = vgName.replace("-", "--");
String snapshotDevice = vgNameEscaped + "-" + md5Hash;

Script dmRemoveCmd = new Script("/usr/sbin/dmsetup", 30000, logger);
dmRemoveCmd.add("remove");
dmRemoveCmd.add(snapshotDevice);
String dmResult = dmRemoveCmd.execute();
if (dmResult != null) {
logger.debug("dmsetup remove returned: {} (may already be removed)", dmResult);
}

// Remove the COW (copy-on-write) volume: /dev/vgname/md5hash-cow
String cowLvPath = "/dev/" + vgName + "/" + md5Hash + "-cow";
Script removeCowCmd = new Script("/usr/sbin/lvremove", 30000, logger);
removeCowCmd.add("-f");
removeCowCmd.add(cowLvPath);

String cowResult = removeCowCmd.execute();
if (cowResult != null) {
logger.warn("Failed to remove CLVM COW volume {} : {}",cowLvPath, cowResult);
} else {
logger.debug("Successfully deleted CLVM snapshot COW volume: {}", cowLvPath);
}

} catch (Exception ex) {
logger.error("Exception while deleting CLVM snapshot {}", snapshotPath, ex);
}
}

/**
* Compute MD5 hash of a string, matching what managesnapshot.sh does:
* echo "${snapshot}" | md5sum -t | awk '{ print $1 }'
*/
private String computeMd5Hash(String input) {
try {
java.security.MessageDigest md = java.security.MessageDigest.getInstance("MD5");
byte[] array = md.digest((input + "\n").getBytes("UTF-8"));
StringBuilder sb = new StringBuilder();
for (byte b : array) {
sb.append(String.format("%02x", b));
}
return sb.toString();
} catch (Exception e) {
logger.error("Failed to compute MD5 hash for: {}", input, e);
return input;
}
}

protected synchronized void attachOrDetachISO(final Connect conn, final String vmName, String isoPath, final boolean isAttach, Map<String, String> params, DataStoreTO store) throws
LibvirtException, InternalErrorException {
DiskDef iso = new DiskDef();
Expand Down Expand Up @@ -1842,8 +1928,14 @@ public Answer createSnapshot(final CreateObjectCommand cmd) {
}
}

if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state) && volume.requiresEncryption()) {
throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported");
if (DomainInfo.DomainState.VIR_DOMAIN_RUNNING.equals(state)) {
if (volume.requiresEncryption()) {
throw new CloudRuntimeException("VM is running, encrypted volume snapshots aren't supported");
}

if (StoragePoolType.CLVM.name().equals(primaryStore.getType())) {
throw new CloudRuntimeException("VM is running, live snapshots aren't supported with CLVM primary storage");
}
}

KVMStoragePool primaryPool = storagePoolMgr.getStoragePool(primaryStore.getPoolType(), primaryStore.getUuid());
Expand Down
169 changes: 165 additions & 4 deletions ...hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStorageAdaptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import com.cloud.agent.properties.AgentProperties;
import com.cloud.agent.properties.AgentPropertiesFileHandler;
import com.cloud.utils.script.OutputInterpreter;
import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.utils.cryptsetup.KeyFile;
import org.apache.cloudstack.utils.qemu.QemuImageOptions;
Expand Down Expand Up @@ -254,9 +255,12 @@ public StorageVol getVolume(StoragePool pool, String volName) {

try {
vol = pool.storageVolLookupByName(volName);
logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool");
if (vol != null) {
logger.debug("Found volume " + volName + " in storage pool " + pool.getName() + " after refreshing the pool");
}
} catch (LibvirtException e) {
throw new CloudRuntimeException("Could not find volume " + volName + ": " + e.getMessage());
logger.debug("Volume " + volName + " still not found after pool refresh: " + e.getMessage());
return null;
}
}

Expand Down Expand Up @@ -663,6 +667,17 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) {

try {
StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid);

// Check if volume was found - if null, treat as not found and trigger fallback for CLVM
if (vol == null) {
logger.debug("Volume " + volumeUuid + " not found in libvirt, will check for CLVM fallback");
if (pool.getType() == StoragePoolType.CLVM) {
return getPhysicalDisk(volumeUuid, pool, libvirtPool);
}

throw new CloudRuntimeException("Volume " + volumeUuid + " not found in libvirt pool");
}

KVMPhysicalDisk disk;
LibvirtStorageVolumeDef voldef = getStorageVolumeDef(libvirtPool.getPool().getConnect(), vol);
disk = new KVMPhysicalDisk(vol.getPath(), vol.getName(), pool);
Expand Down Expand Up @@ -693,11 +708,153 @@ public KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool) {
}
return disk;
} catch (LibvirtException e) {
logger.debug("Failed to get physical disk:", e);
logger.debug("Failed to get volume from libvirt: " + e.getMessage());
// For CLVM, try direct block device access as fallback
if (pool.getType() == StoragePoolType.CLVM) {
return getPhysicalDisk(volumeUuid, pool, libvirtPool);
}

throw new CloudRuntimeException(e.toString());
}
}

private KVMPhysicalDisk getPhysicalDisk(String volumeUuid, KVMStoragePool pool, LibvirtStoragePool libvirtPool) {
logger.info("CLVM volume not visible to libvirt, attempting direct block device access for volume: {}", volumeUuid);

try {
logger.debug("Refreshing libvirt storage pool: {}", pool.getUuid());
libvirtPool.getPool().refresh(0);

StorageVol vol = getVolume(libvirtPool.getPool(), volumeUuid);
if (vol != null) {
logger.info("Volume found after pool refresh: {}", volumeUuid);
KVMPhysicalDisk disk;
LibvirtStorageVolumeDef voldef = getStorageVolumeDef(libvirtPool.getPool().getConnect(), vol);
disk = new KVMPhysicalDisk(vol.getPath(), vol.getName(), pool);
disk.setSize(vol.getInfo().allocation);
disk.setVirtualSize(vol.getInfo().capacity);
disk.setFormat(voldef.getFormat() == LibvirtStorageVolumeDef.VolumeFormat.QCOW2 ?
PhysicalDiskFormat.QCOW2 : PhysicalDiskFormat.RAW);
return disk;
}
} catch (LibvirtException refreshEx) {
logger.debug("Pool refresh failed or volume still not found: {}", refreshEx.getMessage());
}

// Still not found after refresh, try direct block device access
return getPhysicalDiskViaDirectBlockDevice(volumeUuid, pool);
}

/**
* For CLVM volumes that exist in LVM but are not visible to libvirt,
* access them directly via block device path.
*/
private KVMPhysicalDisk getPhysicalDiskViaDirectBlockDevice(String volumeUuid, KVMStoragePool pool) {
try {
// For CLVM, pool sourceDir contains the VG path (e.g., "/dev/acsvg")
// Extract the VG name
String sourceDir = pool.getLocalPath();
if (sourceDir == null || sourceDir.isEmpty()) {
throw new CloudRuntimeException("CLVM pool sourceDir is not set, cannot determine VG name");
}

String vgName = sourceDir;
if (vgName.startsWith("/")) {
String[] parts = vgName.split("/");
List<String> tokens = Arrays.stream(parts)
.filter(s -> !s.isEmpty()).collect(Collectors.toList());

vgName = tokens.size() > 1 ? tokens.get(1)
: tokens.size() == 1 ? tokens.get(0)
: "";
}

logger.debug("Using VG name: {} (from sourceDir: {}) ", vgName, sourceDir);

// Check if the LV exists in LVM using lvs command
logger.debug("Checking if volume {} exsits in VG {}", volumeUuid, vgName);
Script checkLvCmd = new Script("/usr/sbin/lvs", 5000, logger);
checkLvCmd.add("--noheadings");
checkLvCmd.add("--unbuffered");
checkLvCmd.add(vgName + "/" + volumeUuid);

String checkResult = checkLvCmd.execute();
if (checkResult != null) {
logger.debug("Volume {} does not exist in VG {}: {}", volumeUuid, vgName, checkResult);
throw new CloudRuntimeException(String.format("Storage volume not found: no storage vol with matching name '%s'", volumeUuid));
}

logger.info("Volume {} exists in LVM but not visible to libvirt, accessing directly", volumeUuid);

// Try standard device path first
String lvPath = "/dev/" + vgName + "/" + volumeUuid;
File lvDevice = new File(lvPath);

if (!lvDevice.exists()) {
// Try device-mapper path with escaped hyphens
String vgNameEscaped = vgName.replace("-", "--");
String volumeUuidEscaped = volumeUuid.replace("-", "--");
lvPath = "/dev/mapper/" + vgNameEscaped + "-" + volumeUuidEscaped;
lvDevice = new File(lvPath);

if (!lvDevice.exists()) {
logger.warn("Volume exists in LVM but device node not found: {}", volumeUuid);
throw new CloudRuntimeException(String.format("Could not find volume %s " +
"in VG %s - volume exists in LVM but device node not accessible", volumeUuid, vgName));
}
}

long size = 0;
try {
Script lvsCmd = new Script("/usr/sbin/lvs", 5000, logger);
lvsCmd.add("--noheadings");
lvsCmd.add("--units");
lvsCmd.add("b");
lvsCmd.add("-o");
lvsCmd.add("lv_size");
lvsCmd.add(lvPath);

OutputInterpreter.AllLinesParser parser = new OutputInterpreter.AllLinesParser();
String result = lvsCmd.execute(parser);

String output = null;
if (result == null) {
output = parser.getLines();
} else {
output = result;
}

if (output != null && !output.isEmpty()) {
String sizeStr = output.trim().replaceAll("[^0-9]", "");
if (!sizeStr.isEmpty()) {
size = Long.parseLong(sizeStr);
}
}
} catch (Exception sizeEx) {
logger.warn("Failed to get size for CLVM volume via lvs: {}", sizeEx.getMessage());
if (lvDevice.isFile()) {
size = lvDevice.length();
}
}

KVMPhysicalDisk disk = new KVMPhysicalDisk(lvPath, volumeUuid, pool);
disk.setFormat(PhysicalDiskFormat.RAW);
disk.setSize(size);
disk.setVirtualSize(size);

logger.info("Successfully accessed CLVM volume via direct block device: {} " +
"with size: {} bytes",lvPath, size);

return disk;

} catch (CloudRuntimeException ex) {
throw ex;
} catch (Exception ex) {
logger.error("Failed to access CLVM volume via direct block device: {}",volumeUuid, ex);
throw new CloudRuntimeException(String.format("Could not find volume %s: %s ",volumeUuid, ex.getMessage()));
}
}

/**
* adjust refcount
*/
Expand Down Expand Up @@ -1227,7 +1384,11 @@ public boolean deletePhysicalDisk(String uuid, KVMStoragePool pool, Storage.Imag
LibvirtStoragePool libvirtPool = (LibvirtStoragePool)pool;
try {
StorageVol vol = getVolume(libvirtPool.getPool(), uuid);
logger.debug("Instructing libvirt to remove volume " + uuid + " from pool " + pool.getUuid());
if (vol == null) {
logger.warn("Volume %s not found in libvirt pool %s, it may have been already deleted", uuid, pool.getUuid());
return true;
}
logger.debug("Instructing libvirt to remove volume %s from pool %s", uuid, pool.getUuid());
if(Storage.ImageFormat.DIR.equals(format)){
deleteDirVol(libvirtPool, vol);
} else {
Expand Down
16 changes: 8 additions & 8 deletions scripts/storage/qcow2/managesnapshot.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,12 +212,12 @@ backup_snapshot() {
return 1
fi

qemuimg_ret=$($qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}")
qemuimg_ret=$($qemu_img convert $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}" 2>&1)
ret_code=$?
if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]]
if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]])
then
forceShareFlag=""
$qemu_img $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}"
$qemu_img convert $forceShareFlag -f raw -O qcow2 "/dev/mapper/${vg_dm}-${snapshotname}" "${destPath}/${destName}"
ret_code=$?
fi
if [ $ret_code -gt 0 ]
Expand All @@ -240,9 +240,9 @@ backup_snapshot() {
# Backup VM snapshot
qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk 2>&1)
ret_code=$?
if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"snapshot: invalid option -- 'U'"* ]]; then
if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]]); then
forceShareFlag=""
qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk)
qemuimg_ret=$($qemu_img snapshot $forceShareFlag -l $disk 2>&1)
ret_code=$?
fi

Expand All @@ -251,11 +251,11 @@ backup_snapshot() {
return 1
fi

qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null)
qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1)
ret_code=$?
if [ $ret_code -gt 0 ] && [[ $qemuimg_ret == *"convert: invalid option -- 'U'"* ]]; then
if [ $ret_code -gt 0 ] && ([[ $qemuimg_ret == *"invalid option"*"'U'"* ]] || [[ $qemuimg_ret == *"unrecognized option"*"'-U'"* ]]); then
forceShareFlag=""
qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1 > /dev/null)
qemuimg_ret=$($qemu_img convert $forceShareFlag -f qcow2 -O qcow2 -l snapshot.name=$snapshotname $disk $destPath/$destName 2>&1)
ret_code=$?
fi

Expand Down
Loading