From a581db922b1448f0574744b87a2b2a75420a8254 Mon Sep 17 00:00:00 2001 From: "zhong.zhou" Date: Thu, 16 Jul 2026 19:33:08 +0800 Subject: [PATCH 1/3] [storage]: ZSV-12657 wait zbs clients before revert Wait for active ZBS volume clients to disappear before rolling back a volume snapshot. Add ZBS primary storage regression coverage for delayed client release before revert. Test: git diff --check Test: cbok zsv groovy_test ZbsPrimaryStorageCase Resolves: ZSV-12657 Change-Id: I3792c66a658e9cec77c3c322ce8f4e135414b534 --- .../zstack/storage/zbs/ZbsGlobalConfig.java | 19 ++ .../storage/zbs/ZbsStorageController.java | 78 ++++++-- .../addon/zbs/ZbsPrimaryStorageCase.groovy | 182 ++++++++++++++++++ 3 files changed, 268 insertions(+), 11 deletions(-) create mode 100644 plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java new file mode 100644 index 00000000000..ddfbeec8990 --- /dev/null +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java @@ -0,0 +1,19 @@ +package org.zstack.storage.zbs; + +import org.zstack.core.config.GlobalConfig; +import org.zstack.core.config.GlobalConfigDef; +import org.zstack.core.config.GlobalConfigDefinition; +import org.zstack.core.config.GlobalConfigValidation; + +@GlobalConfigDefinition +public class ZbsGlobalConfig { + public static final String CATEGORY = "zbs"; + + @GlobalConfigValidation(numberGreaterThan = 0) + @GlobalConfigDef(defaultValue = "100", type = Long.class, description = "timeout in seconds for waiting volume clients released before reverting snapshot") + public static GlobalConfig VOLUME_CLIENT_RELEASE_TIMEOUT = new GlobalConfig(CATEGORY, "volume.client.release.timeout"); + + @GlobalConfigValidation(numberGreaterThan = 0) + @GlobalConfigDef(defaultValue = "1", type = Long.class, description = "poll interval in seconds for waiting volume clients released before reverting snapshot") + public static GlobalConfig VOLUME_CLIENT_RELEASE_POLL_INTERVAL = new GlobalConfig(CATEGORY, "volume.client.release.pollInterval"); +} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 70a5420c544..bc5ef01034b 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -15,6 +15,7 @@ import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.SQL; +import org.zstack.core.thread.ThreadFacade; import org.zstack.core.workflow.FlowChainBuilder; import org.zstack.core.workflow.ShareFlow; import org.zstack.header.HasThreadContext; @@ -80,6 +81,8 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar @Autowired private CloudBus bus; @Autowired + private ThreadFacade thdf; + @Autowired private PluginRegistry pluginRgty; private ExternalPrimaryStorageVO self; @@ -1029,18 +1032,28 @@ public void expungeSnapshot(String installPath, Completion comp) { @Override public void revertVolumeSnapshot(String snapshotInstallPath, ReturnValueCompletion comp) { - RollbackSnapshotCmd cmd = new RollbackSnapshotCmd(); - cmd.setPath(snapshotInstallPath); - - httpCall(ROLLBACK_SNAPSHOT_PATH, cmd, RollbackSnapshotRsp.class, new ReturnValueCompletion(comp) { + waitVolumeClientsReleased(snapshotInstallPath, new Completion(comp) { @Override - public void success(RollbackSnapshotRsp returnValue) { - VolumeStats stats = new VolumeStats(); - stats.setInstallPath(returnValue.getInstallPath()); - stats.setSize(returnValue.getSize()); - stats.setActualSize(returnValue.getActualSize()); - stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); - comp.success(stats); + public void success() { + RollbackSnapshotCmd cmd = new RollbackSnapshotCmd(); + cmd.setPath(snapshotInstallPath); + + httpCall(ROLLBACK_SNAPSHOT_PATH, cmd, RollbackSnapshotRsp.class, new ReturnValueCompletion(comp) { + @Override + public void success(RollbackSnapshotRsp returnValue) { + VolumeStats stats = new VolumeStats(); + stats.setInstallPath(returnValue.getInstallPath()); + stats.setSize(returnValue.getSize()); + stats.setActualSize(returnValue.getActualSize()); + stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + comp.success(stats); + } + + @Override + public void fail(ErrorCode errorCode) { + comp.fail(errorCode); + } + }); } @Override @@ -1050,6 +1063,49 @@ public void fail(ErrorCode errorCode) { }); } + private void waitVolumeClientsReleased(String snapshotInstallPath, Completion comp) { + int index = snapshotInstallPath.indexOf("@snapshot_"); + if (index < 0) { + comp.fail(operr("unable to revert snapshot[path:%s], invalid zbs snapshot install path", snapshotInstallPath)); + return; + } + String volumeInstallPath = snapshotInstallPath.substring(0, index); + long timeout = ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_TIMEOUT.value(Long.class); + long pollInterval = ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_POLL_INTERVAL.value(Long.class); + long expired = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(timeout); + + thdf.submitTimeoutTask(() -> waitVolumeClientsReleased(volumeInstallPath, snapshotInstallPath, expired, timeout, pollInterval, comp), + TimeUnit.SECONDS, 0); + } + + private void waitVolumeClientsReleased(String volumeInstallPath, String snapshotInstallPath, long expired, long timeout, long pollInterval, Completion comp) { + List clients; + try { + clients = getActiveClients(volumeInstallPath, VolumeProtocol.CBD.toString()); + } catch (OperationFailureException e) { + comp.fail(e.getErrorCode()); + return; + } + + if (clients.isEmpty()) { + comp.success(); + return; + } + + if (System.currentTimeMillis() >= expired) { + comp.fail(operr("unable to revert volume[path:%s] to snapshot[path:%s], active clients[%s] were not released within %s seconds", + volumeInstallPath, snapshotInstallPath, + clients.stream().map(ActiveVolumeClient::getManagerIp).collect(Collectors.joining(",")), + timeout)); + return; + } + + long remaining = expired - System.currentTimeMillis(); + long nextDelay = Math.min(pollInterval, Math.max(1, TimeUnit.MILLISECONDS.toSeconds(remaining))); + thdf.submitTimeoutTask(() -> waitVolumeClientsReleased(volumeInstallPath, snapshotInstallPath, expired, timeout, pollInterval, comp), + TimeUnit.SECONDS, nextDelay); + } + @Override public void validateConfig(String config) { Config old = JSONObjectUtil.toObject(self.getConfig(), Config.class); diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 5fc8f14aa97..f549649e91d 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -2,18 +2,23 @@ package org.zstack.test.integration.storage.primary.addon.zbs import org.springframework.http.HttpEntity import org.zstack.core.db.Q +import org.zstack.header.core.ReturnValueCompletion +import org.zstack.header.errorcode.ErrorCode import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_ import org.zstack.header.storage.primary.PrimaryStorageHostRefVO import org.zstack.header.storage.primary.PrimaryStorageHostRefVO_ import org.zstack.header.storage.primary.PrimaryStorageStatus +import org.zstack.header.volume.VolumeStats import org.zstack.cbd.MdsUri import org.zstack.kvm.KVMAgentCommands import org.zstack.sdk.* +import org.zstack.storage.addon.primary.ExternalPrimaryStorageFactory import org.zstack.storage.primary.PrimaryStorageGlobalConfig import org.zstack.header.storage.primary.PrimaryStorageHostStatus import org.zstack.storage.volume.VolumeGlobalConfig import org.zstack.storage.zbs.ZbsConstants +import org.zstack.storage.zbs.ZbsGlobalConfig import org.zstack.storage.zbs.ZbsPrimaryStorageMdsBase import org.zstack.storage.zbs.ZbsStorageController import org.zstack.test.integration.storage.StorageTest @@ -23,6 +28,12 @@ import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference + /** * @author Xingwei Yu * @date 2024/4/19 10:09 @@ -157,6 +168,8 @@ class ZbsPrimaryStorageCase extends SubCase { testUpdateExternalPrimaryStorage() testLifecycle() testDataVolumeLifecycle() + testRevertVolumeSnapshotDoesNotBlockWhileWaitingForActiveClients() + testRevertVolumeSnapshotFailsWhenActiveClientsAreNotReleasedBeforeTimeout() testMdsPing() testCheckHostStorageConnection() testNegativeScenario() @@ -386,6 +399,175 @@ class ZbsPrimaryStorageCase extends SubCase { deleteVolume(vol.uuid) } + void testRevertVolumeSnapshotDoesNotBlockWhileWaitingForActiveClients() { + VolumeInventory volume = null + VolumeSnapshotInventory snapshot = null + AtomicBoolean clientsReleased = new AtomicBoolean(false) + AtomicInteger rollbackCount = new AtomicInteger() + CountDownLatch revertMethodReturned = new CountDownLatch(1) + CountDownLatch completionDone = new CountDownLatch(1) + AtomicReference volumeStats = new AtomicReference<>() + AtomicReference errorCode = new AtomicReference<>() + Thread revertThread = null + + try { + volume = createDataVolume { + name = "test-revert-waits-active-client-release-async" + diskOfferingUuid = diskOffering.uuid + primaryStorageUuid = ps.uuid + } as VolumeInventory + + snapshot = createVolumeSnapshot { + name = "test-revert-waits-active-client-release-async" + volumeUuid = volume.uuid + } as VolumeSnapshotInventory + + String volumeInstallPath = volume.installPath + + env.simulator(ZbsStorageController.GET_VOLUME_CLIENTS_PATH) { HttpEntity e, EnvSpec spec -> + ZbsStorageController.GetVolumeClientsCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.GetVolumeClientsCmd.class) + assert cmd.path == volumeInstallPath: "revert must query active clients by the target volume install path" + + ZbsStorageController.GetVolumeClientsRsp rsp = new ZbsStorageController.GetVolumeClientsRsp() + if (!clientsReleased.get()) { + rsp.clients = [new ZbsStorageController.ClientInfo("127.0.0.1", 7700)] + } + return rsp + } + + env.simulator(ZbsStorageController.ROLLBACK_SNAPSHOT_PATH) { HttpEntity e, EnvSpec spec -> + ZbsStorageController.RollbackSnapshotCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.RollbackSnapshotCmd.class) + assert cmd.path == snapshot.primaryStorageInstallPath: "rollback must use the requested snapshot install path" + assert clientsReleased.get(): "rollback must not run before active clients are released" + rollbackCount.incrementAndGet() + + ZbsStorageController.RollbackSnapshotRsp rsp = new ZbsStorageController.RollbackSnapshotRsp() + rsp.installPath = volumeInstallPath + rsp.size = volume.size + rsp.actualSize = volume.actualSize + return rsp + } + + ZbsStorageController controller = bean(ExternalPrimaryStorageFactory.class).getControllerSvc(ps.uuid) as ZbsStorageController + revertThread = Thread.start { + try { + controller.revertVolumeSnapshot(snapshot.primaryStorageInstallPath, new ReturnValueCompletion(null) { + @Override + void success(VolumeStats returnValue) { + volumeStats.set(returnValue) + completionDone.countDown() + } + + @Override + void fail(ErrorCode error) { + errorCode.set(error) + completionDone.countDown() + } + }) + } finally { + revertMethodReturned.countDown() + } + } + + assert revertMethodReturned.await(2, TimeUnit.SECONDS): "revertVolumeSnapshot must return without synchronously sleeping while active clients remain" + assert rollbackCount.get() == 0: "rollback must not run before active clients are released" + assert completionDone.getCount() == 1: "completion must wait for active clients to be released" + + clientsReleased.set(true) + + assert completionDone.await(5, TimeUnit.SECONDS): "completion must finish after active clients are released" + assert errorCode.get() == null: "revert must not fail after active clients are released" + assert volumeStats.get() != null: "revert must return volume stats after rollback" + assert rollbackCount.get() == 1: "rollback must run exactly once after active clients are released" + } finally { + clientsReleased.set(true) + if (revertThread != null && revertThread.isAlive()) { + revertThread.interrupt() + revertThread.join(5000) + } + if (snapshot != null) { + deleteVolumeSnapshot { + uuid = snapshot.uuid + } + } + if (volume != null) { + deleteVolume(volume.uuid) + } + + env.cleanSimulatorAndMessageHandlers() + } + } + + void testRevertVolumeSnapshotFailsWhenActiveClientsAreNotReleasedBeforeTimeout() { + VolumeInventory volume = null + VolumeSnapshotInventory snapshot = null + AtomicInteger rollbackCount = new AtomicInteger() + CountDownLatch completionDone = new CountDownLatch(1) + AtomicReference errorCode = new AtomicReference<>() + + try { + ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_TIMEOUT.updateValue(1) + + volume = createDataVolume { + name = "test-revert-fails-when-active-client-not-released" + diskOfferingUuid = diskOffering.uuid + primaryStorageUuid = ps.uuid + } as VolumeInventory + + snapshot = createVolumeSnapshot { + name = "test-revert-fails-when-active-client-not-released" + volumeUuid = volume.uuid + } as VolumeSnapshotInventory + + String volumeInstallPath = volume.installPath + + env.simulator(ZbsStorageController.GET_VOLUME_CLIENTS_PATH) { HttpEntity e, EnvSpec spec -> + ZbsStorageController.GetVolumeClientsCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.GetVolumeClientsCmd.class) + assert cmd.path == volumeInstallPath: "revert must keep polling the target volume install path before timeout" + + ZbsStorageController.GetVolumeClientsRsp rsp = new ZbsStorageController.GetVolumeClientsRsp() + rsp.clients = [new ZbsStorageController.ClientInfo("127.0.0.1", 7700)] + return rsp + } + + env.simulator(ZbsStorageController.ROLLBACK_SNAPSHOT_PATH) { HttpEntity e, EnvSpec spec -> + rollbackCount.incrementAndGet() + return new ZbsStorageController.RollbackSnapshotRsp() + } + + ZbsStorageController controller = bean(ExternalPrimaryStorageFactory.class).getControllerSvc(ps.uuid) as ZbsStorageController + controller.revertVolumeSnapshot(snapshot.primaryStorageInstallPath, new ReturnValueCompletion(null) { + @Override + void success(VolumeStats returnValue) { + completionDone.countDown() + } + + @Override + void fail(ErrorCode error) { + errorCode.set(error) + completionDone.countDown() + } + }) + + assert completionDone.await(5, TimeUnit.SECONDS): "revert must fail when active clients are not released before the configured timeout" + assert errorCode.get() != null: "timeout must fail the completion" + assert errorCode.get().details.contains("were not released within 1 seconds") + assert rollbackCount.get() == 0: "rollback must not run after client-release wait times out" + } finally { + ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_TIMEOUT.resetValue() + if (snapshot != null) { + deleteVolumeSnapshot { + uuid = snapshot.uuid + } + } + if (volume != null) { + deleteVolume(volume.uuid) + } + + env.cleanSimulatorAndMessageHandlers() + } + } + void testNegativeScenario() { expect(AssertionError.class) { addExternalPrimaryStorage { From e9b92db9e785de0fde023e38405c4c640820394e Mon Sep 17 00:00:00 2001 From: "zhong.zhou" Date: Fri, 17 Jul 2026 10:50:57 +0800 Subject: [PATCH 2/3] [storage]: remove revert client release wait Release the storage client during VM shutdown instead of polling in the snapshot revert path. Resolves: ZSV-12657 Change-Id: I274b3d6acf1dbfba53665ca75d53c0b5df62901b --- .../zstack/storage/zbs/ZbsGlobalConfig.java | 19 -- .../storage/zbs/ZbsStorageController.java | 78 ++------ .../addon/zbs/ZbsPrimaryStorageCase.groovy | 182 ------------------ 3 files changed, 11 insertions(+), 268 deletions(-) delete mode 100644 plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java deleted file mode 100644 index ddfbeec8990..00000000000 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsGlobalConfig.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.zstack.storage.zbs; - -import org.zstack.core.config.GlobalConfig; -import org.zstack.core.config.GlobalConfigDef; -import org.zstack.core.config.GlobalConfigDefinition; -import org.zstack.core.config.GlobalConfigValidation; - -@GlobalConfigDefinition -public class ZbsGlobalConfig { - public static final String CATEGORY = "zbs"; - - @GlobalConfigValidation(numberGreaterThan = 0) - @GlobalConfigDef(defaultValue = "100", type = Long.class, description = "timeout in seconds for waiting volume clients released before reverting snapshot") - public static GlobalConfig VOLUME_CLIENT_RELEASE_TIMEOUT = new GlobalConfig(CATEGORY, "volume.client.release.timeout"); - - @GlobalConfigValidation(numberGreaterThan = 0) - @GlobalConfigDef(defaultValue = "1", type = Long.class, description = "poll interval in seconds for waiting volume clients released before reverting snapshot") - public static GlobalConfig VOLUME_CLIENT_RELEASE_POLL_INTERVAL = new GlobalConfig(CATEGORY, "volume.client.release.pollInterval"); -} diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index bc5ef01034b..70a5420c544 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -15,7 +15,6 @@ import org.zstack.core.componentloader.PluginRegistry; import org.zstack.core.db.DatabaseFacade; import org.zstack.core.db.SQL; -import org.zstack.core.thread.ThreadFacade; import org.zstack.core.workflow.FlowChainBuilder; import org.zstack.core.workflow.ShareFlow; import org.zstack.header.HasThreadContext; @@ -81,8 +80,6 @@ public class ZbsStorageController implements PrimaryStorageControllerSvc, Primar @Autowired private CloudBus bus; @Autowired - private ThreadFacade thdf; - @Autowired private PluginRegistry pluginRgty; private ExternalPrimaryStorageVO self; @@ -1032,28 +1029,18 @@ public void expungeSnapshot(String installPath, Completion comp) { @Override public void revertVolumeSnapshot(String snapshotInstallPath, ReturnValueCompletion comp) { - waitVolumeClientsReleased(snapshotInstallPath, new Completion(comp) { - @Override - public void success() { - RollbackSnapshotCmd cmd = new RollbackSnapshotCmd(); - cmd.setPath(snapshotInstallPath); - - httpCall(ROLLBACK_SNAPSHOT_PATH, cmd, RollbackSnapshotRsp.class, new ReturnValueCompletion(comp) { - @Override - public void success(RollbackSnapshotRsp returnValue) { - VolumeStats stats = new VolumeStats(); - stats.setInstallPath(returnValue.getInstallPath()); - stats.setSize(returnValue.getSize()); - stats.setActualSize(returnValue.getActualSize()); - stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); - comp.success(stats); - } + RollbackSnapshotCmd cmd = new RollbackSnapshotCmd(); + cmd.setPath(snapshotInstallPath); - @Override - public void fail(ErrorCode errorCode) { - comp.fail(errorCode); - } - }); + httpCall(ROLLBACK_SNAPSHOT_PATH, cmd, RollbackSnapshotRsp.class, new ReturnValueCompletion(comp) { + @Override + public void success(RollbackSnapshotRsp returnValue) { + VolumeStats stats = new VolumeStats(); + stats.setInstallPath(returnValue.getInstallPath()); + stats.setSize(returnValue.getSize()); + stats.setActualSize(returnValue.getActualSize()); + stats.setFormat(VolumeConstant.VOLUME_FORMAT_RAW); + comp.success(stats); } @Override @@ -1063,49 +1050,6 @@ public void fail(ErrorCode errorCode) { }); } - private void waitVolumeClientsReleased(String snapshotInstallPath, Completion comp) { - int index = snapshotInstallPath.indexOf("@snapshot_"); - if (index < 0) { - comp.fail(operr("unable to revert snapshot[path:%s], invalid zbs snapshot install path", snapshotInstallPath)); - return; - } - String volumeInstallPath = snapshotInstallPath.substring(0, index); - long timeout = ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_TIMEOUT.value(Long.class); - long pollInterval = ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_POLL_INTERVAL.value(Long.class); - long expired = System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(timeout); - - thdf.submitTimeoutTask(() -> waitVolumeClientsReleased(volumeInstallPath, snapshotInstallPath, expired, timeout, pollInterval, comp), - TimeUnit.SECONDS, 0); - } - - private void waitVolumeClientsReleased(String volumeInstallPath, String snapshotInstallPath, long expired, long timeout, long pollInterval, Completion comp) { - List clients; - try { - clients = getActiveClients(volumeInstallPath, VolumeProtocol.CBD.toString()); - } catch (OperationFailureException e) { - comp.fail(e.getErrorCode()); - return; - } - - if (clients.isEmpty()) { - comp.success(); - return; - } - - if (System.currentTimeMillis() >= expired) { - comp.fail(operr("unable to revert volume[path:%s] to snapshot[path:%s], active clients[%s] were not released within %s seconds", - volumeInstallPath, snapshotInstallPath, - clients.stream().map(ActiveVolumeClient::getManagerIp).collect(Collectors.joining(",")), - timeout)); - return; - } - - long remaining = expired - System.currentTimeMillis(); - long nextDelay = Math.min(pollInterval, Math.max(1, TimeUnit.MILLISECONDS.toSeconds(remaining))); - thdf.submitTimeoutTask(() -> waitVolumeClientsReleased(volumeInstallPath, snapshotInstallPath, expired, timeout, pollInterval, comp), - TimeUnit.SECONDS, nextDelay); - } - @Override public void validateConfig(String config) { Config old = JSONObjectUtil.toObject(self.getConfig(), Config.class); diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index f549649e91d..5fc8f14aa97 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -2,23 +2,18 @@ package org.zstack.test.integration.storage.primary.addon.zbs import org.springframework.http.HttpEntity import org.zstack.core.db.Q -import org.zstack.header.core.ReturnValueCompletion -import org.zstack.header.errorcode.ErrorCode import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_ import org.zstack.header.storage.primary.PrimaryStorageHostRefVO import org.zstack.header.storage.primary.PrimaryStorageHostRefVO_ import org.zstack.header.storage.primary.PrimaryStorageStatus -import org.zstack.header.volume.VolumeStats import org.zstack.cbd.MdsUri import org.zstack.kvm.KVMAgentCommands import org.zstack.sdk.* -import org.zstack.storage.addon.primary.ExternalPrimaryStorageFactory import org.zstack.storage.primary.PrimaryStorageGlobalConfig import org.zstack.header.storage.primary.PrimaryStorageHostStatus import org.zstack.storage.volume.VolumeGlobalConfig import org.zstack.storage.zbs.ZbsConstants -import org.zstack.storage.zbs.ZbsGlobalConfig import org.zstack.storage.zbs.ZbsPrimaryStorageMdsBase import org.zstack.storage.zbs.ZbsStorageController import org.zstack.test.integration.storage.StorageTest @@ -28,12 +23,6 @@ import org.zstack.testlib.SubCase import org.zstack.utils.data.SizeUnit import org.zstack.utils.gson.JSONObjectUtil -import java.util.concurrent.CountDownLatch -import java.util.concurrent.TimeUnit -import java.util.concurrent.atomic.AtomicBoolean -import java.util.concurrent.atomic.AtomicInteger -import java.util.concurrent.atomic.AtomicReference - /** * @author Xingwei Yu * @date 2024/4/19 10:09 @@ -168,8 +157,6 @@ class ZbsPrimaryStorageCase extends SubCase { testUpdateExternalPrimaryStorage() testLifecycle() testDataVolumeLifecycle() - testRevertVolumeSnapshotDoesNotBlockWhileWaitingForActiveClients() - testRevertVolumeSnapshotFailsWhenActiveClientsAreNotReleasedBeforeTimeout() testMdsPing() testCheckHostStorageConnection() testNegativeScenario() @@ -399,175 +386,6 @@ class ZbsPrimaryStorageCase extends SubCase { deleteVolume(vol.uuid) } - void testRevertVolumeSnapshotDoesNotBlockWhileWaitingForActiveClients() { - VolumeInventory volume = null - VolumeSnapshotInventory snapshot = null - AtomicBoolean clientsReleased = new AtomicBoolean(false) - AtomicInteger rollbackCount = new AtomicInteger() - CountDownLatch revertMethodReturned = new CountDownLatch(1) - CountDownLatch completionDone = new CountDownLatch(1) - AtomicReference volumeStats = new AtomicReference<>() - AtomicReference errorCode = new AtomicReference<>() - Thread revertThread = null - - try { - volume = createDataVolume { - name = "test-revert-waits-active-client-release-async" - diskOfferingUuid = diskOffering.uuid - primaryStorageUuid = ps.uuid - } as VolumeInventory - - snapshot = createVolumeSnapshot { - name = "test-revert-waits-active-client-release-async" - volumeUuid = volume.uuid - } as VolumeSnapshotInventory - - String volumeInstallPath = volume.installPath - - env.simulator(ZbsStorageController.GET_VOLUME_CLIENTS_PATH) { HttpEntity e, EnvSpec spec -> - ZbsStorageController.GetVolumeClientsCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.GetVolumeClientsCmd.class) - assert cmd.path == volumeInstallPath: "revert must query active clients by the target volume install path" - - ZbsStorageController.GetVolumeClientsRsp rsp = new ZbsStorageController.GetVolumeClientsRsp() - if (!clientsReleased.get()) { - rsp.clients = [new ZbsStorageController.ClientInfo("127.0.0.1", 7700)] - } - return rsp - } - - env.simulator(ZbsStorageController.ROLLBACK_SNAPSHOT_PATH) { HttpEntity e, EnvSpec spec -> - ZbsStorageController.RollbackSnapshotCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.RollbackSnapshotCmd.class) - assert cmd.path == snapshot.primaryStorageInstallPath: "rollback must use the requested snapshot install path" - assert clientsReleased.get(): "rollback must not run before active clients are released" - rollbackCount.incrementAndGet() - - ZbsStorageController.RollbackSnapshotRsp rsp = new ZbsStorageController.RollbackSnapshotRsp() - rsp.installPath = volumeInstallPath - rsp.size = volume.size - rsp.actualSize = volume.actualSize - return rsp - } - - ZbsStorageController controller = bean(ExternalPrimaryStorageFactory.class).getControllerSvc(ps.uuid) as ZbsStorageController - revertThread = Thread.start { - try { - controller.revertVolumeSnapshot(snapshot.primaryStorageInstallPath, new ReturnValueCompletion(null) { - @Override - void success(VolumeStats returnValue) { - volumeStats.set(returnValue) - completionDone.countDown() - } - - @Override - void fail(ErrorCode error) { - errorCode.set(error) - completionDone.countDown() - } - }) - } finally { - revertMethodReturned.countDown() - } - } - - assert revertMethodReturned.await(2, TimeUnit.SECONDS): "revertVolumeSnapshot must return without synchronously sleeping while active clients remain" - assert rollbackCount.get() == 0: "rollback must not run before active clients are released" - assert completionDone.getCount() == 1: "completion must wait for active clients to be released" - - clientsReleased.set(true) - - assert completionDone.await(5, TimeUnit.SECONDS): "completion must finish after active clients are released" - assert errorCode.get() == null: "revert must not fail after active clients are released" - assert volumeStats.get() != null: "revert must return volume stats after rollback" - assert rollbackCount.get() == 1: "rollback must run exactly once after active clients are released" - } finally { - clientsReleased.set(true) - if (revertThread != null && revertThread.isAlive()) { - revertThread.interrupt() - revertThread.join(5000) - } - if (snapshot != null) { - deleteVolumeSnapshot { - uuid = snapshot.uuid - } - } - if (volume != null) { - deleteVolume(volume.uuid) - } - - env.cleanSimulatorAndMessageHandlers() - } - } - - void testRevertVolumeSnapshotFailsWhenActiveClientsAreNotReleasedBeforeTimeout() { - VolumeInventory volume = null - VolumeSnapshotInventory snapshot = null - AtomicInteger rollbackCount = new AtomicInteger() - CountDownLatch completionDone = new CountDownLatch(1) - AtomicReference errorCode = new AtomicReference<>() - - try { - ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_TIMEOUT.updateValue(1) - - volume = createDataVolume { - name = "test-revert-fails-when-active-client-not-released" - diskOfferingUuid = diskOffering.uuid - primaryStorageUuid = ps.uuid - } as VolumeInventory - - snapshot = createVolumeSnapshot { - name = "test-revert-fails-when-active-client-not-released" - volumeUuid = volume.uuid - } as VolumeSnapshotInventory - - String volumeInstallPath = volume.installPath - - env.simulator(ZbsStorageController.GET_VOLUME_CLIENTS_PATH) { HttpEntity e, EnvSpec spec -> - ZbsStorageController.GetVolumeClientsCmd cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.GetVolumeClientsCmd.class) - assert cmd.path == volumeInstallPath: "revert must keep polling the target volume install path before timeout" - - ZbsStorageController.GetVolumeClientsRsp rsp = new ZbsStorageController.GetVolumeClientsRsp() - rsp.clients = [new ZbsStorageController.ClientInfo("127.0.0.1", 7700)] - return rsp - } - - env.simulator(ZbsStorageController.ROLLBACK_SNAPSHOT_PATH) { HttpEntity e, EnvSpec spec -> - rollbackCount.incrementAndGet() - return new ZbsStorageController.RollbackSnapshotRsp() - } - - ZbsStorageController controller = bean(ExternalPrimaryStorageFactory.class).getControllerSvc(ps.uuid) as ZbsStorageController - controller.revertVolumeSnapshot(snapshot.primaryStorageInstallPath, new ReturnValueCompletion(null) { - @Override - void success(VolumeStats returnValue) { - completionDone.countDown() - } - - @Override - void fail(ErrorCode error) { - errorCode.set(error) - completionDone.countDown() - } - }) - - assert completionDone.await(5, TimeUnit.SECONDS): "revert must fail when active clients are not released before the configured timeout" - assert errorCode.get() != null: "timeout must fail the completion" - assert errorCode.get().details.contains("were not released within 1 seconds") - assert rollbackCount.get() == 0: "rollback must not run after client-release wait times out" - } finally { - ZbsGlobalConfig.VOLUME_CLIENT_RELEASE_TIMEOUT.resetValue() - if (snapshot != null) { - deleteVolumeSnapshot { - uuid = snapshot.uuid - } - } - if (volume != null) { - deleteVolume(volume.uuid) - } - - env.cleanSimulatorAndMessageHandlers() - } - } - void testNegativeScenario() { expect(AssertionError.class) { addExternalPrimaryStorage { From 74cfe66e579116ea3336d0eb9814036c93285047 Mon Sep 17 00:00:00 2001 From: "zhong.zhou" Date: Fri, 17 Jul 2026 11:47:36 +0800 Subject: [PATCH 3/3] [kvm]: update no-os stop config description Update the Java global config description to match the cold stop strategy. Test: git diff --check Resolves: ZSV-12657 Change-Id: I47b852c437f0c7baa61203d3256b35bdee5cf60a --- plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java index 4c2ff3a9175..8d220a1ffb8 100755 --- a/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java +++ b/plugin/kvm/src/main/java/org/zstack/kvm/KVMGlobalConfig.java @@ -132,7 +132,7 @@ public class KVMGlobalConfig { @BindResourceConfig({HostVO.class}) public static GlobalConfig HOST_KSM = new GlobalConfig(CATEGORY, "host.ksm"); - @GlobalConfigDef(defaultValue = "true", type = Boolean.class, description = "use 'force' mode when stop VM without operating system") + @GlobalConfigDef(defaultValue = "true", type = Boolean.class, description = "use 'cold' mode when stop VM without operating system") public static GlobalConfig STOP_VM_WITHOUT_OS_BY_FORCE_MODE = new GlobalConfig(CATEGORY, "stop.vm.without.os.by.force.mode");