From 782c2d259fb789461388c884254d47eccccb363d Mon Sep 17 00:00:00 2001 From: "ye.zou" Date: Thu, 12 Feb 2026 13:52:13 +0800 Subject: [PATCH 1/2] [zbs]: enable tryNext and 30s timeout for getActiveClients MDS call When anti-split-brain check selects a disconnected MDS node, the HTTP call now times out after 30s instead of 5+ minutes, and automatically retries the next available MDS via tryNext mechanism. Resolves: ZSTAC-80595 Change-Id: I1be80f1b70cad1606eb38d1f0078c8f2781e6941 --- .../org/zstack/storage/zbs/ZbsStorageController.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java index 4db9baf0d1d..068ce63bbcf 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java @@ -168,7 +168,10 @@ public List getActiveClients(String installPath, String prot if (VolumeProtocol.CBD.toString().equals(protocol)) { GetVolumeClientsCmd cmd = new GetVolumeClientsCmd(); cmd.setPath(installPath); - GetVolumeClientsRsp rsp = syncHttpCall(GET_VOLUME_CLIENTS_PATH, cmd, GetVolumeClientsRsp.class); + GetVolumeClientsRsp rsp = new HttpCaller<>(GET_VOLUME_CLIENTS_PATH, cmd, GetVolumeClientsRsp.class, + null, TimeUnit.SECONDS, 30, true) + .setTryNext(true) + .syncCall(); List clients = new ArrayList<>(); if (!rsp.isSuccess()) { @@ -1314,6 +1317,11 @@ public class HttpCaller { private boolean tryNext = false; + HttpCaller setTryNext(boolean tryNext) { + this.tryNext = tryNext; + return this; + } + public HttpCaller(String path, AgentCommand cmd, Class retClass, ReturnValueCompletion callback) { this(path, cmd, retClass, callback, null, 0, false); } From 839f494bf211530ea563136573e576447ff4c249 Mon Sep 17 00:00:00 2001 From: "haidong.pang" Date: Mon, 29 Jun 2026 17:45:03 +0800 Subject: [PATCH 2/2] [zbs]: try next mds on sync failure Catch sync REST failures in the ZBS MDS HTTP caller so getActiveClients can try the next connected MDS when the selected MDS is unreachable. Add a regression test for the volume clients path where the first MDS throws an operation failure and the second returns active clients. Resolves: ZSTAC-80595 Change-Id: Id891693b033ac3d4588243d2dd25750f63110db6 --- .../org/zstack/storage/zbs/ZbsMdsBase.java | 9 +++++- .../addon/zbs/ZbsPrimaryStorageCase.groovy | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsMdsBase.java b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsMdsBase.java index 1d4f286233a..b4565eab2ed 100644 --- a/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsMdsBase.java +++ b/plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsMdsBase.java @@ -12,6 +12,7 @@ import org.zstack.header.errorcode.OperationFailureException; import org.zstack.header.rest.JsonAsyncRESTCallback; import org.zstack.header.rest.RESTFacade; +import org.zstack.utils.gson.JSONObjectUtil; import org.zstack.utils.ssh.Ssh; import org.zstack.utils.ssh.SshException; import org.zstack.utils.ssh.SshResult; @@ -77,7 +78,13 @@ protected void checkStorageHealth() { } public T syncCall(final String path, final Object cmd, final Class retClass, TimeUnit unit, long timeout) { - return restf.syncJsonPost(makeHttpPath(self.getAddr(), path), cmd, retClass, unit, timeout); + try { + return restf.syncJsonPost(makeHttpPath(self.getAddr(), path), cmd, retClass, unit, timeout); + } catch (OperationFailureException e) { + T ret = JSONObjectUtil.toObject("{}", retClass); + ret.setError(e.getErrorCode().getDetails()); + return ret; + } } public void httpCall(final String path, final Object cmd, final Class retClass, final ReturnValueCompletion completion) { diff --git a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy index 6f4fb8db734..877ac5d0f74 100644 --- a/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy +++ b/test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy @@ -5,6 +5,8 @@ import org.zstack.cbd.AddonInfo import org.zstack.core.cloudbus.EventCallback import org.zstack.core.cloudbus.EventFacade import org.zstack.core.db.Q +import org.zstack.header.errorcode.ErrorCode +import org.zstack.header.errorcode.OperationFailureException import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_ import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO @@ -171,6 +173,7 @@ class ZbsPrimaryStorageCase extends SubCase { testMdsPing() testCheckHostStorageConnection() testNegativeScenario() + testDeleteVolumeTriesNextMdsAfterSyncFailure() testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor() testDataVolumeNegativeScenario() testDecodeMdsUriWithSpecialPassword() @@ -466,6 +469,35 @@ class ZbsPrimaryStorageCase extends SubCase { deleteVolume(vol.uuid) } + void testDeleteVolumeTriesNextMdsAfterSyncFailure() { + AtomicInteger getVolumeClientsCallCount = new AtomicInteger(0) + List getVolumeClientsMds = Collections.synchronizedList(new ArrayList<>()) + env.simulator(ZbsStorageController.GET_VOLUME_CLIENTS_PATH) { HttpEntity e, EnvSpec spec -> + def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.GetVolumeClientsCmd.class) + getVolumeClientsMds.add(cmd.addr) + if (getVolumeClientsCallCount.incrementAndGet() == 1) { + throw new OperationFailureException(new ErrorCode("TEST.1000", "test error", "simulated MDS network failure")) + } + + return new ZbsStorageController.GetVolumeClientsRsp() + } + + VolumeInventory volume = createDataVolume { + name = "delete-after-mds-sync-failure" + diskOfferingUuid = diskOffering.uuid + primaryStorageUuid = ps.uuid + } as VolumeInventory + + deleteVolume(volume.uuid) + + assert getVolumeClientsCallCount.get() == 2 : + "deleteVolume expunge should retry GET_VOLUME_CLIENTS on next MDS after sync failure: expectedCalls=2 actualCalls=${getVolumeClientsCallCount.get()} mds=${getVolumeClientsMds}" + assert getVolumeClientsMds.toSet().size() == 2 : + "deleteVolume expunge should use two different MDS nodes after first sync failure: expectedDistinctMds=2 actualMds=${getVolumeClientsMds}" + + env.cleanSimulatorHandlers() + } + void testNegativeScenario() { expect(AssertionError.class) { addExternalPrimaryStorage {