Draft: <fix>[zbs]: ZSTAC-86782 backport MDS failover

plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsMdsBase.java

@@ -12,6 +12,7 @@
 import org.zstack.header.errorcode.OperationFailureException;
 import org.zstack.header.rest.JsonAsyncRESTCallback;
 import org.zstack.header.rest.RESTFacade;
+import org.zstack.utils.gson.JSONObjectUtil;
 import org.zstack.utils.ssh.Ssh;
 import org.zstack.utils.ssh.SshException;
 import org.zstack.utils.ssh.SshResult;
@@ -77,7 +78,13 @@ protected void checkStorageHealth() {
     }
 
     public <T extends AgentResponse> T syncCall(final String path, final Object cmd, final Class<T> retClass, TimeUnit unit, long timeout) {
-        return restf.syncJsonPost(makeHttpPath(self.getAddr(), path), cmd, retClass, unit, timeout);
+        try {
+            return restf.syncJsonPost(makeHttpPath(self.getAddr(), path), cmd, retClass, unit, timeout);
+        } catch (OperationFailureException e) {
+            T ret = JSONObjectUtil.toObject("{}", retClass);
+            ret.setError(e.getErrorCode().getDetails());
+            return ret;
+        }
     }
 
     public <T extends AgentResponse> void httpCall(final String path, final Object cmd, final Class<T> retClass, final ReturnValueCompletion<T> completion) {

plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsStorageController.java

@@ -168,7 +168,10 @@ public List<ActiveVolumeClient> getActiveClients(String installPath, String prot
         if (VolumeProtocol.CBD.toString().equals(protocol)) {
             GetVolumeClientsCmd cmd = new GetVolumeClientsCmd();
             cmd.setPath(installPath);
-            GetVolumeClientsRsp rsp = syncHttpCall(GET_VOLUME_CLIENTS_PATH, cmd, GetVolumeClientsRsp.class);
+            GetVolumeClientsRsp rsp = new HttpCaller<>(GET_VOLUME_CLIENTS_PATH, cmd, GetVolumeClientsRsp.class,
+                    null, TimeUnit.SECONDS, 30, true)
+                    .setTryNext(true)
+                    .syncCall();
             List<ActiveVolumeClient> clients = new ArrayList<>();
 
             if (!rsp.isSuccess()) {
@@ -1314,6 +1317,11 @@ public class HttpCaller<T extends AgentResponse> {
 
         private boolean tryNext = false;
 
+        HttpCaller<T> setTryNext(boolean tryNext) {
+            this.tryNext = tryNext;
+            return this;
+        }
+
         public HttpCaller(String path, AgentCommand cmd, Class<T> retClass, ReturnValueCompletion<T> callback) {
             this(path, cmd, retClass, callback, null, 0, false);
         }

test/src/test/groovy/org/zstack/test/integration/storage/primary/addon/zbs/ZbsPrimaryStorageCase.groovy

@@ -5,6 +5,8 @@ import org.zstack.cbd.AddonInfo
 import org.zstack.core.cloudbus.EventCallback
 import org.zstack.core.cloudbus.EventFacade
 import org.zstack.core.db.Q
+import org.zstack.header.errorcode.ErrorCode
+import org.zstack.header.errorcode.OperationFailureException
 import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO
 import org.zstack.header.storage.addon.primary.ExternalPrimaryStorageVO_
 import org.zstack.header.storage.addon.primary.PrimaryStorageOutputProtocolRefVO
@@ -171,6 +173,7 @@ class ZbsPrimaryStorageCase extends SubCase {
             testMdsPing()
             testCheckHostStorageConnection()
             testNegativeScenario()
+            testDeleteVolumeTriesNextMdsAfterSyncFailure()
             testAddExternalPrimaryStorageWithMalformedJsonRejectedByInterceptor()
             testDataVolumeNegativeScenario()
             testDecodeMdsUriWithSpecialPassword()
@@ -466,6 +469,35 @@ class ZbsPrimaryStorageCase extends SubCase {
         deleteVolume(vol.uuid)
     }
 
+    void testDeleteVolumeTriesNextMdsAfterSyncFailure() {
+        AtomicInteger getVolumeClientsCallCount = new AtomicInteger(0)
+        List<String> getVolumeClientsMds = Collections.synchronizedList(new ArrayList<>())
+        env.simulator(ZbsStorageController.GET_VOLUME_CLIENTS_PATH) { HttpEntity<String> e, EnvSpec spec ->
+            def cmd = JSONObjectUtil.toObject(e.body, ZbsStorageController.GetVolumeClientsCmd.class)
+            getVolumeClientsMds.add(cmd.addr)
+            if (getVolumeClientsCallCount.incrementAndGet() == 1) {
+                throw new OperationFailureException(new ErrorCode("TEST.1000", "test error", "simulated MDS network failure"))
+            }
+
+            return new ZbsStorageController.GetVolumeClientsRsp()
+        }
+
+        VolumeInventory volume = createDataVolume {
+            name = "delete-after-mds-sync-failure"
+            diskOfferingUuid = diskOffering.uuid
+            primaryStorageUuid = ps.uuid
+        } as VolumeInventory
+
+        deleteVolume(volume.uuid)
+
+        assert getVolumeClientsCallCount.get() == 2 :
+                "deleteVolume expunge should retry GET_VOLUME_CLIENTS on next MDS after sync failure: expectedCalls=2 actualCalls=${getVolumeClientsCallCount.get()} mds=${getVolumeClientsMds}"
+        assert getVolumeClientsMds.toSet().size() == 2 :
+                "deleteVolume expunge should use two different MDS nodes after first sync failure: expectedDistinctMds=2 actualMds=${getVolumeClientsMds}"
+
+        env.cleanSimulatorHandlers()
+    }
+
     void testNegativeScenario() {
         expect(AssertionError.class) {
             addExternalPrimaryStorage {