<fix>[core]: register IPv6 prefer config

core/src/main/java/org/zstack/core/CoreGlobalProperty.java

@@ -79,9 +79,6 @@ public class CoreGlobalProperty {
     public static List<String> CHRONY_SERVERS;
     @GlobalProperty(name="management.server.vip")
     public static String MN_VIP;
-    @GlobalProperty(name = "management.server.prefer.ipv6", defaultValue = "false")
-    @AvailableValues(value = {"true", "false"})
-    public static boolean MANAGEMENT_SERVER_PREFER_IPV6;
     @GlobalProperty(name = "simulatorsOn", defaultValue = "false")
     public static boolean SIMULATORS_ON;
     @GlobalProperty(name = "startMode", defaultValue = "")

core/src/main/java/org/zstack/core/Platform.java

@@ -83,7 +83,6 @@ public class Platform {
     private static MessageSource messageSource;
     private static String encryptionKey = EncryptRSA.generateKeyString("ZStack open source");
     private static final String MANAGEMENT_SERVER_IP_PROPERTY = "management.server.ip";
-    private static final String MANAGEMENT_SERVER_PREFER_IPV6_PROPERTY = "management.server.prefer.ipv6";
     private static final String ZSTACK_MANAGEMENT_SERVER_IP_ENV = "ZSTACK_MANAGEMENT_SERVER_IP";
     private static final String IPV4_ADDRESS_COMMAND = "ip -4 add";
     private static final String IPV6_ADDRESS_COMMAND = "ip -6 addr";
@@ -994,7 +993,7 @@ private static String getManagementServerIpInternal() {
             for (NetworkInterface iface : Collections.list(nets)) {
                 String name = iface.getName();
                 if (defaultLine.contains(name)) {
-                    ip = selectManagementServerIp(Collections.list(iface.getInetAddresses()), isManagementServerPreferIpv6());
+                    ip = selectManagementServerIp(Collections.list(iface.getInetAddresses()));
                 }
             }
         } catch (SocketException e) {
@@ -1070,7 +1069,74 @@ public static List<String> getManagementServerIps() {
         return new ArrayList<>(ips);
     }
 
-    public static String selectManagementServerIp(Collection<InetAddress> addresses, boolean preferIpv6) {
+    public static List<String> getManagementServerIpsWithLocalFallback() {
+        LinkedHashSet<String> ips = new LinkedHashSet<>(getManagementServerIps());
+        ips.addAll(getLocalNonLoopbackIps());
+        ips.remove(null);
+        return new ArrayList<>(ips);
+    }
+
+    private static List<String> getLocalNonLoopbackIps() {
+        List<String> ips = new ArrayList<>();
+        try {
+            Enumeration<NetworkInterface> nets = NetworkInterface.getNetworkInterfaces();
+            for (NetworkInterface iface : Collections.list(nets)) {
+                if (!iface.isUp()) {
+                    continue;
+                }
+                for (InetAddress address : Collections.list(iface.getInetAddresses())) {
+                    if (address.isLoopbackAddress() || address.isLinkLocalAddress()) {
+                        continue;
+                    }
+                    ips.add(normalizeManagementIp(address.getHostAddress()));
+                }
+            }
+        } catch (SocketException e) {
+            logger.warn("failed to list local non-loopback IPs", e);
+        }
+        return ips;
+    }
+
+    public static String getRouteSourceIp(String remoteIp) {
+        if (StringUtils.isBlank(remoteIp)) {
+            return null;
+        }
+
+        remoteIp = normalizeManagementIp(remoteIp);
+        String family;
+        if (IPv6NetworkUtils.isIpv6Address(remoteIp)) {
+            family = "-6";
+        } else if (NetworkUtils.isIpv4Address(remoteIp)) {
+            family = "-4";
+        } else {
+            return null;
+        }
+
+        Linux.ShellResult ret = Linux.shell(String.format("/sbin/ip %s route get %s", family, remoteIp));
+        if (ret.getExitCode() != 0) {
+            logger.warn(String.format("failed to get route source IP for remote[%s], stdout[%s], stderr[%s]",
+                    remoteIp, ret.getStdout(), ret.getStderr()));
+            return null;
+        }
+
+        String[] tokens = ret.getStdout().trim().split("\\s+");
+        for (int i = 0; i < tokens.length - 1; i++) {
+            if (!"src".equals(tokens[i])) {
+                continue;
+            }
+            String sourceIp = normalizeManagementIp(tokens[i + 1]);
+            if (IPv6NetworkUtils.isIpv6Address(remoteIp) && IPv6NetworkUtils.isIpv6Address(sourceIp)) {
+                return sourceIp;
+            }
+            if (NetworkUtils.isIpv4Address(remoteIp) && NetworkUtils.isIpv4Address(sourceIp)) {
+                return sourceIp;
+            }
+        }
+
+        return null;
+    }
+
+    public static String selectManagementServerIp(Collection<InetAddress> addresses) {
         String ipv4 = null;
         String ipv6 = null;
 
@@ -1087,22 +1153,9 @@ public static String selectManagementServerIp(Collection<InetAddress> addresses,
             }
         }
 
-        if (preferIpv6 && ipv6 != null) {
-            return ipv6;
-        }
-
         return ipv4 != null ? ipv4 : ipv6;
     }
 
-    public static boolean isManagementServerPreferIpv6() {
-        String propertyValue = System.getProperty(MANAGEMENT_SERVER_PREFER_IPV6_PROPERTY);
-        if (propertyValue != null) {
-            return Boolean.parseBoolean(propertyValue);
-        }
-
-        return CoreGlobalProperty.MANAGEMENT_SERVER_PREFER_IPV6;
-    }
-
     public static String formatJGroupsInitialHosts(String nodeIp, String peerIp, int port) {
         return String.format(JGROUPS_INITIAL_HOST_FORMAT,
                 IPv6NetworkUtils.formatHostForUrl(nodeIp), port,

core/src/main/java/org/zstack/core/ansible/CallBackNetworkChecker.java

@@ -10,6 +10,7 @@
 import org.zstack.utils.ssh.SshCmdHelper;
 import org.zstack.utils.ssh.SshException;
 import org.zstack.utils.ssh.SshResult;
+import org.zstack.utils.network.IPv6NetworkUtils;
 
 import static org.zstack.core.Platform.operr;
 import static org.zstack.utils.StringDSL.ln;
@@ -30,8 +31,11 @@ public class CallBackNetworkChecker implements AnsibleChecker {
     private String callbackIp = Platform.getManagementServerIp();
     private int callBackPort = Platform.getManagementNodeServicePort();
 
-    private static StringDSL.StringWrapper script = ln(
-            "cat /dev/null | nc {2} {1} || echo {0} | sudo -S nmap -sS -P0 -n -p {1} {2} 2>/dev/null | grep \"1 host up\""
+    private static final String EMPTY_COMMAND_OPTION = "";
+    private static final String IPV6_COMMAND_OPTION = "-6 ";
+    private static final String HOST_UP_PATTERN = "1 host up";
+    private static final StringDSL.StringWrapper CALLBACK_CHECK_SCRIPT = ln(
+            "cat /dev/null | nc {3}{2} {1} || echo {0} | sudo -S nmap {4}-sS -P0 -n -p {1} {2} 2>/dev/null | grep \"{5}\""
     );
 
     @Override
@@ -49,7 +53,7 @@ public void deleteDestFile() {
      * if failed, use nmap to try again.
      */
     private ErrorCode useNcatAndNmapToTestConnection(Ssh ssh) {
-        String srcScript = script.format(SshCmdHelper.shellQuote(password), callBackPort, callbackIp);
+        String srcScript = buildCallbackCheckScript(SshCmdHelper.shellQuote(password), callBackPort, callbackIp);
 
         ssh.sudoCommand(srcScript);
         SshResult ret = ssh.run();
@@ -58,6 +62,13 @@ private ErrorCode useNcatAndNmapToTestConnection(Ssh ssh) {
         return null;
     }
 
+    public static String buildCallbackCheckScript(String password, int port, String callbackIp) {
+        String callbackHost = IPv6NetworkUtils.stripHostUrlBrackets(callbackIp);
+        String ipVersionOption = IPv6NetworkUtils.isIpv6Address(callbackHost) ? IPV6_COMMAND_OPTION : EMPTY_COMMAND_OPTION;
+
+        return CALLBACK_CHECK_SCRIPT.format(password, port, callbackHost, ipVersionOption, ipVersionOption, HOST_UP_PATTERN);
+    }
+
     @Override
     public ErrorCode stopAnsible() {
         if (CoreGlobalProperty.UNIT_TEST_ON || !AnsibleGlobalConfig.CHECK_MANAGEMENT_CALLBACK.value(Boolean.class)) {

network/src/main/java/org/zstack/network/l3/L3NetworkApiInterceptor.java

@@ -445,10 +445,6 @@ private void validateIpv6Range(IpRangeInventory ipr) {
 
         L3NetworkVO l3Vo = Q.New(L3NetworkVO.class).eq(L3NetworkVO_.uuid, ipr.getL3NetworkUuid()).find();
 
-        if (l3Vo.getCategory().equals(L3NetworkCategory.System)) {
-            throw new ApiMessageInterceptionException(argerr(ORG_ZSTACK_NETWORK_L3_10034, "can not add ip range, because system network doesn't support ipv6 yet"));
-        }
-
         List<NormalIpRangeVO> rangeVOS = Q.New(NormalIpRangeVO.class).eq(NormalIpRangeVO_.l3NetworkUuid, ipr.getL3NetworkUuid()).eq(NormalIpRangeVO_.ipVersion, IPv6Constants.IPv6).list();
         if (rangeVOS != null && !rangeVOS.isEmpty()) {
             if (!rangeVOS.get(0).getAddressMode().equals(ipr.getAddressMode())) {

plugin/applianceVm/src/main/java/org/zstack/appliancevm/ApplianceVmFacadeImpl.java

@@ -467,7 +467,7 @@ public Map<String, Object> prepareBootstrapInformation(VmInstanceSpec spec) {
         ret.put(ApplianceVmConstant.BootstrapParams.publicKey.toString(), publicKey);
         ret.put(BootstrapParams.uuid.toString(), spec.getVmInventory().getUuid());
         putManagementNodeBootstrapParams(ret,
-                Platform.getManagementServerIps(),
+                Platform.getManagementServerIpsWithLocalFallback(),
                 getVrManagementCidrs(mgmtNic),
                 Platform.getManagementServerIp(),
                 Platform.getManagementServerVip(),

plugin/ceph/src/main/java/org/zstack/storage/ceph/primary/CephPrimaryStorageBase.java

@@ -5869,7 +5869,7 @@ public MonMigrateIpInfo(String psUuid) {
                 final String extraIps = CephMonSystemTags.EXTRA_IPS
                         .getTokenByResourceUuid(mon.getUuid(), CephMonSystemTags.EXTRA_IPS_TOKEN);
                 Optional.ofNullable(extraIps).ifPresent(it -> ips.addAll(Arrays.asList(it.split(","))));
-                List<String> cidrIps = NetworkUtils.filterIpv4sInCidr(ips, migrateCidr);
+                List<String> cidrIps = NetworkUtils.filterIpsInCidr(ips, migrateCidr);
                 if (!cidrIps.isEmpty()) {
                     monMigrateIpMap.put(mon.getUuid(), cidrIps.get(0));
                 }

plugin/kvm/src/main/java/org/zstack/kvm/KVMHost.java

@@ -2726,10 +2726,10 @@ public void done() {
                 );
     }
 
-    private String buildUrl(String path) {
+    public static String buildAgentUrl(String host, String path) {
         UriComponentsBuilder ub = UriComponentsBuilder.newInstance();
         ub.scheme(KVMGlobalProperty.AGENT_URL_SCHEME);
-        ub.host(KVMHostUtils.formatHostForUrl(self.getManagementIp()));
+        ub.host(KVMHostUtils.formatHostForUrl(host));
         ub.port(KVMGlobalProperty.AGENT_PORT);
         if (!"".equals(KVMGlobalProperty.AGENT_URL_ROOT_PATH)) {
             ub.path(KVMGlobalProperty.AGENT_URL_ROOT_PATH);
@@ -2738,6 +2738,10 @@ private String buildUrl(String path) {
         return ub.build().toUriString();
     }
 
+    private String buildUrl(String path) {
+        return buildAgentUrl(self.getManagementIp(), path);
+    }
+
     private void executeAsyncHttpCall(final KVMHostAsyncHttpCallMsg msg, final NoErrorCompletion completion) {
         if (!msg.isNoStatusCheck()) {
             checkStatus();
@@ -3154,10 +3158,7 @@ public void run(final FlowTrigger trigger, Map data) {
                         CleanVmFirmwareFlashCmd cmd = new CleanVmFirmwareFlashCmd();
                         cmd.vmUuid = vmUuid;
 
-                        UriComponentsBuilder ub = UriComponentsBuilder.fromHttpUrl(baseUrl);
-                        ub.host(dstHostMnIp);
-                        ub.path(KVMConstant.CLEAN_FIRMWARE_FLASH);
-                        String url = ub.build().toString();
+                        String url = buildAgentUrl(dstHostMnIp, KVMConstant.CLEAN_FIRMWARE_FLASH);
                         new Http<>(url, cmd, AgentResponse.class).call(dstHostUuid, new ReturnValueCompletion<AgentResponse>(trigger) {
                             @Override
                             public void success(AgentResponse ret) {
@@ -3227,9 +3228,9 @@ protected void scripts() {
                             cmd.setDisks(diskMigrationMap);
                         }
 
-                        UriComponentsBuilder ub = UriComponentsBuilder.fromHttpUrl(migrateVmPath);
-                        ub.host(migrateFromDestination ? dstHostMnIp : srcHostMnIp);
-                        String migrateUrl = ub.build().toString();
+                        String migrateUrl = buildAgentUrl(
+                                migrateFromDestination ? dstHostMnIp : srcHostMnIp,
+                                KVMConstant.KVM_MIGRATE_VM_PATH);
                         new Http<>(migrateUrl, cmd, MigrateVmResponse.class).call(migrateFromDestination ? dstHostUuid : srcHostUuid, new ReturnValueCompletion<MigrateVmResponse>(trigger) {
                             @Override
                             public void success(MigrateVmResponse ret) {
@@ -3268,10 +3269,7 @@ public void run(final FlowTrigger trigger, Map data) {
                         cmd.vmUuid = vmUuid;
                         cmd.hostManagementIp = dstHostMnIp;
 
-                        UriComponentsBuilder ub = UriComponentsBuilder.fromHttpUrl(baseUrl);
-                        ub.host(dstHostMnIp);
-                        ub.path(KVMConstant.KVM_HARDEN_CONSOLE_PATH);
-                        String url = ub.build().toString();
+                        String url = buildAgentUrl(dstHostMnIp, KVMConstant.KVM_HARDEN_CONSOLE_PATH);
                         new Http<>(url, cmd, AgentResponse.class).call(dstHostUuid, new ReturnValueCompletion<AgentResponse>(trigger) {
                             @Override
                             public void success(AgentResponse ret) {
@@ -3304,10 +3302,7 @@ public void run(final FlowTrigger trigger, Map data) {
                         cmd.vmUuid = vmUuid;
                         cmd.hostManagementIp = srcHostMnIp;
 
-                        UriComponentsBuilder ub = UriComponentsBuilder.fromHttpUrl(baseUrl);
-                        ub.host(srcHostMnIp);
-                        ub.path(KVMConstant.KVM_DELETE_CONSOLE_FIREWALL_PATH);
-                        String url = ub.build().toString();
+                        String url = buildAgentUrl(srcHostMnIp, KVMConstant.KVM_DELETE_CONSOLE_FIREWALL_PATH);
                         new Http<>(url, cmd, AgentResponse.class).call(new ReturnValueCompletion<AgentResponse>(trigger) {
                             @Override
                             public void success(AgentResponse ret) {

plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterApiInterceptor.java

@@ -212,11 +212,6 @@ private void validate(APICreateVirtualRouterOfferingMsg msg) {
             throw new ApiMessageInterceptionException(argerr(ORG_ZSTACK_NETWORK_SERVICE_VIRTUALROUTER_10030, "management network[uuid:%s] is not in the same zone[uuid:%s] this offering is going to create",
                             msg.getManagementNetworkUuid(), msg.getZoneUuid()));
         }
-        /* mgt network does not support ipv6 yet, TODO, will be implemented soon */
-        if (mgtL3.getIpVersions().contains(IPv6Constants.IPv6) && !mgtL3.getIpVersions().contains(IPv6Constants.IPv4)) {
-            throw new ApiMessageInterceptionException(argerr(ORG_ZSTACK_NETWORK_SERVICE_VIRTUALROUTER_10031, "can not create virtual router offering, because management network doesn't support ipv6 yet"));
-        }
-
         if (!CoreGlobalProperty.UNIT_TEST_ON) {
             checkIfManagementNetworkReachable(msg.getManagementNetworkUuid());
         }

plugin/virtualRouterProvider/src/main/java/org/zstack/network/service/virtualrouter/VirtualRouterManagerImpl.java

@@ -960,7 +960,7 @@ public String buildUrl(String mgmtNicIp, String subPath) {
         if (CoreGlobalProperty.UNIT_TEST_ON) {
             ub.host("localhost");
         } else {
-            ub.host(mgmtNicIp);
+            ub.host(IPv6NetworkUtils.formatHostForUrl(mgmtNicIp));
         }
 
         ub.port(VirtualRouterGlobalProperty.AGENT_PORT);
@@ -978,6 +978,11 @@ public Map<String, String> buildAgentCallbackUrlHeaders(String mgmtNicIp) {
     }
 
     private String selectManagementIpForAgent(String agentIp) {
+        String routeSourceIp = Platform.getRouteSourceIp(agentIp);
+        if (routeSourceIp != null) {
+            return routeSourceIp;
+        }
+
         if (IPv6NetworkUtils.isIpv6Address(agentIp)) {
             return Platform.getManagementServerIps().stream()
                     .filter(IPv6NetworkUtils::isIpv6Address)

plugin/zbs/src/main/java/org/zstack/storage/zbs/MdsUri.java

@@ -5,6 +5,7 @@
 import org.zstack.header.errorcode.ErrorCode;
 import org.zstack.header.errorcode.OperationFailureException;
 import org.zstack.header.exception.CloudRuntimeException;
+import org.zstack.utils.network.IPv6NetworkUtils;
 
 import java.net.URI;
 import java.net.URISyntaxException;
@@ -26,7 +27,7 @@ public class MdsUri {
     private String username;
     private String password;
 
-    private static final String MDS_URL_FORMAT = "sshUsername:sshPassword@hostname:[sshPort]/?[mdsPort=]";
+    private static final String MDS_URL_FORMAT = "sshUsername:sshPassword@hostname[:sshPort]/?[mdsPort=], IPv6 hostname must be bracketed";
     private static final Integer DEFAULT_MDS_PORT = 6666;
     private static final Integer DEFAULT_SSH_PORT = 22;
 
@@ -80,7 +81,7 @@ public MdsUri(String url) {
             password = ssh[1];
 
             URI uri = new URI(String.format("ssh://%s", rest));
-            hostname = uri.getHost();
+            hostname = IPv6NetworkUtils.stripHostUrlBrackets(uri.getHost());
             if (hostname == null) {
                 throw new OperationFailureException(operr(ORG_ZSTACK_STORAGE_ZBS_10004, "invalid mdsUrl[%s], hostname cannot be null. A valid mdsUrl is" +
                                 " in format of %s", url, MDS_URL_FORMAT)

plugin/zbs/src/main/java/org/zstack/storage/zbs/ZbsAgentUrl.java

@@ -1,21 +1,27 @@
 package org.zstack.storage.zbs;
 
-import org.springframework.web.util.UriComponentsBuilder;
+import org.zstack.utils.network.IPv6NetworkUtils;
 
 /**
  * @author Xingwei Yu
  * @date 2024/3/27 17:39
  */
 public class ZbsAgentUrl {
-    public static String primaryStorageUrl(String ip, String path) {
-        UriComponentsBuilder ub = UriComponentsBuilder.newInstance();
-        ub.scheme("http");
-        ub.host(ip);
-        ub.port(ZbsGlobalProperty.PRIMARY_STORAGE_AGENT_PORT);
-        if (!"".equals(ZbsGlobalProperty.PRIMARY_STORAGE_AGENT_URL_ROOT_PATH)) {
-            ub.path(ZbsGlobalProperty.PRIMARY_STORAGE_AGENT_URL_ROOT_PATH);
+    private static void appendPath(StringBuilder sb, String path) {
+        if (path == null || path.isEmpty()) {
+            return;
+        }
+
+        if (!path.startsWith("/")) {
+            sb.append("/");
         }
-        ub.path(path);
-        return ub.build().toUriString();
+        sb.append(path);
+    }
+
+    public static String primaryStorageUrl(String ip, String path) {
+        StringBuilder sb = new StringBuilder(IPv6NetworkUtils.buildHttpUrl(ip, ZbsGlobalProperty.PRIMARY_STORAGE_AGENT_PORT));
+        appendPath(sb, ZbsGlobalProperty.PRIMARY_STORAGE_AGENT_URL_ROOT_PATH);
+        appendPath(sb, path);
+        return sb.toString();
     }
 }