USHIFT-6808: C2CC Chaos Tests (Restarting Services, toggling ethernet)

test/bin/c2cc_common.sh

@@ -111,7 +111,7 @@ c2cc_create_vms() {
     local -r boot_commit_ref="${1}"
     local -r boot_blueprint="${2}"
     local -r network="${3:-default}"
-    local -r ip_family="${4:-ipv4}" 
+    local -r ip_family="${4:-ipv4}"
 
     # Prepare kickstart for all hosts
     local ipv6_args=""
@@ -161,6 +161,12 @@ c2cc_run_tests() {
     local -r suites_dir="${1}"
     local -r foreign_cidr="${2:-}"
     local -r ip_family="${3:-}"
+    local -r include_tags="${4:-c2cc}"
+
+    local include_args=""
+    for tag in ${include_tags}; do
+        include_args+="--include ${tag} "
+    done
 
     local foreign_cidr_var=""
     if [ -n "${foreign_cidr}" ]; then
@@ -177,15 +183,16 @@ c2cc_run_tests() {
         target_ref_var="--variable TARGET_REF:${C2CC_TARGET_REF}"
     fi
 
+    local host2_vm host3_vm
+    host2_vm=$(full_vm_name host2) || return 1
+    host3_vm=$(full_vm_name host3) || return 1
+
     local host2_ip host3_ip
     host2_ip=$(get_host_ip host2) || return 1
     host3_ip=$(get_host_ip host3) || return 1
     readonly host2_ip host3_ip
 
-    # Retrieve host2's kubeconfig
     local -r kubeconfig_b="${SCENARIO_INFO_DIR}/${SCENARIO}/kubeconfig-b"
-
-    # Retrieve host3's kubeconfig
     local -r kubeconfig_c="${SCENARIO_INFO_DIR}/${SCENARIO}/kubeconfig-c"
 
     # Wait for host2 and host3 to be fully ready (run_tests only waits for host1)
@@ -210,10 +217,13 @@ c2cc_run_tests() {
         --variable "CLUSTER_C_SVC_CIDR:${CLUSTER_C_SVC_CIDR}" \
         --variable "CLUSTER_C_DOMAIN:${CLUSTER_C_DOMAIN}" \
         --variable "KUBECONFIG_C:${kubeconfig_c}" \
+        --variable "HOST2_VM_NAME:${host2_vm}" \
+        --variable "HOST3_VM_NAME:${host3_vm}" \
         ${foreign_cidr_var} \
         ${ip_family_var} \
         ${target_ref_var} \
         --variable "BOOTC_REGISTRY:${MIRROR_REGISTRY_URL}" \
+        ${include_args} \
         "${suites_dir}"
 }
 

test/resources/c2cc.resource

@@ -8,6 +8,7 @@ Library             Process
 Library             String
 Library             SSHLibrary
 Resource            common.resource
+Resource            libvirt.resource
 Resource            microshift-host.resource
 
 
@@ -39,6 +40,15 @@ ${IP_CMD}                   ${{'ip -6' if '${IP_FAMILY}' == 'ipv6' else 'ip -4'}
 ...                         cluster-a=${CLUSTER_A_DOMAIN}
 ...                         cluster-b=${CLUSTER_B_DOMAIN}
 ...                         cluster-c=${CLUSTER_C_DOMAIN}
+&{POD_CIDR_MAP}
+...                         cluster-a=${CLUSTER_A_POD_CIDR}
+...                         cluster-b=${CLUSTER_B_POD_CIDR}
+...                         cluster-c=${CLUSTER_C_POD_CIDR}
+&{SVC_CIDR_MAP}
+...                         cluster-a=${CLUSTER_A_SVC_CIDR}
+...                         cluster-b=${CLUSTER_B_SVC_CIDR}
+...                         cluster-c=${CLUSTER_C_SVC_CIDR}
+@{ALL_CLUSTERS}             cluster-a    cluster-b    cluster-c
 
 
 *** Keywords ***
@@ -162,19 +172,28 @@ Verify IP Rules For Table 200
     Should Contain    ${stdout}    to ${remote_pod_cidr} lookup 200
     Should Contain    ${stdout}    to ${remote_svc_cidr} lookup 200
 
-Verify Routes In Table 201
-    [Documentation]    Check that service routes exist in table 201 for the local service CIDR.
-    [Arguments]    ${alias}    ${local_svc_cidr}
-    ${stdout}=    Command On Cluster    ${alias}    ${IP_CMD} route show table 201
-    Should Contain    ${stdout}    ${local_svc_cidr}
-
 Verify Service IP Rules
     [Documentation]    Check that IP rules at priority 99 exist for cross-cluster service routing.
     [Arguments]    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}    ${local_svc_cidr}
     ${stdout}=    Command On Cluster    ${alias}    ${IP_CMD} rule show
     Should Contain    ${stdout}    from ${remote_pod_cidr} to ${local_svc_cidr} lookup 201
     Should Contain    ${stdout}    from ${remote_svc_cidr} to ${local_svc_cidr} lookup 201
 
+Verify All IP Rules
+    [Documentation]    Check all IP rules (table 200 and 201) in a single SSH call.
+    [Arguments]    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}    ${local_svc_cidr}
+    ${stdout}=    Command On Cluster    ${alias}    ${IP_CMD} rule show
+    Should Contain    ${stdout}    to ${remote_pod_cidr} lookup 200
+    Should Contain    ${stdout}    to ${remote_svc_cidr} lookup 200
+    Should Contain    ${stdout}    from ${remote_pod_cidr} to ${local_svc_cidr} lookup 201
+    Should Contain    ${stdout}    from ${remote_svc_cidr} to ${local_svc_cidr} lookup 201
+
+Verify Routes In Table 201
+    [Documentation]    Check that service routes exist in table 201 for the local service CIDR.
+    [Arguments]    ${alias}    ${local_svc_cidr}
+    ${stdout}=    Command On Cluster    ${alias}    ${IP_CMD} route show table 201
+    Should Contain    ${stdout}    ${local_svc_cidr}
+
 Verify NFTables Bypass Rules
     [Documentation]    Check that nftables masquerade bypass rules exist for remote CIDRs.
     [Arguments]    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}
@@ -393,19 +412,6 @@ DNS Lookup Should Succeed
     ...    oc exec curl-pod -n ${NAMESPACES}[${alias}] -- getent hosts ${fqdn}
     Should Not Be Empty    ${stdout}
 
-Verify RemoteCluster State
-    [Documentation]    Check that all RemoteCluster CRs on this cluster have the expected state.
-    [Arguments]    ${alias}    ${expected_state}
-    ${stdout}=    Oc On Cluster    ${alias}
-    ...    oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}'
-    Should Not Be Empty    ${stdout}
-    @{states}=    Split String    ${stdout}
-    ${count}=    Get Length    ${states}
-    Should Be Equal As Integers    ${count}    2    Expected 2 RemoteCluster states, got ${count}
-    FOR    ${state}    IN    @{states}
-        Should Be Equal As Strings    ${state}    ${expected_state}
-    END
-
 Verify All RemoteClusters Healthy
     [Documentation]    Wait for all RemoteCluster CRs on all clusters to report Healthy.
     FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
@@ -434,3 +440,142 @@ Curl Remote Service Via DNS
     ${stdout}=    Curl DNS From Cluster    ${source}
     ...    hello-microshift.${NAMESPACES}[${destination}].svc.${DOMAIN_MAP}[${destination}]    8080
     Should Contain    ${stdout}    Hello from
+
+Verify RemoteCluster State
+    [Documentation]    Check that all RemoteCluster CRs on this cluster have the expected state.
+    [Arguments]    ${alias}    ${expected_state}
+    ${stdout}=    Oc On Cluster    ${alias}
+    ...    oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.state}'
+    Should Not Be Empty    ${stdout}
+    @{states}=    Split String    ${stdout}
+    ${count}=    Get Length    ${states}
+    ${expected_count}=    Evaluate    len($ALL_CLUSTERS) - 1
+    Should Be Equal As Integers    ${count}    ${expected_count}
+    ...    Expected ${expected_count} RemoteCluster states, got ${count}
+    FOR    ${state}    IN    @{states}
+        Should Be Equal As Strings    ${state}    ${expected_state}
+    END
+
+Verify RemoteCluster State By Name
+    [Documentation]    Check that a specific RemoteCluster CR has the expected state.
+    [Arguments]    ${alias}    ${cr_name}    ${expected_state}
+    ${stdout}=    Oc On Cluster
+    ...    ${alias}
+    ...    oc get remoteclusters.microshift.io ${cr_name} -o jsonpath='{.status.state}'
+    Should Be Equal As Strings    ${stdout}    ${expected_state}
+
+RemoteCluster CR Name From IP
+    [Documentation]    Compute the RemoteCluster CR name from a host IP (e.g. 192.168.1.2 -> c2cc-192-168-1-2).
+    [Arguments]    ${ip}
+    ${dashed}=    Replace String    ${ip}    .    -
+    ${dashed}=    Replace String    ${dashed}    :    -
+    RETURN    c2cc-${dashed}
+
+Verify RemoteCluster Unhealthy On Observers
+    [Documentation]    Wait for observer clusters to report a disrupted cluster's CR as Unhealthy.
+    [Arguments]    ${disrupted_ip}    @{observers}
+    ${cr_name}=    RemoteCluster CR Name From IP    ${disrupted_ip}
+    FOR    ${observer}    IN    @{observers}
+        Wait Until Keyword Succeeds    3m    10s
+        ...    Verify RemoteCluster State By Name    ${observer}    ${cr_name}    Unhealthy
+    END
+
+Ensure All Clusters Healthy
+    [Documentation]    Pre-condition: all clusters must be Healthy before fault injection.
+    FOR    ${alias}    IN    cluster-a    cluster-b    cluster-c
+        Wait Until Keyword Succeeds    3m    10s
+        ...    Verify RemoteCluster State    ${alias}    Healthy
+    END
+
+Verify C2CC Infrastructure For Remote
+    [Documentation]    Verify C2CC infrastructure for one remote cluster's CIDRs.
+    [Arguments]    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}    ${local_svc_cidr}
+    Verify Routes In Table 200    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}
+    Verify All IP Rules    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}    ${local_svc_cidr}
+    Verify NFTables Bypass Rules    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}
+    Verify OVN Static Routes    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}
+    Verify Node SNAT Annotation    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}
+
+Verify C2CC Infrastructure On Cluster
+    [Documentation]    Run all C2CC infrastructure checks for all remote CIDRs on the given cluster.
+    [Arguments]    ${alias}
+    ${local_svc_cidr}=    Get From Dictionary    ${SVC_CIDR_MAP}    ${alias}
+    FOR    ${remote}    IN    @{ALL_CLUSTERS}
+        IF    '${remote}' != '${alias}'
+            ${remote_pod_cidr}=    Get From Dictionary    ${POD_CIDR_MAP}    ${remote}
+            ${remote_svc_cidr}=    Get From Dictionary    ${SVC_CIDR_MAP}    ${remote}
+            Verify C2CC Infrastructure For Remote
+            ...    ${alias}    ${remote_pod_cidr}    ${remote_svc_cidr}    ${local_svc_cidr}
+        END
+    END
+    Verify Routes In Table 201    ${alias}    ${local_svc_cidr}
+
+Verify Cross Cluster Connectivity
+    [Documentation]    Verify a representative subset of cross-cluster connectivity.
+    ...    Tests one direction per cluster pair, alternating pod/service.
+    Test Connectivity Between Clusters    cluster-a    cluster-b    pod
+    Test Connectivity Between Clusters    cluster-a    cluster-c    service
+    Test Connectivity Between Clusters    cluster-b    cluster-a    service
+    Test Connectivity Between Clusters    cluster-b    cluster-c    pod
+    Test Connectivity Between Clusters    cluster-c    cluster-a    pod
+    Test Connectivity Between Clusters    cluster-c    cluster-b    service
+
+Verify Cross Cluster DNS
+    [Documentation]    Verify a representative subset of cross-cluster DNS resolution.
+    Curl Remote Service Via DNS    cluster-a    cluster-b
+    Curl Remote Service Via DNS    cluster-b    cluster-c
+    Curl Remote Service Via DNS    cluster-c    cluster-a
+
+Wait For OVN-K Pods Ready On Cluster
+    [Documentation]    Wait for all OVN-K pods to be Running and Ready on the given cluster.
+    [Arguments]    ${alias}    ${timeout}=5m    ${retry}=15s
+    Wait Until Keyword Succeeds    ${timeout}    ${retry}
+    ...    OVN-K Pods Should Be Ready    ${alias}
+
+OVN-K Pods Should Be Ready
+    [Documentation]    Assert all pods in openshift-ovn-kubernetes are Ready.
+    [Arguments]    ${alias}
+    Oc On Cluster    ${alias}
+    ...    oc wait pods -n openshift-ovn-kubernetes --all --for=condition=Ready --timeout=30s
+
+Disable All NICs For VM
+    [Documentation]    Disable all vnet interfaces for the given VM via virsh.
+    ...    Returns the list of disabled interfaces for later re-enabling.
+    [Arguments]    ${vm_name}
+    ${vnet_ifaces}=    Get Vnet Devices For MicroShift Host    ${vm_name}
+    Should Not Be Empty    ${vnet_ifaces}    No vnet interfaces found for VM ${vm_name}
+    FOR    ${iface}    IN    @{vnet_ifaces}
+        Disable NIC For MicroShift Host    ${vm_name}    ${iface}
+    END
+    RETURN    ${vnet_ifaces}
+
+Enable All NICs For VM
+    [Documentation]    Re-enable all specified vnet interfaces for the given VM via virsh.
+    [Arguments]    ${vm_name}    ${vnet_ifaces}
+    FOR    ${iface}    IN    @{vnet_ifaces}
+        Enable NIC For MicroShift Host    ${vm_name}    ${iface}
+    END
+
+Deregister Remote Cluster
+    [Documentation]    Close SSH connection and remove a remote cluster from all tracking structures.
+    ...    Safe to call even if the alias was already deregistered.
+    [Arguments]    ${alias}
+    ${switch_status}    ${_}=    Run Keyword And Ignore Error    SSHLibrary.Switch Connection    ${alias}
+    IF    '${switch_status}' == 'PASS'
+        Run Keyword And Ignore Error    SSHLibrary.Close Connection
+    END
+    ${present}=    Evaluate    '${alias}' in $C2CC_REMOTE_ALIASES
+    IF    ${present}
+        ${idx}=    Evaluate    $C2CC_REMOTE_ALIASES.index('${alias}')
+        Remove From List    ${C2CC_REMOTE_ALIASES}    ${idx}
+    END
+    Run Keyword And Ignore Error    Remove From Dictionary    ${C2CC_SSH_IDS}    ${alias}
+    Run Keyword And Ignore Error    Remove From Dictionary    ${C2CC_KUBECONFIGS}    ${alias}
+
+Reconnect To Cluster
+    [Documentation]    Re-establish SSH connection to a remote cluster after network disruption.
+    ...    Deregisters the stale connection and waits for SSH to come back.
+    [Arguments]    ${alias}    ${host}    ${ssh_port}    ${kubeconfig}    ${timeout}=5m
+    Deregister Remote Cluster    ${alias}
+    Wait Until Keyword Succeeds    ${timeout}    15s
+    ...    Register Remote Cluster    ${alias}    ${host}    ${ssh_port}    ${kubeconfig}

test/scenarios-bootc/c2cc/el98-src@c2cc-chaos.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Sourced from scenario.sh and uses functions defined there.
+
+# shellcheck source=test/bin/c2cc_common.sh
+source "${SCRIPTDIR}/c2cc_common.sh"
+
+# Chaos tests assume a healthy baseline between tests,
+# so disable randomization to ensure predictable ordering.
+export TEST_RANDOMIZATION=none
+
+scenario_create_vms() {
+    c2cc_create_vms rhel98-bootc-source rhel98-bootc
+}
+
+scenario_remove_vms() {
+    c2cc_remove_vms
+}
+
+scenario_run_tests() {
+    # shellcheck disable=SC2119
+    configure_c2cc_hosts
+    c2cc_run_tests "suites/c2cc/chaos.robot" "" "" "chaos"
+}