diff --git a/test/bin/c2cc_common.sh b/test/bin/c2cc_common.sh index 454722dfd3..9268aa2811 100644 --- a/test/bin/c2cc_common.sh +++ b/test/bin/c2cc_common.sh @@ -238,12 +238,6 @@ c2cc_run_tests() { local -r suites_dir="${1}" local -r foreign_cidr="${2:-}" local -r ip_family="${3:-}" - local -r include_tags="${4:-c2cc}" - - local include_args="" - for tag in ${include_tags}; do - include_args+="--include ${tag} " - done local foreign_cidr_var="" if [ -n "${foreign_cidr}" ]; then @@ -324,7 +318,6 @@ c2cc_run_tests() { ${target_ref_var} \ ${dual_cidr_vars} \ --variable "BOOTC_REGISTRY:${MIRROR_REGISTRY_URL}" \ - ${include_args} \ "${suites_dir}" } diff --git a/test/resources/c2cc.resource b/test/resources/c2cc.resource index f0f860fb74..827b4a9fbf 100644 --- a/test/resources/c2cc.resource +++ b/test/resources/c2cc.resource @@ -323,10 +323,10 @@ Wait For Service Endpoints ... Ensures OVN-K has synced the EndpointSlice and programmed the OVN load balancer. FOR ${alias} IN cluster-a cluster-b cluster-c Wait Until Keyword Succeeds 120s 5s - ... Service Endpoints Should Exist ${alias} ${NAMESPACES}[${alias}] + ... Verify Service Endpoints Exist ${alias} ${NAMESPACES}[${alias}] END -Service Endpoints Should Exist +Verify Service Endpoints Exist [Documentation] Verify the hello-microshift endpoint has at least one address. [Arguments] ${alias} ${ns} ${stdout}= Oc On Cluster ${alias} @@ -418,7 +418,7 @@ Curl From Cluster ... oc exec curl-pod -n ${NAMESPACES}[${alias}] -- curl -sS --max-time 10 ${url} RETURN ${stdout} -Test Connectivity Between Clusters +Verify Connectivity Between Clusters [Documentation] Verify pod on ${source} can reach ${endpoint_type} IP on ${destination}. [Arguments] ${source} ${destination} ${endpoint_type} IF '${endpoint_type}' == 'pod' @@ -431,7 +431,7 @@ Test Connectivity Between Clusters ${stdout}= Curl From Cluster ${source} ${ip_dest} 8080 Should Contain ${stdout} Hello from -Test Source IP Preserved Between Clusters +Verify Source IP Preserved Between Clusters [Documentation] Verify ${source} to ${destination} pod-to-${endpoint_type} traffic preserves the source pod IP (no SNAT). [Arguments] ${source} ${destination} ${endpoint_type} IF '${endpoint_type}' == 'pod' @@ -445,7 +445,7 @@ Test Source IP Preserved Between Clusters ${stdout}= Curl From Cluster ${source} ${ip_dest} 8080 Should Contain ${stdout} source: ${curl_pod_ip} -Test Dual Stack Connectivity Between Clusters +Verify Dual Stack Connectivity Between Clusters [Documentation] Verify pod on ${source} can reach ${endpoint_type} dual-stack IP on ${destination}. [Arguments] ${source} ${destination} ${endpoint_type} IF '${endpoint_type}' == 'pod' @@ -458,7 +458,7 @@ Test Dual Stack Connectivity Between Clusters ${stdout}= Curl From Cluster ${source} ${ip_dest} 8080 Should Contain ${stdout} Hello from -Test Dual Stack Source IP Preserved Between Clusters +Verify Dual Stack Source IP Preserved Between Clusters [Documentation] Verify dual-stack traffic preserves the source pod IP (no SNAT). [Arguments] ${source} ${destination} ${endpoint_type} IF '${endpoint_type}' == 'pod' @@ -472,13 +472,13 @@ Test Dual Stack Source IP Preserved Between Clusters ${stdout}= Curl From Cluster ${source} ${ip_dest} 8080 Should Contain ${stdout} source: ${curl_pod_ip} -DNS Resolve From Cluster +Verify DNS Resolution From Cluster [Documentation] Resolve a DNS name from curl-pod on the given cluster. Retries for up to 60s. [Arguments] ${alias} ${fqdn} Wait Until Keyword Succeeds 12x 5s - ... DNS Lookup Should Succeed ${alias} ${fqdn} + ... Verify DNS Lookup ${alias} ${fqdn} -DNS Lookup Should Succeed +Verify DNS Lookup [Documentation] Resolve a DNS name from curl-pod using getent hosts. [Arguments] ${alias} ${fqdn} ${stdout}= Oc On Cluster ${alias} @@ -492,14 +492,14 @@ Verify All RemoteClusters Healthy ... Verify RemoteCluster State ${alias} Healthy END -Curl DNS From Cluster +Verify Curl DNS From Cluster [Documentation] Curl a service by DNS name from curl-pod on the given cluster. [Arguments] ${alias} ${fqdn} ${port} ${stdout}= Wait Until Keyword Succeeds 12x 5s - ... Curl DNS Should Succeed ${alias} ${fqdn} ${port} + ... Verify Curl DNS ${alias} ${fqdn} ${port} RETURN ${stdout} -Curl DNS Should Succeed +Verify Curl DNS [Documentation] Single attempt to curl a DNS name from curl-pod. [Arguments] ${alias} ${fqdn} ${port} ${stdout}= Oc On Cluster ${alias} @@ -507,10 +507,10 @@ Curl DNS Should Succeed Should Contain ${stdout} Hello from RETURN ${stdout} -Curl Remote Service Via DNS +Verify Remote Service Via DNS [Documentation] Verify pod on ${source} can reach a service on ${destination} using the remote DNS name. [Arguments] ${source} ${destination} - ${stdout}= Curl DNS From Cluster ${source} + ${stdout}= Verify Curl DNS From Cluster ${source} ... hello-microshift.${NAMESPACES}[${destination}].svc.${DOMAIN_MAP}[${destination}] 8080 Should Contain ${stdout} Hello from @@ -576,13 +576,14 @@ IP Command For CIDR RETURN ${cmd} Verify C2CC Infrastructure For Remote - [Documentation] Verify C2CC infrastructure for one remote cluster's CIDRs. + [Documentation] Verify all C2CC infrastructure for one remote cluster's CIDRs. [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} ${ip_cmd}=${IP_CMD} Verify Routes In Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${ip_cmd} Verify All IP Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} ${ip_cmd} Verify NFTables Bypass Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} Verify OVN Static Routes ${alias} ${remote_pod_cidr} ${remote_svc_cidr} Verify Node SNAT Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} + Verify C2CC Tracking Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} Verify C2CC Infrastructure On Cluster [Documentation] Run all C2CC infrastructure checks for all remote CIDRs on the given cluster. @@ -608,29 +609,13 @@ Verify C2CC Infrastructure For CIDR Family END Verify Routes In Table 201 ${alias} ${local_svc_cidr} ${ip_cmd} -Verify Cross Cluster Connectivity - [Documentation] Verify a representative subset of cross-cluster connectivity. - ... Tests one direction per cluster pair, alternating pod/service. - Test Connectivity Between Clusters cluster-a cluster-b pod - Test Connectivity Between Clusters cluster-a cluster-c service - Test Connectivity Between Clusters cluster-b cluster-a service - Test Connectivity Between Clusters cluster-b cluster-c pod - Test Connectivity Between Clusters cluster-c cluster-a pod - Test Connectivity Between Clusters cluster-c cluster-b service - -Verify Cross Cluster DNS - [Documentation] Verify a representative subset of cross-cluster DNS resolution. - Curl Remote Service Via DNS cluster-a cluster-b - Curl Remote Service Via DNS cluster-b cluster-c - Curl Remote Service Via DNS cluster-c cluster-a - Wait For OVN-K Pods Ready On Cluster [Documentation] Wait for all OVN-K pods to be Running and Ready on the given cluster. [Arguments] ${alias} ${timeout}=5m ${retry}=15s Wait Until Keyword Succeeds ${timeout} ${retry} - ... OVN-K Pods Should Be Ready ${alias} + ... Verify OVN-K Pods Ready ${alias} -OVN-K Pods Should Be Ready +Verify OVN-K Pods Ready [Documentation] Assert all pods in openshift-ovn-kubernetes are Ready. [Arguments] ${alias} Oc On Cluster ${alias} @@ -698,10 +683,10 @@ Reboot Clusters Simultaneously FOR ${alias} IN @{cluster_aliases} ${old_bootid}= Get From Dictionary ${boot_ids} ${alias} Wait Until Keyword Succeeds 10m 15s - ... Cluster Should Be Rebooted ${alias} ${old_bootid} + ... Verify Cluster Has Rebooted ${alias} ${old_bootid} END -Cluster Should Be Rebooted +Verify Cluster Has Rebooted [Documentation] Assert that a cluster has rebooted and greenboot health check has passed. [Arguments] ${alias} ${old_bootid} Reconnect Cluster ${alias} @@ -731,3 +716,87 @@ Reconnect Cluster END ${kubeconfig}= Get From Dictionary ${C2CC_KUBECONFIGS} ${alias} Reconnect To Cluster ${alias} ${host} ${port} ${kubeconfig} timeout=10m + +C2CC Suite Setup + [Documentation] Standard 3-cluster suite setup: registers all three clusters + ... as remote and optionally deploys test workloads. + [Arguments] ${deploy_workloads}=${FALSE} + Check Required Env Variables + Register All C2CC Clusters + IF ${deploy_workloads} Deploy Test Workloads + +C2CC Suite Teardown + [Documentation] Standard suite teardown: optionally cleans up workloads + ... and tears down remote clusters. + [Arguments] ${cleanup_workloads}=${FALSE} + IF ${cleanup_workloads} Cleanup Test Workloads + Teardown All Remote Clusters + +Verify Full C2CC Connectivity + [Documentation] Verify pod-to-pod, pod-to-service connectivity and source IP preservation + ... across all 6 cluster pairs. + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + Verify Connectivity Between Clusters ${src} ${dst} pod + Verify Connectivity Between Clusters ${src} ${dst} service + Verify Source IP Preserved Between Clusters ${src} ${dst} pod + Verify Source IP Preserved Between Clusters ${src} ${dst} service + END + +Verify Full C2CC DNS + [Documentation] Verify CoreDNS Corefile contains C2CC server blocks and + ... cross-cluster DNS resolution works for all pairs. + FOR ${alias} IN @{ALL_CLUSTERS} + FOR ${remote} IN @{ALL_CLUSTERS} + IF '${remote}' != '${alias}' + ${domain}= Get From Dictionary ${DOMAIN_MAP} ${remote} + Verify Corefile Contains C2CC Server Block ${alias} ${domain} + END + END + END + FOR ${src} ${dst} IN + ... cluster-a cluster-b + ... cluster-a cluster-c + ... cluster-b cluster-a + ... cluster-b cluster-c + ... cluster-c cluster-a + ... cluster-c cluster-b + Verify Remote Service Via DNS ${src} ${dst} + END + +Verify C2CC Health Probes + [Documentation] Verify all RemoteCluster CRs report Healthy with populated timestamps. + FOR ${alias} IN @{ALL_CLUSTERS} + Verify RemoteCluster State ${alias} Healthy + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastProbeTime}' + Should Not Be Empty ${stdout} + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastSuccessfulProbe}' + Should Not Be Empty ${stdout} + END + +Verify All Clusters Healthy + [Documentation] Wait for all clusters to pass the MicroShift health check. + [Arguments] ${timeout}=5m ${retry}=15s + FOR ${cluster} IN @{ALL_CLUSTERS} + Wait Until Keyword Succeeds ${timeout} ${retry} + ... Verify Cluster Is Healthy ${cluster} + END + +Verify Full C2CC Stack + [Documentation] Comprehensive verification of all C2CC components across all clusters. + ... Wraps connectivity, infrastructure, health probes, and DNS checks + ... with retry logic for post-disruption convergence. + Wait Until Keyword Succeeds 10m 10s Verify Full C2CC Connectivity + FOR ${cluster} IN @{ALL_CLUSTERS} + Wait Until Keyword Succeeds 10m 10s + ... Verify C2CC Infrastructure On Cluster ${cluster} + END + Wait Until Keyword Succeeds 10m 10s Verify C2CC Health Probes + Wait Until Keyword Succeeds 10m 10s Verify Full C2CC DNS diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh index 35c1c30644..fea38e9eb1 100755 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-disruptive.sh @@ -20,5 +20,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/" "" "" "disruptive" + c2cc_run_tests "suites/c2cc/disruptive/" } diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh index 48cd060200..794586d645 100755 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-dual-stack.sh @@ -37,5 +37,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/" "192.0.2.0/24" dual-stack + c2cc_run_tests "suites/c2cc/basic/" "192.0.2.0/24" dual-stack } diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec.sh index a38f0dd05f..2c6e13d50d 100644 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipsec.sh @@ -26,5 +26,5 @@ scenario_run_tests() { configure_c2cc_hosts "c2cc_ipsec_pre_greenboot" "c2cc_ipsec_greenboot" configure_ipsec - c2cc_run_tests "suites/c2cc-ipsec/" + c2cc_run_tests "suites/c2cc/ipsec/" } diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh index 250dd129d2..93040a4bdd 100644 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc-ipv6.sh @@ -43,5 +43,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/" "2001:db8::/64" ipv6 + c2cc_run_tests "suites/c2cc/basic/" "2001:db8::/64" ipv6 } diff --git a/test/scenarios-bootc/c2cc/el102-src@c2cc.sh b/test/scenarios-bootc/c2cc/el102-src@c2cc.sh index 37c747c3d8..ff4b4fb75a 100644 --- a/test/scenarios-bootc/c2cc/el102-src@c2cc.sh +++ b/test/scenarios-bootc/c2cc/el102-src@c2cc.sh @@ -16,5 +16,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/" "192.0.2.0/24" + c2cc_run_tests "suites/c2cc/basic/" "192.0.2.0/24" } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh index 79cab102e4..3e25709b2d 100755 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-disruptive.sh @@ -20,5 +20,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/" "" "" "disruptive" + c2cc_run_tests "suites/c2cc/disruptive/" } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh index 657dbe64bf..962f559299 100755 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-dual-stack-v6.sh @@ -39,5 +39,5 @@ scenario_run_tests() { configure_c2cc_hosts # IP_FAMILY=ipv6 so IP_CMD defaults to 'ip -6' for primary (IPv6) CIDRs. # Dual-stack (IPv4) tests derive ip_cmd from CIDR content via IP Command For CIDR. - c2cc_run_tests "suites/c2cc/" "2001:db8::/64" ipv6 + c2cc_run_tests "suites/c2cc/basic/" "2001:db8::/64" ipv6 } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec.sh index 348e66cdfb..e9900c4c7a 100644 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipsec.sh @@ -26,5 +26,5 @@ scenario_run_tests() { configure_c2cc_hosts "c2cc_ipsec_pre_greenboot" "c2cc_ipsec_greenboot" configure_ipsec - c2cc_run_tests "suites/c2cc-ipsec/" + c2cc_run_tests "suites/c2cc/ipsec/" } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh index 50a781ee25..349cebf36b 100644 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc-ipv6.sh @@ -43,5 +43,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/" "2001:db8::/64" ipv6 + c2cc_run_tests "suites/c2cc/basic/" "2001:db8::/64" ipv6 } diff --git a/test/scenarios-bootc/c2cc/el98-src@c2cc.sh b/test/scenarios-bootc/c2cc/el98-src@c2cc.sh index 9d756b5e81..be9cc79ecb 100644 --- a/test/scenarios-bootc/c2cc/el98-src@c2cc.sh +++ b/test/scenarios-bootc/c2cc/el98-src@c2cc.sh @@ -16,5 +16,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/c2cc/" "192.0.2.0/24" + c2cc_run_tests "suites/c2cc/basic/" "192.0.2.0/24" } diff --git a/test/scenarios-bootc/c2cc/el98-src@el102-src@c2cc-upgrade-ok.sh b/test/scenarios-bootc/c2cc/el98-src@el102-src@c2cc-upgrade-ok.sh index d083dd6e98..eaaa59bab4 100755 --- a/test/scenarios-bootc/c2cc/el98-src@el102-src@c2cc-upgrade-ok.sh +++ b/test/scenarios-bootc/c2cc/el98-src@el102-src@c2cc-upgrade-ok.sh @@ -21,5 +21,5 @@ scenario_remove_vms() { scenario_run_tests() { # shellcheck disable=SC2119 configure_c2cc_hosts - c2cc_run_tests "suites/upgrade/upgrade-c2cc.robot" + c2cc_run_tests "suites/c2cc/upgrade/" } diff --git a/test/suites/c2cc/cleanup.robot b/test/suites/c2cc/basic/cleanup.robot similarity index 95% rename from test/suites/c2cc/cleanup.robot rename to test/suites/c2cc/basic/cleanup.robot index 83df15d9e9..2e2feeac8f 100644 --- a/test/suites/c2cc/cleanup.robot +++ b/test/suites/c2cc/basic/cleanup.robot @@ -3,10 +3,10 @@ Documentation Verify C2CC cleanup when the feature is disabled. ... Removes the C2CC config drop-in on Cluster A, restarts MicroShift, ... and verifies all C2CC networking state has been cleaned up. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource Suite Setup Setup Suite Teardown Teardown @@ -142,14 +142,13 @@ C2CC Controller Logged Cleanup *** Keywords *** Setup [Documentation] Register clusters, then disable C2CC on Cluster A and wait for restart. - Check Required Env Variables - Register All C2CC Clusters + C2CC Suite Setup Disable C2CC On Cluster cluster-a Teardown [Documentation] Re-enable C2CC on Cluster A, then close connections. Enable C2CC On Cluster cluster-a - Teardown All Remote Clusters + C2CC Suite Teardown Disable C2CC On Cluster [Documentation] Move the C2CC config drop-in aside and restart MicroShift. diff --git a/test/suites/c2cc/basic/connectivity.robot b/test/suites/c2cc/basic/connectivity.robot new file mode 100644 index 0000000000..8dd91b3e76 --- /dev/null +++ b/test/suites/c2cc/basic/connectivity.robot @@ -0,0 +1,80 @@ +*** Settings *** +Documentation Cross-cluster connectivity tests for C2CC. +... Deploys test workloads on all clusters and verifies pod-to-pod +... and pod-to-service communication in both directions. + +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource + +Suite Setup C2CC Suite Setup deploy_workloads=${TRUE} +Suite Teardown C2CC Suite Teardown cleanup_workloads=${TRUE} + +Test Tags c2cc + + +*** Test Cases *** +Verify Cross Cluster Connectivity + [Documentation] Verify pods on all clusters can reach pods/services on all other clusters. + [Template] Verify Connectivity Between Clusters + cluster-a cluster-b pod + cluster-a cluster-b service + cluster-a cluster-c pod + cluster-a cluster-c service + cluster-b cluster-a pod + cluster-b cluster-a service + cluster-b cluster-c pod + cluster-b cluster-c service + cluster-c cluster-a pod + cluster-c cluster-a service + cluster-c cluster-b pod + cluster-c cluster-b service + +Verify Cross Cluster Source IP Preservation + [Documentation] Verify cross cluster traffic preserves source pod IP (no SNAT). + [Template] Verify Source IP Preserved Between Clusters + cluster-a cluster-b pod + cluster-a cluster-b service + cluster-a cluster-c pod + cluster-a cluster-c service + cluster-b cluster-a pod + cluster-b cluster-a service + cluster-b cluster-c pod + cluster-b cluster-c service + cluster-c cluster-a pod + cluster-c cluster-a service + cluster-c cluster-b pod + cluster-c cluster-b service + +Verify Dual Stack Cross Cluster Connectivity + [Documentation] Verify dual-stack pods on all clusters can reach pods/services on all other clusters. + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + Verify Dual Stack Connectivity Between Clusters cluster-a cluster-b pod + Verify Dual Stack Connectivity Between Clusters cluster-a cluster-b service + Verify Dual Stack Connectivity Between Clusters cluster-a cluster-c pod + Verify Dual Stack Connectivity Between Clusters cluster-a cluster-c service + Verify Dual Stack Connectivity Between Clusters cluster-b cluster-a pod + Verify Dual Stack Connectivity Between Clusters cluster-b cluster-a service + Verify Dual Stack Connectivity Between Clusters cluster-b cluster-c pod + Verify Dual Stack Connectivity Between Clusters cluster-b cluster-c service + Verify Dual Stack Connectivity Between Clusters cluster-c cluster-a pod + Verify Dual Stack Connectivity Between Clusters cluster-c cluster-a service + Verify Dual Stack Connectivity Between Clusters cluster-c cluster-b pod + Verify Dual Stack Connectivity Between Clusters cluster-c cluster-b service + +Verify Dual Stack Cross Cluster Source IP Preservation + [Documentation] Verify dual-stack cross cluster traffic preserves source pod IP (no SNAT). + Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured + Verify Dual Stack Source IP Preserved Between Clusters cluster-a cluster-b pod + Verify Dual Stack Source IP Preserved Between Clusters cluster-a cluster-b service + Verify Dual Stack Source IP Preserved Between Clusters cluster-a cluster-c pod + Verify Dual Stack Source IP Preserved Between Clusters cluster-a cluster-c service + Verify Dual Stack Source IP Preserved Between Clusters cluster-b cluster-a pod + Verify Dual Stack Source IP Preserved Between Clusters cluster-b cluster-a service + Verify Dual Stack Source IP Preserved Between Clusters cluster-b cluster-c pod + Verify Dual Stack Source IP Preserved Between Clusters cluster-b cluster-c service + Verify Dual Stack Source IP Preserved Between Clusters cluster-c cluster-a pod + Verify Dual Stack Source IP Preserved Between Clusters cluster-c cluster-a service + Verify Dual Stack Source IP Preserved Between Clusters cluster-c cluster-b pod + Verify Dual Stack Source IP Preserved Between Clusters cluster-c cluster-b service diff --git a/test/suites/c2cc/custom-route-tables.robot b/test/suites/c2cc/basic/custom-route-tables.robot similarity index 92% rename from test/suites/c2cc/custom-route-tables.robot rename to test/suites/c2cc/basic/custom-route-tables.robot index c7adb1dccd..bfe01ff176 100644 --- a/test/suites/c2cc/custom-route-tables.robot +++ b/test/suites/c2cc/basic/custom-route-tables.robot @@ -10,10 +10,10 @@ Documentation Verify C2CC respects custom routing table IDs from configura ... table IDs while C2CC is enabled results in duplicated rules in both ... old and new tables. The old tables are only cleaned up by rebooting. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource Suite Setup Setup Suite Teardown Teardown @@ -75,9 +75,7 @@ Old Default Tables Are Not Cleaned Up *** Keywords *** Setup [Documentation] Register clusters, apply custom routing drop-in, restart MicroShift. - Check Required Env Variables - Register Remote Cluster cluster-a ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG_A} - Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} + C2CC Suite Setup Apply Custom Routing Config Restart And Wait For Healthy @@ -86,7 +84,7 @@ Teardown Remove Custom Routing Config Restart And Wait For Healthy Flush Custom Tables - Teardown All Remote Clusters + C2CC Suite Teardown Apply Custom Routing Config [Documentation] Write a drop-in that overrides routing table IDs. diff --git a/test/suites/c2cc/dns.robot b/test/suites/c2cc/basic/dns.robot similarity index 66% rename from test/suites/c2cc/dns.robot rename to test/suites/c2cc/basic/dns.robot index 3a2f0ec282..dcb7f6e748 100644 --- a/test/suites/c2cc/dns.robot +++ b/test/suites/c2cc/basic/dns.robot @@ -4,19 +4,19 @@ Documentation Cross-cluster DNS tests for C2CC. ... DNS resolution works across clusters, and service access via ... DNS names works end-to-end. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource -Suite Setup Setup -Suite Teardown Teardown +Suite Setup C2CC Suite Setup deploy_workloads=${TRUE} +Suite Teardown C2CC Suite Teardown cleanup_workloads=${TRUE} Test Tags c2cc *** Test Cases *** -Test Corefile Contains C2CC Server Block +Verify Corefile Contains C2CC Server Block [Documentation] Verify every cluster's Corefile has a server block for every other cluster domain. [Template] Verify Corefile Contains C2CC Server Block cluster-a ${CLUSTER_B_DOMAIN} @@ -26,9 +26,9 @@ Test Corefile Contains C2CC Server Block cluster-c ${CLUSTER_A_DOMAIN} cluster-c ${CLUSTER_B_DOMAIN} -Test Resolve Remote Service DNS +Verify Resolve Remote Service DNS [Documentation] Verify pods can resolve a service on all clusters via DNS. - [Template] DNS Resolve From Cluster + [Template] Verify DNS Resolution From Cluster cluster-a hello-microshift.${NAMESPACES}[cluster-b].svc.${DOMAIN_MAP}[cluster-b] cluster-a hello-microshift.${NAMESPACES}[cluster-c].svc.${DOMAIN_MAP}[cluster-c] cluster-b hello-microshift.${NAMESPACES}[cluster-a].svc.${DOMAIN_MAP}[cluster-a] @@ -36,25 +36,12 @@ Test Resolve Remote Service DNS cluster-c hello-microshift.${NAMESPACES}[cluster-a].svc.${DOMAIN_MAP}[cluster-a] cluster-c hello-microshift.${NAMESPACES}[cluster-b].svc.${DOMAIN_MAP}[cluster-b] -Test Curl Remote Service Via DNS +Verify Remote Service Reachable Via DNS [Documentation] Verify pod on a cluster can reach a service on all clusters using the remote DNS name. - [Template] Curl Remote Service Via DNS + [Template] Verify Remote Service Via DNS cluster-a cluster-b cluster-a cluster-c cluster-b cluster-a cluster-b cluster-c cluster-c cluster-a cluster-c cluster-b - - -*** Keywords *** -Setup - [Documentation] Set up clusters and deploy test workloads on all. - Check Required Env Variables - Register All C2CC Clusters - Deploy Test Workloads - -Teardown - [Documentation] Remove test workloads and close connections. - Cleanup Test Workloads - Teardown All Remote Clusters diff --git a/test/suites/c2cc/healthcheck.robot b/test/suites/c2cc/basic/healthcheck.robot similarity index 90% rename from test/suites/c2cc/healthcheck.robot rename to test/suites/c2cc/basic/healthcheck.robot index ca68b4e290..21dbe9a505 100644 --- a/test/suites/c2cc/healthcheck.robot +++ b/test/suites/c2cc/basic/healthcheck.robot @@ -3,13 +3,13 @@ Documentation Verify C2CC RemoteCluster CRD and CR lifecycle. ... Checks that the CRD is registered, CRs are created per remote cluster, ... and CR specs match the expected probe targets. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource -Suite Setup Setup -Suite Teardown Teardown +Suite Setup C2CC Suite Setup +Suite Teardown C2CC Suite Teardown Test Tags c2cc @@ -58,15 +58,6 @@ Correct Dual Stack RemoteCluster CR Spec *** Keywords *** -Setup - [Documentation] Set up SSH connections and kubeconfigs for all clusters. - Check Required Env Variables - Register All C2CC Clusters - -Teardown - [Documentation] Close all connections and clean up kubeconfigs. - Teardown All Remote Clusters - Wait For RemoteCluster CRD [Documentation] Waits for remoteclusters.microshift.io CRD to be registered. [Arguments] ${alias} diff --git a/test/suites/c2cc/infrastructure.robot b/test/suites/c2cc/basic/infrastructure.robot similarity index 94% rename from test/suites/c2cc/infrastructure.robot rename to test/suites/c2cc/basic/infrastructure.robot index b39724f7e9..b92ba322e2 100644 --- a/test/suites/c2cc/infrastructure.robot +++ b/test/suites/c2cc/basic/infrastructure.robot @@ -1,15 +1,15 @@ *** Settings *** Documentation Verify C2CC controller sets up all networking infrastructure correctly. ... Checks Linux routes, IP rules, nftables bypass, OVN static routes, -... and node annotations on both clusters. +... and node annotations on all clusters. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource -Suite Setup Setup -Suite Teardown Teardown +Suite Setup C2CC Suite Setup +Suite Teardown C2CC Suite Teardown Test Tags c2cc @@ -182,14 +182,3 @@ Dual Stack Node Annotation Set Verify Node SNAT Annotation cluster-b ${CLUSTER_C_POD_CIDR_DUAL} ${CLUSTER_C_SVC_CIDR_DUAL} Verify Node SNAT Annotation cluster-c ${CLUSTER_A_POD_CIDR_DUAL} ${CLUSTER_A_SVC_CIDR_DUAL} Verify Node SNAT Annotation cluster-c ${CLUSTER_B_POD_CIDR_DUAL} ${CLUSTER_B_SVC_CIDR_DUAL} - - -*** Keywords *** -Setup - [Documentation] Set up SSH connections and kubeconfigs for all clusters. - Check Required Env Variables - Register All C2CC Clusters - -Teardown - [Documentation] Close all connections and clean up kubeconfigs. - Teardown All Remote Clusters diff --git a/test/suites/c2cc/probe.robot b/test/suites/c2cc/basic/probe.robot similarity index 83% rename from test/suites/c2cc/probe.robot rename to test/suites/c2cc/basic/probe.robot index 427e9991ac..6840d9b381 100644 --- a/test/suites/c2cc/probe.robot +++ b/test/suites/c2cc/basic/probe.robot @@ -4,13 +4,13 @@ Documentation Verify C2CC probe pod deployment and health status reporting ... ClusterIP, RemoteCluster CRs transition to Healthy, and the ... deployment self-heals after deletion. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource -Suite Setup Setup -Suite Teardown Teardown +Suite Setup C2CC Suite Setup +Suite Teardown C2CC Suite Teardown Test Tags c2cc @@ -50,31 +50,20 @@ RemoteCluster Status Becomes Healthy Wait Until Keyword Succeeds 3m 10s ... Verify RemoteCluster State cluster-c Healthy -RemoteCluster Status Has LastProbeTime - [Documentation] Verify that LastProbeTime is populated on all RemoteCluster CRs. +RemoteCluster Status Has Probe Timestamps + [Documentation] Verify that LastProbeTime and LastSuccessfulProbe are populated + ... on all RemoteCluster CRs across all clusters. FOR ${alias} IN cluster-a cluster-b cluster-c - ${stdout}= Oc On Cluster ${alias} - ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastProbeTime}' - Should Not Be Empty ${stdout} - @{timestamps}= Split String ${stdout} - ${count}= Get Length ${timestamps} - Should Be Equal As Integers ${count} 2 Expected 2 RemoteCluster states, got ${count} - FOR ${t} IN @{timestamps} - Should Not Be Empty ${t} - END - END - -RemoteCluster Status Has LastSuccessfulProbe - [Documentation] Verify that LastSuccessfulProbe is populated on all RemoteCluster CRs. - FOR ${alias} IN cluster-a cluster-b cluster-c - ${stdout}= Oc On Cluster ${alias} - ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastSuccessfulProbe}' - Should Not Be Empty ${stdout} - @{timestamps}= Split String ${stdout} - ${count}= Get Length ${timestamps} - Should Be Equal As Integers ${count} 2 Expected 2 RemoteCluster states, got ${count} - FOR ${t} IN @{timestamps} - Should Not Be Empty ${t} + FOR ${field} IN lastProbeTime lastSuccessfulProbe + ${stdout}= Oc On Cluster ${alias} + ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.${field}}' + Should Not Be Empty ${stdout} + @{timestamps}= Split String ${stdout} + ${count}= Get Length ${timestamps} + Should Be Equal As Integers ${count} 2 Expected 2 RemoteCluster ${field} values, got ${count} + FOR ${t} IN @{timestamps} + Should Not Be Empty ${t} + END END END @@ -137,15 +126,6 @@ RemoteCluster Has TargetResults In Dual Stack *** Keywords *** -Setup - [Documentation] Set up SSH connections and kubeconfigs for all clusters. - Check Required Env Variables - Register All C2CC Clusters - -Teardown - [Documentation] Close all connections and clean up kubeconfigs. - Teardown All Remote Clusters - Verify Probe Pod Is Ready [Documentation] Check that the probe deployment has 1 available replica. [Arguments] ${alias} diff --git a/test/suites/c2cc/reconciliation.robot b/test/suites/c2cc/basic/reconciliation.robot similarity index 93% rename from test/suites/c2cc/reconciliation.robot rename to test/suites/c2cc/basic/reconciliation.robot index d4fe3c45a5..79686139e9 100644 --- a/test/suites/c2cc/reconciliation.robot +++ b/test/suites/c2cc/basic/reconciliation.robot @@ -3,13 +3,13 @@ Documentation Negative/fault-injection tests for C2CC controller reconcili ... Each test deletes or corrupts a specific piece of C2CC networking state, ... then waits for the controller to detect the disruption and self-heal. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource -Suite Setup Setup -Suite Teardown Teardown +Suite Setup C2CC Suite Setup +Suite Teardown C2CC Suite Teardown Test Tags c2cc @@ -132,16 +132,6 @@ Reconcile Dual Stack Service IP Rule After Deletion *** Keywords *** -Setup - [Documentation] Set up SSH connections and kubeconfigs for all clusters. - Check Required Env Variables - Register Remote Cluster cluster-a ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG_A} - Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} - -Teardown - [Documentation] Close all connections and clean up kubeconfigs. - Teardown All Remote Clusters - Get Node Name On Cluster [Documentation] Get the name of the first node on the given cluster. [Arguments] ${alias} diff --git a/test/suites/c2cc/sanity.robot b/test/suites/c2cc/basic/sanity.robot similarity index 79% rename from test/suites/c2cc/sanity.robot rename to test/suites/c2cc/basic/sanity.robot index f647d3877a..573f73e757 100644 --- a/test/suites/c2cc/sanity.robot +++ b/test/suites/c2cc/basic/sanity.robot @@ -2,13 +2,13 @@ Documentation Sanity checks for a multi-cluster C2CC environment. ... Verifies that all clusters are running and MicroShift is healthy. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource -Suite Setup Setup -Suite Teardown Teardown +Suite Setup C2CC Suite Setup +Suite Teardown C2CC Suite Teardown Test Tags c2cc @@ -44,15 +44,6 @@ C2CC Controller Is Running On Cluster *** Keywords *** -Setup - [Documentation] Set up SSH connections and kubeconfigs for all clusters. - Check Required Env Variables - Register All C2CC Clusters - -Teardown - [Documentation] Close all connections and clean up kubeconfigs. - Teardown All Remote Clusters - Verify Cluster Is Running [Documentation] Check the /readyz endpoint on the given cluster. [Arguments] ${alias} diff --git a/test/suites/c2cc/connectivity.robot b/test/suites/c2cc/connectivity.robot deleted file mode 100644 index 1f8bad248b..0000000000 --- a/test/suites/c2cc/connectivity.robot +++ /dev/null @@ -1,93 +0,0 @@ -*** Settings *** -Documentation Cross-cluster connectivity tests for C2CC. -... Deploys test workloads on all clusters and verifies pod-to-pod -... and pod-to-service communication in both directions. - -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource - -Suite Setup Setup -Suite Teardown Teardown - -Test Tags c2cc - - -*** Test Cases *** -Test Cross Cluster Connectivity - [Documentation] Verify pods on all clusters can reach pods/services on all other clusters. - [Template] Test Connectivity Between Clusters - cluster-a cluster-b pod - cluster-a cluster-b service - cluster-a cluster-c pod - cluster-a cluster-c service - cluster-b cluster-a pod - cluster-b cluster-a service - cluster-b cluster-c pod - cluster-b cluster-c service - cluster-c cluster-a pod - cluster-c cluster-a service - cluster-c cluster-b pod - cluster-c cluster-b service - -Test Cross Cluster Source IP Preservation - [Documentation] Verify cross cluster traffic preserves source pod IP (no SNAT). - [Template] Test Source IP Preserved Between Clusters - cluster-a cluster-b pod - cluster-a cluster-b service - cluster-a cluster-c pod - cluster-a cluster-c service - cluster-b cluster-a pod - cluster-b cluster-a service - cluster-b cluster-c pod - cluster-b cluster-c service - cluster-c cluster-a pod - cluster-c cluster-a service - cluster-c cluster-b pod - cluster-c cluster-b service - -Test Dual Stack Cross Cluster Connectivity - [Documentation] Verify dual-stack pods on all clusters can reach pods/services on all other clusters. - Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured - Test Dual Stack Connectivity Between Clusters cluster-a cluster-b pod - Test Dual Stack Connectivity Between Clusters cluster-a cluster-b service - Test Dual Stack Connectivity Between Clusters cluster-a cluster-c pod - Test Dual Stack Connectivity Between Clusters cluster-a cluster-c service - Test Dual Stack Connectivity Between Clusters cluster-b cluster-a pod - Test Dual Stack Connectivity Between Clusters cluster-b cluster-a service - Test Dual Stack Connectivity Between Clusters cluster-b cluster-c pod - Test Dual Stack Connectivity Between Clusters cluster-b cluster-c service - Test Dual Stack Connectivity Between Clusters cluster-c cluster-a pod - Test Dual Stack Connectivity Between Clusters cluster-c cluster-a service - Test Dual Stack Connectivity Between Clusters cluster-c cluster-b pod - Test Dual Stack Connectivity Between Clusters cluster-c cluster-b service - -Test Dual Stack Cross Cluster Source IP Preservation - [Documentation] Verify dual-stack cross cluster traffic preserves source pod IP (no SNAT). - Skip If '${CLUSTER_A_POD_CIDR_DUAL}' == '' Dual-stack CIDRs not configured - Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-b pod - Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-b service - Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-c pod - Test Dual Stack Source IP Preserved Between Clusters cluster-a cluster-c service - Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-a pod - Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-a service - Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-c pod - Test Dual Stack Source IP Preserved Between Clusters cluster-b cluster-c service - Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-a pod - Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-a service - Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-b pod - Test Dual Stack Source IP Preserved Between Clusters cluster-c cluster-b service - - -*** Keywords *** -Setup - [Documentation] Set up clusters and deploy test workloads on all. - Check Required Env Variables - Register All C2CC Clusters - Deploy Test Workloads - -Teardown - [Documentation] Remove test workloads and close connections. - Cleanup Test Workloads - Teardown All Remote Clusters diff --git a/test/suites/c2cc/disruptive.robot b/test/suites/c2cc/disruptive/disruptive.robot similarity index 78% rename from test/suites/c2cc/disruptive.robot rename to test/suites/c2cc/disruptive/disruptive.robot index e715b7c615..612c84a10e 100644 --- a/test/suites/c2cc/disruptive.robot +++ b/test/suites/c2cc/disruptive/disruptive.robot @@ -4,26 +4,24 @@ Documentation Disruptive/resilience tests for C2CC. ... NIC outage), waits for recovery, and verifies that C2CC ... infrastructure and cross-cluster connectivity are restored. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource -Suite Setup Setup -Suite Teardown Teardown +Suite Setup C2CC Suite Setup deploy_workloads=${TRUE} +Suite Teardown C2CC Suite Teardown cleanup_workloads=${TRUE} Test Tags disruptive *** Variables *** -${RECOVERY_TIMEOUT} 5m -${RECOVERY_RETRY} 15s -${INFRA_VERIFY_TIMEOUT} 3m -${INFRA_VERIFY_RETRY} 10s -${HOST2_VM_NAME} ${EMPTY} -${HOST3_VM_NAME} ${EMPTY} -${DISABLED_VM} ${EMPTY} -@{DISABLED_IFACES} @{EMPTY} +${RECOVERY_TIMEOUT} 5m +${RECOVERY_RETRY} 15s +${HOST2_VM_NAME} ${EMPTY} +${HOST3_VM_NAME} ${EMPTY} +${DISABLED_VM} ${EMPTY} +@{DISABLED_IFACES} @{EMPTY} *** Test Cases *** @@ -121,22 +119,8 @@ Recovery After NM Restart On One Cluster And MicroShift Restart On Another Clust *** Keywords *** Verify All Clusters Are Healthy [Documentation] Wait for each cluster's healthcheck, then verify full C2CC recovery. - FOR ${cluster} IN @{ALL_CLUSTERS} - Wait Until Keyword Succeeds ${RECOVERY_TIMEOUT} ${RECOVERY_RETRY} - ... Verify Cluster Is Healthy ${cluster} - END - Verify Full Recovery On All Clusters - -Verify Full Recovery On All Clusters - [Documentation] Wait for RemoteCluster CRs to converge, verify C2CC infrastructure - ... on all clusters, then verify cross-cluster connectivity and DNS. - Verify All RemoteClusters Healthy - FOR ${cluster} IN @{ALL_CLUSTERS} - Wait Until Keyword Succeeds ${INFRA_VERIFY_TIMEOUT} ${INFRA_VERIFY_RETRY} - ... Verify C2CC Infrastructure On Cluster ${cluster} - END - Verify Cross Cluster Connectivity - Verify Cross Cluster DNS + Verify All Clusters Healthy timeout=${RECOVERY_TIMEOUT} retry=${RECOVERY_RETRY} + Verify Full C2CC Stack Restore NICs And Reconnect [Documentation] Re-enable NICs if they were left disabled by a failed NIC-outage test. @@ -146,14 +130,3 @@ Restore NICs And Reconnect Enable All NICs For VM ${vm_name} ${DISABLED_IFACES} Reconnect To Cluster ${alias} ${host_ip} ${ssh_port} ${kubeconfig} END - -Setup - [Documentation] Set up clusters and deploy test workloads. - Check Required Env Variables - Register All C2CC Clusters - Deploy Test Workloads - -Teardown - [Documentation] Remove test workloads and close connections. - Cleanup Test Workloads - Teardown All Remote Clusters diff --git a/test/suites/c2cc/disruptive/reboot.robot b/test/suites/c2cc/disruptive/reboot.robot new file mode 100644 index 0000000000..49e868a7ca --- /dev/null +++ b/test/suites/c2cc/disruptive/reboot.robot @@ -0,0 +1,59 @@ +*** Settings *** +Documentation Verify C2CC survives VM reboots in escalating scenarios. +... Tests single cluster, two clusters simultaneously, and all three +... clusters simultaneously. After each reboot cycle, full-stack +... verification confirms connectivity, infrastructure, health probes, +... and DNS all recover. + +Resource ../../../resources/c2cc.resource + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags disruptive + + +*** Test Cases *** +Reboot Single Cluster + [Documentation] Reboot cluster-a while cluster-b and cluster-c remain up. + ... Verifies that the rebooted cluster re-establishes C2CC connectivity + ... with both peers. + [Setup] Verify All RemoteClusters Healthy + Reboot Clusters Simultaneously cluster-a + Wait For Clusters Ready + Verify Full C2CC Stack + +Reboot Two Clusters Simultaneously + [Documentation] Reboot cluster-b and cluster-c at the same time. + ... The surviving cluster-a must wait for both peers to recover. + ... The two rebooted clusters must also reconnect with each other. + [Setup] Verify All RemoteClusters Healthy + Reboot Clusters Simultaneously cluster-b cluster-c + Wait For Clusters Ready + Verify Full C2CC Stack + +Reboot All Three Clusters Simultaneously + [Documentation] Reboot all three clusters at once. + ... Every cluster starts from scratch simultaneously - no running peer + ... to reference. All must independently reconstruct C2CC state. + [Setup] Verify All RemoteClusters Healthy + Reboot Clusters Simultaneously cluster-a cluster-b cluster-c + Wait For Clusters Ready + Verify Full C2CC Stack + + +*** Keywords *** +Setup + [Documentation] Set up clusters, deploy test workloads, and verify full stack. + C2CC Suite Setup deploy_workloads=${TRUE} + Verify Full C2CC Stack + +Teardown + [Documentation] Remove test workloads and close connections. + C2CC Suite Teardown cleanup_workloads=${TRUE} + +Wait For Clusters Ready + [Documentation] Wait for test pods and service endpoints to become ready + ... after a reboot cycle. + Wait For Test Pods + Wait For Service Endpoints diff --git a/test/suites/c2cc-ipsec/ipsec.robot b/test/suites/c2cc/ipsec/ipsec.robot similarity index 90% rename from test/suites/c2cc-ipsec/ipsec.robot rename to test/suites/c2cc/ipsec/ipsec.robot index d567ca7466..04717e4e76 100644 --- a/test/suites/c2cc-ipsec/ipsec.robot +++ b/test/suites/c2cc/ipsec/ipsec.robot @@ -7,13 +7,13 @@ Documentation IPsec E2E tests for C2CC. ... source IP preservation, policy enforcement (SA flush/restore), ... plaintext rejection, host-to-pod rejection, and MTU validation. -Resource ../../resources/microshift-process.resource -Resource ../../resources/kubeconfig.resource -Resource ../../resources/oc.resource -Resource ../../resources/c2cc.resource -Resource ../../resources/ipsec.resource +Resource ../../../resources/microshift-process.resource +Resource ../../../resources/kubeconfig.resource +Resource ../../../resources/oc.resource +Resource ../../../resources/c2cc.resource +Resource ../../../resources/ipsec.resource -Suite Setup Setup +Suite Setup C2CC Suite Setup deploy_workloads=${TRUE} Suite Teardown Teardown Test Tags c2cc ipsec @@ -45,7 +45,7 @@ ESP Encapsulation On Wire Cross Cluster Connectivity Through IPsec [Documentation] Verify pods on all clusters can reach pods/services on all ... other clusters through the IPsec tunnel. - [Template] Test Connectivity Between Clusters + [Template] Verify Connectivity Between Clusters cluster-a cluster-b pod cluster-a cluster-b service cluster-a cluster-c pod @@ -62,7 +62,7 @@ Cross Cluster Connectivity Through IPsec Source IP Preserved Through IPsec [Documentation] Verify cross-cluster traffic through IPsec preserves the ... source pod IP (no SNAT). - [Template] Test Source IP Preserved Between Clusters + [Template] Verify Source IP Preserved Between Clusters cluster-a cluster-b pod cluster-a cluster-b service cluster-a cluster-c pod @@ -106,7 +106,7 @@ Near MTU Packet Through IPsec Tunnel Cross Cluster DNS Through IPsec [Documentation] Verify pods can resolve and reach services on remote ... clusters via DNS name through the IPsec tunnel. - [Template] Curl Remote Service Via DNS + [Template] Verify Remote Service Via DNS cluster-a cluster-b cluster-a cluster-c cluster-b cluster-a @@ -138,17 +138,10 @@ IPsec Tunnel Recovers After Remote Stop Start *** Keywords *** -Setup - [Documentation] Register all clusters, verify IPsec tunnels, deploy test workloads. - Check Required Env Variables - Register All C2CC Clusters - Deploy Test Workloads - Teardown - [Documentation] Remove test workloads, ensure IPsec is running, close connections. - Cleanup Test Workloads + [Documentation] Ensure IPsec is running, then remove test workloads and close connections. Ensure IPsec Running On All Clusters - Teardown All Remote Clusters + C2CC Suite Teardown cleanup_workloads=${TRUE} Ensure IPsec Running On All Clusters [Documentation] Make sure ipsec service is running on all clusters. diff --git a/test/suites/c2cc/reboot.robot b/test/suites/c2cc/reboot.robot deleted file mode 100644 index 25df402ad6..0000000000 --- a/test/suites/c2cc/reboot.robot +++ /dev/null @@ -1,173 +0,0 @@ -*** Settings *** -Documentation Verify C2CC survives VM reboots in escalating scenarios. -... Tests single cluster, two clusters simultaneously, and all three -... clusters simultaneously. After each reboot cycle, full-stack -... verification confirms connectivity, infrastructure, health probes, -... and DNS all recover. - -Resource ../../resources/c2cc.resource - -Suite Setup Setup -Suite Teardown Teardown - -Test Tags disruptive - - -*** Test Cases *** -Reboot Single Cluster - [Documentation] Reboot cluster-a while cluster-b and cluster-c remain up. - ... Verifies that the rebooted cluster re-establishes C2CC connectivity - ... with both peers. - [Setup] Verify All RemoteClusters Healthy - Reboot Clusters Simultaneously cluster-a - Wait For Clusters Ready - Verify Full C2CC Stack - -Reboot Two Clusters Simultaneously - [Documentation] Reboot cluster-b and cluster-c at the same time. - ... The surviving cluster-a must wait for both peers to recover. - ... The two rebooted clusters must also reconnect with each other. - [Setup] Verify All RemoteClusters Healthy - Reboot Clusters Simultaneously cluster-b cluster-c - Wait For Clusters Ready - Verify Full C2CC Stack - -Reboot All Three Clusters Simultaneously - [Documentation] Reboot all three clusters at once. - ... Every cluster starts from scratch simultaneously — no running peer - ... to reference. All must independently reconstruct C2CC state. - [Setup] Verify All RemoteClusters Healthy - Reboot Clusters Simultaneously cluster-a cluster-b cluster-c - Wait For Clusters Ready - Verify Full C2CC Stack - - -*** Keywords *** -Setup - [Documentation] Set up clusters and deploy test workloads on all. - Check Required Env Variables - Login MicroShift Host - Setup Kubeconfig - Logout MicroShift Host - - Register Remote Cluster cluster-a ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG} - Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} - Register Remote Cluster cluster-c ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} - Deploy Test Workloads - Verify Full C2CC Stack - -Teardown - [Documentation] Remove test workloads and close connections. - Cleanup Test Workloads - Teardown All Remote Clusters - Remove Kubeconfig - -Wait For Clusters Ready - [Documentation] Wait for test pods and service endpoints to become ready - ... after a reboot cycle. - Wait For Test Pods - Wait For Service Endpoints - -Verify Full C2CC Stack - [Documentation] Comprehensive verification of all C2CC components across all clusters. - Wait Until Keyword Succeeds 10m 10s Verify C2CC Connectivity - Wait Until Keyword Succeeds 10m 10s Verify C2CC Infrastructure - Wait Until Keyword Succeeds 10m 10s Verify C2CC Health Probes - Wait Until Keyword Succeeds 10m 10s Verify C2CC DNS - -Verify C2CC Connectivity - [Documentation] Verify pod-to-pod, pod-to-service connectivity and source IP preservation - ... across all 6 cluster pairs. - FOR ${src} ${dst} IN - ... cluster-a cluster-b - ... cluster-a cluster-c - ... cluster-b cluster-a - ... cluster-b cluster-c - ... cluster-c cluster-a - ... cluster-c cluster-b - Test Connectivity Between Clusters ${src} ${dst} pod - Test Connectivity Between Clusters ${src} ${dst} service - Test Source IP Preserved Between Clusters ${src} ${dst} pod - Test Source IP Preserved Between Clusters ${src} ${dst} service - END - -Verify C2CC Infrastructure - [Documentation] Verify routes, IP rules, nftables, OVN static routes, - ... and node annotations for all cluster-peer combinations. - Verify Infra For Remote Peer - ... cluster-a - ... ${CLUSTER_B_POD_CIDR} - ... ${CLUSTER_B_SVC_CIDR} - ... ${CLUSTER_A_SVC_CIDR} - Verify Infra For Remote Peer - ... cluster-a - ... ${CLUSTER_C_POD_CIDR} - ... ${CLUSTER_C_SVC_CIDR} - ... ${CLUSTER_A_SVC_CIDR} - Verify Infra For Remote Peer - ... cluster-b - ... ${CLUSTER_A_POD_CIDR} - ... ${CLUSTER_A_SVC_CIDR} - ... ${CLUSTER_B_SVC_CIDR} - Verify Infra For Remote Peer - ... cluster-b - ... ${CLUSTER_C_POD_CIDR} - ... ${CLUSTER_C_SVC_CIDR} - ... ${CLUSTER_B_SVC_CIDR} - Verify Infra For Remote Peer - ... cluster-c - ... ${CLUSTER_A_POD_CIDR} - ... ${CLUSTER_A_SVC_CIDR} - ... ${CLUSTER_C_SVC_CIDR} - Verify Infra For Remote Peer - ... cluster-c - ... ${CLUSTER_B_POD_CIDR} - ... ${CLUSTER_B_SVC_CIDR} - ... ${CLUSTER_C_SVC_CIDR} - -Verify Infra For Remote Peer - [Documentation] Verify all infrastructure components on a cluster for one remote peer. - [Arguments] ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} - Verify Routes In Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - Verify IP Rules For Table 200 ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - Verify Routes In Table 201 ${alias} ${local_svc_cidr} - Verify Service IP Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} ${local_svc_cidr} - Verify NFTables Bypass Rules ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - Verify OVN Static Routes ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - Verify Node SNAT Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - Verify C2CC Tracking Annotation ${alias} ${remote_pod_cidr} ${remote_svc_cidr} - -Verify C2CC Health Probes - [Documentation] Verify all RemoteCluster CRs report Healthy with populated timestamps. - FOR ${alias} IN cluster-a cluster-b cluster-c - Verify RemoteCluster State ${alias} Healthy - ${stdout}= Oc On Cluster ${alias} - ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastProbeTime}' - Should Not Be Empty ${stdout} - ${stdout}= Oc On Cluster ${alias} - ... oc get remoteclusters.microshift.io -o jsonpath='{.items[*].status.lastSuccessfulProbe}' - Should Not Be Empty ${stdout} - END - -Verify C2CC DNS - [Documentation] Verify CoreDNS Corefile contains C2CC server blocks and - ... cross-cluster DNS resolution works for all pairs. - Verify Corefile Contains C2CC Server Block cluster-a ${CLUSTER_B_DOMAIN} - Verify Corefile Contains C2CC Server Block cluster-a ${CLUSTER_C_DOMAIN} - Verify Corefile Contains C2CC Server Block cluster-b ${CLUSTER_A_DOMAIN} - Verify Corefile Contains C2CC Server Block cluster-b ${CLUSTER_C_DOMAIN} - Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_A_DOMAIN} - Verify Corefile Contains C2CC Server Block cluster-c ${CLUSTER_B_DOMAIN} - Verify DNS Connectivity All Pairs - -Verify DNS Connectivity All Pairs - [Documentation] Verify cross-cluster DNS-based service access for all cluster pairs. - FOR ${src} ${dst} IN - ... cluster-a cluster-b - ... cluster-a cluster-c - ... cluster-b cluster-a - ... cluster-b cluster-c - ... cluster-c cluster-a - ... cluster-c cluster-b - Curl Remote Service Via DNS ${src} ${dst} - END diff --git a/test/suites/c2cc/upgrade/upgrade-c2cc.robot b/test/suites/c2cc/upgrade/upgrade-c2cc.robot new file mode 100644 index 0000000000..662499309d --- /dev/null +++ b/test/suites/c2cc/upgrade/upgrade-c2cc.robot @@ -0,0 +1,101 @@ +*** Settings *** +Documentation Tests RHEL 9.8 to RHEL 10.2 upgrade with C2CC enabled across 3 clusters. +... Upgrades each cluster one by one and verifies C2CC connectivity +... survives at each stage. + +Resource ../../../resources/common.resource +Resource ../../../resources/c2cc.resource +Resource ../../../resources/microshift-host.resource +Library Collections +Library SSHLibrary + +Suite Setup Setup +Suite Teardown Teardown + +Test Tags c2cc ostree + + +*** Variables *** +${TARGET_REF} ${EMPTY} +${BOOTC_REGISTRY} ${EMPTY} +${CLUSTER_A_POD_CIDR} ${EMPTY} +${CLUSTER_A_SVC_CIDR} ${EMPTY} +${CLUSTER_A_DOMAIN} ${EMPTY} +${CLUSTER_B_POD_CIDR} ${EMPTY} +${CLUSTER_B_SVC_CIDR} ${EMPTY} +${CLUSTER_B_DOMAIN} ${EMPTY} +${KUBECONFIG_B} ${EMPTY} +${CLUSTER_C_POD_CIDR} ${EMPTY} +${CLUSTER_C_SVC_CIDR} ${EMPTY} +${CLUSTER_C_DOMAIN} ${EMPTY} +${KUBECONFIG_C} ${EMPTY} + + +*** Test Cases *** +Upgrade C2CC Clusters And Verify + [Documentation] Upgrades 3 C2CC-connected clusters one by one + ... and verifies health and C2CC connectivity after each upgrade. + + Verify All Clusters Healthy + Verify All RemoteClusters Healthy + Deploy Test Workloads + Wait Until Keyword Succeeds 10m 10s Verify Full C2CC Connectivity + + FOR ${alias} IN cluster-a cluster-b cluster-c + Log To Console Upgrading ${alias} to ${TARGET_REF} + Upgrade Cluster ${alias} + Verify All Clusters Healthy + Verify All RemoteClusters Healthy + Wait For Test Pods + Wait For Service Endpoints + Wait Until Keyword Succeeds 10m 10s Verify Full C2CC Connectivity + END + + [Teardown] Cleanup Test Workloads + + +*** Keywords *** +Setup + [Documentation] Validate required variables and register all three clusters. + Should Not Be Empty ${TARGET_REF} TARGET_REF variable is required + Should Not Be Empty ${BOOTC_REGISTRY} BOOTC_REGISTRY variable is required + C2CC Suite Setup + +Teardown + [Documentation] Close all connections and clean up. + C2CC Suite Teardown + +Upgrade Cluster + [Documentation] Upgrade a specific cluster to the target bootc image + ... and verify it booted into the new deployment without rollback. + [Arguments] ${alias} + + ${initial_deploy_id}= Get Deployment Id On Cluster ${alias} + + Command On Cluster + ... ${alias} + ... printf '[[registry]]\nlocation = "${BOOTC_REGISTRY}"\ninsecure = true\n' | sudo tee /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf > /dev/null + + Command On Cluster ${alias} bootc switch --quiet ${BOOTC_REGISTRY}/${TARGET_REF} + + Command On Cluster ${alias} + ... rm -f /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf + + Reboot Cluster And Wait ${alias} + + ${current_deploy_id}= Get Deployment Id On Cluster ${alias} + Should Not Be Equal As Strings ${current_deploy_id} ${initial_deploy_id} + ... msg=${alias} rolled back to initial deployment + +Get Deployment Id On Cluster + [Documentation] Get the booted image digest from a specific cluster. + [Arguments] ${alias} + ${stdout}= Command On Cluster + ... ${alias} + ... bootc status --booted --json | python3 -c "import sys,json; print(json.load(sys.stdin)['status']['booted']['image']['imageDigest'])" + RETURN ${stdout} + +Reboot Cluster And Wait + [Documentation] Reboot a cluster and wait for it to come back with greenboot healthy. + [Arguments] ${alias} + Reboot Clusters Simultaneously ${alias} diff --git a/test/suites/upgrade/upgrade-c2cc.robot b/test/suites/upgrade/upgrade-c2cc.robot deleted file mode 100644 index 964769637a..0000000000 --- a/test/suites/upgrade/upgrade-c2cc.robot +++ /dev/null @@ -1,170 +0,0 @@ -*** Settings *** -Documentation Tests RHEL 9.8 to RHEL 10.2 upgrade with C2CC enabled across 3 clusters. -... Upgrades each cluster one by one and verifies C2CC connectivity -... survives at each stage. - -Resource ../../resources/common.resource -Resource ../../resources/c2cc.resource -Resource ../../resources/microshift-host.resource -Library Collections -Library SSHLibrary - -Suite Setup Setup -Suite Teardown Teardown - -Test Tags c2cc ostree - - -*** Variables *** -${TARGET_REF} ${EMPTY} -${BOOTC_REGISTRY} ${EMPTY} -${CLUSTER_A_POD_CIDR} ${EMPTY} -${CLUSTER_A_SVC_CIDR} ${EMPTY} -${CLUSTER_A_DOMAIN} ${EMPTY} -${CLUSTER_B_POD_CIDR} ${EMPTY} -${CLUSTER_B_SVC_CIDR} ${EMPTY} -${CLUSTER_B_DOMAIN} ${EMPTY} -${KUBECONFIG_B} ${EMPTY} -${CLUSTER_C_POD_CIDR} ${EMPTY} -${CLUSTER_C_SVC_CIDR} ${EMPTY} -${CLUSTER_C_DOMAIN} ${EMPTY} -${KUBECONFIG_C} ${EMPTY} - - -*** Test Cases *** -Upgrade C2CC Clusters And Verify - [Documentation] Upgrades 3 C2CC-connected clusters one by one - ... and verifies health and C2CC connectivity after each upgrade. - - Verify All Clusters Healthy - Verify All RemoteClusters Healthy - Deploy Test Workloads - Verify Full C2CC Connectivity - - FOR ${alias} IN cluster-a cluster-b cluster-c - Log To Console Upgrading ${alias} to ${TARGET_REF} - Upgrade Cluster ${alias} - Verify All Clusters Healthy - Verify All RemoteClusters Healthy - Wait For Test Pods - Wait For Service Endpoints - Verify Full C2CC Connectivity - END - - [Teardown] Cleanup Test Workloads - - -*** Keywords *** -Setup - [Documentation] Register all three clusters for SSH and oc access - ... and store connection details for reconnection after reboots. - Check Required Env Variables - Should Not Be Empty ${TARGET_REF} TARGET_REF variable is required - Should Not Be Empty ${BOOTC_REGISTRY} BOOTC_REGISTRY variable is required - Login MicroShift Host - Setup Kubeconfig - Logout MicroShift Host - - Register Remote Cluster cluster-a ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG} - Register Remote Cluster cluster-b ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} - Register Remote Cluster cluster-c ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} - -Teardown - [Documentation] Close all connections and clean up. - Teardown All Remote Clusters - Remove Kubeconfig - -Verify All Clusters Healthy - [Documentation] Verify all clusters are running and API server is reachable. - FOR ${alias} IN cluster-a cluster-b cluster-c - ${stdout}= Oc On Cluster ${alias} oc get --raw='/readyz' - Should Be Equal As Strings ${stdout} ok strip_spaces=True - END - -Verify Full C2CC Connectivity - [Documentation] Verify pod-to-pod and pod-to-service connectivity between all cluster pairs. - VAR @{clusters}= cluster-a cluster-b cluster-c - FOR ${src} IN @{clusters} - FOR ${dst} IN @{clusters} - IF '${src}' != '${dst}' - Test Connectivity Between Clusters ${src} ${dst} pod - Test Connectivity Between Clusters ${src} ${dst} service - END - END - END - -Upgrade Cluster - [Documentation] Upgrade a specific cluster to the target bootc image - ... and verify it booted into the new deployment without rollback. - [Arguments] ${alias} - - ${initial_deploy_id}= Get Deployment Id On Cluster ${alias} - - Command On Cluster - ... ${alias} - ... printf '[[registry]]\nlocation = "${BOOTC_REGISTRY}"\ninsecure = true\n' | sudo tee /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf > /dev/null - - Command On Cluster ${alias} bootc switch --quiet ${BOOTC_REGISTRY}/${TARGET_REF} - - Command On Cluster ${alias} - ... rm -f /etc/containers/registries.conf.d/999-microshift-insecure-registry.conf - - Reboot Cluster And Wait ${alias} - - ${current_deploy_id}= Get Deployment Id On Cluster ${alias} - Should Not Be Equal As Strings ${current_deploy_id} ${initial_deploy_id} - ... msg=${alias} rolled back to initial deployment - -Get Deployment Id On Cluster - [Documentation] Get the booted image digest from a specific cluster. - [Arguments] ${alias} - ${stdout}= Command On Cluster - ... ${alias} - ... bootc status --booted --json | python3 -c "import sys,json; print(json.load(sys.stdin)['status']['booted']['image']['imageDigest'])" - RETURN ${stdout} - -Reboot Cluster And Wait - [Documentation] Reboot a cluster and wait for it to come back with greenboot healthy. - [Arguments] ${alias} - - ${boot_id}= Command On Cluster ${alias} - ... cat /proc/sys/kernel/random/boot_id sudo_mode=False - - Disruptive Command On Cluster ${alias} reboot - - Wait Until Keyword Succeeds 10m 15s - ... Cluster Rebooted And Healthy ${alias} ${boot_id} - -Cluster Rebooted And Healthy - [Documentation] Verify cluster has rebooted and greenboot health check passed. - [Arguments] ${alias} ${old_boot_id} - - ${old_conn_id}= Get From Dictionary ${C2CC_SSH_IDS} ${alias} - ${status}= Run Keyword And Return Status - ... SSHLibrary.Switch Connection ${old_conn_id} - IF ${status} SSHLibrary.Close Connection - - ${host} ${port} ${kc}= Get Cluster Connection Info ${alias} - Remove Values From List ${C2CC_REMOTE_ALIASES} ${alias} - Register Remote Cluster ${alias} ${host} ${port} ${kc} - - ${new_boot_id}= Command On Cluster ${alias} - ... cat /proc/sys/kernel/random/boot_id sudo_mode=False - Should Not Be Equal As Strings ${old_boot_id} ${new_boot_id} strip_spaces=True - - ${stdout}= Command On Cluster ${alias} - ... systemctl show -p SubState greenboot-healthcheck.service --value - Should Be Equal As Strings ${stdout} exited strip_spaces=True - -Get Cluster Connection Info - [Documentation] Return host, port, and kubeconfig for a given cluster alias. - [Arguments] ${alias} - IF '${alias}' == 'cluster-a' - RETURN ${USHIFT_HOST} ${SSH_PORT} ${KUBECONFIG} - ELSE IF '${alias}' == 'cluster-b' - RETURN ${HOST2_IP} ${HOST2_SSH_PORT} ${KUBECONFIG_B} - ELSE IF '${alias}' == 'cluster-c' - RETURN ${HOST3_IP} ${HOST3_SSH_PORT} ${KUBECONFIG_C} - ELSE - Fail Unknown cluster alias: ${alias} - END