From c1af6029232cf6227695731c84a83b5afa071dcf Mon Sep 17 00:00:00 2001 From: I583364 Date: Mon, 21 Sep 2026 18:33:13 +0800 Subject: [PATCH 1/3] test data for e2e bulk create and delete of bricks Signed-off-by: I583364 --- .../bulk-brick-lifecycle-lab.md | 429 ++++++++++++++++++ docs/developer-guide/index.md | 1 + test/lab/main_test.go | 89 +++- test/lab/testdata/bgp_default.txt | 32 ++ test/lab/testdata/bgp_vrf.txt | 94 ++++ 5 files changed, 640 insertions(+), 5 deletions(-) create mode 100644 docs/developer-guide/bulk-brick-lifecycle-lab.md create mode 100644 test/lab/testdata/bgp_default.txt create mode 100644 test/lab/testdata/bgp_vrf.txt diff --git a/docs/developer-guide/bulk-brick-lifecycle-lab.md b/docs/developer-guide/bulk-brick-lifecycle-lab.md new file mode 100644 index 000000000..8615f91b4 --- /dev/null +++ b/docs/developer-guide/bulk-brick-lifecycle-lab.md @@ -0,0 +1,429 @@ +# Bulk Brick Lifecycle Lab Runbook + +This runbook describes how to manually run the bulk brick lifecycle test against +real or virtual Cisco NX-OS devices in the ORA showroom clabernetes cluster. +It is intended for evidence-driven validation: every Kubernetes result must be +confirmed from the device CLI. + +## Goal + +Validate that bulk-created bricks converge on a multi-device fleet, and that +bulk deletion leaves no device configuration behind. Pay special attention to +NX-OS BGP cleanup: + +- The default BGP instance must disappear after the last managed BGP brick is + deleted. +- Deleting one VRF-scoped BGP brick must remove only that VRF's BGP domain while + leaving other VRF BGP domains untouched. +- Internal ownership marker peer templates named + `__operator-managed--__` must not be left behind. + +## Prerequisites + +- Access to the ORA showroom Kubernetes cluster that runs clabernetes. +- `kubectl` configured with the `clabernetes` context from the ORA dashboard + Setup flow. +- `clabverter` installed and available in `PATH`. +- `gnmic` installed for out-of-band gNMI checks. +- `helm` installed if deploying the operator from the chart. +- A network-operator image that contains the code under test. + +Verify local tooling: + +```sh +kubectl config get-contexts +command -v clabverter +command -v gnmic +command -v helm +``` + +Use an explicit context for all cluster commands: + +```sh +export KCTX=clabernetes +export LAB_NS=lab--i000000-bulk-brick +export TEST_NS=network-operator-bulk-brick +``` + +## 1. Create The clabernetes Lab + +Use the existing EVPN/VXLAN Containerlab topology as the fleet baseline. The +topology contains two spines, three leaves, and two Linux hosts. By default it +uses the existing Cisco NX-OS image from the example: + +```yaml +topology: + kinds: + cisco_n9kv: + image: ${IMAGE:=vrnetlab/cisco_n9kv:9300-10.4.6} +``` + +Only override `IMAGE` if the lab owner explicitly asks for a different NX-OS +image. + +In ORA showroom, the public `vrnetlab/cisco_n9kv:9300-10.4.6` image may not be +pullable by the clabernetes workers. If the NX-OS pods fail with `pull access +denied for vrnetlab/cisco_n9kv`, keep the repository topology unchanged and use +a temporary copy with an ORA-internal image that is already present in existing +labs: + +```sh +mkdir -p /tmp/network-operator-bulk-brick/clab +perl -pe 's#image: \$\{IMAGE:=vrnetlab/cisco_n9kv:9300-10\.4\.6\}#image: keppel.eu-de-1.cloud.sap/ccloud/containerlab/cisco_n9kv:9300-10.4.3#' \ + examples/cisco-n9k-evpn-vxlan/topology.clab.yml \ + > /tmp/network-operator-bulk-brick/clab/topology.clab.yml +cd /tmp/network-operator-bulk-brick/clab +``` + +Deploy the lab: + +```sh +kubectl --context "$KCTX" create namespace "$LAB_NS" +mkdir -p /tmp/network-operator-bulk-brick/clab +cp examples/cisco-n9k-evpn-vxlan/topology.clab.yml /tmp/network-operator-bulk-brick/clab/topology.clab.yml +cd /tmp/network-operator-bulk-brick/clab +clabverter --debug --destinationNamespace "$LAB_NS" --stdout | kubectl --context "$KCTX" apply -f - +kubectl --context "$KCTX" -n "$LAB_NS" get pods,svc -w +``` + +NX-OS images can take about 10 minutes to become usable. Continue only when all +switch services exist and SSH/gNMI respond. + +## 2. Confirm Device Access + +Find service names and ports: + +```sh +kubectl --context "$KCTX" -n "$LAB_NS" get svc +``` + +For human CLI evidence, port-forward one device at a time, or use one terminal +per device with unique local ports: + +```sh +kubectl --context "$KCTX" -n "$LAB_NS" port-forward svc/evpn-vxlan-fabric-leaf1 8022:22 9339:9339 8443:443 +ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null admin@localhost -p 8022 +gnmic -a localhost:9339 -u admin -p admin --skip-verify get --path 'System/bgp-items' +``` + +For the operator, do not use `localhost` unless the controller also runs on the +same machine. When the operator runs inside the cluster, configure each `Device` +with the clabernetes service DNS name: + +```yaml +spec: + endpoint: + address: evpn-vxlan-fabric-leaf1..svc.cluster.local:9339 + secretRef: + name: fabric-credentials +``` + +## 3. Deploy Network Operator With NX-OS Provider + +The controller must run with the Cisco NX-OS provider: + +```sh +helm upgrade --install network-operator ./charts/network-operator \ + --kube-context "$KCTX" \ + --namespace network-operator \ + --create-namespace \ + --set manager.args='{--leader-elect,--provider=cisco-nxos-gnmi}' +``` + +If testing local changes, build and push an image first, then override the chart +image repository and tag: + +```sh +make docker-build IMG=/network-operator: +docker push /network-operator: +helm upgrade --install network-operator ./charts/network-operator \ + --kube-context "$KCTX" \ + --namespace network-operator \ + --create-namespace \ + --set manager.image.repository=/network-operator \ + --set manager.image.tag= \ + --set manager.args='{--leader-elect,--provider=cisco-nxos-gnmi}' +``` + +Check readiness: + +```sh +kubectl --context "$KCTX" -n network-operator rollout status deploy/network-operator-controller-manager +kubectl --context "$KCTX" get crds | grep networking.metal.ironcore.dev +``` + +## 4. Prepare Test Manifests + +Copy the EVPN/VXLAN Kubernetes manifests to a scratch directory and patch the +five `Device` endpoints to the clabernetes service DNS names. Keep credentials +as `admin/admin` unless the image was customized. + +```sh +mkdir -p /tmp/network-operator-bulk-brick +cp -R examples/cisco-n9k-evpn-vxlan/kubernetes /tmp/network-operator-bulk-brick/kubernetes +``` + +Patch these files in the scratch copy: + +- `01-devices/leaf1.yaml` +- `01-devices/leaf2.yaml` +- `01-devices/leaf3.yaml` +- `01-devices/spine1.yaml` +- `01-devices/spine2.yaml` + +Use addresses like: + +```text +evpn-vxlan-fabric-leaf1..svc.cluster.local:9339 +evpn-vxlan-fabric-leaf2..svc.cluster.local:9339 +evpn-vxlan-fabric-leaf3..svc.cluster.local:9339 +evpn-vxlan-fabric-spine1..svc.cluster.local:9339 +evpn-vxlan-fabric-spine2..svc.cluster.local:9339 +``` + +Create the namespace used by the test CRs: + +```sh +kubectl --context "$KCTX" create namespace "$TEST_NS" +``` + +## 5. Evidence Commands + +Run these commands before apply, after apply, after partial deletion, and after +bulk deletion. Save the output in the tracking issue. + +Kubernetes state: + +```sh +kubectl --context "$KCTX" -n "$TEST_NS" get devices,interfaces,vrfs,bgp,bgppeers,vlans,networkvirtualizationedges,ospf,pim -o wide +kubectl --context "$KCTX" -n "$TEST_NS" get events --sort-by=.lastTimestamp +kubectl --context "$KCTX" -n network-operator logs deploy/network-operator-controller-manager --since=30m +``` + +NX-OS CLI state for each switch: + +```text +show running-config bgp +show running-config bgp | include __operator-managed +show bgp sessions +show vrf +show running-config interface +show vlan brief +show nve vni +``` + +The exact command set can be narrowed per test case, but BGP cleanup evidence +must always include both `show running-config bgp` and the marker check. + +## 6. Test Case 1: Bulk Create + +Start from a clean device state. If the lab was reused, run the bulk delete case +first and confirm no residual config remains. + +Apply the full fleet configuration: + +```sh +kubectl --context "$KCTX" -n "$TEST_NS" apply -k /tmp/network-operator-bulk-brick/kubernetes +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=condition=Ready devices --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=condition=Ready interfaces --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=condition=Ready vrfs --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=condition=Ready bgp --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=condition=Ready bgppeers --all --timeout=10m +``` + +Record PASS only if: + +- All CRs report `Ready=True`. +- Every expected interface, VRF, VLAN, NVE, OSPF, BGP, and BGP peer is visible + in the device CLI. +- No unexpected operator errors appear in logs. + +## 7. Test Case 2: Bulk Delete + +Delete all bricks and wait for finalizers to finish: + +Copy the manifest tree and remove `01-devices` from the copied +`kustomization.yaml`. The bulk delete case validates brick cleanup while keeping +the devices registered and reachable for CLI checks. + +```sh +cp -R /tmp/network-operator-bulk-brick/kubernetes /tmp/network-operator-bulk-brick/kubernetes-bricks-only +$EDITOR /tmp/network-operator-bulk-brick/kubernetes-bricks-only/kustomization.yaml +kubectl --context "$KCTX" -n "$TEST_NS" delete -k /tmp/network-operator-bulk-brick/kubernetes-bricks-only --ignore-not-found +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete interfaces --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete vrfs --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete bgp --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete bgppeers --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete vlans --all --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete networkvirtualizationedges --all --timeout=10m +``` + +Record PASS only if device CLI shows no operator-created residual config. For +NX-OS BGP, `show running-config bgp` must not contain the deleted domains, +neighbors, address families, or `__operator-managed--` markers. + +## 8. Test Case 3: Delete Device With Dependents + +Re-apply at least the resources targeting `leaf1`, then delete the device with +foreground cascading deletion: + +```sh +kubectl --context "$KCTX" -n "$TEST_NS" apply -k /tmp/network-operator-bulk-brick/kubernetes +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=condition=Ready devices/leaf1 --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" delete device leaf1 --cascade=foreground +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete device/leaf1 --timeout=10m +kubectl --context "$KCTX" -n "$TEST_NS" get bgp,bgppeers,vrfs,interfaces,vlans,networkvirtualizationedges,ospf,pim -l networking.metal.ironcore.dev/device=leaf1 +``` + +Record PASS only if all leaf1-owned bricks are deleted and leaf1's CLI has no +remaining operator-created config. Other devices must remain configured. + +## 9. Test Case 4: NX-OS Default BGP Instance Removal + +Run this on a clean leaf or after bulk delete. Apply only a `Device` and one +default-scoped `BGP` brick: + +```yaml +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: leaf1-default-bgp +spec: + deviceRef: + name: leaf1 + asNumber: 65000 + routerId: 10.255.0.1 + addressFamilies: + ipv4Unicast: + enabled: true +``` + +Verify from CLI: + +```text +show running-config bgp +show running-config bgp | include __operator-managed--default__ +``` + +Delete the BGP CR and wait for deletion: + +```sh +kubectl --context "$KCTX" -n "$TEST_NS" delete bgp leaf1-default-bgp +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete bgp/leaf1-default-bgp --timeout=10m +``` + +Record PASS only if the global `router bgp` configuration and +`__operator-managed--default__` marker are gone from the CLI. + +## 10. Test Case 5: NX-OS Per-VRF BGP Removal + +Run this on a clean leaf or after the default BGP cleanup test. Apply two VRFs +and two VRF-scoped BGP bricks on the same device: + +```yaml +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: leaf1-tenant-a +spec: + deviceRef: + name: leaf1 + name: TENANT_A +--- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: leaf1-tenant-b +spec: + deviceRef: + name: leaf1 + name: TENANT_B +--- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: leaf1-tenant-a-bgp +spec: + deviceRef: + name: leaf1 + vrfRef: + name: leaf1-tenant-a + asNumber: 65000 + routerId: 10.255.0.11 + addressFamilies: + ipv4Unicast: + enabled: true +--- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: leaf1-tenant-b-bgp +spec: + deviceRef: + name: leaf1 + vrfRef: + name: leaf1-tenant-b + asNumber: 65000 + routerId: 10.255.0.12 + addressFamilies: + ipv4Unicast: + enabled: true +``` + +Verify both VRF BGP domains and both markers exist: + +```text +show running-config bgp +show running-config bgp | include __operator-managed--TENANT_A__ +show running-config bgp | include __operator-managed--TENANT_B__ +``` + +Delete only one VRF-scoped BGP brick: + +```sh +kubectl --context "$KCTX" -n "$TEST_NS" delete bgp leaf1-tenant-a-bgp +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete bgp/leaf1-tenant-a-bgp --timeout=10m +``` + +Record PASS only if: + +- The `TENANT_A` BGP domain and `__operator-managed--TENANT_A__` marker are gone. +- The `TENANT_B` BGP domain and `__operator-managed--TENANT_B__` marker remain. +- The global BGP instance remains while `TENANT_B` is still managed. + +Then delete the second BGP brick and verify global cleanup: + +```sh +kubectl --context "$KCTX" -n "$TEST_NS" delete bgp leaf1-tenant-b-bgp +kubectl --context "$KCTX" -n "$TEST_NS" wait --for=delete bgp/leaf1-tenant-b-bgp --timeout=10m +``` + +Record PASS only if no BGP config and no ownership markers remain. + +## Result Template + +Use this table in the tracking issue: + +| Case | CRs applied/removed | Operator result | Device CLI evidence | Result | Issue | +| ---- | ------------------- | --------------- | ------------------- | ------ | ----- | +| Bulk create | | | | PASS/FAIL | | +| Bulk delete | | | | PASS/FAIL | | +| Delete device with dependents | | | | PASS/FAIL | | +| NX-OS default BGP removal | | | | PASS/FAIL | | +| NX-OS per-VRF BGP removal | | | | PASS/FAIL | | + +For every leftover or discrepancy, file a separate bug issue and link it from +the `Issue` column. + +## Cleanup + +Remove test CRs, the operator, and the lab namespace when finished: + +```sh +kubectl --context "$KCTX" -n "$TEST_NS" delete -k /tmp/network-operator-bulk-brick/kubernetes --ignore-not-found +kubectl --context "$KCTX" delete namespace "$TEST_NS" --ignore-not-found +helm --kube-context "$KCTX" -n network-operator uninstall network-operator +cd /tmp/network-operator-bulk-brick/clab +clabverter --debug --destinationNamespace "$LAB_NS" --stdout | kubectl --context "$KCTX" delete -f - --ignore-not-found +kubectl --context "$KCTX" delete namespace "$LAB_NS" +``` \ No newline at end of file diff --git a/docs/developer-guide/index.md b/docs/developer-guide/index.md index a71a23888..e6390c9c6 100644 --- a/docs/developer-guide/index.md +++ b/docs/developer-guide/index.md @@ -3,3 +3,4 @@ This section covers workflows for contributing to and developing the Network Operator. - [Local Debug Environment](./local-debug-environment.md) — Run and debug the controller locally against a Kind cluster from your IDE. +- [Bulk Brick Lifecycle Lab Runbook](./bulk-brick-lifecycle-lab.md) — Manually validate bulk brick create/delete and NX-OS BGP cleanup against a clabernetes fleet. diff --git a/test/lab/main_test.go b/test/lab/main_test.go index bc5ae791d..b1c0141cd 100644 --- a/test/lab/main_test.go +++ b/test/lab/main_test.go @@ -56,6 +56,7 @@ func DefaultCmds() map[string]script.Cmd { cmds := scripttest.DefaultCmds() cmds["vty"] = Vty() cmds["apply"] = Apply() + cmds["delete"] = Delete() return cmds } @@ -155,19 +156,79 @@ func Apply() script.Cmd { ) } +// Delete returns a script command that deletes a Kubernetes resource from the cluster +// and waits for the resource to be fully removed (finalizers completed). +func Delete() script.Cmd { + return script.Command( + script.CmdUsage{ + Summary: "delete a Kubernetes resource and wait for it to be fully removed", + Args: "file", + Async: true, + }, + func(s *script.State, args ...string) (script.WaitFunc, error) { + if len(args) != 1 { + return nil, script.ErrUsage + } + data, err := os.ReadFile(s.Path(args[0])) + if err != nil { + return nil, fmt.Errorf("failed to read file %s: %w", args[0], err) + } + json, err := yaml.YAMLToJSON(data) + if err != nil { + return nil, fmt.Errorf("failed to convert YAML to JSON: %w", err) + } + dec := serializer.NewCodecFactory(scheme.Scheme).UniversalDeserializer() + obj, _, err := dec.Decode(json, nil, nil) + if err != nil { + return nil, fmt.Errorf("failed to decode resource: %w", err) + } + res, ok := obj.(client.Object) + if !ok { + return nil, fmt.Errorf("decoded object is not a client.Object: %T", obj) + } + res.SetNamespace(metav1.NamespaceDefault) + if err := k8sClient.Delete(s.Context(), res); client.IgnoreNotFound(err) != nil { + return nil, fmt.Errorf("failed to delete resource: %w", err) + } + wait := func(s *script.State) (stdout, stderr string, reterr error) { + err := k8sClient.Get(s.Context(), client.ObjectKeyFromObject(res), res) + if err == nil { + return "", "", fmt.Errorf("resource %s still exists", res.GetName()) + } + if client.IgnoreNotFound(err) != nil { + return "", "", fmt.Errorf("failed to check resource deletion: %w", err) + } + return "", "", nil + } + return WaitTimeout(wait, timeout, interval), nil + }, + ) +} + // TODO(felix-kaestner): Load endpoint configuration from a config file. var Endpoint = struct { - Addr string - User string - Pass string `json:"-"` + Addr string + GNMIAddr string + User string + Pass string `json:"-"` + SSHPort string + GNMIPort string }{} // ReadEnv reads required environment variables and populates the global Endpoint struct. +// SSH_PORT and GNMI_PORT are optional and default to 22 and 9339 respectively, +// which allows overriding when using port-forwarding (e.g. SSH_PORT=8022 GNMI_PORT=9339). +// GNMI_ADDR overrides the address used for the Device gNMI endpoint (defaults to ADDR). +// This allows SSH to reach the device via a local port-forward while the operator +// connects directly via the cluster-internal address. func ReadEnv(t *testing.T) { t.Helper() Endpoint.Addr = MustGetEnv(t, "ADDR") + Endpoint.GNMIAddr = GetEnvOrDefault(t, "GNMI_ADDR", Endpoint.Addr) Endpoint.User = MustGetEnv(t, "USER") Endpoint.Pass = MustGetEnv(t, "PASS") + Endpoint.SSHPort = GetEnvOrDefault(t, "SSH_PORT", "22") + Endpoint.GNMIPort = GetEnvOrDefault(t, "GNMI_PORT", "9339") } var sshClient *ssh.Client @@ -177,7 +238,7 @@ var sshClient *ssh.Client func SetupSSH(t *testing.T) { t.Helper() var err error - sshClient, err = ssh.Dial("tcp", net.JoinHostPort(Endpoint.Addr, "22"), &ssh.ClientConfig{ + sshClient, err = ssh.Dial("tcp", net.JoinHostPort(Endpoint.Addr, Endpoint.SSHPort), &ssh.ClientConfig{ User: Endpoint.User, Auth: []ssh.AuthMethod{ssh.Password(Endpoint.Pass)}, HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec @@ -224,8 +285,9 @@ func SetupK8s(t *testing.T) { Name: "device", Namespace: metav1.NamespaceDefault, Spec: v1alpha1.DeviceSpec{ + Provider: "nx.cisco.networking.metal.ironcore.dev", Endpoint: v1alpha1.Endpoint{ - Address: net.JoinHostPort(Endpoint.Addr, "9339"), + Address: net.JoinHostPort(ResolveAddr(t, Endpoint.GNMIAddr), Endpoint.GNMIPort), SecretRef: &v1alpha1.SecretReference{Name: "secret", Namespace: metav1.NamespaceDefault}, }, }, @@ -248,6 +310,23 @@ func Create(t *testing.T, obj client.Object) { }) } +// ResolveAddr resolves a hostname to its first IPv4 address string. +// The Device address field requires IPv4 format, so hostnames like "localhost" must be resolved. +func ResolveAddr(t *testing.T, host string) string { + t.Helper() + addrs, err := net.LookupHost(host) + if err != nil { + t.Fatalf("failed to resolve host %q: %v", host, err) + } + for _, addr := range addrs { + if ip := net.ParseIP(addr); ip != nil && ip.To4() != nil { + return ip.String() + } + } + t.Fatalf("no IPv4 address found for host %q", host) + return "" +} + // MustGetEnv retrieves the value of an environment variable // and fails the test if it is not set. func MustGetEnv(t *testing.T, key string) string { diff --git a/test/lab/testdata/bgp_default.txt b/test/lab/testdata/bgp_default.txt new file mode 100644 index 000000000..fead81f71 --- /dev/null +++ b/test/lab/testdata/bgp_default.txt @@ -0,0 +1,32 @@ +# Test Case 4: NX-OS default BGP instance — create and delete +# +# Verifies that: +# - The global BGP instance and ownership marker appear after apply +# - Both are fully removed after deletion (no residual config) + +apply bgp.yaml +sleep 30s +vty show running-config bgp +stdout 'router bgp 65000' +stdout 'router-id 10\.255\.0\.1' +stdout '__operator-managed--default__' + +delete bgp.yaml +sleep 30s +vty show running-config bgp +! stdout 'router bgp 65000' +! stdout '__operator-managed' + +-- bgp.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: default-bgp +spec: + deviceRef: + name: device + asNumber: 65000 + routerId: 10.255.0.1 + addressFamilies: + ipv4Unicast: + enabled: true diff --git a/test/lab/testdata/bgp_vrf.txt b/test/lab/testdata/bgp_vrf.txt new file mode 100644 index 000000000..50f8b2cfd --- /dev/null +++ b/test/lab/testdata/bgp_vrf.txt @@ -0,0 +1,94 @@ +# Test Case 5: NX-OS per-VRF BGP instance — create and selective delete +# +# Verifies that: +# - Both VRF BGP domains and their ownership markers appear after apply +# - Deleting one VRF-scoped BGP removes only that VRF domain and marker +# - The other VRF domain and the global BGP instance remain untouched +# - After deleting the second VRF-scoped BGP, the global instance is removed too + +apply vrf-a.yaml +apply vrf-b.yaml +apply bgp-a.yaml +apply bgp-b.yaml +sleep 30s + +vty show running-config bgp +stdout 'router bgp 65000' +stdout 'vrf TENANT_A' +stdout '__operator-managed--TENANT_A__' +stdout 'vrf TENANT_B' +stdout '__operator-managed--TENANT_B__' + +# Delete only TENANT_A BGP — TENANT_B must remain +delete bgp-a.yaml +sleep 30s +vty show running-config bgp +! stdout '__operator-managed--TENANT_A__' +! stdout 'vrf TENANT_A' +stdout 'router bgp 65000' +stdout 'vrf TENANT_B' +stdout '__operator-managed--TENANT_B__' + +# Delete TENANT_B BGP — global instance must be gone too +delete bgp-b.yaml +sleep 30s +vty show running-config bgp +! stdout 'router bgp 65000' +! stdout '__operator-managed' + +delete vrf-a.yaml +delete vrf-b.yaml + +-- vrf-a.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: tenant-a +spec: + deviceRef: + name: device + name: TENANT_A + routeDistinguisher: "65000:101" + +-- vrf-b.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: tenant-b +spec: + deviceRef: + name: device + name: TENANT_B + routeDistinguisher: "65000:102" + +-- bgp-a.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: tenant-a-bgp +spec: + deviceRef: + name: device + vrfRef: + name: tenant-a + asNumber: 65000 + routerId: 10.255.0.11 + addressFamilies: + ipv4Unicast: + enabled: true + +-- bgp-b.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: tenant-b-bgp +spec: + deviceRef: + name: device + vrfRef: + name: tenant-b + asNumber: 65000 + routerId: 10.255.0.12 + addressFamilies: + ipv4Unicast: + enabled: true From 57812b654575a7131c8a5ab3ac3abedcbdeaac7d Mon Sep 17 00:00:00 2001 From: I583364 Date: Tue, 22 Sep 2026 17:14:30 +0800 Subject: [PATCH 2/3] Add NAMESPACE support and BGP lab test cases - Support NAMESPACE env var so tests run in a dedicated namespace instead of always defaulting to "default" - Delete pre-existing objects in Create() to recover from interrupted runs - Increase timeout from 10s to 60s for real-device reconciliation - Add precondition check to BGP test cases (verify clean state before test) - Add bgp.txt combining test cases 4 and 5 into a single script Signed-off-by: I583364 --- test/lab/main_test.go | 21 +++-- test/lab/testdata/bgp.txt | 130 ++++++++++++++++++++++++++++++ test/lab/testdata/bgp_default.txt | 3 + test/lab/testdata/bgp_vrf.txt | 3 + 4 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 test/lab/testdata/bgp.txt diff --git a/test/lab/main_test.go b/test/lab/main_test.go index b1c0141cd..9e0ea8949 100644 --- a/test/lab/main_test.go +++ b/test/lab/main_test.go @@ -30,10 +30,12 @@ import ( ) const ( - timeout = 10 * time.Second + timeout = 60 * time.Second interval = time.Second ) +var testNamespace string + // TestAll runs all lab tests by setting up the environment, SSH connection, // and Kubernetes client, then executing all test cases from the testdata directory. func TestAll(t *testing.T) { @@ -136,7 +138,7 @@ func Apply() script.Cmd { if !ok { return nil, fmt.Errorf("decoded object is not a client.Object: %T", obj) } - res.SetNamespace(metav1.NamespaceDefault) + res.SetNamespace(testNamespace) res.SetLabels(map[string]string{v1alpha1.DeviceLabel: "device"}) if err := k8sClient.Create(s.Context(), res); err != nil { return nil, fmt.Errorf("failed to apply resource: %w", err) @@ -186,7 +188,7 @@ func Delete() script.Cmd { if !ok { return nil, fmt.Errorf("decoded object is not a client.Object: %T", obj) } - res.SetNamespace(metav1.NamespaceDefault) + res.SetNamespace(testNamespace) if err := k8sClient.Delete(s.Context(), res); client.IgnoreNotFound(err) != nil { return nil, fmt.Errorf("failed to delete resource: %w", err) } @@ -229,6 +231,7 @@ func ReadEnv(t *testing.T) { Endpoint.Pass = MustGetEnv(t, "PASS") Endpoint.SSHPort = GetEnvOrDefault(t, "SSH_PORT", "22") Endpoint.GNMIPort = GetEnvOrDefault(t, "GNMI_PORT", "9339") + testNamespace = GetEnvOrDefault(t, "NAMESPACE", metav1.NamespaceDefault) } var sshClient *ssh.Client @@ -274,7 +277,7 @@ func SetupK8s(t *testing.T) { } Create(t, &corev1.Secret{ Name: "secret", - Namespace: metav1.NamespaceDefault, + Namespace: testNamespace, StringData: map[string]string{ "username": Endpoint.User, "password": Endpoint.Pass, @@ -283,21 +286,25 @@ func SetupK8s(t *testing.T) { }) Create(t, &v1alpha1.Device{ Name: "device", - Namespace: metav1.NamespaceDefault, + Namespace: testNamespace, Spec: v1alpha1.DeviceSpec{ Provider: "nx.cisco.networking.metal.ironcore.dev", Endpoint: v1alpha1.Endpoint{ Address: net.JoinHostPort(ResolveAddr(t, Endpoint.GNMIAddr), Endpoint.GNMIPort), - SecretRef: &v1alpha1.SecretReference{Name: "secret", Namespace: metav1.NamespaceDefault}, + SecretRef: &v1alpha1.SecretReference{Name: "secret", Namespace: testNamespace}, }, }, }) } // Create creates a Kubernetes object in the cluster and registers a cleanup function -// to delete it after the test completes. It fails the test if the creation fails. +// to delete it after the test completes. If the object already exists from a +// previous interrupted run, it is deleted first so the test starts from a clean state. func Create(t *testing.T, obj client.Object) { t.Helper() + if err := k8sClient.Delete(t.Context(), obj); client.IgnoreNotFound(err) != nil { + t.Fatalf("failed to delete existing %T: %v", obj, err) + } if err := k8sClient.Create(t.Context(), obj); err != nil { t.Fatalf("failed to create %T: %v", obj, err) } diff --git a/test/lab/testdata/bgp.txt b/test/lab/testdata/bgp.txt new file mode 100644 index 000000000..68fb8c180 --- /dev/null +++ b/test/lab/testdata/bgp.txt @@ -0,0 +1,130 @@ +# Test Case 4: NX-OS default BGP instance — create and delete +# +# Verifies that: +# - The global BGP instance and ownership marker appear after apply +# - Both are fully removed after deletion (no residual config) + +? vty show running-config bgp +! stdout 'router bgp' + +apply bgp.yaml +sleep 30s +vty show running-config bgp +stdout 'router bgp 65000' +stdout 'router-id 10\.255\.0\.1' +stdout '__operator-managed--default__' + +delete bgp.yaml +sleep 30s +? vty show running-config bgp +! stdout 'router bgp 65000' +! stdout '__operator-managed' + +# Test Case 5: NX-OS per-VRF BGP instance — create and selective delete +# +# Verifies that: +# - Both VRF BGP domains and their ownership markers appear after apply +# - Deleting one VRF-scoped BGP removes only that VRF domain and marker +# - The other VRF domain and the global BGP instance remain untouched +# - After deleting the second VRF-scoped BGP, the global instance is removed too + +apply vrf-a.yaml +apply vrf-b.yaml +apply bgp-a.yaml +apply bgp-b.yaml +sleep 30s + +vty show running-config bgp +stdout 'router bgp 65000' +stdout 'vrf TENANT_A' +stdout '__operator-managed--TENANT_A__' +stdout 'vrf TENANT_B' +stdout '__operator-managed--TENANT_B__' + +# Delete only TENANT_A BGP — TENANT_B must remain +delete bgp-a.yaml +sleep 30s +vty show running-config bgp +! stdout '__operator-managed--TENANT_A__' +! stdout 'vrf TENANT_A' +stdout 'router bgp 65000' +stdout 'vrf TENANT_B' +stdout '__operator-managed--TENANT_B__' + +# Delete TENANT_B BGP — global instance must be gone too +delete bgp-b.yaml +sleep 30s +? vty show running-config bgp +! stdout 'router bgp 65000' +! stdout '__operator-managed' + +delete vrf-a.yaml +delete vrf-b.yaml + +-- bgp.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: default-bgp +spec: + deviceRef: + name: device + asNumber: 65000 + routerId: 10.255.0.1 + addressFamilies: + ipv4Unicast: + enabled: true + +-- vrf-a.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: tenant-a +spec: + deviceRef: + name: device + name: TENANT_A + routeDistinguisher: "65000:101" + +-- vrf-b.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: VRF +metadata: + name: tenant-b +spec: + deviceRef: + name: device + name: TENANT_B + routeDistinguisher: "65000:102" + +-- bgp-a.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: tenant-a-bgp +spec: + deviceRef: + name: device + vrfRef: + name: tenant-a + asNumber: 65000 + routerId: 10.255.0.11 + addressFamilies: + ipv4Unicast: + enabled: true + +-- bgp-b.yaml -- +apiVersion: networking.metal.ironcore.dev/v1alpha1 +kind: BGP +metadata: + name: tenant-b-bgp +spec: + deviceRef: + name: device + vrfRef: + name: tenant-b + asNumber: 65000 + routerId: 10.255.0.12 + addressFamilies: + ipv4Unicast: + enabled: true diff --git a/test/lab/testdata/bgp_default.txt b/test/lab/testdata/bgp_default.txt index fead81f71..4bcbc18f3 100644 --- a/test/lab/testdata/bgp_default.txt +++ b/test/lab/testdata/bgp_default.txt @@ -4,6 +4,9 @@ # - The global BGP instance and ownership marker appear after apply # - Both are fully removed after deletion (no residual config) +? vty show running-config bgp +! stdout 'router bgp' + apply bgp.yaml sleep 30s vty show running-config bgp diff --git a/test/lab/testdata/bgp_vrf.txt b/test/lab/testdata/bgp_vrf.txt index 50f8b2cfd..dee0b55aa 100644 --- a/test/lab/testdata/bgp_vrf.txt +++ b/test/lab/testdata/bgp_vrf.txt @@ -6,6 +6,9 @@ # - The other VRF domain and the global BGP instance remain untouched # - After deleting the second VRF-scoped BGP, the global instance is removed too +? vty show running-config bgp +! stdout 'router bgp' + apply vrf-a.yaml apply vrf-b.yaml apply bgp-a.yaml From de177fb587994b726ccb6bf6e3ef21e261400a74 Mon Sep 17 00:00:00 2001 From: I583364 Date: Tue, 22 Sep 2026 17:59:45 +0800 Subject: [PATCH 3/3] Fix lint errors in lab test harness Signed-off-by: I583364 --- test/lab/main_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/lab/main_test.go b/test/lab/main_test.go index 9e0ea8949..af960dc56 100644 --- a/test/lab/main_test.go +++ b/test/lab/main_test.go @@ -189,7 +189,7 @@ func Delete() script.Cmd { return nil, fmt.Errorf("decoded object is not a client.Object: %T", obj) } res.SetNamespace(testNamespace) - if err := k8sClient.Delete(s.Context(), res); client.IgnoreNotFound(err) != nil { + if err := client.IgnoreNotFound(k8sClient.Delete(s.Context(), res)); err != nil { return nil, fmt.Errorf("failed to delete resource: %w", err) } wait := func(s *script.State) (stdout, stderr string, reterr error) { @@ -302,7 +302,7 @@ func SetupK8s(t *testing.T) { // previous interrupted run, it is deleted first so the test starts from a clean state. func Create(t *testing.T, obj client.Object) { t.Helper() - if err := k8sClient.Delete(t.Context(), obj); client.IgnoreNotFound(err) != nil { + if err := client.IgnoreNotFound(k8sClient.Delete(t.Context(), obj)); err != nil { t.Fatalf("failed to delete existing %T: %v", obj, err) } if err := k8sClient.Create(t.Context(), obj); err != nil { @@ -321,7 +321,7 @@ func Create(t *testing.T, obj client.Object) { // The Device address field requires IPv4 format, so hostnames like "localhost" must be resolved. func ResolveAddr(t *testing.T, host string) string { t.Helper() - addrs, err := net.LookupHost(host) + addrs, err := (&net.Resolver{}).LookupHost(t.Context(), host) if err != nil { t.Fatalf("failed to resolve host %q: %v", host, err) }