Isolate per-device provider state in controller tests

Makefile

@@ -66,7 +66,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate setup-envtest ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /lab | grep -v /gnmi/) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e | grep -v /lab | grep -v /gnmi) -coverprofile cover.out
 
 .PHONY: coverage
 coverage: test ## Run tests and generate coverage report.

internal/controller/cisco/nx/bordergateway_controller_test.go

@@ -8,6 +8,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -77,6 +78,12 @@ var _ = Describe("BorderGateway Controller", func() {
 			bg.Namespace = metav1.NamespaceDefault
 			Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, bg))).To(Succeed())
 
+			By("Waiting for BorderGateway to be fully deleted")
+			Eventually(func(g Gomega) {
+				err := k8sClient.Get(ctx, key, &nxv1alpha1.BorderGateway{})
+				g.Expect(errors.IsNotFound(err)).To(BeTrue())
+			}).Should(Succeed())
+
 			By("Ensuring the resource is deleted from the provider")
 			Eventually(func(g Gomega) {
 				g.Expect(testProvider.BorderGateway).To(BeNil(), "Provider BorderGateway settings should be reset after deletion")

internal/controller/cisco/nx/suite_test.go

@@ -13,6 +13,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"go.uber.org/zap/zapcore"
 
 	coordinationv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -55,7 +56,11 @@ func TestControllers(t *testing.T) {
 }
 
 var _ = BeforeSuite(func() {
-	logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
+	logf.SetLogger(zap.New(
+		zap.WriteTo(GinkgoWriter),
+		zap.UseDevMode(true),
+		zap.Level(zapcore.Level(-3)),
+	))
 
 	SetDefaultEventuallyTimeout(time.Minute)
 	SetDefaultEventuallyPollingInterval(200 * time.Millisecond)
@@ -90,7 +95,6 @@ var _ = BeforeSuite(func() {
 
 	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:  scheme.Scheme,
-		Logger:  GinkgoLogr,
 		Metrics: metricsserver.Options{BindAddress: "0"},
 	})
 	Expect(err).ToNot(HaveOccurred())
@@ -160,10 +164,9 @@ var _ = BeforeSuite(func() {
 		Expect(err).ToNot(HaveOccurred(), "failed to run manager")
 	}()
 
-	Eventually(func() error {
-		var namespace corev1.Namespace
-		return k8sClient.Get(ctx, client.ObjectKey{Name: metav1.NamespaceDefault}, &namespace)
-	}).Should(Succeed())
+	Eventually(func() bool {
+		return k8sManager.GetCache().WaitForCacheSync(ctx)
+	}).Should(BeTrue())
 })
 
 var _ = AfterSuite(func() {

internal/controller/cisco/nx/system_controller_test.go

@@ -6,6 +6,7 @@ package nx
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -58,6 +59,12 @@ var _ = Describe("System Controller", func() {
 			system.Namespace = metav1.NamespaceDefault
 			Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, system))).To(Succeed())
 
+			By("Waiting for System to be fully deleted")
+			Eventually(func(g Gomega) {
+				err := k8sClient.Get(ctx, key, &nxv1alpha1.System{})
+				g.Expect(errors.IsNotFound(err)).To(BeTrue())
+			}).Should(Succeed())
+
 			By("Ensuring the resource is deleted from the provider")
 			Eventually(func(g Gomega) {
 				g.Expect(testProvider.Settings).To(BeNil(), "Provider System settings should be reset after deletion")

internal/controller/cisco/nx/vpcdomain_controller_test.go

@@ -6,6 +6,7 @@ package nx
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -115,6 +116,12 @@ var _ = Describe("VPCDomain Controller", func() {
 			By("Cleanup the specific resource instance VPCDomain")
 			Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
 
+			By("Waiting for VPCDomain to be fully deleted")
+			Eventually(func(g Gomega) {
+				err := k8sClient.Get(ctx, vpcdomainKey, &nxv1.VPCDomain{})
+				g.Expect(errors.IsNotFound(err)).To(BeTrue())
+			}).Should(Succeed())
+
 			By("Ensuring the resource is deleted from the provider")
 			Eventually(func(g Gomega) {
 				g.Expect(testProvider.VPCDomain).To(BeNil(), "Provider VPCDomain should be nil")
@@ -293,19 +300,37 @@ var _ = Describe("VPCDomain Controller", func() {
 			By("Cleanup the VPCDomain")
 			Expect(k8sClient.Delete(ctx, resource)).To(Succeed())
 
+			By("Waiting for VPCDomain to be fully deleted")
+			Eventually(func(g Gomega) {
+				err := k8sClient.Get(ctx, vpcdomainKey, &nxv1.VPCDomain{})
+				g.Expect(errors.IsNotFound(err)).To(BeTrue())
+			}).Should(Succeed())
+
 			By("Cleanup Interface and VRF resources")
 			for _, ifName := range []string{name + "-phys", name + "-po", name + "-phys-b", name + "-po-b", name + "-lo0"} {
 				intf := &corev1.Interface{}
 				intf.Name = ifName
 				intf.Namespace = metav1.NamespaceDefault
 				Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, intf))).To(Succeed())
 			}
+			Eventually(func(g Gomega) {
+				for _, ifName := range []string{name + "-phys", name + "-po", name + "-phys-b", name + "-po-b", name + "-lo0"} {
+					err := k8sClient.Get(ctx, client.ObjectKey{Name: ifName, Namespace: metav1.NamespaceDefault}, &corev1.Interface{})
+					g.Expect(errors.IsNotFound(err)).To(BeTrue())
+				}
+			}).Should(Succeed())
 			for _, vrfName := range []string{name + "-vrf-a", name + "-vrf-b"} {
 				vrf := &corev1.VRF{}
 				vrf.Name = vrfName
 				vrf.Namespace = metav1.NamespaceDefault
 				Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, vrf))).To(Succeed())
 			}
+			Eventually(func(g Gomega) {
+				for _, vrfName := range []string{name + "-vrf-a", name + "-vrf-b"} {
+					err := k8sClient.Get(ctx, client.ObjectKey{Name: vrfName, Namespace: metav1.NamespaceDefault}, &corev1.VRF{})
+					g.Expect(errors.IsNotFound(err)).To(BeTrue())
+				}
+			}).Should(Succeed())
 
 			By("Ensuring the resource is deleted from the provider")
 			Eventually(func(g Gomega) {

internal/controller/core/acl_controller_test.go

@@ -8,6 +8,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -77,7 +78,13 @@ var _ = Describe("AccessControlList Controller", func() {
 
 			By("Verifying the resource is removed from the provider")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.ACLs.Has(name)).To(BeFalse(), "Provider shouldn't have AccessControlList configured anymore")
+				g.Expect(testDevices.StateFor(name).ACLs.Has(name)).To(BeFalse(), "Provider shouldn't have AccessControlList configured anymore")
+			}).Should(Succeed())
+
+			By("Waiting for the AccessControlList to be fully deleted")
+			Eventually(func(g Gomega) {
+				err := k8sClient.Get(ctx, key, &v1alpha1.AccessControlList{})
+				g.Expect(apierrors.IsNotFound(err)).To(BeTrue())
 			}).Should(Succeed())
 
 			By("Cleaning up the Device resource")
@@ -124,7 +131,7 @@ var _ = Describe("AccessControlList Controller", func() {
 
 			By("Ensuring the resource is created in the provider")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.ACLs.Has(name)).To(BeTrue(), "Provider should have AccessControlList configured")
+				g.Expect(testDevices.StateFor(name).ACLs.Has(name)).To(BeTrue(), "Provider should have AccessControlList configured")
 			}).Should(Succeed())
 		})
 	})

internal/controller/core/banner_controller_test.go

@@ -6,6 +6,7 @@ package core
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -46,8 +47,14 @@ var _ = Describe("Banner Controller", func() {
 
 			By("Verifying the resource is removed from the provider")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.PreLoginBanner).To(BeNil(), "Provider PreLogin Banner should be nil")
-				g.Expect(testProvider.PostLoginBanner).To(BeNil(), "Provider PostLogin Banner should be nil")
+				g.Expect(testDevices.StateFor(name).PreLoginBanner).To(BeNil(), "Provider PreLogin Banner should be nil")
+				g.Expect(testDevices.StateFor(name).PostLoginBanner).To(BeNil(), "Provider PostLogin Banner should be nil")
+			}).Should(Succeed())
+
+			By("Waiting for the Banner to be fully deleted")
+			Eventually(func(g Gomega) {
+				err := k8sClient.Get(ctx, key, &v1alpha1.Banner{})
+				g.Expect(apierrors.IsNotFound(err)).To(BeTrue())
 			}).Should(Succeed())
 
 			By("Cleaning up the Device resource")
@@ -108,10 +115,10 @@ var _ = Describe("Banner Controller", func() {
 
 			By("Ensuring the resource is created in the provider")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.PreLoginBanner).ToNot(BeNil(), "Provider Banner should not be nil")
-				g.Expect(testProvider.PostLoginBanner).To(BeNil(), "Provider PostLogin Banner should be nil")
-				if testProvider.PreLoginBanner != nil {
-					g.Expect(*testProvider.PreLoginBanner).To(Equal("Test Banner"))
+				g.Expect(testDevices.StateFor(name).PreLoginBanner).ToNot(BeNil(), "Provider Banner should not be nil")
+				g.Expect(testDevices.StateFor(name).PostLoginBanner).To(BeNil(), "Provider PostLogin Banner should be nil")
+				if testDevices.StateFor(name).PreLoginBanner != nil {
+					g.Expect(*testDevices.StateFor(name).PreLoginBanner).To(Equal("Test Banner"))
 				}
 			}).Should(Succeed())
 		})
@@ -167,10 +174,10 @@ var _ = Describe("Banner Controller", func() {
 
 			By("Ensuring the resource is created in the provider")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.PreLoginBanner).To(BeNil(), "Provider PreLogin Banner should be nil")
-				g.Expect(testProvider.PostLoginBanner).ToNot(BeNil(), "Provider PostLogin Banner should not be nil")
-				if testProvider.PostLoginBanner != nil {
-					g.Expect(*testProvider.PostLoginBanner).To(Equal("Test Banner"))
+				g.Expect(testDevices.StateFor(name).PreLoginBanner).To(BeNil(), "Provider PreLogin Banner should be nil")
+				g.Expect(testDevices.StateFor(name).PostLoginBanner).ToNot(BeNil(), "Provider PostLogin Banner should not be nil")
+				if testDevices.StateFor(name).PostLoginBanner != nil {
+					g.Expect(*testDevices.StateFor(name).PostLoginBanner).To(Equal("Test Banner"))
 				}
 			}).Should(Succeed())
 		})

internal/controller/core/bgp_controller_test.go

@@ -35,37 +35,50 @@ var _ = Describe("BGP Controller", func() {
 		})
 
 		AfterEach(func() {
+			// Use the manager client for MatchingFields queries — the direct k8sClient
+			// does not have the custom field indexes registered on the API server.
 			By("Cleaning up BGP resources for this device")
 			bgpList := &v1alpha1.BGPList{}
-			Expect(k8sClient.List(ctx, bgpList, client.InNamespace(metav1.NamespaceDefault), client.MatchingLabels{v1alpha1.DeviceLabel: device.Name})).To(Succeed())
+			Expect(k8sManager.GetClient().List(ctx, bgpList, client.InNamespace(metav1.NamespaceDefault), client.MatchingFields{v1alpha1.DeviceRefIndexKey: device.Name})).To(Succeed())
 			for i := range bgpList.Items {
 				Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, &bgpList.Items[i]))).To(Succeed())
 			}
+			By("Waiting for BGP resources to be fully deleted")
+			Eventually(func(g Gomega) {
+				list := &v1alpha1.BGPList{}
+				g.Expect(k8sManager.GetClient().List(ctx, list, client.InNamespace(metav1.NamespaceDefault), client.MatchingFields{v1alpha1.DeviceRefIndexKey: device.Name})).To(Succeed())
+				g.Expect(list.Items).To(BeEmpty())
+			}).Should(Succeed())
 
 			By("Cleaning up VRF resources for this device")
 			vrfList := &v1alpha1.VRFList{}
-			Expect(k8sClient.List(ctx, vrfList, client.InNamespace(metav1.NamespaceDefault), client.MatchingLabels{v1alpha1.DeviceLabel: device.Name})).To(Succeed())
+			Expect(k8sManager.GetClient().List(ctx, vrfList, client.InNamespace(metav1.NamespaceDefault), client.MatchingFields{v1alpha1.DeviceRefIndexKey: device.Name})).To(Succeed())
 			for i := range vrfList.Items {
 				Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, &vrfList.Items[i]))).To(Succeed())
 			}
+			By("Waiting for VRF resources to be fully deleted")
+			Eventually(func(g Gomega) {
+				list := &v1alpha1.VRFList{}
+				g.Expect(k8sManager.GetClient().List(ctx, list, client.InNamespace(metav1.NamespaceDefault), client.MatchingFields{v1alpha1.DeviceRefIndexKey: device.Name})).To(Succeed())
+				g.Expect(list.Items).To(BeEmpty())
+			}).Should(Succeed())
 
 			By("Cleaning up RoutingPolicy resources for this device")
 			rpList := &v1alpha1.RoutingPolicyList{}
-			Expect(k8sClient.List(ctx, rpList, client.InNamespace(metav1.NamespaceDefault), client.MatchingLabels{v1alpha1.DeviceLabel: device.Name})).To(Succeed())
+			Expect(k8sManager.GetClient().List(ctx, rpList, client.InNamespace(metav1.NamespaceDefault), client.MatchingFields{v1alpha1.DeviceRefIndexKey: device.Name})).To(Succeed())
 			for i := range rpList.Items {
 				Expect(client.IgnoreNotFound(k8sClient.Delete(ctx, &rpList.Items[i]))).To(Succeed())
 			}
-
-			By("Waiting for BGP resources to be fully deleted")
+			By("Waiting for RoutingPolicy resources to be fully deleted")
 			Eventually(func(g Gomega) {
-				list := &v1alpha1.BGPList{}
-				g.Expect(k8sClient.List(ctx, list, client.InNamespace(metav1.NamespaceDefault), client.MatchingLabels{v1alpha1.DeviceLabel: device.Name})).To(Succeed())
+				list := &v1alpha1.RoutingPolicyList{}
+				g.Expect(k8sManager.GetClient().List(ctx, list, client.InNamespace(metav1.NamespaceDefault), client.MatchingFields{v1alpha1.DeviceRefIndexKey: device.Name})).To(Succeed())
 				g.Expect(list.Items).To(BeEmpty())
 			}).Should(Succeed())
 
 			By("Verifying BGP is removed from the provider")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.BGP).To(BeNil(), "Provider should not have BGP instance configured")
+				g.Expect(testDevices.StateFor(device.Name).BGP).To(BeNil(), "Provider should not have BGP instance configured")
 			}).Should(Succeed())
 
 			By("Deleting the Device resource")
@@ -121,7 +134,7 @@ var _ = Describe("BGP Controller", func() {
 
 			By("Ensuring the resource is created in the provider")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.BGP).ToNot(BeNil(), "Provider should have BGP instance configured")
+				g.Expect(testDevices.StateFor(device.Name).BGP).ToNot(BeNil(), "Provider should have BGP instance configured")
 			}).Should(Succeed())
 		})
 
@@ -177,8 +190,8 @@ var _ = Describe("BGP Controller", func() {
 
 			By("Ensuring the provider receives the VRF")
 			Eventually(func(g Gomega) {
-				g.Expect(testProvider.BGPVRF).ToNot(BeNil())
-				g.Expect(testProvider.BGPVRF.Spec.Name).To(Equal("CC-MGMT"))
+				g.Expect(testDevices.StateFor(device.Name).BGPVRF).ToNot(BeNil())
+				g.Expect(testDevices.StateFor(device.Name).BGPVRF.Spec.Name).To(Equal("CC-MGMT"))
 			}).Should(Succeed())
 
 			By("Ensuring ReadyCondition is True")