Ensure pool member reaches active state

When (re)creating a pool member, wait until its
provisioning_status is ACTIVE. This avoids scenarios
were CAPO contrinues removing members before new ones
are actually active

Author: nikParasyr

pkg/clients/loadbalancer.go

@@ -51,6 +51,7 @@ type LbClient interface {
 	DeletePool(id string) error
 	CreatePoolMember(poolID string, opts pools.CreateMemberOptsBuilder) (*pools.Member, error)
 	ListPoolMember(poolID string, opts pools.ListMembersOptsBuilder) ([]pools.Member, error)
+	GetPoolMember(poolID string, lbMemberID string) (*pools.Member, error)
 	DeletePoolMember(poolID string, lbMemberID string) error
 	CreateMonitor(opts monitors.CreateOptsBuilder) (*monitors.Monitor, error)
 	ListMonitors(opts monitors.ListOptsBuilder) ([]monitors.Monitor, error)
@@ -213,6 +214,15 @@ func (l lbClient) ListPoolMember(poolID string, opts pools.ListMembersOptsBuilde
 	return pools.ExtractMembers(allPages)
 }
 
+func (l lbClient) GetPoolMember(poolID string, lbMemberID string) (*pools.Member, error) {
+	mc := metrics.NewMetricPrometheusContext("loadbalancer_member", "get")
+	member, err := pools.GetMember(context.TODO(), l.serviceClient, poolID, lbMemberID).Extract()
+	if mc.ObserveRequest(err) != nil {
+		return nil, fmt.Errorf("error getting lbmember: %s", err)
+	}
+	return member, nil
+}
+
 func (l lbClient) DeletePoolMember(poolID string, lbMemberID string) error {
 	mc := metrics.NewMetricPrometheusContext("loadbalancer_member", "delete")
 	err := pools.DeleteMember(context.TODO(), l.serviceClient, poolID, lbMemberID).ExtractErr()

pkg/clients/mock/loadbalancer.go

@@ -52,6 +52,7 @@ const (
 const (
 	loadBalancerProvisioningStatusActive        = "ACTIVE"
 	loadBalancerProvisioningStatusPendingDelete = "PENDING_DELETE"
+	poolMembeProvisioningStatusActive           = "ACTIVE"
 )
 
 // Default values for Monitor, sync with `kubebuilder:default` annotations on APIServerLoadBalancerMonitor object.
@@ -714,7 +715,12 @@ func (s *Service) ReconcileLoadBalancerMember(openStackCluster *infrav1.OpenStac
 			return err
 		}
 
-		if _, err := s.loadbalancerClient.CreatePoolMember(pool.ID, lbMemberOpts); err != nil {
+		member, err := s.loadbalancerClient.CreatePoolMember(pool.ID, lbMemberOpts)
+		if err != nil {
+			return err
+		}
+
+		if _, err := s.waitForPoolMemberActive(pool.ID, member.ID); err != nil {
 			return err
 		}
 
@@ -928,6 +934,25 @@ func (s *Service) waitForLoadBalancerActive(id string) (*loadbalancers.LoadBalan
 	return lb, nil
 }
 
+// Possible Pool Member states are documented here: https://docs.openstack.org/api-ref/load-balancer/v2/#prov-status
+func (s *Service) waitForPoolMemberActive(poolID, memberID string) (*pools.Member, error) {
+	var member *pools.Member
+
+	s.scope.Logger().Info("Waiting for pool member", "pool_id", poolID, "member_id", memberID, "targetStatus", "ACTIVE")
+	err := wait.ExponentialBackoff(backoff, func() (bool, error) {
+		var err error
+		member, err = s.loadbalancerClient.GetPoolMember(poolID, memberID)
+		if err != nil {
+			return false, err
+		}
+		return member.ProvisioningStatus == poolMembeProvisioningStatusActive, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return member, nil
+}
+
 func (s *Service) waitForListener(id, target string) error {
 	s.scope.Logger().Info("Waiting for load balancer listener", "id", id, "targetStatus", target)
 	return wait.ExponentialBackoff(backoff, func() (bool, error) {