Skip to content

Improved diags for kube-controllers CI failure. #11184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Improved diags for kube-controllers CI failure. #11184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
73e4003
Improved diags for kube-controllers CI failure.
fasaxc Oct 13, 2025
ec5d8e2
Update kube-controllers/pkg/controllers/loadbalancer/loadbalancer_con…
fasaxc Oct 14, 2025
e6a46bc
Update kube-controllers/pkg/controllers/loadbalancer/loadbalancer_con…
fasaxc Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -408,7 +408,7 @@ func (c *loadBalancerController) syncService(svcKey serviceKey) {
if len(c.ipPools) == 0 {
if _, ok := c.allocationTracker.ipsByService[svcKey]; ok {
// Last LoadBalancer IPPool was deleted, and we have previously assigned IPs to this service. We need to release the IPs now and update the service status
log.Debugf("No ippools with allowedUse LoadBalancer found. Releasing previously assigned IPs for Service %s/%s", svcKey.namespace, svcKey.name)
log.Warnf("No ippools with allowedUse LoadBalancer found. Releasing previously assigned IPs for Service %s/%s", svcKey.namespace, svcKey.name)
err := c.releaseIPsByHandle(svcKey)
if err != nil {
log.WithError(err).Errorf("Error releasing previously assigned IPs for Service %s/%s", svcKey.namespace, svcKey.name)
Expand All @@ -431,7 +431,7 @@ func (c *loadBalancerController) syncService(svcKey serviceKey) {
}
} else {
// We can skip service sync if there are no ippools defined that can be used for Service LoadBalancer
log.Debugf("No ippools with allowedUse LoadBalancer found. Skipping IP assignment for Service %s/%s", svcKey.namespace, svcKey.name)
log.Warnf("No ippools with allowedUse LoadBalancer found. Skipping IP assignment for Service %s/%s", svcKey.namespace, svcKey.name)
}
return
}
Expand Down Expand Up @@ -491,6 +491,7 @@ func (c *loadBalancerController) syncService(svcKey serviceKey) {
}
for ip := range c.allocationTracker.ipsByService[svcKey] {
if _, ok := lbIPs[ip]; !ok {
log.Infof("Removing IP assignment (%s) for Service %s/%s; no longer in annotations.", ip, svc.Namespace, svc.Name)
err = c.releaseIP(svcKey, ip)
if err != nil {
log.WithError(err).Errorf("Failed to release IP for %s/%s", svc.Namespace, svc.Name)
Expand All @@ -502,6 +503,7 @@ func (c *loadBalancerController) syncService(svcKey serviceKey) {
// If pool annotations are specified, we need to check that the IPs assigned are from the specified pools
for ip := range c.allocationTracker.ipsByService[svcKey] {
if !poolContains(ip, ipv4pools) && !poolContains(ip, ipv6pools) {
log.Infof("Removing IP assignment (%s) for Service %s/%s: not from specified pools (%v, %v).", ip, svc.Namespace, svc.Name, ipv4pools, ipv6pools)
err = c.releaseIP(svcKey, ip)
if err != nil {
log.WithError(err).Errorf("Failed to release IP for %s/%s", svc.Namespace, svc.Name)
Expand All @@ -523,6 +525,7 @@ func (c *loadBalancerController) syncService(svcKey serviceKey) {
//
// AssignmentMode should never be nil due to defaulting, but we check it just in case.
if pool.Spec.AssignmentMode != nil && *pool.Spec.AssignmentMode == api.Manual {
log.Infof("Removing IP assignment (%s) for Service %s/%s. No annotations but IP is from a 'Manual' IP pool.", ip, svc.Namespace, svc.Name)
err = c.releaseIP(svcKey, ip)
if err != nil {
log.WithError(err).Errorf("Failed to release IP for %s/%s", svc.Namespace, svc.Name)
Expand All @@ -534,9 +537,12 @@ func (c *loadBalancerController) syncService(svcKey serviceKey) {
}

if c.needsIPsAssigned(svc, svcKey) {
_, err = c.assignIP(svc)
log.Infof("Service requires an IP assignment %s/%s", svc.Namespace, svc.Name)
ips, err := c.assignIP(svc)
if err != nil {
log.WithError(err).Errorf("Failed to assign IP for %s/%s", svc.Namespace, svc.Name)
} else {
log.Infof("Assigned IPs %s for Service %s/%s", ips, svc.Namespace, svc.Name)
}
}

Expand Down Expand Up @@ -642,6 +648,7 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) ([]string, error) {

if loadBalancerIPs != nil {
// User requested specific IP, attempt to allocate
log.Infof("Trying to assign requested IPs %v to Service %s/%s", loadBalancerIPs, svc.Namespace, svc.Name)
for _, addr := range loadBalancerIPs {
if _, exists := c.allocationTracker.ipsByService[*svcKey][addr.String()]; exists {
// We must be trying to assign missing address due to an error,
Expand Down Expand Up @@ -680,15 +687,18 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) ([]string, error) {
num6++
}
}
log.Infof("Service %s/%s requires %v IPv4 and %v IPv6 addresses.", svc.Namespace, svc.Name, num4, num6)

// Check if IP from ipFamily is already assigned, skip it as we're trying to assign only the missing one.
// This can happen when error happened during the initial assignment, and now we're trying to assign ip again from the syncIPAM func
for ingress := range c.allocationTracker.ipsByService[*svcKey] {
if ip := cnet.ParseIP(ingress); ip != nil {
if ip.To4() != nil {
log.Infof("Service already has an IPv4 address: %s", ip.String())
num4 = 0
}
if ip.To16() != nil {
log.Infof("Service already has an IPv6 address: %s", ip.String())
num6 = 0
}
}
Expand Down Expand Up @@ -732,13 +742,15 @@ func (c *loadBalancerController) assignIP(svc *v1.Service) ([]string, error) {

if v4Assignments != nil {
for _, assignment := range v4Assignments.IPs {
log.Infof("Service %s/%s now has IP: %s", svcKey.namespace, svcKey.name, assignment.IP.String())
assignedIPs = append(assignedIPs, assignment.IP.String())
c.allocationTracker.assignAddressToService(*svcKey, assignment.IP.String())
}
}

if v6assignments != nil {
for _, assignment := range v6assignments.IPs {
log.Infof("Service %s/%s now has IP: %s", svcKey.namespace, svcKey.name, assignment.IP.String())
assignedIPs = append(assignedIPs, assignment.IP.String())
c.allocationTracker.assignAddressToService(*svcKey, assignment.IP.String())
}
Expand Down
2 changes: 1 addition & 1 deletion kube-controllers/pkg/controllers/loadbalancer/loadbalancer_controller_fv_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -477,8 +477,8 @@ var _ = Describe("Calico loadbalancer controller FV tests (etcd mode)", func() {

service, err = k8sClient.CoreV1().Services(testNamespace).Get(context.Background(), serviceIpv4PoolSpecified.Name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(service.Status.LoadBalancer.Ingress).NotTo(BeEmpty(), "saw service.Status.LoadBalancer.Ingress non-empty and then empty again!")
Expect(service.Status.LoadBalancer.Ingress[0].IP).Should(Equal(specificIpFromAutomaticPool))

})
})
})