Skip to content

Commit 60faea1

Browse files
pippolo84pippolo84
committed
ipam: Wait for ENI netlink iface before configuring routes
In ENI IPAM mode, the ENIs are created by the operator. At the same time, on each node, the agents configure the v4 and v6 rules and routes for the Cilium router, after retrieving its IPs from either k8s or the filesystem. In order to do so, each agent query netlink to get the ifindex of the interface with the router IP, given its MAC address. Unfortunately this behavior is racy, since the agent might query netlink too soon, when the ENI is not yet up and running. This leads to the following error from netlink: "daemon creation failed: failed to configure router IP rules and routes: unable to find ifindex for interface MAC: interface with MAC ... not found" that ultimately stops the daemon startup. To address this, let's poll netlink and wait for the ENI netlink interface to show up before going ahead with routes and rules configuration. Fixes: #37948 Signed-off-by: Fabio Falzoi <[email protected]>
1 parent 17afaef commit 60faea1

1 file changed

Lines changed: 44 additions & 0 deletions

File tree

‎daemon/cmd/ipam.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,12 @@ import (
1212
"github.com/cilium/hive/cell"
1313
"github.com/cilium/hive/job"
1414
"github.com/cilium/statedb"
15+
"golang.org/x/sys/unix"
16+
"k8s.io/apimachinery/pkg/util/wait"
1517

1618
"github.com/cilium/cilium/pkg/cidr"
1719
linuxrouting "github.com/cilium/cilium/pkg/datapath/linux/routing"
20+
"github.com/cilium/cilium/pkg/datapath/linux/safenetlink"
1821
"github.com/cilium/cilium/pkg/datapath/tables"
1922
"github.com/cilium/cilium/pkg/datapath/types"
2023
iputil "github.com/cilium/cilium/pkg/ip"
@@ -146,6 +149,35 @@ func reallocateDatapathIPs(logger *slog.Logger, alloc ipamAllocateIP, fromK8s, f
146149
return result
147150
}
148151

152+
func waitForENI(ctx context.Context, macAddr string) error {
153+
bo := wait.Backoff{
154+
Duration: 250 * time.Millisecond,
155+
Factor: 2,
156+
Jitter: 0.2,
157+
Steps: 5,
158+
}
159+
160+
findENIByMAC := func(ctx context.Context) (bool, error) {
161+
links, err := safenetlink.LinkList()
162+
if err != nil {
163+
return false, fmt.Errorf("unable to list interfaces: %w", err)
164+
}
165+
166+
for _, l := range links {
167+
// filter out slave devices
168+
if l.Attrs().RawFlags&unix.IFF_SLAVE != 0 {
169+
continue
170+
}
171+
if l.Attrs().HardwareAddr.String() == macAddr {
172+
return true, nil
173+
}
174+
}
175+
return false, nil
176+
}
177+
178+
return wait.ExponentialBackoffWithContext(ctx, bo, findENIByMAC)
179+
}
180+
149181
func (d *Daemon) allocateDatapathIPs(family types.NodeAddressingFamily, fromK8s, fromFS net.IP) (routerIP net.IP, err error) {
150182
// Avoid allocating external IP
151183
d.params.IPAM.ExcludeIP(family.PrimaryExternal(), "node-ip", ipam.PoolDefault())
@@ -189,6 +221,18 @@ func (d *Daemon) allocateDatapathIPs(family types.NodeAddressingFamily, fromK8s,
189221
if err != nil {
190222
return nil, fmt.Errorf("failed to create router info: %w", err)
191223
}
224+
225+
// wait for ENI to be up and running before configuring routes and rules.
226+
// This avoids spurious errors where netlink is not able to find
227+
// the ifindex by its MAC because the ENI is not showing up yet.
228+
if option.Config.IPAM == ipamOption.IPAMENI {
229+
if err := waitForENI(context.TODO(), result.PrimaryMAC); err != nil {
230+
d.params.Logger.Warn("unable to find ENI netlink interface, this will likely lead to an error in configuring the router routes and rules",
231+
logfields.MACAddr, result.PrimaryMAC,
232+
)
233+
}
234+
}
235+
192236
if err = routingInfo.Configure(
193237
result.IP,
194238
d.params.MTU.GetDeviceMTU(),

0 commit comments

Comments
 (0)