Kernel 4.9: race condition seen with port channel creation. #1981
Description
Description
Portchannel creation failed partially with config reload or system reboot.
root@sonic:/home/admin# config reload
Clear current config and reload config from the file /etc/sonic/config_db.json? [y/N]: y
Running command: systemctl stop dhcp_relay
Running command: systemctl stop swss
Running command: systemctl stop snmp
Warning: Stopping snmp.service, but it can still be activated by:
snmp.timer
Running command: systemctl stop lldp
Running command: systemctl stop pmon
Running command: systemctl stop bgp
Running command: systemctl stop teamd
Running command: sonic-cfggen -j /etc/sonic/config_db.json --write-to-db
Running command: systemctl restart hostname-config
Running command: systemctl restart interfaces-config
Running command: systemctl restart ntp-config
Running command: systemctl restart rsyslog-config
Running command: systemctl restart swss
Running command: systemctl restart bgp
Running command: systemctl restart teamd
Running command: systemctl restart pmon
Running command: systemctl restart lldp
Running command: systemctl restart snmp
Running command: systemctl restart dhcp_relay
root@sonic:/home/admin#
root@sonic:/home/admin# team show
bash: team: command not found
root@sonic:/home/admin# teamshow
Flags: A - active, I - inactive, Up - up, Dw - Down, N/A - not available, S - selected, D - deselected
No. Team Dev Protocol Ports
----- ------------- ----------- -------------
1 PortChannel1 N/A
2 PortChannel2 N/A
3 PortChannel3 N/A
4 PortChannel4 N/A
5 PortChannel5 N/A
6 PortChannel6 N/A
7 PortChannel7 N/A
8 PortChannel8 N/A
9 PortChannel9 N/A
10 PortChannel10 LACP(A)(Up) Ethernet10(S)
11 PortChannel11 LACP(A)(Up) Ethernet11(S)
12 PortChannel12 LACP(A)(Up) Ethernet12(S)
13 PortChannel13 LACP(A)(Up) Ethernet13(S)
14 PortChannel14 N/A
15 PortChannel15 N/A
16 PortChannel16 N/A
17 PortChannel17 N/A
18 PortChannel18 LACP(A)(Up) Ethernet18(S)
19 PortChannel19 N/A
20 PortChannel20 LACP(A)(Up) Ethernet20(S)
Restart teamd service, portchannel got created succefully.
root@sonic:/home/admin# systemctl restart teamd
root@sonic:/home/admin# teamshow
Flags: A - active, I - inactive, Up - up, Dw - Down, N/A - not available, S - selected, D - deselected
No. Team Dev Protocol Ports
----- ------------- ------------ -------------
1 PortChannel1 LACP(A)(N/A) Ethernet1(D)
2 PortChannel2 LACP(A)(N/A) Ethernet2(S)
3 PortChannel3 LACP(A)(N/A) Ethernet3(D)
4 PortChannel4 LACP(A)(N/A) Ethernet4(D)
5 PortChannel5 LACP(A)(N/A) Ethernet5(D)
6 PortChannel6 LACP(A)(N/A) Ethernet6(D)
7 PortChannel7 LACP(A)(N/A) Ethernet7(D)
8 PortChannel8 LACP(A)(N/A) Ethernet8(D)
9 PortChannel9 LACP(A)(N/A) Ethernet9(D)
10 PortChannel10 LACP(A)(N/A) Ethernet10(D)
11 PortChannel11 LACP(A)(Up) Ethernet11(D)
12 PortChannel12 LACP(A)(Up) Ethernet12(D)
13 PortChannel13 LACP(A)(Up) Ethernet13(D)
14 PortChannel14 LACP(A)(N/A) Ethernet14(D)
15 PortChannel15 LACP(A)(N/A) Ethernet15(D)
16 PortChannel16 LACP(A)(N/A) Ethernet16(D)
17 PortChannel17 LACP(A)(N/A) Ethernet17(D)
18 PortChannel18 LACP(A)(Up) Ethernet18(D)
19 PortChannel19 LACP(A)(N/A) Ethernet19(D)
20 PortChannel20 LACP(A)(Up) Ethernet20(D)
root@sonic:/home/admin#
It looks the portchannel config in /etc/network/interface will bring up the member Ethernet interface.
While if the admin status of member Ethernet interface is up, portchannel create will fail.
root@sonic:/home/admin# teamshow
Flags: A - active, I - inactive, Up - up, Dw - Down, N/A - not available, S - selected, D - deselected
No. Team Dev Protocol Ports
----- ------------- ----------- -------------
1 PortChannel1 N/A
2 PortChannel2 N/A
3 PortChannel3 N/A
4 PortChannel4 N/A
5 PortChannel5 N/A
6 PortChannel6 N/A
7 PortChannel7 N/A
8 PortChannel8 N/A
9 PortChannel9 N/A
10 PortChannel10 LACP(A)(Up) Ethernet10(S)
11 PortChannel11 LACP(A)(Up) Ethernet11(S)
12 PortChannel12 LACP(A)(Up) Ethernet12(S)
13 PortChannel13 LACP(A)(Up) Ethernet13(S)
14 PortChannel14 N/A
15 PortChannel15 N/A
16 PortChannel16 N/A
17 PortChannel17 N/A
18 PortChannel18 LACP(A)(Up) Ethernet18(S)
19 PortChannel19 N/A
20 PortChannel20 LACP(A)(Up) Ethernet20(S)
root@sonic:/home/admin# ip link show Ethernet1 Ethernet2
Error: either "dev" is duplicate, or "Ethernet2" is a garbage.
root@sonic:/home/admin# ip link show Ethernet1
947: Ethernet1: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc pfifo_fast state DOWN mode DEFAULT group default qlen 1000
link/ether 00:05:64:30:73:c0 brd ff:ff:ff:ff:ff:ff
root@sonic:/home/admin# ip link show Ethernet2
948: Ethernet2: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
link/ether 00:05:64:30:73:c0 brd ff:ff:ff:ff:ff:ff
root@sonic:/home/admin# ip link show Ethernet3
949: Ethernet3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP mode DEFAULT group default qlen 1000
link/ether 00:05:64:30:73:c0 brd ff:ff:ff:ff:ff:ff
root@sonic:/home/admin# ip link show Ethernet4
950: Ethernet4: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc pfifo_fast state DOWN mode DEFAULT group default qlen 1000
link/ether 00:05:64:30:73:c0 brd ff:ff:ff:ff:ff:ff
root@sonic:/home/admin#
root@sonic:/home/admin# cat /etc/network/interfaces
#
# =============== Managed by SONiC Config Engine DO NOT EDIT! ===============
# generated from /usr/share/sonic/templates/interfaces.j2 using sonic-cfggen
# file: /etc/network/interfaces
#
# The loopback network interface
auto lo
iface lo inet loopback
# Use command 'ip addr list dev lo' to check all addresses
iface lo inet static
address 11.161.62.196
netmask 255.255.255.255
#
# The management network interface
auto eth0
iface eth0 inet static
address 30.57.185.38
netmask 255.255.255.0
########## management network policy routing rules
# management port up rules
up ip -4 route add default via 30.57.185.254 dev eth0 table default
up ip -4 route add 30.57.185.0/24 dev eth0 table default
up ip -4 rule add from 30.57.185.38/32 table default
# management port down rules
down ip -4 route delete default via 30.57.185.254 dev eth0 table default
down ip -4 route delete 30.57.185.0/24 dev eth0 table default
down ip -4 rule delete from 30.57.185.38/32 table default
#
# "|| true" is added to suppress the error when interface is already a member of LAG
# "ip link show | grep -q master" is added to ensure interface is enslaved
allow-hotplug Ethernet1
iface Ethernet1 inet manual
pre-up teamdctl PortChannel1 port add Ethernet1 || true
post-up ip link show Ethernet1 | grep -q master && ifconfig Ethernet1 up
post-down ifconfig Ethernet1 down
#
allow-hotplug Ethernet10
iface Ethernet10 inet manual
pre-up teamdctl PortChannel10 port add Ethernet10 || true
post-up ip link show Ethernet10 | grep -q master && ifconfig Ethernet10 up
post-down ifconfig Ethernet10 down
#
allow-hotplug Ethernet11
iface Ethernet11 inet manual
pre-up teamdctl PortChannel11 port add Ethernet11 || true
post-up ip link show Ethernet11 | grep -q master && ifconfig Ethernet11 up
post-down ifconfig Ethernet11 down
#
allow-hotplug Ethernet12
iface Ethernet12 inet manual
pre-up teamdctl PortChannel12 port add Ethernet12 || true
post-up ip link show Ethernet12 | grep -q master && ifconfig Ethernet12 up
post-down ifconfig Ethernet12 down
#
allow-hotplug Ethernet13
iface Ethernet13 inet manual
pre-up teamdctl PortChannel13 port add Ethernet13 || true
post-up ip link show Ethernet13 | grep -q master && ifconfig Ethernet13 up
post-down ifconfig Ethernet13 down
#
allow-hotplug Ethernet14
iface Ethernet14 inet manual
pre-up teamdctl PortChannel14 port add Ethernet14 || true
post-up ip link show Ethernet14 | grep -q master && ifconfig Ethernet14 up
post-down ifconfig Ethernet14 down
#
allow-hotplug Ethernet15
iface Ethernet15 inet manual
pre-up teamdctl PortChannel15 port add Ethernet15 || true
post-up ip link show Ethernet15 | grep -q master && ifconfig Ethernet15 up
post-down ifconfig Ethernet15 down
#
allow-hotplug Ethernet16
iface Ethernet16 inet manual
pre-up teamdctl PortChannel16 port add Ethernet16 || true
post-up ip link show Ethernet16 | grep -q master && ifconfig Ethernet16 up
post-down ifconfig Ethernet16 down
#
allow-hotplug Ethernet17
iface Ethernet17 inet manual
pre-up teamdctl PortChannel17 port add Ethernet17 || true
post-up ip link show Ethernet17 | grep -q master && ifconfig Ethernet17 up
post-down ifconfig Ethernet17 down
#
allow-hotplug Ethernet18
iface Ethernet18 inet manual
pre-up teamdctl PortChannel18 port add Ethernet18 || true
post-up ip link show Ethernet18 | grep -q master && ifconfig Ethernet18 up
post-down ifconfig Ethernet18 down
#
allow-hotplug Ethernet19
iface Ethernet19 inet manual
pre-up teamdctl PortChannel19 port add Ethernet19 || true
post-up ip link show Ethernet19 | grep -q master && ifconfig Ethernet19 up
post-down ifconfig Ethernet19 down
#
allow-hotplug Ethernet2
iface Ethernet2 inet manual
pre-up teamdctl PortChannel2 port add Ethernet2 || true
post-up ip link show Ethernet2 | grep -q master && ifconfig Ethernet2 up
post-down ifconfig Ethernet2 down
#
allow-hotplug Ethernet20
iface Ethernet20 inet manual
pre-up teamdctl PortChannel20 port add Ethernet20 || true
post-up ip link show Ethernet20 | grep -q master && ifconfig Ethernet20 up
post-down ifconfig Ethernet20 down
#
allow-hotplug Ethernet3
iface Ethernet3 inet manual
pre-up teamdctl PortChannel3 port add Ethernet3 || true
post-up ip link show Ethernet3 | grep -q master && ifconfig Ethernet3 up
post-down ifconfig Ethernet3 down
#
allow-hotplug Ethernet4
iface Ethernet4 inet manual
pre-up teamdctl PortChannel4 port add Ethernet4 || true
post-up ip link show Ethernet4 | grep -q master && ifconfig Ethernet4 up
post-down ifconfig Ethernet4 down
#
allow-hotplug Ethernet5
iface Ethernet5 inet manual
pre-up teamdctl PortChannel5 port add Ethernet5 || true
post-up ip link show Ethernet5 | grep -q master && ifconfig Ethernet5 up
post-down ifconfig Ethernet5 down
#
allow-hotplug Ethernet6
iface Ethernet6 inet manual
pre-up teamdctl PortChannel6 port add Ethernet6 || true
post-up ip link show Ethernet6 | grep -q master && ifconfig Ethernet6 up
post-down ifconfig Ethernet6 down
#
allow-hotplug Ethernet7
iface Ethernet7 inet manual
pre-up teamdctl PortChannel7 port add Ethernet7 || true
post-up ip link show Ethernet7 | grep -q master && ifconfig Ethernet7 up
post-down ifconfig Ethernet7 down
#
allow-hotplug Ethernet8
iface Ethernet8 inet manual
pre-up teamdctl PortChannel8 port add Ethernet8 || true
post-up ip link show Ethernet8 | grep -q master && ifconfig Ethernet8 up
post-down ifconfig Ethernet8 down
#
allow-hotplug Ethernet9
iface Ethernet9 inet manual
pre-up teamdctl PortChannel9 port add Ethernet9 || true
post-up ip link show Ethernet9 | grep -q master && ifconfig Ethernet9 up
post-down ifconfig Ethernet9 down
#
admin@sonic:~$ docker exec -it teamd /bin/bash
root@sonic:/# ps -x
PID TTY STAT TIME COMMAND
1 ? Ss+ 0:00 /usr/bin/python /usr/bin/supervisord
273 ? Sl 0:00 /usr/sbin/rsyslogd -n
278 ? S 0:00 bash /usr/bin/teamd.sh
286 ? S 0:00 teamd -f /etc/teamd/PortChannel10.conf -d
289 ? S 0:00 teamd -f /etc/teamd/PortChannel11.conf -d
292 ? S 0:00 teamd -f /etc/teamd/PortChannel12.conf -d
295 ? S 0:00 teamd -f /etc/teamd/PortChannel13.conf -d
310 ? S 0:00 teamd -f /etc/teamd/PortChannel18.conf -d
319 ? S 0:00 teamd -f /etc/teamd/PortChannel20.conf -d
341 ? Sl 0:00 teamsyncd
457 ? Ss 0:00 /bin/bash
463 ? R+ 0:00 ps -x
root@sonic:/# teamd -f /etc/teamd/PortChannel10.conf -d^C
root@sonic:/# ip link show Ethernet6
952: Ethernet6: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc pfifo_fast state DOWN mode DEFAULT group default qlen 1000
link/ether 00:05:64:30:73:c0 brd ff:ff:ff:ff:ff:ff
root@sonic:/# teamd -f /etc/teamd/PortChannel6.conf -d
This program is not intended to be run as root.
Daemon process failed.
Failed: Device or resource busy
root@sonic:/#
root@sonic:/#
root@sonic:/# ip link set down Ethernet6
root@sonic:/# teamd -f /etc/teamd/PortChannel6.conf -d
This program is not intended to be run as root.
root@sonic:/#