0

I have an Ubuntu server as my gateway to the internet, with 2 82574LM Eth cards on the LAN side. I wish to aggregate both. My Eth switch supports 802.3ad, and is configured in dynamic mode, which I have enabled only on the ports that these Eth ports are connected to. The router supports OpenVPN bridged, therefore the LAN ports need be configured for promiscuous mode. I use dnsmasq to serve DHCP and DNS to my LAN.

I have added bonding to /etc/modules, and installed ifenslave.

I have configured /etc/network/interfaces thus:

# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).

# Localhost
auto lo
iface lo inet loopback

##########################
# WAN - connected to MODEM
##########################
# We use allow-hotplug here as otherwise get boot hang if modem down
auto eth0
allow-hotplug eth0
iface eth0 inet dhcp
  up ethtool -s eth0 wol d
  up ethtool -G eth0 rx 1024 tx 1024

# static multicast route for LAN DLNA/uPNP
up route add -net 239.0.0.0 netmask 255.0.0.0 dev eth1


########################
# LAN config...single port
########################
#auto eth1
#allow-hotplug eth1
#iface eth1 inet manual
#  up ip link set $IFACE up promisc on
#  up ethtool -s eth1 wol d
#  up ethtool -G eth1 rx 1024 tx 1024
#  down ip link set $IFACE down promisc off
## with bridge...
#auto br0
#iface br0 inet static
#       address 192.168.10.2
#       netmask 255.255.255.0
#       broadcast 192.168.10.255
#       network 192.168.10.0
#       bridge_ports eth1
#iface br0 inet6 static
#        address 2a01:348:1a3::1
#        netmask 48
## Following is for IPv6 default DNS servers
#        up ip addr add fec0:0:0:ffff::1 dev $IFACE label $IFACE:0
#        up ip addr add fec0:0:0:ffff::2 dev $IFACE label $IFACE:1
#        up ip addr add fec0:0:0:ffff::3 dev $IFACE label $IFACE:2

####################################
# LAN Config...link aggregated
#
# Bond eth1 & eth2 together to bond0.
# Note that 802.3ad is the ONLY one
# that can work for aggregation, and
# the connected Eth switch ports must
# be configured for LACP
####################################
auto eth1
allow-hotplug eth1
iface eth1 inet manual
  up ethtool -s eth1 wol d
  up ethtool -G eth1 rx 1024 tx 1024
  bond-master bond0
auto eth2
allow-hotplug eth2
iface eth2 inet manual
  up ethtool -s eth2 wol d
  up ethtool -G eth2 rx 1024 tx 1024
  bond-master bond0
#
# Create bond...
auto bond0
iface bond0 inet static
  up ip link set $IFACE up promisc on
  down ip link set $IFACE down promisc off
  bond-downdelay 200
  bond-updelay 200
  bond-miimon 100
  bond_mode 802.3ad
  bond-slaves eth1 eth2
  bond-ad_select bandwidth
  bond-min_links 1
  address 192.168.10.2
  netmask 255.255.255.0
  broadcast 192.168.10.255
  network 192.168.10.0
iface bond0 inet6 static
  address 2a01:348:1a3::1
  netmask 48
# Following is for IPv6 default DNS servers
  up ip addr add fec0:0:0:ffff::1 dev $IFACE label $IFACE:0
  up ip addr add fec0:0:0:ffff::2 dev $IFACE label $IFACE:1
  up ip addr add fec0:0:0:ffff::3 dev $IFACE label $IFACE:2
#
# with bridge...
auto br0
iface br0 inet static
  address 192.168.10.2
  netmask 255.255.255.0
  broadcast 192.168.10.255
  network 192.168.10.0
  bridge_ports bond0
iface br0 inet6 static
  address 2a01:348:1a3::1
  netmask 48
# Following is for IPv6 default DNS servers
  up ip addr add fec0:0:0:ffff::1 dev $IFACE label $IFACE:0
  up ip addr add fec0:0:0:ffff::2 dev $IFACE label $IFACE:1
  up ip addr add fec0:0:0:ffff::3 dev $IFACE label $IFACE:2

dmesg output:

[   15.336548] e1000e 0000:01:00.0 rename3: renamed from eth1
[   15.353819] systemd-udevd[398]: renamed network interface eth1 to rename3
[   15.377579] e1000e 0000:03:00.0 eth1: renamed from eth2
[   15.405802] e1000e 0000:01:00.0 eth2: renamed from rename3
[   15.405805] systemd-udevd[396]: renamed network interface eth2 to eth1
[   15.429822] systemd-udevd[398]: renamed network interface rename3 to eth2

[   16.688845] IPv6: ADDRCONF(NETDEV_UP): bond0: link is not ready
[   16.689371] device bond0 entered promiscuous mode
[   16.703777] bond0: Setting MII monitoring interval to 100
[   16.703812] bond0: Setting down delay to 200
[   16.703828] bond0: Setting up delay to 200
[   16.721013] br0: port 1(bond0) entered disabled state
[   16.736648] bond0: Setting ad_select to bandwidth (1)
[   16.757080] IPv6: ADDRCONF(NETDEV_UP): bond0: link is not ready
[   16.767130] bond0: Adding slave eth1
[   16.848898] bond0: Enslaving eth1 as a backup interface with a down link
[   16.849033] bond0: Adding slave eth2
[   16.928246] bond0: Enslaving eth2 as a backup interface with a down link
[   19.568416] bond0: link status up for interface eth1, enabling it in 0 ms
[   19.568419] bond0: link status up for interface eth2, enabling it in 200 ms
[   19.568420] bond0: link status definitely up for interface eth1, 1000 Mbps full duplex
[   19.568432] bond0: now running without any active interface!
[   19.568557] IPv6: ADDRCONF(NETDEV_CHANGE): bond0: link becomes ready
[   19.568576] br0: port 1(bond0) entered forwarding state
[   19.568582] br0: port 1(bond0) entered forwarding state
[   19.768379] bond0: link status definitely up for interface eth2, 1000 Mbps full duplex
[   34.587447] br0: port 1(bond0) entered forwarding state
[   37.668468] audit: type=1400 audit(1452280972.561:100): apparmor="ALLOWED" operation="open" profile="/usr/sbin/dnsmasq" name="/proc/sys/net/ipv6/conf/bond0/mtu" pid=5214 comm="dnsmasq" requested_mask="r" denied_mask="r" fsuid=105 ouid=0

I am using UFW to ensure ports are blocked on the WAN, rather than using binding for the services (as binding to devices is a little bit hit/miss on Linux due to system startup order). Here's the config:

ufw status
Status: active

To                         Action      From
--                         ------      ----
80/tcp                     ALLOW       Anywhere
443/tcp                    ALLOW       Anywhere
993/tcp                    ALLOW       Anywhere
67/udp on br0              ALLOW       68/udp
Anywhere                   ALLOW       192.168.0.0/16
Anywhere                   ALLOW       10.0.0.0/8
5631/tcp                   ALLOW       Anywhere
80/tcp (v6)                ALLOW       Anywhere (v6)
443/tcp (v6)               ALLOW       Anywhere (v6)
Anywhere (v6)              ALLOW       2a01:348:1a3::/48
993/tcp (v6)               ALLOW       Anywhere (v6)
67/udp (v6) on br0         ALLOW       68/udp (v6)
5631/tcp (v6)              ALLOW       Anywhere (v6)

/proc/net/bonding/bond0:

Ethernet Channel Bonding Driver: v3.7.1 (April 27, 2011)

Bonding Mode: IEEE 802.3ad Dynamic link aggregation
Transmit Hash Policy: layer2 (0)
MII Status: up
MII Polling Interval (ms): 100
Up Delay (ms): 200
Down Delay (ms): 200

802.3ad info
LACP rate: slow
Min links: 0
Aggregator selection policy (ad_select): bandwidth
Active Aggregator Info:
        Aggregator ID: 1
        Number of ports: 2
        Actor Key: 9
        Partner Key: 3
        Partner Mac Address: 00:23:47:71:14:a0

Slave Interface: eth1
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 68:05:ca:35:c7:10
Aggregator ID: 1
Slave queue ID: 0

Slave Interface: eth2
MII Status: up
Speed: 1000 Mbps
Duplex: full
Link Failure Count: 0
Permanent HW addr: 68:05:ca:35:c7:0f
Aggregator ID: 1
Slave queue ID: 0

ifconfig output:

bond0     Link encap:Ethernet  HWaddr 68:05:ca:35:c7:10
          inet addr:192.168.10.2  Bcast:192.168.10.255  Mask:255.255.255.0
          inet6 addr: fec0:0:0:ffff::3/128 Scope:Site
          inet6 addr: fec0:0:0:ffff::2/128 Scope:Site
          inet6 addr: fec0:0:0:ffff::1/128 Scope:Site
          inet6 addr: fe80::6a05:caff:fe35:c710/64 Scope:Link
          inet6 addr: 2a01:348:1a3::1/48 Scope:Global
          UP BROADCAST RUNNING PROMISC MASTER MULTICAST  MTU:1500  Metric:1
          RX packets:2945 errors:0 dropped:93 overruns:0 frame:0
          TX packets:2049 errors:0 dropped:117 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:362102 (362.1 KB)  TX bytes:347834 (347.8 KB)

br0       Link encap:Ethernet  HWaddr 68:05:ca:35:c7:10
          inet addr:192.168.10.2  Bcast:192.168.10.255  Mask:255.255.255.0
          inet6 addr: fec0:0:0:ffff::3/128 Scope:Site
          inet6 addr: fec0:0:0:ffff::2/128 Scope:Site
          inet6 addr: fec0:0:0:ffff::1/128 Scope:Site
          inet6 addr: fe80::6a05:caff:fe35:c710/64 Scope:Link
          inet6 addr: 2a01:348:1a3::1/48 Scope:Global
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:2311 errors:0 dropped:0 overruns:0 frame:0
          TX packets:469 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:0
          RX bytes:239930 (239.9 KB)  TX bytes:172190 (172.1 KB)

eth1      Link encap:Ethernet  HWaddr 68:05:ca:35:c7:10
          UP BROADCAST RUNNING PROMISC SLAVE MULTICAST  MTU:1500  Metric:1
          RX packets:1744 errors:0 dropped:13 overruns:0 frame:0
          TX packets:1141 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:195847 (195.8 KB)  TX bytes:222128 (222.1 KB)
          Interrupt:17 Memory:f7cc0000-f7ce0000

eth2      Link encap:Ethernet  HWaddr 68:05:ca:35:c7:10
          UP BROADCAST RUNNING SLAVE MULTICAST  MTU:1500  Metric:1
          RX packets:1201 errors:0 dropped:64 overruns:0 frame:0
          TX packets:908 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000
          RX bytes:166255 (166.2 KB)  TX bytes:125706 (125.7 KB)
          Interrupt:16 Memory:f7dc0000-f7de0000

My Eth switch confirms that it can see the link aggregation. When I boot there's no warnings or errors, and ifconfig and /proc/net/bonding/bond0 looks correct. But I am unable to ping it, and looking at a wireshark from a remote PC, I never get any ARP replies back. There is some odd UDP packets being emitted from the server though (which I am unsure what it is). I have also tried to ping the server from the Eth switch, which also fails.

Does anyone have any idea what I am doing wrong?

1 Answer 1

0

Finally found an answer here - https://help.nceas.ucsb.edu/ubuntu_network_configuration. Changed my interfaces file to:

####################################
# LAN Config...link aggregated
#
# Bond eth1 & eth2 together to bond0.
# Note that 802.3ad is the ONLY one
# that can work for aggregation, and
# the connected Eth switch ports must
# be configured for LACP
####################################
auto eth1
allow-hotplug eth1
iface eth1 inet manual
  up ip link set $IFACE up promisc on
  down ip link set $IFACE down promisc off
  up ethtool -s eth1 wol d
  up ethtool -G eth1 rx 1024 tx 1024
  bond-master bond0
# 
auto eth2
allow-hotplug eth2
iface eth2 inet manual
  up ip link set $IFACE up promisc on
  down ip link set $IFACE down promisc off
  up ethtool -s eth2 wol d
  up ethtool -G eth2 rx 1024 tx 1024
  bond-master bond0
#
auto bond0
iface bond0 inet manual
  bond-downdelay 200
  bond-updelay 200
  bond-miimon 100
  bond_mode 802.3ad
  bond-slaves eth1 eth2
  bond-ad_select bandwidth
  bond-min_links 1
iface bond0 inet6 manual
#
auto br0
iface br0 inet static
  address 192.168.10.2
  netmask 255.255.255.0
  broadcast 192.168.10.255
  network 192.168.10.0
  bridge_ports bond0
iface br0 inet6 static
  address 2a01:348:1a3::1
  netmask 48
# Following is for IPv6 default DNS servers
  up ip addr add fec0:0:0:ffff::1 dev $IFACE label $IFACE:0
  up ip addr add fec0:0:0:ffff::2 dev $IFACE label $IFACE:1
  up ip addr add fec0:0:0:ffff::3 dev $IFACE label $IFACE:2

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .