feat: auto-detect WireGuard MTU from underlay interface

Change the --mtu flag default from a fixed 1420 to "auto", which detects
the MTU of the default route interface and subtracts the WireGuard
overhead (80 bytes). Re-detect MTU on each reconciliation loop to handle
interface changes. The flag still accepts numeric values for manual
override.

Co-Authored-By: Claude <noreply@anthropic.com>
Signed-off-by: Andrei Kvapil <kvapss@gmail.com>

cmd/kg/main.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -116,7 +117,7 @@ var (
 	listen                string
 	local                 bool
 	master                string
-	mtu                   uint
+	mtuFlag               string
 	topologyLabel         string
 	port                  int
 	serviceCIDRsRaw       []string
@@ -149,7 +150,7 @@ func init() {
 	cmd.Flags().StringVar(&listen, "listen", ":1107", "The address at which to listen for health and metrics.")
 	cmd.Flags().BoolVar(&local, "local", true, "Should Kilo manage routes within a location?")
 	cmd.Flags().StringVar(&master, "master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
-	cmd.Flags().UintVar(&mtu, "mtu", wireguard.DefaultMTU, "The MTU of the WireGuard interface created by Kilo.")
+	cmd.Flags().StringVar(&mtuFlag, "mtu", "auto", "The MTU of the WireGuard interface created by Kilo. Set to 'auto' to detect from the underlay interface.")
 	cmd.Flags().StringVar(&topologyLabel, "topology-label", k8s.RegionLabelKey, "Kubernetes node label used to group nodes into logical locations.")
 	cmd.Flags().IntVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
 	cmd.Flags().StringSliceVar(&serviceCIDRsRaw, "service-cidr", nil, "The service CIDR for the Kubernetes cluster. Can be provided optionally to avoid masquerading packets sent to service IPs. Can be specified multiple times.")
@@ -277,7 +278,20 @@ func runRoot(_ *cobra.Command, _ []string) error {
 		internalCIDRs = append(internalCIDRs, s)
 	}
 
-	m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, internalCIDRs, serviceCIDRs, log.With(logger, "component", "kilo"), registry)
+	var mtu uint
+	var autoMTU bool
+	if mtuFlag == "auto" {
+		autoMTU = true
+		mtu = wireguard.DefaultMTU
+	} else {
+		v, err := strconv.ParseUint(mtuFlag, 10, 32)
+		if err != nil {
+			return fmt.Errorf("failed to parse MTU %q: %v", mtuFlag, err)
+		}
+		mtu = uint(v)
+	}
+
+	m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, autoMTU, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, internalCIDRs, serviceCIDRs, log.With(logger, "component", "kilo"), registry)
 	if err != nil {
 		return fmt.Errorf("failed to create Kilo mesh: %v", err)
 	}

docs/kg.md

@@ -51,7 +51,7 @@ Flags:
       --log-level string               Log level to use. Possible values: all, debug, info, warn, error, none (default "info")
       --master string                  The address of the Kubernetes API server (overrides any value in kubeconfig).
       --mesh-granularity string        The granularity of the network mesh to create. Possible values: location, full (default "location")
-      --mtu uint                       The MTU of the WireGuard interface created by Kilo. (default 1420)
+      --mtu string                     The MTU of the WireGuard interface created by Kilo. Set to 'auto' to detect from the underlay interface. (default "auto")
       --port int                       The port over which WireGuard peers should communicate. (default 51820)
       --prioritise-private-addresses   Prefer to assign a private IP address to the node's endpoint.
       --resync-period duration         How often should the Kilo controllers reconcile? (default 30s)

pkg/mesh/mesh.go

@@ -63,6 +63,8 @@ type Mesh struct {
 	kiloIface           int
 	kiloIfaceName       string
 	local               bool
+	mtu                 uint
+	autoMTU             bool
 	port                int
 	priv                wgtypes.Key
 	privIface           int
@@ -89,7 +91,7 @@ type Mesh struct {
 }
 
 // New returns a new Mesh instance.
-func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, allowedInternalCIDRs []*net.IPNet, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) {
+func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, autoMTU bool, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, allowedInternalCIDRs []*net.IPNet, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) {
 	if err := os.MkdirAll(kiloPath, 0700); err != nil {
 		return nil, fmt.Errorf("failed to create directory to store configuration: %v", err)
 	}
@@ -155,6 +157,9 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 		enc = encapsulation.Noop(enc.Strategy())
 		_ = level.Debug(logger).Log("msg", "running without a private IP address")
 	}
+	if autoMTU {
+		mtu = detectMTU(logger)
+	}
 	var externalIP *net.IPNet
 	if prioritisePrivateAddr && privateIP != nil {
 		externalIP = privateIP
@@ -189,6 +194,8 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit
 		resyncPeriod:        resyncPeriod,
 		iptablesForwardRule: iptablesForwardRule,
 		local:               local,
+		mtu:                 mtu,
+		autoMTU:             autoMTU,
 		serviceCIDRs:        serviceCIDRs,
 		subnet:              subnet,
 		table:               route.NewTable(),
@@ -479,6 +486,16 @@ func (m *Mesh) applyTopology() {
 	if nodes[m.hostname] == nil {
 		return
 	}
+	// Re-detect MTU if auto mode is enabled.
+	if m.autoMTU {
+		m.mtu = detectMTU(m.logger)
+	}
+	// Ensure the WireGuard interface has the correct MTU.
+	if err := wireguard.SetMTU(m.kiloIface, m.mtu); err != nil {
+		_ = level.Error(m.logger).Log("error", fmt.Errorf("failed to set MTU on WireGuard interface: %v", err))
+		m.errorCounter.WithLabelValues("apply").Inc()
+		return
+	}
 	// Find the Kilo interface name.
 	link, err := linkByIndex(m.kiloIface)
 	if err != nil {
@@ -795,6 +812,24 @@ func discoveredEndpointsAreEqual(a, b map[string]*net.UDPAddr) bool {
 	return true
 }
 
+func detectMTU(logger log.Logger) uint {
+	iface, err := defaultInterface()
+	if err != nil {
+		_ = level.Warn(logger).Log("msg", "failed to get default interface for MTU detection, using default MTU", "error", err)
+		return wireguard.DefaultMTU
+	}
+	link, err := netlink.LinkByIndex(iface.Index)
+	if err != nil {
+		_ = level.Warn(logger).Log("msg", "failed to get default interface link for MTU detection, using default MTU", "error", err)
+		return wireguard.DefaultMTU
+	}
+	baseMTU := link.Attrs().MTU
+	_ = level.Info(logger).Log("msg", fmt.Sprintf("detected underlay MTU %d on default interface %s", baseMTU, link.Attrs().Name))
+	mtu := uint(baseMTU) - wireguard.WireGuardOverhead
+	_ = level.Info(logger).Log("msg", fmt.Sprintf("auto-detected WireGuard MTU: %d (underlay %d - overhead %d)", mtu, baseMTU, wireguard.WireGuardOverhead))
+	return mtu
+}
+
 func linkByIndex(index int) (netlink.Link, error) {
 	link, err := netlink.LinkByIndex(index)
 	if err != nil {

pkg/wireguard/wireguard.go

@@ -26,6 +26,23 @@ import (
 // DefaultMTU is the the default MTU used by WireGuard.
 const DefaultMTU = 1420
 
+// WireGuardOverhead is the overhead in bytes added by WireGuard encapsulation.
+// IPv4 header (20) + UDP header (8) + WireGuard header (32) + MAC (16) + padding (4) = 80 bytes.
+const WireGuardOverhead = 80
+
+// SetMTU sets the MTU on the interface with the given index.
+// If the current MTU already matches, no change is made.
+func SetMTU(index int, mtu uint) error {
+	link, err := netlink.LinkByIndex(index)
+	if err != nil {
+		return fmt.Errorf("failed to get link: %v", err)
+	}
+	if link.Attrs().MTU == int(mtu) {
+		return nil
+	}
+	return netlink.LinkSetMTU(link, int(mtu))
+}
+
 type wgLink struct {
 	a netlink.LinkAttrs
 	t string