pkg/mesh,cmd: add WireGuard IP to Nodes

This allows admins or users to have some easy visibility into the
configuration of the Kilo cluster.

cmd/kg/main.go

@@ -87,7 +87,7 @@ func Main() error {
 	master := flag.String("master", "", "The address of the Kubernetes API server (overrides any value in kubeconfig).")
 	var port uint
 	flag.UintVar(&port, "port", mesh.DefaultKiloPort, "The port over which WireGuard peers should communicate.")
-	subnet := flag.String("subnet", "10.4.0.0/16", "CIDR from which to allocate addresses for WireGuard interfaces.")
+	subnet := flag.String("subnet", mesh.DefaultKiloSubnet.String(), "CIDR from which to allocate addresses for WireGuard interfaces.")
 	printVersion := flag.Bool("version", false, "Print version and exit")
 	flag.Parse()
 

cmd/kgctl/graph.go

@@ -35,17 +35,22 @@ func runGraph(_ *cobra.Command, _ []string) error {
 		return fmt.Errorf("failed to list nodes: %v", err)
 	}
 	var hostname string
+	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
 	for _, n := range ns {
 		if n.Ready() {
 			nodes[n.Name] = n
 			hostname = n.Name
 		}
+		if n.WireGuardIP != nil {
+			subnet = n.WireGuardIP
+		}
 	}
+	subnet.IP = subnet.IP.Mask(subnet.Mask)
 	if len(nodes) == 0 {
 		return fmt.Errorf("did not find any valid Kilo nodes in the cluster")
 	}
-	t, err := mesh.NewTopology(nodes, nil, opts.granularity, hostname, 0, []byte{}, opts.subnet)
+	t, err := mesh.NewTopology(nodes, nil, opts.granularity, hostname, 0, []byte{}, subnet)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}

cmd/kgctl/main.go

@@ -16,7 +16,6 @@ package main
 
 import (
 	"fmt"
-	"net"
 	"os"
 	"strings"
 
@@ -59,21 +58,13 @@ var (
 	opts struct {
 		backend     mesh.Backend
 		granularity mesh.Granularity
-		subnet      *net.IPNet
 	}
 	backend     string
 	granularity string
 	kubeconfig  string
-	subnet      string
 )
 
 func runRoot(_ *cobra.Command, _ []string) error {
-	_, s, err := net.ParseCIDR(subnet)
-	if err != nil {
-		return fmt.Errorf("failed to parse %q as CIDR: %v", subnet, err)
-	}
-	opts.subnet = s
-
 	opts.granularity = mesh.Granularity(granularity)
 	switch opts.granularity {
 	case mesh.LogicalGranularity:
@@ -117,7 +108,6 @@ func main() {
 	cmd.PersistentFlags().StringVar(&backend, "backend", k8s.Backend, fmt.Sprintf("The backend for the mesh. Possible values: %s", availableBackends))
 	cmd.PersistentFlags().StringVar(&granularity, "mesh-granularity", string(mesh.LogicalGranularity), fmt.Sprintf("The granularity of the network mesh to create. Possible values: %s", availableGranularities))
 	cmd.PersistentFlags().StringVar(&kubeconfig, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig.")
-	cmd.PersistentFlags().StringVar(&subnet, "subnet", "10.4.0.0/16", "CIDR from which to allocate addressees to WireGuard interfaces.")
 
 	for _, subCmd := range []*cobra.Command{
 		graph(),

cmd/kgctl/showconf.go

@@ -121,12 +121,17 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to list peers: %v", err)
 	}
 	hostname := args[0]
+	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
 	for _, n := range ns {
 		if n.Ready() {
 			nodes[n.Name] = n
 		}
+		if n.WireGuardIP != nil {
+			subnet = n.WireGuardIP
+		}
 	}
+	subnet.IP = subnet.IP.Mask(subnet.Mask)
 	if len(nodes) == 0 {
 		return errors.New("did not find any valid Kilo nodes in the cluster")
 	}
@@ -141,7 +146,7 @@ func runShowConfNode(_ *cobra.Command, args []string) error {
 		}
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, opts.subnet)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}
@@ -192,13 +197,18 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to list peers: %v", err)
 	}
 	var hostname string
+	subnet := mesh.DefaultKiloSubnet
 	nodes := make(map[string]*mesh.Node)
 	for _, n := range ns {
 		if n.Ready() {
 			nodes[n.Name] = n
 			hostname = n.Name
 		}
+		if n.WireGuardIP != nil {
+			subnet = n.WireGuardIP
+		}
 	}
+	subnet.IP = subnet.IP.Mask(subnet.Mask)
 	if len(nodes) == 0 {
 		return errors.New("did not find any valid Kilo nodes in the cluster")
 	}
@@ -214,7 +224,7 @@ func runShowConfPeer(_ *cobra.Command, args []string) error {
 		return fmt.Errorf("did not find any peer named %q in the cluster", peer)
 	}
 
-	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, opts.subnet)
+	t, err := mesh.NewTopology(nodes, peers, opts.granularity, hostname, mesh.DefaultKiloPort, []byte{}, subnet)
 	if err != nil {
 		return fmt.Errorf("failed to create topology: %v", err)
 	}

pkg/k8s/backend.go

@@ -55,9 +55,11 @@ const (
 	lastSeenAnnotationKey        = "kilo.squat.ai/last-seen"
 	leaderAnnotationKey          = "kilo.squat.ai/leader"
 	locationAnnotationKey        = "kilo.squat.ai/location"
-	regionLabelKey               = "failure-domain.beta.kubernetes.io/region"
-	jsonPatchSlash               = "~1"
-	jsonRemovePatch              = `{"op": "remove", "path": "%s"}`
+	wireGuardIPAnnotationKey     = "kilo.squat.ai/wireguard-ip"
+
+	regionLabelKey  = "failure-domain.beta.kubernetes.io/region"
+	jsonPatchSlash  = "~1"
+	jsonRemovePatch = `{"op": "remove", "path": "%s"}`
 )
 
 type backend struct {
@@ -119,6 +121,7 @@ func (nb *nodeBackend) CleanUp(name string) error {
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(internalIPAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(keyAnnotationKey, "/", jsonPatchSlash, 1))),
 		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(lastSeenAnnotationKey, "/", jsonPatchSlash, 1))),
+		fmt.Sprintf(jsonRemovePatch, path.Join("/metadata", "annotations", strings.Replace(wireGuardIPAnnotationKey, "/", jsonPatchSlash, 1))),
 	}, ",") + "]")
 	if _, err := nb.client.CoreV1().Nodes().Patch(name, types.JSONPatchType, patch); err != nil {
 		return fmt.Errorf("failed to patch node: %v", err)
@@ -204,6 +207,11 @@ func (nb *nodeBackend) Set(name string, node *mesh.Node) error {
 	n.ObjectMeta.Annotations[internalIPAnnotationKey] = node.InternalIP.String()
 	n.ObjectMeta.Annotations[keyAnnotationKey] = string(node.Key)
 	n.ObjectMeta.Annotations[lastSeenAnnotationKey] = strconv.FormatInt(node.LastSeen, 10)
+	if node.WireGuardIP == nil {
+		n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = ""
+	} else {
+		n.ObjectMeta.Annotations[wireGuardIPAnnotationKey] = node.WireGuardIP.String()
+	}
 	oldData, err := json.Marshal(old)
 	if err != nil {
 		return err
@@ -270,6 +278,10 @@ func translateNode(node *v1.Node) *mesh.Node {
 		Location:   location,
 		Name:       node.Name,
 		Subnet:     subnet,
+		// WireGuardIP can fail to parse if the node is not a leader or if
+		// the node's agent has not yet reconciled. In either case, the IP
+		// will parse as nil.
+		WireGuardIP: normalizeIP(node.ObjectMeta.Annotations[wireGuardIPAnnotationKey]),
 	}
 }
 

pkg/k8s/backend_test.go

@@ -128,18 +128,20 @@ func TestTranslateNode(t *testing.T) {
 				lastSeenAnnotationKey:        "1000000000",
 				leaderAnnotationKey:          "",
 				locationAnnotationKey:        "b",
+				wireGuardIPAnnotationKey:     "10.4.0.1/16",
 			},
 			labels: map[string]string{
 				regionLabelKey: "a",
 			},
 			out: &mesh.Node{
-				ExternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
-				InternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
-				Key:        []byte("foo"),
-				LastSeen:   1000000000,
-				Leader:     true,
-				Location:   "b",
-				Subnet:     &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				ExternalIP:  &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
+				InternalIP:  &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
+				Key:         []byte("foo"),
+				LastSeen:    1000000000,
+				Leader:      true,
+				Location:    "b",
+				Subnet:      &net.IPNet{IP: net.ParseIP("10.2.1.0"), Mask: net.CIDRMask(24, 32)},
+				WireGuardIP: &net.IPNet{IP: net.ParseIP("10.4.0.1"), Mask: net.CIDRMask(16, 32)},
 			},
 			subnet: "10.2.1.0/24",
 		},

pkg/mesh/mesh.go

@@ -49,6 +49,9 @@ const (
 	DefaultCNIPath = "/etc/cni/net.d/10-kilo.conflist"
 )
 
+// DefaultKiloSubnet is the default CIDR for Kilo.
+var DefaultKiloSubnet = &net.IPNet{IP: []byte{10, 4, 0, 0}, Mask: []byte{255, 255, 0, 0}}
+
 // Granularity represents the abstraction level at which the network
 // should be meshed.
 type Granularity string
@@ -86,14 +89,16 @@ type Node struct {
 	LastSeen int64
 	// Leader is a suggestion to Kilo that
 	// the node wants to lead its segment.
-	Leader   bool
-	Location string
-	Name     string
-	Subnet   *net.IPNet
+	Leader      bool
+	Location    string
+	Name        string
+	Subnet      *net.IPNet
+	WireGuardIP *net.IPNet
 }
 
 // Ready indicates whether or not the node is ready.
 func (n *Node) Ready() bool {
+	// Nodes that are not leaders will not have WireGuardIPs, so it is not required.
 	return n != nil && n.ExternalIP != nil && n.Key != nil && n.InternalIP != nil && n.Subnet != nil && time.Now().Unix()-n.LastSeen < int64(resyncPeriod)*2/int64(time.Second)
 }
 
@@ -194,6 +199,7 @@ type Mesh struct {
 	subnet      *net.IPNet
 	table       *route.Table
 	tunlIface   int
+	wireGuardIP *net.IPNet
 
 	// nodes and peers are mutable fields in the struct
 	// and needs to be guarded.
@@ -514,14 +520,15 @@ func (m *Mesh) handleLocal(n *Node) {
 	// Take leader, location, and subnet from the argument, as these
 	// are not determined by kilo.
 	local := &Node{
-		ExternalIP: n.ExternalIP,
-		Key:        m.pub,
-		InternalIP: m.internalIP,
-		LastSeen:   time.Now().Unix(),
-		Leader:     n.Leader,
-		Location:   n.Location,
-		Name:       m.hostname,
-		Subnet:     n.Subnet,
+		ExternalIP:  n.ExternalIP,
+		Key:         m.pub,
+		InternalIP:  m.internalIP,
+		LastSeen:    time.Now().Unix(),
+		Leader:      n.Leader,
+		Location:    n.Location,
+		Name:        m.hostname,
+		Subnet:      n.Subnet,
+		WireGuardIP: m.wireGuardIP,
 	}
 	if !nodesAreEqual(n, local) {
 		level.Debug(m.logger).Log("msg", "local node differs from backend")
@@ -583,6 +590,8 @@ func (m *Mesh) applyTopology() {
 		m.errorCounter.WithLabelValues("apply").Inc()
 		return
 	}
+	// Update the node's WireGuard IP.
+	m.wireGuardIP = t.wireGuardCIDR
 	conf := t.Conf()
 	buf, err := conf.Bytes()
 	if err != nil {
@@ -740,7 +749,7 @@ func nodesAreEqual(a, b *Node) bool {
 	// Ignore LastSeen when comparing equality we want to check if the nodes are
 	// equivalent. However, we do want to check if LastSeen has transitioned
 	// between valid and invalid.
-	return ipNetsEqual(a.ExternalIP, b.ExternalIP) && string(a.Key) == string(b.Key) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready()
+	return ipNetsEqual(a.ExternalIP, b.ExternalIP) && string(a.Key) == string(b.Key) && ipNetsEqual(a.WireGuardIP, b.WireGuardIP) && ipNetsEqual(a.InternalIP, b.InternalIP) && a.Leader == b.Leader && a.Location == b.Location && a.Name == b.Name && subnetsEqual(a.Subnet, b.Subnet) && a.Ready() == b.Ready()
 }
 
 func peersAreEqual(a, b *Peer) bool {

pkg/mesh/topology_test.go

@@ -29,9 +29,8 @@ func allowedIPs(ips ...string) string {
 	return strings.Join(ips, ", ")
 }
 
-func setup(t *testing.T) (map[string]*Node, map[string]*Peer, []byte, uint32, *net.IPNet) {
+func setup(t *testing.T) (map[string]*Node, map[string]*Peer, []byte, uint32) {
 	key := []byte("private")
-	kiloNet := &net.IPNet{IP: net.ParseIP("10.4.0.0").To4(), Mask: net.CIDRMask(16, 32)}
 	e1 := &net.IPNet{IP: net.ParseIP("10.1.0.1").To4(), Mask: net.CIDRMask(16, 32)}
 	e2 := &net.IPNet{IP: net.ParseIP("10.1.0.2").To4(), Mask: net.CIDRMask(16, 32)}
 	e3 := &net.IPNet{IP: net.ParseIP("10.1.0.3").To4(), Mask: net.CIDRMask(16, 32)}
@@ -89,11 +88,11 @@ func setup(t *testing.T) (map[string]*Node, map[string]*Peer, []byte, uint32, *n
 			},
 		},
 	}
-	return nodes, peers, key, DefaultKiloPort, kiloNet
+	return nodes, peers, key, DefaultKiloPort
 }
 
 func TestNewTopology(t *testing.T) {
-	nodes, peers, key, port, kiloNet := setup(t)
+	nodes, peers, key, port := setup(t)
 
 	w1 := net.ParseIP("10.4.0.1").To4()
 	w2 := net.ParseIP("10.4.0.2").To4()
@@ -112,7 +111,7 @@ func TestNewTopology(t *testing.T) {
 				hostname:      nodes["a"].Name,
 				leader:        true,
 				location:      nodes["a"].Location,
-				subnet:        kiloNet,
+				subnet:        DefaultKiloSubnet,
 				privateIP:     nodes["a"].InternalIP,
 				wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)},
 				segments: []*segment{
@@ -148,7 +147,7 @@ func TestNewTopology(t *testing.T) {
 				hostname:      nodes["b"].Name,
 				leader:        true,
 				location:      nodes["b"].Location,
-				subnet:        kiloNet,
+				subnet:        DefaultKiloSubnet,
 				privateIP:     nodes["b"].InternalIP,
 				wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)},
 				segments: []*segment{
@@ -184,7 +183,7 @@ func TestNewTopology(t *testing.T) {
 				hostname:      nodes["c"].Name,
 				leader:        false,
 				location:      nodes["b"].Location,
-				subnet:        kiloNet,
+				subnet:        DefaultKiloSubnet,
 				privateIP:     nodes["c"].InternalIP,
 				wireGuardCIDR: nil,
 				segments: []*segment{
@@ -220,7 +219,7 @@ func TestNewTopology(t *testing.T) {
 				hostname:      nodes["a"].Name,
 				leader:        true,
 				location:      nodes["a"].Name,
-				subnet:        kiloNet,
+				subnet:        DefaultKiloSubnet,
 				privateIP:     nodes["a"].InternalIP,
 				wireGuardCIDR: &net.IPNet{IP: w1, Mask: net.CIDRMask(16, 32)},
 				segments: []*segment{
@@ -266,7 +265,7 @@ func TestNewTopology(t *testing.T) {
 				hostname:      nodes["b"].Name,
 				leader:        true,
 				location:      nodes["b"].Name,
-				subnet:        kiloNet,
+				subnet:        DefaultKiloSubnet,
 				privateIP:     nodes["b"].InternalIP,
 				wireGuardCIDR: &net.IPNet{IP: w2, Mask: net.CIDRMask(16, 32)},
 				segments: []*segment{
@@ -312,7 +311,7 @@ func TestNewTopology(t *testing.T) {
 				hostname:      nodes["c"].Name,
 				leader:        true,
 				location:      nodes["c"].Name,
-				subnet:        kiloNet,
+				subnet:        DefaultKiloSubnet,
 				privateIP:     nodes["c"].InternalIP,
 				wireGuardCIDR: &net.IPNet{IP: w3, Mask: net.CIDRMask(16, 32)},
 				segments: []*segment{
@@ -353,7 +352,7 @@ func TestNewTopology(t *testing.T) {
 	} {
 		tc.result.key = key
 		tc.result.port = port
-		topo, err := NewTopology(nodes, peers, tc.granularity, tc.hostname, port, key, kiloNet)
+		topo, err := NewTopology(nodes, peers, tc.granularity, tc.hostname, port, key, DefaultKiloSubnet)
 		if err != nil {
 			t.Errorf("test case %q: failed to generate Topology: %v", tc.name, err)
 		}
@@ -372,12 +371,12 @@ func mustTopo(t *testing.T, nodes map[string]*Node, peers map[string]*Peer, gran
 }
 
 func TestRoutes(t *testing.T) {
-	nodes, peers, key, port, kiloNet := setup(t)
+	nodes, peers, key, port := setup(t)
 	kiloIface := 0
 	privIface := 1
 	pubIface := 2
 	mustTopoForGranularityAndHost := func(granularity Granularity, hostname string) *Topology {
-		return mustTopo(t, nodes, peers, granularity, hostname, port, key, kiloNet)
+		return mustTopo(t, nodes, peers, granularity, hostname, port, key, DefaultKiloSubnet)
 	}
 
 	for _, tc := range []struct {
@@ -987,7 +986,7 @@ func TestRoutes(t *testing.T) {
 }
 
 func TestConf(t *testing.T) {
-	nodes, peers, key, port, kiloNet := setup(t)
+	nodes, peers, key, port := setup(t)
 	for _, tc := range []struct {
 		name     string
 		topology *Topology
@@ -995,7 +994,7 @@ func TestConf(t *testing.T) {
 	}{
 		{
 			name:     "logical from a",
-			topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["a"].Name, port, key, kiloNet),
+			topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["a"].Name, port, key, DefaultKiloSubnet),
 			result: `[Interface]
 PrivateKey = private
 ListenPort = 51820
@@ -1019,7 +1018,7 @@ AllowedIPs = 10.5.0.3/24
 		},
 		{
 			name:     "logical from b",
-			topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["b"].Name, port, key, kiloNet),
+			topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet),
 			result: `[Interface]
 		PrivateKey = private
 		ListenPort = 51820
@@ -1043,7 +1042,7 @@ AllowedIPs = 10.5.0.3/24
 		},
 		{
 			name:     "logical from c",
-			topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["c"].Name, port, key, kiloNet),
+			topology: mustTopo(t, nodes, peers, LogicalGranularity, nodes["c"].Name, port, key, DefaultKiloSubnet),
 			result: `[Interface]
 		PrivateKey = private
 		ListenPort = 51820
@@ -1067,7 +1066,7 @@ AllowedIPs = 10.5.0.3/24
 		},
 		{
 			name:     "full from a",
-			topology: mustTopo(t, nodes, peers, FullGranularity, nodes["a"].Name, port, key, kiloNet),
+			topology: mustTopo(t, nodes, peers, FullGranularity, nodes["a"].Name, port, key, DefaultKiloSubnet),
 			result: `[Interface]
 		PrivateKey = private
 		ListenPort = 51820
@@ -1096,7 +1095,7 @@ AllowedIPs = 10.5.0.3/24
 		},
 		{
 			name:     "full from b",
-			topology: mustTopo(t, nodes, peers, FullGranularity, nodes["b"].Name, port, key, kiloNet),
+			topology: mustTopo(t, nodes, peers, FullGranularity, nodes["b"].Name, port, key, DefaultKiloSubnet),
 			result: `[Interface]
 		PrivateKey = private
 		ListenPort = 51820
@@ -1125,7 +1124,7 @@ AllowedIPs = 10.5.0.3/24
 		},
 		{
 			name:     "full from c",
-			topology: mustTopo(t, nodes, peers, FullGranularity, nodes["c"].Name, port, key, kiloNet),
+			topology: mustTopo(t, nodes, peers, FullGranularity, nodes["c"].Name, port, key, DefaultKiloSubnet),
 			result: `[Interface]
 		PrivateKey = private
 		ListenPort = 51820