pkg: allow overriding internal IP

This addresses the request for enhancement in
https://github.com/squat/kilo/issues/7.

docs/annotations.md

@@ -5,6 +5,7 @@ The following annotations can be added to any Kubernetes Node object to configur
 |Name|type|example|
 |----|----|-------|
 |[kilo.squat.ai/force-external-ip](#force-external-ip)|CIDR|`"55.55.55.55/32"`|
+|[kilo.squat.ai/force-internal-ip](#force-internal-ip)|CIDR|`"55.55.55.55/32"`|
 |[kilo.squat.ai/leader](#leader)|string|`""`|
 |[kilo.squat.ai/location](#location)|string|`"gcp-east"`|
 
@@ -13,7 +14,13 @@ Kilo requires at least one node in each location to have a publicly accessible I
 The Kilo agent running on each node will use heuristics to automatically detect an external IP address for the node; however, in some circumstances it may be necessary to explicitly configure the IP address, for example:
  * _no automatic public IP on ethernet device_: on some cloud providers it is common for nodes to be allocated a public IP address but for the Ethernet devices to only be automatically configured with the private network address; in this case the allocated public IP address should be specified;
  * _multiple public IP addresses_: if a node has multiple public IPs but one is preferred, then the preferred IP address should be specified;
- * _IPv6_: if a node has both public IPv4 and IPv6 addresses and the Kilo network should operate over IPv6, then the IPv6 address should be specified;
+ * _IPv6_: if a node has both public IPv4 and IPv6 addresses and the Kilo network should operate over IPv6, then the IPv6 address should be specified.
+
+### force-internal-ip
+Kilo routes packets destined for nodes inside the same logical location using the node's internal IP address.
+The Kilo agent running on each node will use heuristics to automatically detect a private IP address for the node; however, in some circumstances it may be necessary to explicitly configure the IP address, for example:
+ * _multiple private IP addresses_: if a node has multiple private IPs but one is preferred, then the preferred IP address should be specified;
+ * _IPv6_: if a node has both private IPv4 and IPv6 addresses and the Kilo network should operate over IPv6, then the IPv6 address should be specified.
 
 ### leader
 By default, Kilo creates a network mesh at the data-center granularity.
@@ -21,7 +28,7 @@ This means that one leader node is selected from each location to be an edge ser
 Kilo automatically selects the leader for each location in a stable and deterministic manner to avoid churn in the network configuration, while giving preference to nodes that are known to have public IP addresses.
 In some situations it may be desirable to manually select the leader for a location, for example:
  * _firewall_: Kilo requires an open UDP port, which defaults to 51820, to communicate between locations; if only one node is configured to have that port open, then that node should be given the leader annotation;
- * _bandwidth_: if certain nodes in the cluster have a higher bandwidth or lower latency Internet connection, then those nodes should be given the leader annotation;
+ * _bandwidth_: if certain nodes in the cluster have a higher bandwidth or lower latency Internet connection, then those nodes should be given the leader annotation.
 
 _Note_: multiple nodes within a single location can be given the leader annotation; in this case, Kilo will select one leader from the set of annotated nodes. 
 

pkg/k8s/backend.go

@@ -50,6 +50,7 @@ const (
 	Backend                      = "kubernetes"
 	externalIPAnnotationKey      = "kilo.squat.ai/external-ip"
 	forceExternalIPAnnotationKey = "kilo.squat.ai/force-external-ip"
+	forceInternalIPAnnotationKey = "kilo.squat.ai/force-internal-ip"
 	internalIPAnnotationKey      = "kilo.squat.ai/internal-ip"
 	keyAnnotationKey             = "kilo.squat.ai/key"
 	lastSeenAnnotationKey        = "kilo.squat.ai/last-seen"
@@ -252,11 +253,15 @@ func translateNode(node *v1.Node) *mesh.Node {
 	if !ok {
 		location = node.ObjectMeta.Labels[regionLabelKey]
 	}
-	// Allow the external IP to be overridden.
+	// Allow the IPs to be overridden.
 	externalIP, ok := node.ObjectMeta.Annotations[forceExternalIPAnnotationKey]
 	if !ok {
 		externalIP = node.ObjectMeta.Annotations[externalIPAnnotationKey]
 	}
+	internalIP, ok := node.ObjectMeta.Annotations[forceInternalIPAnnotationKey]
+	if !ok {
+		internalIP = node.ObjectMeta.Annotations[internalIPAnnotationKey]
+	}
 	var lastSeen int64
 	if ls, ok := node.ObjectMeta.Annotations[lastSeenAnnotationKey]; !ok {
 		lastSeen = 0
@@ -271,7 +276,7 @@ func translateNode(node *v1.Node) *mesh.Node {
 		// in this case the IP will be nil and
 		// the mesh can wait for the node to be updated.
 		ExternalIP: normalizeIP(externalIP),
-		InternalIP: normalizeIP(node.ObjectMeta.Annotations[internalIPAnnotationKey]),
+		InternalIP: normalizeIP(internalIP),
 		Key:        []byte(node.ObjectMeta.Annotations[keyAnnotationKey]),
 		LastSeen:   lastSeen,
 		Leader:     leader,

pkg/k8s/backend_test.go

@@ -111,6 +111,16 @@ func TestTranslateNode(t *testing.T) {
 				ExternalIP: &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
 			},
 		},
+		{
+			name: "internal IP override",
+			annotations: map[string]string{
+				internalIPAnnotationKey:      "10.1.0.1/24",
+				forceInternalIPAnnotationKey: "10.1.0.2/24",
+			},
+			out: &mesh.Node{
+				InternalIP: &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(24, 32)},
+			},
+		},
 		{
 			name: "invalid time",
 			annotations: map[string]string{
@@ -123,7 +133,8 @@ func TestTranslateNode(t *testing.T) {
 			annotations: map[string]string{
 				externalIPAnnotationKey:      "10.0.0.1/24",
 				forceExternalIPAnnotationKey: "10.0.0.2/24",
-				internalIPAnnotationKey:      "10.0.0.2/32",
+				forceInternalIPAnnotationKey: "10.1.0.2/32",
+				internalIPAnnotationKey:      "10.1.0.1/32",
 				keyAnnotationKey:             "foo",
 				lastSeenAnnotationKey:        "1000000000",
 				leaderAnnotationKey:          "",
@@ -135,7 +146,7 @@ func TestTranslateNode(t *testing.T) {
 			},
 			out: &mesh.Node{
 				ExternalIP:  &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(24, 32)},
-				InternalIP:  &net.IPNet{IP: net.ParseIP("10.0.0.2"), Mask: net.CIDRMask(32, 32)},
+				InternalIP:  &net.IPNet{IP: net.ParseIP("10.1.0.2"), Mask: net.CIDRMask(32, 32)},
 				Key:         []byte("foo"),
 				LastSeen:    1000000000,
 				Leader:      true,

pkg/mesh/mesh.go

@@ -505,17 +505,20 @@ func (m *Mesh) checkIn() {
 }
 
 func (m *Mesh) handleLocal(n *Node) {
-	// Allow the external IP to be overridden.
+	// Allow the IPs to be overridden.
 	if n.ExternalIP == nil {
 		n.ExternalIP = m.externalIP
 	}
+	if n.InternalIP == nil {
+		n.InternalIP = m.internalIP
+	}
 	// Compare the given node to the calculated local node.
 	// Take leader, location, and subnet from the argument, as these
 	// are not determined by kilo.
 	local := &Node{
 		ExternalIP:  n.ExternalIP,
 		Key:         m.pub,
-		InternalIP:  m.internalIP,
+		InternalIP:  n.InternalIP,
 		LastSeen:    time.Now().Unix(),
 		Leader:      n.Leader,
 		Location:    n.Location,