feat: cilium add-mode support (#312)

* feat: cilium add-mode support

when cni management by kilo is disable, we can use existing cluster's cni setup thanks to add-on mode

https://kilo.squat.ai/docs/introduction#add-on-mode

* feat: manifest example for cilium addon mode

* fix: apply comment from PR review

* fix: add mutex to interface retrieval into flannel addon mode

cmd/kg/main.go

@@ -213,6 +213,8 @@ func runRoot(_ *cobra.Command, _ []string) error {
 	switch compatibility {
 	case "flannel":
 		enc = encapsulation.NewFlannel(e)
+	case "cilium":
+		enc = encapsulation.NewCilium(e)
 	default:
 		enc = encapsulation.NewIPIP(e)
 	}

manifests/kilo-kubeadm-cilium.yaml

@@ -0,0 +1,142 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kilo
+  namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kilo
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - kilo.squat.ai
+  resources:
+  - peers
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: kilo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kilo
+subjects:
+  - kind: ServiceAccount
+    name: kilo
+    namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: kilo
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/name: kilo
+    app.kubernetes.io/part-of: kilo
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kilo
+      app.kubernetes.io/part-of: kilo
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: kilo
+        app.kubernetes.io/part-of: kilo
+    spec:
+      serviceAccountName: kilo
+      hostNetwork: true
+      containers:
+      - name: kilo
+        image: squat/kilo:0.5.0
+        args:
+        - --kubeconfig=/etc/kubernetes/kubeconfig
+        - --hostname=$(NODE_NAME)
+        - --cni=false
+        - --compatibility=cilium
+        - --local=false
+        # additional and also optional flag
+        - --encapsulate=crosssubnet
+        - --clean-up-interface=true
+        - --subnet=172.31.254.0/24
+        - --log-level=all
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        ports:
+        - containerPort: 1107
+          name: metrics
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: kilo-dir
+          mountPath: /var/lib/kilo
+
+        # with kube-proxy configmap
+        # - name: kubeconfig
+        #   mountPath: /etc/kubernetes
+        #   readOnly: true
+
+        # without kube-proxy host kubeconfig binding
+        - name: kubeconfig
+          mount_path: /etc/kubernetes/kubeconfig
+          sub_path: admin.conf
+          read_only: true
+
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: xtables-lock
+          mountPath: /run/xtables.lock
+          readOnly: false
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+      volumes:
+      - name: kilo-dir
+        hostPath:
+          path: /var/lib/kilo
+
+      # with kube-proxy configmap
+      # - name: kubeconfig
+      #   configMap:
+      #     name: kube-proxy
+      #     items:
+      #       - key: kubeconfig.conf
+      #         path: kubeconfig
+
+      # without kube-proxy host kubeconfig binding
+      - name: kubeconfig
+        host_path:
+          path: /etc/kubernetes
+
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate

pkg/encapsulation/cilium.go

@@ -0,0 +1,111 @@
+// Copyright 2019 the Kilo authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package encapsulation
+
+import (
+	"fmt"
+	"net"
+	"sync"
+
+	"github.com/vishvananda/netlink"
+
+	"github.com/squat/kilo/pkg/iptables"
+)
+
+const ciliumDeviceName = "cilium_host"
+
+type cilium struct {
+	iface    int
+	strategy Strategy
+	ch       chan netlink.LinkUpdate
+	done     chan struct{}
+	// mu guards updates to the iface field.
+	mu sync.Mutex
+}
+
+// NewCilium returns an encapsulator that uses Cilium.
+func NewCilium(strategy Strategy) Encapsulator {
+	return &cilium{
+		ch:       make(chan netlink.LinkUpdate),
+		done:     make(chan struct{}),
+		strategy: strategy,
+	}
+}
+
+// CleanUp close done channel
+func (f *cilium) CleanUp() error {
+	close(f.done)
+	return nil
+}
+
+// Gw returns the correct gateway IP associated with the given node.
+func (f *cilium) Gw(_, _ net.IP, subnet *net.IPNet) net.IP {
+	return subnet.IP
+}
+
+// Index returns the index of the Cilium interface.
+func (f *cilium) Index() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.iface
+}
+
+// Init finds the Cilium interface index.
+func (f *cilium) Init(_ int) error {
+	if err := netlink.LinkSubscribe(f.ch, f.done); err != nil {
+		return fmt.Errorf("failed to subscribe to updates to %s: %v", ciliumDeviceName, err)
+	}
+	go func() {
+		var lu netlink.LinkUpdate
+		for {
+			select {
+			case lu = <-f.ch:
+				if lu.Attrs().Name == ciliumDeviceName {
+					f.mu.Lock()
+					f.iface = lu.Attrs().Index
+					f.mu.Unlock()
+				}
+			case <-f.done:
+				return
+			}
+		}
+	}()
+	i, err := netlink.LinkByName(ciliumDeviceName)
+	if _, ok := err.(netlink.LinkNotFoundError); ok {
+		return nil
+	}
+	if err != nil {
+		return fmt.Errorf("failed to query for Cilium interface: %v", err)
+	}
+	f.mu.Lock()
+	f.iface = i.Attrs().Index
+	f.mu.Unlock()
+	return nil
+}
+
+// Rules is a no-op.
+func (f *cilium) Rules(_ []*net.IPNet) []iptables.Rule {
+	return nil
+}
+
+// Set is a no-op.
+func (f *cilium) Set(_ *net.IPNet) error {
+	return nil
+}
+
+// Strategy returns the configured strategy for encapsulation.
+func (f *cilium) Strategy() Strategy {
+	return f.strategy
+}

pkg/encapsulation/flannel.go

@@ -56,6 +56,8 @@ func (f *flannel) Gw(_, _ net.IP, subnet *net.IPNet) net.IP {
 
 // Index returns the index of the Flannel interface.
 func (f *flannel) Index() int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
 	return f.iface
 }