kilo/manifests/kilo-k3s-userspace-heterogeneous.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: kilo
  namespace: kube-system
  labels:
    app.kubernetes.io/name: kilo
data:
  cni-conf.json: |
    {
       "cniVersion":"0.3.1",
       "name":"kilo",
       "plugins":[
          {
             "name":"kubernetes",
             "type":"bridge",
             "bridge":"kube-bridge",
             "isDefaultGateway":true,
             "forceAddress":true,
             "mtu": 1420,
             "ipam":{
                "type":"host-local"
             }
          },
          {
             "type":"portmap",
             "snat":true,
             "capabilities":{
                "portMappings":true
             }
          }
       ]
    }
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: kilo
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: kilo
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  verbs:
  - list
  - get
  - patch
  - watch
- apiGroups:
  - kilo.squat.ai
  resources:
  - peers
  verbs:
  - list
  - watch
- apiGroups:
  - apiextensions.k8s.io
  resources:
  - customresourcedefinitions
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kilo
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: kilo
subjects:
  - kind: ServiceAccount
    name: kilo
    namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: kilo-scripts
  namespace: kube-system
data:
  init.sh: |
    #!/bin/sh
    cat > /etc/kubernetes/kubeconfig <<EOF
        apiVersion: v1
        kind: Config
        name: kilo
        clusters:
        - cluster:
            server: $(sed -n 's/.*server: \(.*\)/\1/p' /var/lib/rancher/k3s/agent/kubelet.kubeconfig)
            certificate-authority: /var/lib/rancher/k3s/agent/server-ca.crt
        users:
        - name: kilo
          user:
            token: $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)
        contexts:
        - name: kilo
          context:
            cluster: kilo
            namespace: ${NAMESPACE}
            user: kilo
        current-context: kilo
    EOF
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kilo
  namespace: kube-system
  labels:
    app.kubernetes.io/name: kilo
    app.kubernetes.io/part-of: kilo
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: kilo
      app.kubernetes.io/part-of: kilo
  template:
    metadata:
      labels:
        app.kubernetes.io/name: kilo
        app.kubernetes.io/part-of: kilo
    spec:
      nodeSelector:
        nkml.squat.ai/wireguard: "true"
      serviceAccountName: kilo
      hostNetwork: true
      containers:
      - name: kilo
        image: squat/kilo
        args:
        - --kubeconfig=/etc/kubernetes/kubeconfig
        - --hostname=$(NODE_NAME)
        - --interface=kilo0
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        ports:
        - containerPort: 1107
          name: metrics
        securityContext:
          privileged: true
        volumeMounts:
        - name: cni-conf-dir
          mountPath: /etc/cni/net.d
        - name: kilo-dir
          mountPath: /var/lib/kilo
        - name: kubeconfig
          mountPath: /etc/kubernetes
          readOnly: true
        - name: lib-modules
          mountPath: /lib/modules
          readOnly: true
        - name: xtables-lock
          mountPath: /run/xtables.lock
          readOnly: false
      initContainers:
      - name: generate-kubeconfig
        image: squat/kilo
        command:
        - /bin/sh
        args:
        - /scripts/init.sh
        imagePullPolicy: Always
        volumeMounts:
        - name: kubeconfig
          mountPath: /etc/kubernetes
        - name: scripts
          mountPath: /scripts/
          readOnly: true
        - name: k3s-agent
          mountPath: /var/lib/rancher/k3s/agent/
          readOnly: true
        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
      - name: install-cni
        image: squat/kilo
        command:
        - /bin/sh
        - -c
        - set -e -x;
          cp /opt/cni/bin/* /host/opt/cni/bin/;
          TMP_CONF="$CNI_CONF_NAME".tmp;
          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
          rm -f /host/etc/cni/net.d/*;
          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
        env:
        - name: CNI_CONF_NAME
          value: 10-kilo.conflist
        - name: CNI_NETWORK_CONFIG
          valueFrom:
            configMapKeyRef:
              name: kilo
              key: cni-conf.json
        volumeMounts:
        - name: cni-bin-dir
          mountPath: /host/opt/cni/bin
        - name: cni-conf-dir
          mountPath: /host/etc/cni/net.d
      tolerations:
      - effect: NoSchedule
        operator: Exists
      - effect: NoExecute
        operator: Exists
      volumes:
      - name: cni-bin-dir
        hostPath:
          path: /opt/cni/bin
      - name: cni-conf-dir
        hostPath:
          path: /etc/cni/net.d
      - name: kilo-dir
        hostPath:
          path: /var/lib/kilo
      - name: kubeconfig
        emptyDir: {}
      - name: scripts
        configMap:
          name: kilo-scripts
      - name: k3s-agent
        hostPath:
          path: /var/lib/rancher/k3s/agent
      - name: lib-modules
        hostPath:
          path: /lib/modules
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: kilo-userspace
  namespace: kube-system
  labels:
    app.kubernetes.io/name: kilo-userspace
    app.kubernetes.io/part-of: kilo
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: kilo-userspace
      app.kubernetes.io/part-of: kilo
  template:
    metadata:
      labels:
        app.kubernetes.io/name: kilo-userspace
        app.kubernetes.io/part-of: kilo
    spec:
      nodeSelector:
        nkml.squat.ai/wireguard: "false"
      serviceAccountName: kilo
      hostNetwork: true
      containers:
      - name: kilo
        image: squat/kilo 
        args:
        - --kubeconfig=/etc/kubernetes/kubeconfig
        - --hostname=$(NODE_NAME)
        - --create-interface=false
        - --interface=kilo0
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        ports:
        - containerPort: 1107
          name: metrics
        securityContext:
          privileged: true
        volumeMounts:
        - name: cni-conf-dir
          mountPath: /etc/cni/net.d
        - name: kilo-dir
          mountPath: /var/lib/kilo
        - name: kubeconfig
          mountPath: /etc/kubernetes
          readOnly: true
        - name: lib-modules
          mountPath: /lib/modules
          readOnly: true
        - name: xtables-lock
          mountPath: /run/xtables.lock
          readOnly: false
        - name: wireguard
          mountPath: /var/run/wireguard
          readOnly: false
      - name: boringtun
        image: leonnicolas/boringtun
        args:
        - --disable-drop-privileges=true
        - --foreground
        - kilo0
        securityContext:
          privileged: true
        volumeMounts:
        - name: wireguard
          mountPath: /var/run/wireguard
          readOnly: false
      initContainers:
      - name: generate-kubeconfig
        image: squat/kilo
        command:
        - /bin/sh
        args:
        - /scripts/init.sh
        imagePullPolicy: Always
        volumeMounts:
        - name: kubeconfig
          mountPath: /etc/kubernetes
        - name: scripts
          mountPath: /scripts/
          readOnly: true
        - name: k3s-agent
          mountPath: /var/lib/rancher/k3s/agent/
          readOnly: true
        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
      - name: install-cni
        image: squat/kilo
        command:
        - /bin/sh
        - -c
        - set -e -x;
          cp /opt/cni/bin/* /host/opt/cni/bin/;
          TMP_CONF="$CNI_CONF_NAME".tmp;
          echo "$CNI_NETWORK_CONFIG" > $TMP_CONF;
          rm -f /host/etc/cni/net.d/*;
          mv $TMP_CONF /host/etc/cni/net.d/$CNI_CONF_NAME
        env:
        - name: CNI_CONF_NAME
          value: 10-kilo.conflist
        - name: CNI_NETWORK_CONFIG
          valueFrom:
            configMapKeyRef:
              name: kilo
              key: cni-conf.json
        volumeMounts:
        - name: cni-bin-dir
          mountPath: /host/opt/cni/bin
        - name: cni-conf-dir
          mountPath: /host/etc/cni/net.d
      tolerations:
      - effect: NoSchedule
        operator: Exists
      - effect: NoExecute
        operator: Exists
      volumes:
      - name: cni-bin-dir
        hostPath:
          path: /opt/cni/bin
      - name: cni-conf-dir
        hostPath:
          path: /etc/cni/net.d
      - name: kilo-dir
        hostPath:
          path: /var/lib/kilo
      - name: kubeconfig
        emptyDir: {}
      - name: scripts
        configMap:
          name: kilo-scripts
      - name: k3s-agent
        hostPath:
          path: /var/lib/rancher/k3s/agent
      - name: lib-modules
        hostPath:
          path: /lib/modules
      - name: xtables-lock
        hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
      - name: wireguard
        hostPath:
          path: /var/run/wireguard
---
kind: DaemonSet
apiVersion: apps/v1
metadata: 
  name: nkml
  namespace: kube-system
  labels:
    app.kubernetes.io/name: nkml
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: nkml
  template:
    metadata:
      labels:
        app.kubernetes.io/name: nkml
    spec:
      hostNetwork: true
      serviceAccountName: kilo
      containers:
      - name: nkml
        image: leonnicolas/nkml
        args: 
        - --hostname=$(NODE_NAME)
        - --label-mod=wireguard
        - --kubeconfig=/etc/kubernetes/kubeconfig
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        ports:
        - name: http
          containerPort: 8080
        volumeMounts:
        - name: kubeconfig
          mountPath: /etc/kubernetes
          readOnly: true
      initContainers:
      - name: generate-kubeconfig
        image: squat/kilo
        command:
        - /bin/sh
        args:
        - /scripts/init.sh
        imagePullPolicy: Always
        volumeMounts:
        - name: kubeconfig
          mountPath: /etc/kubernetes
        - name: scripts
          mountPath: /scripts/
          readOnly: true
        - name: k3s-agent
          mountPath: /var/lib/rancher/k3s/agent/
          readOnly: true
        env:
        - name: NAMESPACE
          valueFrom:
            fieldRef:
              fieldPath: metadata.namespace
      volumes:
      - name: kubeconfig
        emptyDir: {}
      - name: scripts
        configMap:
          name: kilo-scripts
      - name: k3s-agent
        hostPath:
          path: /var/lib/rancher/k3s/agent