kafka.yaml 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448
  1. ---
  2. # Source: kafka/templates/serviceaccount.yaml
  3. apiVersion: v1
  4. kind: ServiceAccount
  5. metadata:
  6. name: my-release-kafka
  7. labels:
  8. app.kubernetes.io/name: kafka
  9. helm.sh/chart: kafka-11.8.6
  10. app.kubernetes.io/instance: my-release
  11. app.kubernetes.io/managed-by: Helm
  12. app.kubernetes.io/component: kafka
  13. ---
  14. # Source: kafka/templates/scripts-configmap.yaml
  15. apiVersion: v1
  16. kind: ConfigMap
  17. metadata:
  18. name: my-release-kafka-scripts
  19. labels:
  20. app.kubernetes.io/name: kafka
  21. helm.sh/chart: kafka-11.8.6
  22. app.kubernetes.io/instance: my-release
  23. app.kubernetes.io/managed-by: Helm
  24. data:
  25. setup.sh: |-
  26. #!/bin/bash
  27. ID="${MY_POD_NAME#"my-release-kafka-"}"
  28. export KAFKA_CFG_BROKER_ID="$ID"
  29. exec /entrypoint.sh /run.sh
  30. ---
  31. # Source: kafka/charts/zookeeper/templates/svc-headless.yaml
  32. apiVersion: v1
  33. kind: Service
  34. metadata:
  35. name: my-release-zookeeper-headless
  36. namespace: default
  37. labels:
  38. app.kubernetes.io/name: zookeeper
  39. helm.sh/chart: zookeeper-5.21.9
  40. app.kubernetes.io/instance: my-release
  41. app.kubernetes.io/managed-by: Helm
  42. app.kubernetes.io/component: zookeeper
  43. spec:
  44. type: ClusterIP
  45. clusterIP: None
  46. publishNotReadyAddresses: true
  47. ports:
  48. - name: tcp-client
  49. port: 2181
  50. targetPort: client
  51. - name: follower
  52. port: 2888
  53. targetPort: follower
  54. - name: tcp-election
  55. port: 3888
  56. targetPort: election
  57. selector:
  58. app.kubernetes.io/name: zookeeper
  59. app.kubernetes.io/instance: my-release
  60. app.kubernetes.io/component: zookeeper
  61. ---
  62. # Source: kafka/charts/zookeeper/templates/svc.yaml
  63. apiVersion: v1
  64. kind: Service
  65. metadata:
  66. name: my-release-zookeeper
  67. namespace: default
  68. labels:
  69. app.kubernetes.io/name: zookeeper
  70. helm.sh/chart: zookeeper-5.21.9
  71. app.kubernetes.io/instance: my-release
  72. app.kubernetes.io/managed-by: Helm
  73. app.kubernetes.io/component: zookeeper
  74. spec:
  75. type: ClusterIP
  76. ports:
  77. - name: tcp-client
  78. port: 2181
  79. targetPort: client
  80. - name: follower
  81. port: 2888
  82. targetPort: follower
  83. - name: tcp-election
  84. port: 3888
  85. targetPort: election
  86. selector:
  87. app.kubernetes.io/name: zookeeper
  88. app.kubernetes.io/instance: my-release
  89. app.kubernetes.io/component: zookeeper
  90. ---
  91. # Source: kafka/templates/svc-headless.yaml
  92. apiVersion: v1
  93. kind: Service
  94. metadata:
  95. name: my-release-kafka-headless
  96. labels:
  97. app.kubernetes.io/name: kafka
  98. helm.sh/chart: kafka-11.8.6
  99. app.kubernetes.io/instance: my-release
  100. app.kubernetes.io/managed-by: Helm
  101. app.kubernetes.io/component: kafka
  102. spec:
  103. type: ClusterIP
  104. clusterIP: None
  105. ports:
  106. - name: tcp-client
  107. port: 9092
  108. protocol: TCP
  109. targetPort: kafka-client
  110. - name: tcp-internal
  111. port: 9093
  112. protocol: TCP
  113. targetPort: kafka-internal
  114. selector:
  115. app.kubernetes.io/name: kafka
  116. app.kubernetes.io/instance: my-release
  117. app.kubernetes.io/component: kafka
  118. ---
  119. # Source: kafka/templates/svc.yaml
  120. apiVersion: v1
  121. kind: Service
  122. metadata:
  123. name: my-release-kafka
  124. labels:
  125. app.kubernetes.io/name: kafka
  126. helm.sh/chart: kafka-11.8.6
  127. app.kubernetes.io/instance: my-release
  128. app.kubernetes.io/managed-by: Helm
  129. app.kubernetes.io/component: kafka
  130. spec:
  131. type: ClusterIP
  132. ports:
  133. - name: tcp-client
  134. port: 9092
  135. protocol: TCP
  136. targetPort: kafka-client
  137. nodePort: null
  138. selector:
  139. app.kubernetes.io/name: kafka
  140. app.kubernetes.io/instance: my-release
  141. app.kubernetes.io/component: kafka
  142. ---
  143. # Source: kafka/charts/zookeeper/templates/statefulset.yaml
  144. apiVersion: apps/v1
  145. kind: StatefulSet
  146. metadata:
  147. name: my-release-zookeeper
  148. namespace: default
  149. labels:
  150. app.kubernetes.io/name: zookeeper
  151. helm.sh/chart: zookeeper-5.21.9
  152. app.kubernetes.io/instance: my-release
  153. app.kubernetes.io/managed-by: Helm
  154. app.kubernetes.io/component: zookeeper
  155. role: zookeeper
  156. spec:
  157. serviceName: my-release-zookeeper-headless
  158. replicas: 1
  159. podManagementPolicy: Parallel
  160. updateStrategy:
  161. type: RollingUpdate
  162. selector:
  163. matchLabels:
  164. app.kubernetes.io/name: zookeeper
  165. app.kubernetes.io/instance: my-release
  166. app.kubernetes.io/component: zookeeper
  167. template:
  168. metadata:
  169. name: my-release-zookeeper
  170. labels:
  171. app.kubernetes.io/name: zookeeper
  172. helm.sh/chart: zookeeper-5.21.9
  173. app.kubernetes.io/instance: my-release
  174. app.kubernetes.io/managed-by: Helm
  175. app.kubernetes.io/component: zookeeper
  176. spec:
  177. serviceAccountName: default
  178. securityContext:
  179. fsGroup: 1001
  180. containers:
  181. - name: zookeeper
  182. image: docker.io/bitnami/zookeeper:3.6.2-debian-10-r10
  183. imagePullPolicy: "IfNotPresent"
  184. securityContext:
  185. runAsUser: 1001
  186. command:
  187. - bash
  188. - -ec
  189. - |
  190. # Execute entrypoint as usual after obtaining ZOO_SERVER_ID based on POD hostname
  191. HOSTNAME=`hostname -s`
  192. if [[ $HOSTNAME =~ (.*)-([0-9]+)$ ]]; then
  193. ORD=${BASH_REMATCH[2]}
  194. export ZOO_SERVER_ID=$((ORD+1))
  195. else
  196. echo "Failed to get index from hostname $HOST"
  197. exit 1
  198. fi
  199. exec /entrypoint.sh /run.sh
  200. resources:
  201. requests:
  202. cpu: 250m
  203. memory: 256Mi
  204. env:
  205. - name: ZOO_DATA_LOG_DIR
  206. value: ""
  207. - name: ZOO_PORT_NUMBER
  208. value: "2181"
  209. - name: ZOO_TICK_TIME
  210. value: "2000"
  211. - name: ZOO_INIT_LIMIT
  212. value: "10"
  213. - name: ZOO_SYNC_LIMIT
  214. value: "5"
  215. - name: ZOO_MAX_CLIENT_CNXNS
  216. value: "60"
  217. - name: ZOO_4LW_COMMANDS_WHITELIST
  218. value: "srvr, mntr, ruok"
  219. - name: ZOO_LISTEN_ALLIPS_ENABLED
  220. value: "no"
  221. - name: ZOO_AUTOPURGE_INTERVAL
  222. value: "0"
  223. - name: ZOO_AUTOPURGE_RETAIN_COUNT
  224. value: "3"
  225. - name: ZOO_MAX_SESSION_TIMEOUT
  226. value: "40000"
  227. - name: ZOO_SERVERS
  228. value: my-release-zookeeper-0.my-release-zookeeper-headless.default.svc.cluster.local:2888:3888
  229. - name: ZOO_ENABLE_AUTH
  230. value: "no"
  231. - name: ZOO_HEAP_SIZE
  232. value: "1024"
  233. - name: ZOO_LOG_LEVEL
  234. value: "ERROR"
  235. - name: ALLOW_ANONYMOUS_LOGIN
  236. value: "yes"
  237. - name: POD_NAME
  238. valueFrom:
  239. fieldRef:
  240. apiVersion: v1
  241. fieldPath: metadata.name
  242. ports:
  243. - name: client
  244. containerPort: 2181
  245. - name: follower
  246. containerPort: 2888
  247. - name: election
  248. containerPort: 3888
  249. livenessProbe:
  250. exec:
  251. command:
  252. [
  253. "/bin/bash",
  254. "-c",
  255. 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok',
  256. ]
  257. initialDelaySeconds: 30
  258. periodSeconds: 10
  259. timeoutSeconds: 5
  260. successThreshold: 1
  261. failureThreshold: 6
  262. readinessProbe:
  263. exec:
  264. command:
  265. [
  266. "/bin/bash",
  267. "-c",
  268. 'echo "ruok" | timeout 2 nc -w 2 localhost 2181 | grep imok',
  269. ]
  270. initialDelaySeconds: 5
  271. periodSeconds: 10
  272. timeoutSeconds: 5
  273. successThreshold: 1
  274. failureThreshold: 6
  275. volumeMounts:
  276. - name: data
  277. mountPath: /bitnami/zookeeper
  278. volumes:
  279. volumeClaimTemplates:
  280. - metadata:
  281. name: data
  282. annotations:
  283. spec:
  284. accessModes:
  285. - "ReadWriteOnce"
  286. resources:
  287. requests:
  288. storage: "8Gi"
  289. ---
  290. # Source: kafka/templates/statefulset.yaml
  291. apiVersion: apps/v1
  292. kind: StatefulSet
  293. metadata:
  294. name: my-release-kafka
  295. labels:
  296. app.kubernetes.io/name: kafka
  297. helm.sh/chart: kafka-11.8.6
  298. app.kubernetes.io/instance: my-release
  299. app.kubernetes.io/managed-by: Helm
  300. app.kubernetes.io/component: kafka
  301. spec:
  302. podManagementPolicy: Parallel
  303. replicas: 1
  304. selector:
  305. matchLabels:
  306. app.kubernetes.io/name: kafka
  307. app.kubernetes.io/instance: my-release
  308. app.kubernetes.io/component: kafka
  309. serviceName: my-release-kafka-headless
  310. updateStrategy:
  311. type: "RollingUpdate"
  312. template:
  313. metadata:
  314. labels:
  315. app.kubernetes.io/name: kafka
  316. helm.sh/chart: kafka-11.8.6
  317. app.kubernetes.io/instance: my-release
  318. app.kubernetes.io/managed-by: Helm
  319. app.kubernetes.io/component: kafka
  320. spec:
  321. securityContext:
  322. fsGroup: 1001
  323. runAsUser: 1001
  324. serviceAccountName: my-release-kafka
  325. containers:
  326. - name: kafka
  327. image: docker.io/bitnami/kafka:2.6.0-debian-10-r30
  328. imagePullPolicy: "IfNotPresent"
  329. command:
  330. - /scripts/setup.sh
  331. env:
  332. - name: BITNAMI_DEBUG
  333. value: "false"
  334. - name: MY_POD_IP
  335. valueFrom:
  336. fieldRef:
  337. fieldPath: status.podIP
  338. - name: MY_POD_NAME
  339. valueFrom:
  340. fieldRef:
  341. fieldPath: metadata.name
  342. - name: KAFKA_CFG_ZOOKEEPER_CONNECT
  343. value: "my-release-zookeeper"
  344. - name: KAFKA_INTER_BROKER_LISTENER_NAME
  345. value: "INTERNAL"
  346. - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
  347. value: "INTERNAL:PLAINTEXT,CLIENT:PLAINTEXT"
  348. - name: KAFKA_CFG_LISTENERS
  349. value: "INTERNAL://:9093,CLIENT://:9092"
  350. - name: KAFKA_CFG_ADVERTISED_LISTENERS
  351. value: "INTERNAL://$(MY_POD_NAME).my-release-kafka-headless.default.svc.cluster.local:9093,CLIENT://$(MY_POD_NAME).my-release-kafka-headless.default.svc.cluster.local:9092"
  352. - name: ALLOW_PLAINTEXT_LISTENER
  353. value: "yes"
  354. - name: KAFKA_CFG_DELETE_TOPIC_ENABLE
  355. value: "false"
  356. - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE
  357. value: "true"
  358. - name: KAFKA_HEAP_OPTS
  359. value: "-Xmx1024m -Xms1024m"
  360. - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES
  361. value: "10000"
  362. - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS
  363. value: "1000"
  364. - name: KAFKA_CFG_LOG_RETENTION_BYTES
  365. value: "1073741824"
  366. - name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS
  367. value: "300000"
  368. - name: KAFKA_CFG_LOG_RETENTION_HOURS
  369. value: "168"
  370. - name: KAFKA_CFG_MESSAGE_MAX_BYTES
  371. value: "1000012"
  372. - name: KAFKA_CFG_LOG_SEGMENT_BYTES
  373. value: "1073741824"
  374. - name: KAFKA_CFG_LOG_DIRS
  375. value: "/bitnami/kafka/data"
  376. - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR
  377. value: "1"
  378. - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR
  379. value: "1"
  380. - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
  381. value: "1"
  382. - name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR
  383. value: "1"
  384. - name: KAFKA_CFG_NUM_IO_THREADS
  385. value: "8"
  386. - name: KAFKA_CFG_NUM_NETWORK_THREADS
  387. value: "3"
  388. - name: KAFKA_CFG_NUM_PARTITIONS
  389. value: "1"
  390. - name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR
  391. value: "1"
  392. - name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES
  393. value: "102400"
  394. - name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES
  395. value: "104857600"
  396. - name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES
  397. value: "102400"
  398. - name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS
  399. value: "6000"
  400. ports:
  401. - name: kafka-client
  402. containerPort: 9092
  403. - name: kafka-internal
  404. containerPort: 9093
  405. livenessProbe:
  406. tcpSocket:
  407. port: kafka-client
  408. initialDelaySeconds: 10
  409. timeoutSeconds: 5
  410. failureThreshold:
  411. periodSeconds:
  412. successThreshold:
  413. readinessProbe:
  414. tcpSocket:
  415. port: kafka-client
  416. initialDelaySeconds: 5
  417. timeoutSeconds: 5
  418. failureThreshold: 6
  419. periodSeconds:
  420. successThreshold:
  421. resources:
  422. limits: {}
  423. requests: {}
  424. volumeMounts:
  425. - name: data
  426. mountPath: /bitnami/kafka
  427. - name: scripts
  428. mountPath: /scripts/setup.sh
  429. subPath: setup.sh
  430. volumes:
  431. - name: scripts
  432. configMap:
  433. name: my-release-kafka-scripts
  434. defaultMode: 0755
  435. volumeClaimTemplates:
  436. - metadata:
  437. name: data
  438. spec:
  439. accessModes:
  440. - "ReadWriteOnce"
  441. resources:
  442. requests:
  443. storage: "8Gi"