1apiVersion: apps/v1
2kind: StatefulSet
3metadata:
4 name: etcd
5 labels:
6 app: etcd
7spec:
8 serviceName: etcd
9 replicas: 3
10 selector:
11 matchLabels:
12 app: etcd
13 template:
14 metadata:
15 name: etcd
16 labels:
17 app: etcd
18 spec:
19 containers:
20 - name: etcd
21 image: registry.k8s.io/etcd:3.2.24
22 imagePullPolicy: Always
23 ports:
24 - containerPort: 2380
25 name: peer
26 - containerPort: 2379
27 name: client
28 resources:
29 requests:
30 cpu: 100m
31 memory: 512Mi
32 env:
33 - name: INITIAL_CLUSTER_SIZE
34 value: "3"
35 - name: SET_NAME
36 value: etcd
37 volumeMounts:
38 - name: datadir
39 mountPath: /var/run/etcd
40 lifecycle:
41 preStop:
42 exec:
43 command:
44 - "/bin/sh"
45 - "-ec"
46 - |
47 EPS=""
48 for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
49 EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379"
50 done
51
52 HOSTNAME=$(hostname)
53
54 member_hash() {
55 etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1
56 }
57
58 echo "Removing ${HOSTNAME} from etcd cluster"
59
60 ETCDCTL_ENDPOINT=${EPS} etcdctl member remove $(member_hash)
61 if [ $? -eq 0 ]; then
62 # Remove everything otherwise the cluster will no longer scale-up
63 rm -rf /var/run/etcd/*
64 fi
65 command:
66 - "/bin/sh"
67 - "-ec"
68 - |
69 HOSTNAME=$(hostname)
70
71 # store member id into PVC for later member replacement
72 collect_member() {
73 while ! etcdctl member list &>/dev/null; do sleep 1; done
74 etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1 > /var/run/etcd/member_id
75 exit 0
76 }
77
78 eps() {
79 EPS=""
80 for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
81 EPS="${EPS}${EPS:+,}http://${SET_NAME}-${i}.${SET_NAME}:2379"
82 done
83 echo ${EPS}
84 }
85
86 member_hash() {
87 etcdctl member list | grep http://${HOSTNAME}.${SET_NAME}:2380 | cut -d':' -f1 | cut -d'[' -f1
88 }
89
90 # re-joining after failure?
91 if [ -e /var/run/etcd/default.etcd ]; then
92 echo "Re-joining etcd member"
93 member_id=$(cat /var/run/etcd/member_id)
94
95 # re-join member
96 ETCDCTL_ENDPOINT=$(eps) etcdctl member update ${member_id} http://${HOSTNAME}.${SET_NAME}:2380
97 exec etcd --name ${HOSTNAME} \
98 --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \
99 --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \
100 --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \
101 --data-dir /var/run/etcd/default.etcd
102 fi
103
104 # etcd-SET_ID
105 SET_ID=${HOSTNAME:5:${#HOSTNAME}}
106
107 # adding a new member to existing cluster (assuming all initial pods are available)
108 if [ "${SET_ID}" -ge ${INITIAL_CLUSTER_SIZE} ]; then
109 export ETCDCTL_ENDPOINT=$(eps)
110
111 # member already added?
112 MEMBER_HASH=$(member_hash)
113 if [ -n "${MEMBER_HASH}" ]; then
114 # the member hash exists but for some reason etcd failed
115 # as the datadir has not be created, we can remove the member
116 # and retrieve new hash
117 etcdctl member remove ${MEMBER_HASH}
118 fi
119
120 echo "Adding new member"
121 etcdctl member add ${HOSTNAME} http://${HOSTNAME}.${SET_NAME}:2380 | grep "^ETCD_" > /var/run/etcd/new_member_envs
122
123 if [ $? -ne 0 ]; then
124 echo "Exiting"
125 rm -f /var/run/etcd/new_member_envs
126 exit 1
127 fi
128
129 cat /var/run/etcd/new_member_envs
130 source /var/run/etcd/new_member_envs
131
132 collect_member &
133
134 exec etcd --name ${HOSTNAME} \
135 --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \
136 --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \
137 --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \
138 --data-dir /var/run/etcd/default.etcd \
139 --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \
140 --initial-cluster ${ETCD_INITIAL_CLUSTER} \
141 --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE}
142 fi
143
144 for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
145 while true; do
146 echo "Waiting for ${SET_NAME}-${i}.${SET_NAME} to come up"
147 ping -W 1 -c 1 ${SET_NAME}-${i}.${SET_NAME} > /dev/null && break
148 sleep 1s
149 done
150 done
151
152 PEERS=""
153 for i in $(seq 0 $((${INITIAL_CLUSTER_SIZE} - 1))); do
154 PEERS="${PEERS}${PEERS:+,}${SET_NAME}-${i}=http://${SET_NAME}-${i}.${SET_NAME}:2380"
155 done
156
157 collect_member &
158
159 # join member
160 exec etcd --name ${HOSTNAME} \
161 --initial-advertise-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \
162 --listen-peer-urls http://${HOSTNAME}.${SET_NAME}:2380 \
163 --listen-client-urls http://${HOSTNAME}.${SET_NAME}:2379,http://127.0.0.1:2379 \
164 --advertise-client-urls http://${HOSTNAME}.${SET_NAME}:2379 \
165 --initial-cluster-token etcd-cluster-1 \
166 --initial-cluster ${PEERS} \
167 --initial-cluster-state new \
168 --data-dir /var/run/etcd/default.etcd
169 volumeClaimTemplates:
170 - metadata:
171 name: datadir
172 spec:
173 accessModes:
174 - "ReadWriteOnce"
175 resources:
176 requests:
177 # upstream recommended max is 700M
178 storage: 1Gi
View as plain text