1#!/usr/bin/env bash
2
3# Copyright 2018 The Kubernetes Authors.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17# !!!EXPERIMENTAL!!! Upgrade a K8s cluster from routes to IP aliases for
18# node connectivity on GCE. This is only for migration.
19
20set -o errexit
21set -o nounset
22set -o pipefail
23
24if [[ "${KUBERNETES_PROVIDER:-gce}" != "gce" ]]; then
25 echo "ERR: KUBERNETES_PROVIDER must be gce" >&2
26 exit 1
27fi
28
29KUBE_ROOT=$(dirname "${BASH_SOURCE[0]}")/../..
30source "${KUBE_ROOT}/hack/lib/util.sh"
31source "${KUBE_ROOT}/cluster/kube-util.sh"
32
33# Print the number of routes used for K8s cluster node connectivity.
34#
35# Assumed vars:
36# PROJECT
37function get-k8s-node-routes-count() {
38 local k8s_node_routes_count
39 k8s_node_routes_count=$(gcloud compute routes list \
40 --project="${PROJECT}" --filter='description=k8s-node-route' \
41 --format='value(name)' | wc -l)
42 echo -n "${k8s_node_routes_count}"
43}
44
45# Detect the subnetwork where the K8s cluster resides.
46#
47# Assumed vars:
48# KUBE_MASTER
49# PROJECT
50# ZONE
51# Vars set:
52# IP_ALIAS_SUBNETWORK
53function detect-k8s-subnetwork() {
54 local subnetwork_url
55 subnetwork_url=$(gcloud compute instances describe \
56 "${KUBE_MASTER}" --project="${PROJECT}" --zone="${ZONE}" \
57 --format='value(networkInterfaces[0].subnetwork)')
58 if [[ -n ${subnetwork_url} ]]; then
59 IP_ALIAS_SUBNETWORK=${subnetwork_url##*/}
60 fi
61}
62
63# Set IP_ALIAS_SUBNETWORK's allowSubnetCidrRoutesOverlap to a boolean value.
64# $1: true or false for the desired allowSubnetCidrRoutesOverlap.
65#
66# Assumed vars:
67# IP_ALIAS_SUBNETWORK
68# GCE_API_ENDPOINT
69# PROJECT
70# REGION
71function set-allow-subnet-cidr-routes-overlap() {
72 local allow_subnet_cidr_routes_overlap
73 allow_subnet_cidr_routes_overlap=$(gcloud compute networks subnets \
74 describe "${IP_ALIAS_SUBNETWORK}" --project="${PROJECT}" --region="${REGION}" \
75 --format='value(allowSubnetCidrRoutesOverlap)')
76 local allow_overlap=$1
77 if [ "${allow_subnet_cidr_routes_overlap,,}" = "${allow_overlap}" ]; then
78 echo "Subnet ${IP_ALIAS_SUBNETWORK}'s allowSubnetCidrRoutesOverlap is already set as $1"
79 return
80 fi
81
82 echo "Setting subnet \"${IP_ALIAS_SUBNETWORK}\" allowSubnetCidrRoutesOverlap to $1"
83 local fingerprint
84 fingerprint=$(gcloud compute networks subnets describe \
85 "${IP_ALIAS_SUBNETWORK}" --project="${PROJECT}" --region="${REGION}" \
86 --format='value(fingerprint)')
87 local access_token
88 access_token=$(gcloud auth print-access-token)
89 local request="{\"allowSubnetCidrRoutesOverlap\":$1, \"fingerprint\":\"${fingerprint}\"}"
90 local subnetwork_url
91 subnetwork_url="${GCE_API_ENDPOINT}projects/${PROJECT}/regions/${REGION}/subnetworks/${IP_ALIAS_SUBNETWORK}"
92 until curl -s --header "Content-Type: application/json" --header "Authorization: Bearer ${access_token}" \
93 -X PATCH -d "${request}" "${subnetwork_url}" --output /dev/null; do
94 printf "."
95 sleep 1
96 done
97}
98
99# Add secondary ranges to K8s subnet.
100#
101# Assumed vars:
102# IP_ALIAS_SUBNETWORK
103# PROJECT
104# REGION
105# CLUSTER_IP_RANGE
106# SERVICE_CLUSTER_IP_RANGE
107function add-k8s-subnet-secondary-ranges() {
108 local secondary_ranges
109 secondary_ranges=$(gcloud compute networks subnets describe "${IP_ALIAS_SUBNETWORK}" \
110 --project="${PROJECT}" --region="${REGION}" \
111 --format='value(secondaryIpRanges)')
112 if [[ "${secondary_ranges}" =~ "pods-default" && "${secondary_ranges}" =~ "services-default" ]]; then
113 echo "${secondary_ranges} already contains both pods-default and services-default secondary ranges"
114 return
115 fi
116
117 echo "Adding secondary ranges: pods-default (${CLUSTER_IP_RANGE}), services-default (${SERVICE_CLUSTER_IP_RANGE})"
118 until gcloud compute networks subnets update "${IP_ALIAS_SUBNETWORK}" \
119 --project="${PROJECT}" --region="${REGION}" \
120 --add-secondary-ranges="pods-default=${CLUSTER_IP_RANGE},services-default=${SERVICE_CLUSTER_IP_RANGE}"; do
121 printf "."
122 sleep 1
123 done
124}
125
126# Delete all K8s node routes.
127#
128# Assumed vars:
129# PROJECT
130function delete-k8s-node-routes() {
131 local -a routes
132 local -r batch=200
133 routes=()
134 while IFS=$'\n' read -r route; do
135 routes+=( "${route}" )
136 done < <(gcloud compute routes list \
137 --project="${PROJECT}" --filter='description=k8s-node-route' \
138 --format='value(name)')
139 while (( "${#routes[@]}" > 0 )); do
140 echo Deleting k8s node routes "${routes[*]::${batch}}"
141 gcloud compute routes delete --project "${PROJECT}" --quiet "${routes[@]::${batch}}"
142 routes=( "${routes[@]:${batch}}" )
143 done
144}
145
146detect-project
147detect-master
148
149k8s_node_routes_count=$(get-k8s-node-routes-count)
150if [[ "${k8s_node_routes_count}" -eq 0 ]]; then
151 echo "No k8s node routes found and IP alias should already be enabled. Exiting..."
152 exit 0
153fi
154echo "Found ${k8s_node_routes_count} K8s node routes. Proceeding to upgrade them to IP aliases based connectivity..."
155
156detect-k8s-subnetwork
157if [ -z "${IP_ALIAS_SUBNETWORK}" ]; then
158 echo "No k8s cluster subnetwork found. Exiting..."
159 exit 1
160fi
161echo "k8s cluster sits on subnetwork \"${IP_ALIAS_SUBNETWORK}\""
162
163set-allow-subnet-cidr-routes-overlap true
164add-k8s-subnet-secondary-ranges
165
166echo "Changing K8s master envs and restarting..."
167export KUBE_GCE_IP_ALIAS_SUBNETWORK=${IP_ALIAS_SUBNETWORK}
168export KUBE_GCE_NODE_IPAM_MODE="IPAMFromCluster"
169export KUBE_GCE_ENABLE_IP_ALIASES=true
170export SECONDARY_RANGE_NAME="pods-default"
171export STORAGE_BACKEND="etcd3"
172export STORAGE_MEDIA_TYPE="application/vnd.kubernetes.protobuf"
173export ETCD_IMAGE=3.5.12-0
174export ETCD_VERSION=3.5.12
175
176# Upgrade master with updated kube envs
177"${KUBE_ROOT}/cluster/gce/upgrade.sh" -M -l
178
179delete-k8s-node-routes
180set-allow-subnet-cidr-routes-overlap false
View as plain text