apiserver_identity.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2022 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"crypto/sha256"
    22  	"encoding/base32"
    23  	"errors"
    24  	"fmt"
    25  	"net"
    26  	"strings"
    27  	"time"
    28  
    29  	"github.com/onsi/gomega"
    30  	"golang.org/x/crypto/cryptobyte"
    31  
    32  	v1 "k8s.io/api/core/v1"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/util/wait"
    35  	"k8s.io/kubernetes/test/e2e/feature"
    36  	"k8s.io/kubernetes/test/e2e/framework"
    37  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    38  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    39  	e2essh "k8s.io/kubernetes/test/e2e/framework/ssh"
    40  	admissionapi "k8s.io/pod-security-admission/api"
    41  )
    42  
    43  func getControlPlaneHostname(ctx context.Context, node *v1.Node) (string, error) {
    44  	nodeAddresses := e2enode.GetAddresses(node, v1.NodeExternalIP)
    45  	if len(nodeAddresses) == 0 {
    46  		return "", errors.New("no valid addresses to use for SSH")
    47  	}
    48  
    49  	controlPlaneAddress := nodeAddresses[0]
    50  
    51  	host := controlPlaneAddress + ":" + e2essh.SSHPort
    52  	result, err := e2essh.SSH(ctx, "hostname", host, framework.TestContext.Provider)
    53  	if err != nil {
    54  		return "", err
    55  	}
    56  
    57  	if result.Code != 0 {
    58  		return "", fmt.Errorf("encountered non-zero exit code when running hostname command: %d", result.Code)
    59  	}
    60  
    61  	return strings.TrimSpace(result.Stdout), nil
    62  }
    63  
    64  // restartAPIServer attempts to restart the kube-apiserver on a node
    65  func restartAPIServer(ctx context.Context, node *v1.Node) error {
    66  	nodeAddresses := e2enode.GetAddresses(node, v1.NodeExternalIP)
    67  	if len(nodeAddresses) == 0 {
    68  		return errors.New("no valid addresses to use for SSH")
    69  	}
    70  
    71  	controlPlaneAddress := nodeAddresses[0]
    72  	cmd := "pidof kube-apiserver | xargs sudo kill"
    73  	framework.Logf("Restarting kube-apiserver via ssh, running: %v", cmd)
    74  	result, err := e2essh.SSH(ctx, cmd, net.JoinHostPort(controlPlaneAddress, e2essh.SSHPort), framework.TestContext.Provider)
    75  	if err != nil || result.Code != 0 {
    76  		e2essh.LogResult(result)
    77  		return fmt.Errorf("couldn't restart kube-apiserver: %w", err)
    78  	}
    79  	return nil
    80  }
    81  
    82  // This test requires that --feature-gates=APIServerIdentity=true be set on the apiserver
    83  var _ = SIGDescribe("kube-apiserver identity", feature.APIServerIdentity, func() {
    84  	f := framework.NewDefaultFramework("kube-apiserver-identity")
    85  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    86  
    87  	f.It("kube-apiserver identity should persist after restart", f.WithDisruptive(), func(ctx context.Context) {
    88  		e2eskipper.SkipUnlessProviderIs("gce")
    89  
    90  		client := f.ClientSet
    91  
    92  		var controlPlaneNodes []v1.Node
    93  		nodes, err := client.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
    94  		framework.ExpectNoError(err)
    95  
    96  		for _, node := range nodes.Items {
    97  			if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
    98  				controlPlaneNodes = append(controlPlaneNodes, node)
    99  				continue
   100  			}
   101  
   102  			if _, ok := node.Labels["node-role.kubernetes.io/master"]; ok {
   103  				controlPlaneNodes = append(controlPlaneNodes, node)
   104  				continue
   105  			}
   106  
   107  			for _, taint := range node.Spec.Taints {
   108  				if taint.Key == "node-role.kubernetes.io/master" {
   109  					controlPlaneNodes = append(controlPlaneNodes, node)
   110  					break
   111  				}
   112  
   113  				if taint.Key == "node-role.kubernetes.io/control-plane" {
   114  					controlPlaneNodes = append(controlPlaneNodes, node)
   115  					break
   116  				}
   117  			}
   118  		}
   119  
   120  		leases, err := client.CoordinationV1().Leases(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{
   121  			LabelSelector: "apiserver.kubernetes.io/identity=kube-apiserver",
   122  		})
   123  		framework.ExpectNoError(err)
   124  		gomega.Expect(leases.Items).To(gomega.HaveLen(len(controlPlaneNodes)), "unexpected number of leases")
   125  
   126  		for _, node := range controlPlaneNodes {
   127  			hostname, err := getControlPlaneHostname(ctx, &node)
   128  			framework.ExpectNoError(err)
   129  
   130  			b := cryptobyte.NewBuilder(nil)
   131  			b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
   132  				b.AddBytes([]byte(hostname))
   133  			})
   134  			b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
   135  				b.AddBytes([]byte("kube-apiserver"))
   136  			})
   137  
   138  			hashData, err := b.Bytes()
   139  			framework.ExpectNoError(err)
   140  			hash := sha256.Sum256(hashData)
   141  			leaseName := "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16]))
   142  
   143  			lease, err := client.CoordinationV1().Leases(metav1.NamespaceSystem).Get(context.TODO(), leaseName, metav1.GetOptions{})
   144  			framework.ExpectNoError(err)
   145  			oldHolderIdentity := lease.Spec.HolderIdentity
   146  			lastRenewedTime := lease.Spec.RenewTime
   147  
   148  			err = restartAPIServer(ctx, &node)
   149  			framework.ExpectNoError(err)
   150  
   151  			err = wait.PollImmediate(time.Second, wait.ForeverTestTimeout, func() (bool, error) {
   152  				lease, err = client.CoordinationV1().Leases(metav1.NamespaceSystem).Get(context.TODO(), leaseName, metav1.GetOptions{})
   153  				if err != nil {
   154  					return false, nil
   155  				}
   156  
   157  				// expect only the holder identity to change after a restart
   158  				newHolderIdentity := lease.Spec.HolderIdentity
   159  				if newHolderIdentity == oldHolderIdentity {
   160  					return false, nil
   161  				}
   162  
   163  				// wait for at least one lease heart beat after the holder identity changes
   164  				if !lease.Spec.RenewTime.After(lastRenewedTime.Time) {
   165  					return false, nil
   166  				}
   167  
   168  				return true, nil
   169  
   170  			})
   171  			framework.ExpectNoError(err, "holder identity did not change after a restart")
   172  		}
   173  
   174  		// As long as the hostname of kube-apiserver is unchanged, a restart should not result in new Lease objects.
   175  		// Check that the number of lease objects remains the same after restarting kube-apiserver.
   176  		leases, err = client.CoordinationV1().Leases(metav1.NamespaceSystem).List(context.TODO(), metav1.ListOptions{
   177  			LabelSelector: "apiserver.kubernetes.io/identity=kube-apiserver",
   178  		})
   179  		framework.ExpectNoError(err)
   180  		gomega.Expect(leases.Items).To(gomega.HaveLen(len(controlPlaneNodes)), "unexpected number of leases")
   181  	})
   182  })
   183
View as plain text