apiserver_identity_test.go

Documentation: k8s.io/kubernetes/test/integration/controlplane

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package controlplane
    18  
    19  import (
    20  	"context"
    21  	"crypto/sha256"
    22  	"encoding/base32"
    23  	"fmt"
    24  	"os"
    25  	"strings"
    26  	"testing"
    27  	"time"
    28  
    29  	"golang.org/x/crypto/cryptobyte"
    30  
    31  	coordinationv1 "k8s.io/api/coordination/v1"
    32  	corev1 "k8s.io/api/core/v1"
    33  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    34  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    35  	"k8s.io/apimachinery/pkg/util/wait"
    36  	"k8s.io/apiserver/pkg/features"
    37  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    38  	"k8s.io/client-go/kubernetes"
    39  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    40  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    41  	"k8s.io/kubernetes/pkg/controlplane"
    42  	"k8s.io/kubernetes/test/integration/framework"
    43  	"k8s.io/utils/pointer"
    44  )
    45  
    46  const (
    47  	testLeaseName = "apiserver-lease-test"
    48  )
    49  
    50  func expectedAPIServerIdentity(t *testing.T, hostname string) string {
    51  	b := cryptobyte.NewBuilder(nil)
    52  	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
    53  		b.AddBytes([]byte(hostname))
    54  	})
    55  	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
    56  		b.AddBytes([]byte("kube-apiserver"))
    57  	})
    58  	hashData, err := b.Bytes()
    59  	if err != nil {
    60  		t.Fatalf("error building hash data for apiserver identity: %v", err)
    61  	}
    62  
    63  	hash := sha256.Sum256(hashData)
    64  	return "apiserver-" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(hash[:16]))
    65  }
    66  
    67  func TestCreateLeaseOnStart(t *testing.T) {
    68  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)()
    69  	result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
    70  	defer result.TearDownFn()
    71  
    72  	kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
    73  	if err != nil {
    74  		t.Fatalf("Unexpected error: %v", err)
    75  	}
    76  
    77  	hostname, err := os.Hostname()
    78  	if err != nil {
    79  		t.Fatalf("Unexpected error getting apiserver hostname: %v", err)
    80  	}
    81  
    82  	t.Logf(`Waiting the kube-apiserver Lease to be created`)
    83  	if err := wait.PollImmediate(500*time.Millisecond, 10*time.Second, func() (bool, error) {
    84  		leases, err := kubeclient.
    85  			CoordinationV1().
    86  			Leases(metav1.NamespaceSystem).
    87  			List(context.TODO(), metav1.ListOptions{LabelSelector: controlplane.KubeAPIServerIdentityLeaseLabelSelector})
    88  		if err != nil {
    89  			return false, err
    90  		}
    91  
    92  		if leases == nil {
    93  			return false, nil
    94  		}
    95  
    96  		if len(leases.Items) != 1 {
    97  			return false, nil
    98  		}
    99  
   100  		lease := leases.Items[0]
   101  		if lease.Name != expectedAPIServerIdentity(t, hostname) {
   102  			return false, fmt.Errorf("unexpected apiserver identity, got: %v, expected: %v", lease.Name, expectedAPIServerIdentity(t, hostname))
   103  		}
   104  
   105  		if lease.Labels[corev1.LabelHostname] != hostname {
   106  			return false, fmt.Errorf("unexpected hostname label, got: %v, expected: %v", lease.Labels[corev1.LabelHostname], hostname)
   107  		}
   108  
   109  		return true, nil
   110  	}); err != nil {
   111  		t.Fatalf("Failed to see the kube-apiserver lease: %v", err)
   112  	}
   113  }
   114  
   115  func TestLeaseGarbageCollection(t *testing.T) {
   116  	oldIdentityLeaseDurationSeconds := controlplane.IdentityLeaseDurationSeconds
   117  	oldIdentityLeaseGCPeriod := controlplane.IdentityLeaseGCPeriod
   118  	oldIdentityLeaseRenewIntervalPeriod := controlplane.IdentityLeaseRenewIntervalPeriod
   119  	defer func() {
   120  		// reset the default values for leases after this test
   121  		controlplane.IdentityLeaseDurationSeconds = oldIdentityLeaseDurationSeconds
   122  		controlplane.IdentityLeaseGCPeriod = oldIdentityLeaseGCPeriod
   123  		controlplane.IdentityLeaseRenewIntervalPeriod = oldIdentityLeaseRenewIntervalPeriod
   124  	}()
   125  
   126  	// Shorten lease parameters so GC behavior can be exercised in integration tests
   127  	controlplane.IdentityLeaseDurationSeconds = 1
   128  	controlplane.IdentityLeaseGCPeriod = time.Second
   129  	controlplane.IdentityLeaseRenewIntervalPeriod = time.Second
   130  
   131  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIServerIdentity, true)()
   132  	result := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd())
   133  	defer result.TearDownFn()
   134  
   135  	kubeclient, err := kubernetes.NewForConfig(result.ClientConfig)
   136  	if err != nil {
   137  		t.Fatalf("Unexpected error: %v", err)
   138  	}
   139  	expiredLease := newTestLease(time.Now().Add(-2*time.Hour), metav1.NamespaceSystem)
   140  	t.Run("expired apiserver lease should be garbage collected",
   141  		testLeaseGarbageCollected(t, kubeclient, expiredLease))
   142  
   143  	freshLease := newTestLease(time.Now().Add(-2*time.Minute), metav1.NamespaceSystem)
   144  	t.Run("fresh apiserver lease should not be garbage collected",
   145  		testLeaseNotGarbageCollected(t, kubeclient, freshLease))
   146  
   147  	expiredLease.Labels = nil
   148  	t.Run("expired non-identity lease should not be garbage collected",
   149  		testLeaseNotGarbageCollected(t, kubeclient, expiredLease))
   150  
   151  	// identity leases (with apiserver.kubernetes.io/identity label) created in user namespaces should not be GC'ed
   152  	expiredNonKubeSystemLease := newTestLease(time.Now().Add(-2*time.Hour), metav1.NamespaceDefault)
   153  	t.Run("expired non-system identity lease should not be garbage collected",
   154  		testLeaseNotGarbageCollected(t, kubeclient, expiredNonKubeSystemLease))
   155  }
   156  
   157  func testLeaseGarbageCollected(t *testing.T, client kubernetes.Interface, lease *coordinationv1.Lease) func(t *testing.T) {
   158  	return func(t *testing.T) {
   159  		ns := lease.Namespace
   160  		if _, err := client.CoordinationV1().Leases(ns).Create(context.TODO(), lease, metav1.CreateOptions{}); err != nil {
   161  			t.Fatalf("Unexpected error creating lease: %v", err)
   162  		}
   163  		if err := wait.PollImmediate(500*time.Millisecond, 5*time.Second, func() (bool, error) {
   164  			_, err := client.CoordinationV1().Leases(ns).Get(context.TODO(), lease.Name, metav1.GetOptions{})
   165  			if err == nil {
   166  				return false, nil
   167  			}
   168  			if apierrors.IsNotFound(err) {
   169  				return true, nil
   170  			}
   171  			return false, err
   172  		}); err != nil {
   173  			t.Fatalf("Failed to see the expired lease garbage collected: %v", err)
   174  		}
   175  
   176  	}
   177  }
   178  
   179  func testLeaseNotGarbageCollected(t *testing.T, client kubernetes.Interface, lease *coordinationv1.Lease) func(t *testing.T) {
   180  	return func(t *testing.T) {
   181  		ns := lease.Namespace
   182  		if _, err := client.CoordinationV1().Leases(ns).Create(context.TODO(), lease, metav1.CreateOptions{}); err != nil {
   183  			t.Fatalf("Unexpected error creating lease: %v", err)
   184  		}
   185  		if err := wait.PollImmediate(500*time.Millisecond, 5*time.Second, func() (bool, error) {
   186  			_, err := client.CoordinationV1().Leases(ns).Get(context.TODO(), lease.Name, metav1.GetOptions{})
   187  			if err != nil && apierrors.IsNotFound(err) {
   188  				return true, nil
   189  			}
   190  			return false, nil
   191  		}); err == nil {
   192  			t.Fatalf("Unexpected valid lease getting garbage collected")
   193  		}
   194  		if _, err := client.CoordinationV1().Leases(ns).Get(context.TODO(), lease.Name, metav1.GetOptions{}); err != nil {
   195  			t.Fatalf("Failed to retrieve valid lease: %v", err)
   196  		}
   197  		if err := client.CoordinationV1().Leases(ns).Delete(context.TODO(), lease.Name, metav1.DeleteOptions{}); err != nil {
   198  			t.Fatalf("Failed to clean up valid lease: %v", err)
   199  		}
   200  	}
   201  }
   202  
   203  func newTestLease(acquireTime time.Time, namespace string) *coordinationv1.Lease {
   204  	return &coordinationv1.Lease{
   205  		ObjectMeta: metav1.ObjectMeta{
   206  			Name:      testLeaseName,
   207  			Namespace: namespace,
   208  			Labels: map[string]string{
   209  				controlplane.IdentityLeaseComponentLabelKey: controlplane.KubeAPIServer,
   210  			},
   211  		},
   212  		Spec: coordinationv1.LeaseSpec{
   213  			HolderIdentity:       pointer.String(testLeaseName),
   214  			LeaseDurationSeconds: pointer.Int32(3600),
   215  			AcquireTime:          &metav1.MicroTime{Time: acquireTime},
   216  			RenewTime:            &metav1.MicroTime{Time: acquireTime},
   217  		},
   218  	}
   219  }
   220
View as plain text