servicecidr_test.go

Documentation: k8s.io/kubernetes/test/integration/servicecidr

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package servicecidr
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"net/netip"
    23  	"strings"
    24  	"testing"
    25  	"time"
    26  
    27  	v1 "k8s.io/api/core/v1"
    28  	networkingv1alpha1 "k8s.io/api/networking/v1alpha1"
    29  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    30  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    31  	"k8s.io/apimachinery/pkg/util/wait"
    32  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    33  	"k8s.io/client-go/informers"
    34  	"k8s.io/client-go/kubernetes"
    35  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    36  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    37  	"k8s.io/kubernetes/pkg/controller/servicecidrs"
    38  	"k8s.io/kubernetes/pkg/features"
    39  	"k8s.io/kubernetes/test/integration/framework"
    40  )
    41  
    42  func TestServiceAllocNewServiceCIDR(t *testing.T) {
    43  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, true)()
    44  
    45  	etcdOptions := framework.SharedEtcd()
    46  	apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
    47  	s := kubeapiservertesting.StartTestServerOrDie(t,
    48  		apiServerOptions,
    49  		[]string{
    50  			"--runtime-config=networking.k8s.io/v1alpha1=true",
    51  			"--service-cluster-ip-range=192.168.0.0/29",
    52  			"--advertise-address=10.1.1.1",
    53  			"--disable-admission-plugins=ServiceAccount",
    54  		},
    55  		etcdOptions)
    56  	defer s.TearDownFn()
    57  
    58  	client, err := kubernetes.NewForConfig(s.ClientConfig)
    59  	if err != nil {
    60  		t.Fatalf("Unexpected error: %v", err)
    61  	}
    62  	// ServiceCIDR controller
    63  	ctx, cancel := context.WithCancel(context.Background())
    64  	defer cancel()
    65  	resyncPeriod := 12 * time.Hour
    66  	informerFactory := informers.NewSharedInformerFactory(client, resyncPeriod)
    67  	go servicecidrs.NewController(
    68  		ctx,
    69  		informerFactory.Networking().V1alpha1().ServiceCIDRs(),
    70  		informerFactory.Networking().V1alpha1().IPAddresses(),
    71  		client,
    72  	).Run(ctx, 5)
    73  	informerFactory.Start(ctx.Done())
    74  
    75  	// /29 = 6 services, kubernetes.default takes the first address
    76  	// make 5 more services to take up all IPs
    77  	for i := 0; i < 5; i++ {
    78  		if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil {
    79  			t.Fatal(err)
    80  		}
    81  	}
    82  
    83  	// Make another service. It will fail because we're out of cluster IPs
    84  	if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService("fail"), metav1.CreateOptions{}); err != nil {
    85  		if !strings.Contains(err.Error(), "range is full") {
    86  			t.Fatalf("unexpected error text: %v", err)
    87  		}
    88  	} else {
    89  		svcs, err := client.CoreV1().Services(metav1.NamespaceAll).List(context.Background(), metav1.ListOptions{})
    90  		if err != nil {
    91  			t.Fatalf("unexpected error getting the services: %v", err)
    92  		}
    93  		allIPs := []string{}
    94  		for _, s := range svcs.Items {
    95  			allIPs = append(allIPs, s.Spec.ClusterIP)
    96  		}
    97  		t.Fatalf("unexpected creation success. The following IPs exist: %#v. It should only be possible to allocate 6 IP addresses in this cluster.\n\n%#v", allIPs, svcs)
    98  	}
    99  
   100  	// Add a new service CIDR to be able to create new IPs.
   101  	cidr := makeServiceCIDR("test2", "10.168.0.0/24", "")
   102  	if _, err := client.NetworkingV1alpha1().ServiceCIDRs().Create(context.Background(), cidr, metav1.CreateOptions{}); err != nil {
   103  		t.Fatalf("got unexpected error: %v", err)
   104  	}
   105  	// wait ServiceCIDR is ready
   106  	if err := wait.PollUntilContextTimeout(context.Background(), 1*time.Second, time.Minute, false, func(ctx context.Context) (bool, error) {
   107  		cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(context.TODO(), cidr.Name, metav1.GetOptions{})
   108  		if err != nil {
   109  			return false, err
   110  		}
   111  		return isServiceCIDRReady(cidr), nil
   112  	}); err != nil {
   113  		t.Fatalf("waiting for default service cidr ready condition set to false: %v", err)
   114  	}
   115  	// This time creating more Services should work
   116  	for i := 10; i < 150; i++ {
   117  		if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil {
   118  			t.Fatal(err)
   119  		}
   120  	}
   121  }
   122  
   123  // A ServiceCIDR can be deleted if there are no orphan IPs or if the existing IPs are contained in other ServiceCIDR
   124  // that is not being deleted.
   125  // The test starts the apiserver with the range "192.168.0.0/29"
   126  // Create Services to fill the range
   127  // Creates a new ServiceCIDR cidr1 with the same range as the one defined in the apiserver
   128  // Deletes cidr1 object will work since its range is covered by the default ServiceCIDR created by the apiserver flags
   129  // Creates a new cidr2 with a different range than cidr1
   130  // Creates a new service so it picks an IPAddress on this range because "192.168.0.0/29" is full at this point
   131  // Creates a new cidr3 that contains cidr2
   132  // Deletes cidr2 since it is covered by cidr3
   133  // Tries to delete cidr3 but is blocked since there is an IPAddress
   134  // Deletes the Service with the IPAddress blocking the deletion
   135  // cidr3 must not exist at this point
   136  func TestServiceCIDRDeletion(t *testing.T) {
   137  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.MultiCIDRServiceAllocator, true)()
   138  	cidr1 := "192.168.0.0/29" // same as the default
   139  	cidr2 := "10.0.0.0/24"    // new range
   140  	cidr3 := "10.0.0.0/16"    // contains cidr2
   141  
   142  	etcdOptions := framework.SharedEtcd()
   143  	apiServerOptions := kubeapiservertesting.NewDefaultTestServerOptions()
   144  	s := kubeapiservertesting.StartTestServerOrDie(t,
   145  		apiServerOptions,
   146  		[]string{
   147  			"--runtime-config=networking.k8s.io/v1alpha1=true",
   148  			"--service-cluster-ip-range=" + cidr1,
   149  			"--advertise-address=172.16.1.1",
   150  			"--disable-admission-plugins=ServiceAccount",
   151  		},
   152  		etcdOptions)
   153  	defer s.TearDownFn()
   154  
   155  	client, err := kubernetes.NewForConfig(s.ClientConfig)
   156  	if err != nil {
   157  		t.Fatalf("Unexpected error: %v", err)
   158  	}
   159  
   160  	ns := framework.CreateNamespaceOrDie(client, "test-service-cidr-deletion", t)
   161  	defer framework.DeleteNamespaceOrDie(client, ns, t)
   162  
   163  	// ServiceCIDR controller
   164  	ctx, cancel := context.WithCancel(context.Background())
   165  	defer cancel()
   166  	resyncPeriod := 12 * time.Hour
   167  	informerFactory := informers.NewSharedInformerFactory(client, resyncPeriod)
   168  	go servicecidrs.NewController(
   169  		ctx,
   170  		informerFactory.Networking().V1alpha1().ServiceCIDRs(),
   171  		informerFactory.Networking().V1alpha1().IPAddresses(),
   172  		client,
   173  	).Run(ctx, 5)
   174  	informerFactory.Start(ctx.Done())
   175  
   176  	// /29 = 6 services, kubernetes.default takes the first address
   177  	// make 5 more services to take up all IPs
   178  	for i := 0; i < 5; i++ {
   179  		if _, err := client.CoreV1().Services(ns.Name).Create(context.Background(), makeService(fmt.Sprintf("service-%d", i)), metav1.CreateOptions{}); err != nil {
   180  			t.Fatal(err)
   181  		}
   182  	}
   183  	// create a new ServiceCIDRs that overlaps the default one
   184  	_, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr1", cidr1, ""), metav1.CreateOptions{})
   185  	if err != nil {
   186  		t.Fatal((err))
   187  	}
   188  	// Wait until is ready.
   189  	if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
   190  		cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr1", metav1.GetOptions{})
   191  		if err != nil {
   192  			return false, nil
   193  		}
   194  		return isServiceCIDRReady(cidr), nil
   195  	}); err != nil {
   196  		t.Fatalf("cidr1 is not ready")
   197  	}
   198  	// we should be able to delete the ServiceCIDR despite it contains IP addresses as it overlaps with the default ServiceCIDR
   199  	err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr1", metav1.DeleteOptions{})
   200  	if err != nil {
   201  		t.Fatal((err))
   202  	}
   203  
   204  	if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
   205  		_, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr1", metav1.GetOptions{})
   206  		if err != nil && apierrors.IsNotFound(err) {
   207  			return true, nil
   208  		}
   209  		return false, nil
   210  	}); err != nil {
   211  		t.Fatalf("cidr1 has not been deleted")
   212  	}
   213  
   214  	// add a new ServiceCIDR with a new range
   215  	_, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr2", cidr2, ""), metav1.CreateOptions{})
   216  	if err != nil {
   217  		t.Fatal((err))
   218  	}
   219  	// wait the allocator process the new ServiceCIDR
   220  	// Wait until is ready.
   221  	if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
   222  		cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr2", metav1.GetOptions{})
   223  		if err != nil {
   224  			return false, nil
   225  		}
   226  		return isServiceCIDRReady(cidr), nil
   227  	}); err != nil {
   228  		t.Fatalf("cidr2 is not ready")
   229  	}
   230  	// create a new Service so it will take a new IP address from the new range
   231  	svc, err := client.CoreV1().Services(ns.Name).Create(context.Background(), makeService("new-cidr-service"), metav1.CreateOptions{})
   232  	if err != nil {
   233  		t.Fatal(err)
   234  	}
   235  
   236  	if !cidrContainsIP(cidr2, svc.Spec.ClusterIP) {
   237  		t.Fatalf("Service %s expected to have an IP on range %s, got %s", svc.Name, cidr2, svc.Spec.ClusterIP)
   238  	}
   239  
   240  	// add a new ServiceCIDR that overlaps the existing one
   241  	_, err = client.NetworkingV1alpha1().ServiceCIDRs().Create(ctx, makeServiceCIDR("cidr3", cidr3, ""), metav1.CreateOptions{})
   242  	if err != nil {
   243  		t.Fatal((err))
   244  	}
   245  	// Wait until is ready.
   246  	if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
   247  		cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{})
   248  		if err != nil {
   249  			return false, nil
   250  		}
   251  		return isServiceCIDRReady(cidr), nil
   252  	}); err != nil {
   253  		t.Fatalf("cidr3 is not ready")
   254  	}
   255  	// we should be able to delete the ServiceCIDR2 despite it contains IP addresses as it is contained on ServiceCIDR3
   256  	err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr2", metav1.DeleteOptions{})
   257  	if err != nil {
   258  		t.Fatal((err))
   259  	}
   260  
   261  	if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
   262  		_, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr2", metav1.GetOptions{})
   263  		if err != nil && apierrors.IsNotFound(err) {
   264  			return true, nil
   265  		}
   266  		return false, nil
   267  	}); err != nil {
   268  		t.Fatalf("cidr2 has not been deleted")
   269  	}
   270  
   271  	// serviceCIDR3 will not be able to be deleted until the IPAddress is removed
   272  	err = client.NetworkingV1alpha1().ServiceCIDRs().Delete(ctx, "cidr3", metav1.DeleteOptions{})
   273  	if err != nil {
   274  		t.Fatal((err))
   275  	}
   276  	// Wait until is not ready.
   277  	if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
   278  		cidr, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{})
   279  		if err != nil {
   280  			return false, nil
   281  		}
   282  		for _, condition := range cidr.Status.Conditions {
   283  			if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady {
   284  				return condition.Status == metav1.ConditionStatus(metav1.ConditionFalse), nil
   285  			}
   286  		}
   287  		return false, nil
   288  	}); err != nil {
   289  		t.Fatalf("cidr3 is ready")
   290  	}
   291  
   292  	// delete the service blocking the deletion
   293  	if err := client.CoreV1().Services(ns.Name).Delete(context.Background(), "new-cidr-service", metav1.DeleteOptions{}); err != nil {
   294  		t.Fatal(err)
   295  	}
   296  
   297  	// cidr3 must not exist
   298  	if err := wait.PollUntilContextTimeout(context.Background(), 250*time.Millisecond, 30*time.Second, false, func(ctx context.Context) (bool, error) {
   299  		_, err := client.NetworkingV1alpha1().ServiceCIDRs().Get(ctx, "cidr3", metav1.GetOptions{})
   300  		if err != nil && apierrors.IsNotFound(err) {
   301  			return true, nil
   302  		}
   303  		return false, nil
   304  	}); err != nil {
   305  		t.Fatalf("cidr3 has not been deleted")
   306  	}
   307  }
   308  
   309  func makeServiceCIDR(name, primary, secondary string) *networkingv1alpha1.ServiceCIDR {
   310  	serviceCIDR := &networkingv1alpha1.ServiceCIDR{
   311  		ObjectMeta: metav1.ObjectMeta{
   312  			Name: name,
   313  		},
   314  		Spec: networkingv1alpha1.ServiceCIDRSpec{},
   315  	}
   316  	serviceCIDR.Spec.CIDRs = append(serviceCIDR.Spec.CIDRs, primary)
   317  	if secondary != "" {
   318  		serviceCIDR.Spec.CIDRs = append(serviceCIDR.Spec.CIDRs, secondary)
   319  	}
   320  	return serviceCIDR
   321  }
   322  
   323  func makeService(name string) *v1.Service {
   324  	return &v1.Service{
   325  		ObjectMeta: metav1.ObjectMeta{
   326  			Name: name,
   327  		},
   328  		Spec: v1.ServiceSpec{
   329  			Type: v1.ServiceTypeClusterIP,
   330  			Ports: []v1.ServicePort{
   331  				{Port: 80},
   332  			},
   333  		},
   334  	}
   335  }
   336  
   337  // returns true of the ServiceCIDRConditionReady is true
   338  func isServiceCIDRReady(serviceCIDR *networkingv1alpha1.ServiceCIDR) bool {
   339  	if serviceCIDR == nil {
   340  		return false
   341  	}
   342  
   343  	for _, condition := range serviceCIDR.Status.Conditions {
   344  		if condition.Type == networkingv1alpha1.ServiceCIDRConditionReady {
   345  			return condition.Status == metav1.ConditionStatus(metav1.ConditionTrue)
   346  		}
   347  	}
   348  
   349  	return false
   350  }
   351  
   352  func cidrContainsIP(cidr, ip string) bool {
   353  	prefix := netip.MustParsePrefix(cidr)
   354  	address := netip.MustParseAddr(ip)
   355  	return prefix.Contains(address)
   356  }
   357
View as plain text