services_test.go

Documentation: k8s.io/kubernetes/test/integration/network

     1  /*
     2  Copyright 2021 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package network
    18  
    19  import (
    20  	"encoding/json"
    21  	"fmt"
    22  	"testing"
    23  	"time"
    24  
    25  	v1 "k8s.io/api/core/v1"
    26  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    27  	"k8s.io/apimachinery/pkg/types"
    28  	"k8s.io/apimachinery/pkg/util/intstr"
    29  	"k8s.io/apimachinery/pkg/util/strategicpatch"
    30  	"k8s.io/apimachinery/pkg/util/wait"
    31  	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
    32  	"k8s.io/kubernetes/pkg/controlplane"
    33  
    34  	"k8s.io/kubernetes/test/integration/framework"
    35  	"k8s.io/kubernetes/test/utils/ktesting"
    36  )
    37  
    38  // TestServicesFinalizersRepairLoop tests that Services participate in the object
    39  // deletion when using finalizers, and that the Services Repair controller doesn't,
    40  // mistakenly, repair the ClusterIP assigned to the Service that is being deleted.
    41  // https://issues.k8s.io/87603
    42  func TestServicesFinalizersRepairLoop(t *testing.T) {
    43  	serviceCIDR := "10.0.0.0/16"
    44  	clusterIP := "10.0.0.20"
    45  	interval := 5 * time.Second
    46  
    47  	tCtx := ktesting.Init(t)
    48  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
    49  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
    50  			opts.ServiceClusterIPRanges = serviceCIDR
    51  		},
    52  		ModifyServerConfig: func(cfg *controlplane.Config) {
    53  			cfg.ExtraConfig.RepairServicesInterval = interval
    54  		},
    55  	})
    56  	defer tearDownFn()
    57  
    58  	// verify client is working
    59  	if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
    60  		_, err := client.CoreV1().Endpoints(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
    61  		if err != nil {
    62  			t.Logf("error fetching endpoints: %v", err)
    63  			return false, nil
    64  		}
    65  		return true, nil
    66  	}); err != nil {
    67  		t.Errorf("server without enabled endpoints failed to register: %v", err)
    68  	}
    69  
    70  	// Create a NodePort service with one finalizer
    71  	svcNodePort := v1.Service{
    72  		ObjectMeta: metav1.ObjectMeta{
    73  			Name:       "svc",
    74  			Finalizers: []string{"foo.bar/some-finalizer"},
    75  		},
    76  		Spec: v1.ServiceSpec{
    77  			ClusterIP: clusterIP,
    78  			Ports: []v1.ServicePort{{
    79  				Port:       8443,
    80  				NodePort:   30443,
    81  				TargetPort: intstr.FromInt32(8443),
    82  				Protocol:   v1.ProtocolTCP,
    83  			}},
    84  			Type: v1.ServiceTypeNodePort,
    85  		},
    86  	}
    87  
    88  	// Create service
    89  	if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, &svcNodePort, metav1.CreateOptions{}); err != nil {
    90  		t.Errorf("unexpected error creating service: %v", err)
    91  	}
    92  	t.Logf("Created service: %s", svcNodePort.Name)
    93  
    94  	// Check the service has been created correctly
    95  	svc, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svcNodePort.Name, metav1.GetOptions{})
    96  	if err != nil || svc.Spec.ClusterIP != clusterIP {
    97  		t.Errorf("created service is not correct: %v", err)
    98  	}
    99  	t.Logf("Service created successfully: %v", svc)
   100  
   101  	// Delete service
   102  	if err := client.CoreV1().Services(metav1.NamespaceDefault).Delete(tCtx, svcNodePort.Name, metav1.DeleteOptions{}); err != nil {
   103  		t.Errorf("unexpected error deleting service: %v", err)
   104  	}
   105  	t.Logf("Deleted service: %s", svcNodePort.Name)
   106  
   107  	// wait for the repair loop to recover the deleted resources
   108  	time.Sleep(interval + 1)
   109  
   110  	// Check that the service was not deleted and the IP is already allocated
   111  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svcNodePort.Name, metav1.GetOptions{})
   112  	if err != nil || svc.Spec.ClusterIP != clusterIP {
   113  		t.Errorf("created service is not correct: %v", err)
   114  	}
   115  	t.Logf("Service after Delete: %v", svc)
   116  
   117  	// Remove the finalizer
   118  	if _, err = client.CoreV1().Services(metav1.NamespaceDefault).Patch(tCtx, svcNodePort.Name, types.JSONPatchType, []byte(`[{"op":"remove","path":"/metadata/finalizers"}]`), metav1.PatchOptions{}); err != nil {
   119  		t.Errorf("unexpected error removing finalizer: %v", err)
   120  	}
   121  	t.Logf("Removed service finalizer: %s", svcNodePort.Name)
   122  
   123  	// Check that the service was deleted
   124  	_, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svcNodePort.Name, metav1.GetOptions{})
   125  	if err == nil {
   126  		t.Errorf("service was not delete: %v", err)
   127  	}
   128  
   129  	// Try to create service again
   130  	if _, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, &svcNodePort, metav1.CreateOptions{}); err != nil {
   131  		t.Errorf("unexpected error creating service: %v", err)
   132  	}
   133  	t.Logf("Created service: %s", svcNodePort.Name)
   134  }
   135  
   136  func TestServicesFinalizersPatchStatus(t *testing.T) {
   137  	serviceCIDR := "10.0.0.0/16"
   138  	clusterIP := "10.0.0.21"
   139  	nodePort := 30443
   140  	tCtx := ktesting.Init(t)
   141  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   142  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   143  			opts.ServiceClusterIPRanges = serviceCIDR
   144  		},
   145  	})
   146  	defer tearDownFn()
   147  
   148  	for _, testcase := range []string{"spec", "status"} {
   149  		t.Run(testcase, func(t *testing.T) {
   150  			// Create a NodePort service with one finalizer
   151  			svcNodePort := v1.Service{
   152  				ObjectMeta: metav1.ObjectMeta{
   153  					Name:       "svc" + testcase,
   154  					Finalizers: []string{"foo.bar/some-finalizer"},
   155  				},
   156  				Spec: v1.ServiceSpec{
   157  					ClusterIP: clusterIP,
   158  					Ports: []v1.ServicePort{{
   159  						Port:       8443,
   160  						NodePort:   int32(nodePort),
   161  						TargetPort: intstr.FromInt32(8443),
   162  						Protocol:   v1.ProtocolTCP,
   163  					}},
   164  					Type: v1.ServiceTypeNodePort,
   165  				},
   166  			}
   167  
   168  			ns := framework.CreateNamespaceOrDie(client, "test-service-finalizers-"+testcase, t)
   169  			defer framework.DeleteNamespaceOrDie(client, ns, t)
   170  
   171  			// Create service
   172  			if _, err := client.CoreV1().Services(ns.Name).Create(tCtx, &svcNodePort, metav1.CreateOptions{}); err != nil {
   173  				t.Fatalf("unexpected error creating service: %v", err)
   174  			}
   175  			t.Logf("Created service: %s", svcNodePort.Name)
   176  
   177  			// Check the service has been created correctly
   178  			svc, err := client.CoreV1().Services(ns.Name).Get(tCtx, svcNodePort.Name, metav1.GetOptions{})
   179  			if err != nil || svc.Spec.ClusterIP != clusterIP {
   180  				t.Fatalf("created service is not correct: %v", err)
   181  			}
   182  			t.Logf("Service created successfully: %+v", svc)
   183  
   184  			// Delete service
   185  			if err := client.CoreV1().Services(ns.Name).Delete(tCtx, svcNodePort.Name, metav1.DeleteOptions{}); err != nil {
   186  				t.Fatalf("unexpected error deleting service: %v", err)
   187  			}
   188  			t.Logf("Deleted service: %s", svcNodePort.Name)
   189  
   190  			// Check that the service was not deleted and the IP is already allocated
   191  			svc, err = client.CoreV1().Services(ns.Name).Get(tCtx, svcNodePort.Name, metav1.GetOptions{})
   192  			if err != nil ||
   193  				svc.Spec.ClusterIP != clusterIP ||
   194  				int(svc.Spec.Ports[0].NodePort) != nodePort ||
   195  				svc.DeletionTimestamp == nil ||
   196  				len(svc.ObjectMeta.Finalizers) != 1 {
   197  				t.Fatalf("Service expected to be deleting and with the same values: %v", err)
   198  			}
   199  			t.Logf("Service after Delete: %+v", svc)
   200  
   201  			// Remove the finalizer
   202  			updated := svc.DeepCopy()
   203  			updated.ObjectMeta.Finalizers = []string{}
   204  			patchBytes, err := getPatchBytes(svc, updated)
   205  			if err != nil {
   206  				t.Fatalf("unexpected error getting patch bytes: %v", err)
   207  			}
   208  
   209  			if testcase == "spec" {
   210  				if _, err = client.CoreV1().Services(ns.Name).Patch(tCtx, svcNodePort.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}); err != nil {
   211  					t.Fatalf("unexpected error removing finalizer: %v", err)
   212  				}
   213  			} else {
   214  				if _, err = client.CoreV1().Services(ns.Name).Patch(tCtx, svcNodePort.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}, "status"); err != nil {
   215  					t.Fatalf("unexpected error removing finalizer: %v", err)
   216  				}
   217  			}
   218  			t.Logf("Removed service finalizer: %s", svcNodePort.Name)
   219  
   220  			// Check that the service was deleted
   221  			_, err = client.CoreV1().Services(ns.Name).Get(tCtx, svcNodePort.Name, metav1.GetOptions{})
   222  			if err == nil {
   223  				t.Fatalf("service was not delete: %v", err)
   224  			}
   225  
   226  			// Try to create service again without the finalizer to check the ClusterIP and NodePort are deallocated
   227  			svc = svcNodePort.DeepCopy()
   228  			svc.Finalizers = []string{}
   229  			if _, err := client.CoreV1().Services(ns.Name).Create(tCtx, svc, metav1.CreateOptions{}); err != nil {
   230  				t.Fatalf("unexpected error creating service: %v", err)
   231  			}
   232  			// Delete service
   233  			if err := client.CoreV1().Services(ns.Name).Delete(tCtx, svc.Name, metav1.DeleteOptions{}); err != nil {
   234  				t.Fatalf("unexpected error deleting service: %v", err)
   235  			}
   236  		})
   237  	}
   238  }
   239  
   240  // Regresion test for https://issues.k8s.io/115316
   241  func TestServiceCIDR28bits(t *testing.T) {
   242  	serviceCIDR := "10.0.0.0/28"
   243  
   244  	tCtx := ktesting.Init(t)
   245  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   246  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   247  			opts.ServiceClusterIPRanges = serviceCIDR
   248  		},
   249  	})
   250  	defer tearDownFn()
   251  
   252  	// Wait until the default "kubernetes" service is created.
   253  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
   254  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
   255  		if err != nil {
   256  			return false, err
   257  		}
   258  		return true, nil
   259  	}); err != nil {
   260  		t.Fatalf("creating kubernetes service timed out")
   261  	}
   262  
   263  	ns := framework.CreateNamespaceOrDie(client, "test-regression", t)
   264  	defer framework.DeleteNamespaceOrDie(client, ns, t)
   265  
   266  	service := &v1.Service{
   267  		ObjectMeta: metav1.ObjectMeta{
   268  			Name: "test-1234",
   269  		},
   270  		Spec: v1.ServiceSpec{
   271  			Type: v1.ServiceTypeClusterIP,
   272  			Ports: []v1.ServicePort{{
   273  				Port: int32(80),
   274  			}},
   275  			Selector: map[string]string{
   276  				"foo": "bar",
   277  			},
   278  		},
   279  	}
   280  
   281  	_, err := client.CoreV1().Services(ns.Name).Create(tCtx, service, metav1.CreateOptions{})
   282  	if err != nil {
   283  		t.Fatalf("Error creating test service: %v", err)
   284  	}
   285  }
   286  
   287  func getPatchBytes(oldSvc, newSvc *v1.Service) ([]byte, error) {
   288  	oldData, err := json.Marshal(oldSvc)
   289  	if err != nil {
   290  		return nil, fmt.Errorf("failed to Marshal oldData for svc %s/%s: %v", oldSvc.Namespace, oldSvc.Name, err)
   291  	}
   292  
   293  	newData, err := json.Marshal(newSvc)
   294  	if err != nil {
   295  		return nil, fmt.Errorf("failed to Marshal newData for svc %s/%s: %v", newSvc.Namespace, newSvc.Name, err)
   296  	}
   297  
   298  	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Service{})
   299  	if err != nil {
   300  		return nil, fmt.Errorf("failed to CreateTwoWayMergePatch for svc %s/%s: %v", oldSvc.Namespace, oldSvc.Name, err)
   301  	}
   302  	return patchBytes, nil
   303  
   304  }
   305
View as plain text