number_generated_rules_test.go

Documentation: k8s.io/kubernetes/pkg/proxy/iptables

     1  //go:build linux
     2  // +build linux
     3  
     4  /*
     5  Copyright 2022 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package iptables
    21  
    22  import (
    23  	"fmt"
    24  	"testing"
    25  	"time"
    26  
    27  	v1 "k8s.io/api/core/v1"
    28  	discovery "k8s.io/api/discovery/v1"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/util/intstr"
    31  	iptablestest "k8s.io/kubernetes/pkg/util/iptables/testing"
    32  	netutils "k8s.io/utils/net"
    33  	"k8s.io/utils/ptr"
    34  )
    35  
    36  // kube-proxy generates iptables rules to forward traffic from Services to Endpoints
    37  // kube-proxy uses iptables-restore to configure the rules atomically, however,
    38  // this has the downside that large number of rules take a long time to be processed,
    39  // causing disruption.
    40  // There are different parameters than influence the number of rules generated:
    41  // - ServiceType
    42  // - Number of Services
    43  // - Number of Endpoints per Service
    44  // This test will fail when the number of rules change, so the person
    45  // that is modifying the code can have feedback about the performance impact
    46  // on their changes. It also runs multiple number of rules test cases to check
    47  // if the number of rules grows linearly.
    48  func TestNumberIptablesRules(t *testing.T) {
    49  	testCases := []struct {
    50  		name                string
    51  		epsFunc             func(eps *discovery.EndpointSlice)
    52  		svcFunc             func(svc *v1.Service)
    53  		services            int
    54  		epPerService        int
    55  		expectedFilterRules int
    56  		expectedNatRules    int
    57  	}{
    58  		{
    59  			name:                "0 Services 0 EndpointsPerService - ClusterIP",
    60  			services:            0,
    61  			epPerService:        0,
    62  			expectedFilterRules: 4,
    63  			expectedNatRules:    5,
    64  		},
    65  		{
    66  			name:                "1 Services 0 EndpointPerService - ClusterIP",
    67  			services:            1,
    68  			epPerService:        0,
    69  			expectedFilterRules: 5,
    70  			expectedNatRules:    5,
    71  		},
    72  		{
    73  			name:                "1 Services 1 EndpointPerService - ClusterIP",
    74  			services:            1,
    75  			epPerService:        1,
    76  			expectedFilterRules: 4,
    77  			expectedNatRules:    10,
    78  		},
    79  		{
    80  			name:                "1 Services 2 EndpointPerService - ClusterIP",
    81  			services:            1,
    82  			epPerService:        2,
    83  			expectedFilterRules: 4,
    84  			expectedNatRules:    13,
    85  		},
    86  		{
    87  			name:                "1 Services 10 EndpointPerService - ClusterIP",
    88  			services:            1,
    89  			epPerService:        10,
    90  			expectedFilterRules: 4,
    91  			expectedNatRules:    37,
    92  		},
    93  		{
    94  			name:                "10 Services 0 EndpointsPerService - ClusterIP",
    95  			services:            10,
    96  			epPerService:        0,
    97  			expectedFilterRules: 14,
    98  			expectedNatRules:    5,
    99  		},
   100  		{
   101  			name:                "10 Services 1 EndpointPerService - ClusterIP",
   102  			services:            10,
   103  			epPerService:        1,
   104  			expectedFilterRules: 4,
   105  			expectedNatRules:    55,
   106  		},
   107  		{
   108  			name:                "10 Services 2 EndpointPerService - ClusterIP",
   109  			services:            10,
   110  			epPerService:        2,
   111  			expectedFilterRules: 4,
   112  			expectedNatRules:    85,
   113  		},
   114  		{
   115  			name:                "10 Services 10 EndpointPerService - ClusterIP",
   116  			services:            10,
   117  			epPerService:        10,
   118  			expectedFilterRules: 4,
   119  			expectedNatRules:    325,
   120  		},
   121  
   122  		{
   123  			name: "0 Services 0 EndpointsPerService - LoadBalancer",
   124  			svcFunc: func(svc *v1.Service) {
   125  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   126  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   127  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   128  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   129  					IP: "1.2.3.4",
   130  				}}
   131  			},
   132  			services:            0,
   133  			epPerService:        0,
   134  			expectedFilterRules: 4,
   135  			expectedNatRules:    5,
   136  		},
   137  		{
   138  			name: "1 Services 0 EndpointPerService - LoadBalancer",
   139  			svcFunc: func(svc *v1.Service) {
   140  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   141  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   142  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   143  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   144  					IP: "1.2.3.4",
   145  				}}
   146  			},
   147  			services:            1,
   148  			epPerService:        0,
   149  			expectedFilterRules: 8,
   150  			expectedNatRules:    5,
   151  		},
   152  		{
   153  			name: "1 Services 1 EndpointPerService - LoadBalancer",
   154  			svcFunc: func(svc *v1.Service) {
   155  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   156  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   157  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   158  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   159  					IP: "1.2.3.4",
   160  				}}
   161  			},
   162  			services:            1,
   163  			epPerService:        1,
   164  			expectedFilterRules: 5,
   165  			expectedNatRules:    17,
   166  		},
   167  		{
   168  			name: "1 Services 2 EndpointPerService - LoadBalancer",
   169  			svcFunc: func(svc *v1.Service) {
   170  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   171  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   172  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   173  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   174  					IP: "1.2.3.4",
   175  				}}
   176  			},
   177  			services:            1,
   178  			epPerService:        2,
   179  			expectedFilterRules: 5,
   180  			expectedNatRules:    20,
   181  		},
   182  		{
   183  			name: "1 Services 10 EndpointPerService - LoadBalancer",
   184  			svcFunc: func(svc *v1.Service) {
   185  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   186  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   187  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   188  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   189  					IP: "1.2.3.4",
   190  				}}
   191  			},
   192  			services:            1,
   193  			epPerService:        10,
   194  			expectedFilterRules: 5,
   195  			expectedNatRules:    44,
   196  		},
   197  		{
   198  			name: "10 Services 0 EndpointsPerService - LoadBalancer",
   199  			svcFunc: func(svc *v1.Service) {
   200  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   201  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   202  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   203  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   204  					IP: "1.2.3.4",
   205  				}}
   206  			},
   207  			services:            10,
   208  			epPerService:        0,
   209  			expectedFilterRules: 44,
   210  			expectedNatRules:    5,
   211  		},
   212  		{
   213  			name: "10 Services 1 EndpointPerService - LoadBalancer",
   214  			svcFunc: func(svc *v1.Service) {
   215  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   216  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   217  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   218  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   219  					IP: "1.2.3.4",
   220  				}}
   221  			},
   222  			services:            10,
   223  			epPerService:        1,
   224  			expectedFilterRules: 14,
   225  			expectedNatRules:    125,
   226  		},
   227  		{
   228  			name: "10 Services 2 EndpointPerService - LoadBalancer",
   229  			svcFunc: func(svc *v1.Service) {
   230  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   231  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   232  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   233  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   234  					IP: "1.2.3.4",
   235  				}}
   236  			},
   237  			services:            10,
   238  			epPerService:        2,
   239  			expectedFilterRules: 14,
   240  			expectedNatRules:    155,
   241  		},
   242  		{
   243  			name: "10 Services 10 EndpointPerService - LoadBalancer",
   244  			svcFunc: func(svc *v1.Service) {
   245  				svc.Spec.Type = v1.ServiceTypeLoadBalancer
   246  				svc.Spec.ExternalIPs = []string{"1.2.3.4"}
   247  				svc.Spec.LoadBalancerSourceRanges = []string{" 1.2.3.4/28"}
   248  				svc.Status.LoadBalancer.Ingress = []v1.LoadBalancerIngress{{
   249  					IP: "1.2.3.4",
   250  				}}
   251  			},
   252  			services:            10,
   253  			epPerService:        10,
   254  			expectedFilterRules: 14,
   255  			expectedNatRules:    395,
   256  		},
   257  	}
   258  
   259  	for _, test := range testCases {
   260  		t.Run(test.name, func(t *testing.T) {
   261  			ipt := iptablestest.NewFake()
   262  			fp := NewFakeProxier(ipt)
   263  
   264  			svcs, eps := generateServiceEndpoints(test.services, test.epPerService, test.epsFunc, test.svcFunc)
   265  
   266  			makeServiceMap(fp, svcs...)
   267  			populateEndpointSlices(fp, eps...)
   268  
   269  			now := time.Now()
   270  			fp.syncProxyRules()
   271  			t.Logf("time to sync rule: %v", time.Since(now))
   272  			t.Logf("iptables data size: %d bytes", fp.iptablesData.Len())
   273  
   274  			if fp.filterRules.Lines() != test.expectedFilterRules {
   275  				t.Errorf("expected number of Filter rules: %d, got: %d", test.expectedFilterRules, fp.filterRules.Lines())
   276  			}
   277  
   278  			if fp.natRules.Lines() != test.expectedNatRules {
   279  				t.Errorf("expected number of NAT rules: %d, got: %d", test.expectedNatRules, fp.natRules.Lines())
   280  			}
   281  
   282  			// print generated iptables data
   283  			// t.Logf("Generated rules:\n %s", fp.iptablesData.String())
   284  		})
   285  	}
   286  }
   287  
   288  func Test_generateServiceEndpoints(t *testing.T) {
   289  	testCases := []struct {
   290  		name         string
   291  		services     int
   292  		epPerService int
   293  		svcType      v1.ServiceType
   294  	}{
   295  		{
   296  			name:         "Generate 10 Services with 10 Endpoints per Service and LoadBalancer Type",
   297  			services:     10,
   298  			epPerService: 10,
   299  			svcType:      v1.ServiceTypeLoadBalancer,
   300  		},
   301  		{
   302  			name:         "Generate 10 Services with 20 Endpoints per Service and NodePort Type",
   303  			services:     10,
   304  			epPerService: 20,
   305  			svcType:      v1.ServiceTypeNodePort,
   306  		},
   307  	}
   308  
   309  	for _, test := range testCases {
   310  		t.Run(test.name, func(t *testing.T) {
   311  			// test the function to mutate services
   312  			svcFunc := func(svc *v1.Service) {
   313  				svc.Spec.Type = test.svcType
   314  			}
   315  			// test the function to mutate endpoint slices
   316  			epsFunc := func(eps *discovery.EndpointSlice) {
   317  				for i := range eps.Endpoints {
   318  					nodeName := fmt.Sprintf("node-%d", i)
   319  					eps.Endpoints[i].NodeName = &nodeName
   320  				}
   321  			}
   322  
   323  			svcs, eps := generateServiceEndpoints(test.services, test.epPerService, epsFunc, svcFunc)
   324  
   325  			if len(svcs) != test.services {
   326  				t.Fatalf("expected %d service, received %d", test.services, len(svcs))
   327  			}
   328  			if len(eps) != test.services {
   329  				t.Fatalf("expected %d endpoint slice , received %d", test.services, len(eps))
   330  			}
   331  
   332  			for i := 0; i < test.services; i++ {
   333  				if svcs[i].Spec.Type != test.svcType {
   334  					t.Fatalf("expected Service Type %s, got %s", test.svcType, svcs[i].Spec.Type)
   335  				}
   336  				if eps[i].ObjectMeta.Labels[discovery.LabelServiceName] != svcs[i].Name {
   337  					t.Fatalf("endpoint slice reference %s instead of Service %s", eps[i].ObjectMeta.Labels[discovery.LabelServiceName], svcs[i].Name)
   338  				}
   339  				if len(eps[i].Endpoints) != test.epPerService {
   340  					t.Fatalf("expected %d endpoints per slice , received %d", test.epPerService, len(eps[i].Endpoints))
   341  				}
   342  				for j := 0; j < test.epPerService; j++ {
   343  					nodeName := fmt.Sprintf("node-%d", j)
   344  					if *eps[i].Endpoints[j].NodeName != nodeName {
   345  						t.Errorf("Endpoint %d on EndpointSlice %d expected Nodename %s, got %s", j, i, nodeName, *eps[i].Endpoints[j].NodeName)
   346  					}
   347  				}
   348  			}
   349  		})
   350  	}
   351  
   352  }
   353  
   354  // generateServiceEndpoints generate Services with the Type specified and it creates N Endpoints per Service
   355  func generateServiceEndpoints(nServices, nEndpoints int, epsFunc func(eps *discovery.EndpointSlice), svcFunc func(svc *v1.Service)) ([]*v1.Service, []*discovery.EndpointSlice) {
   356  	services := make([]*v1.Service, nServices)
   357  	endpointSlices := make([]*discovery.EndpointSlice, nServices)
   358  
   359  	// base parameters
   360  	basePort := 80
   361  	base := netutils.BigForIP(netutils.ParseIPSloppy("10.0.0.1"))
   362  
   363  	// generate a base endpoint slice object
   364  	baseEp := netutils.BigForIP(netutils.ParseIPSloppy("172.16.0.1"))
   365  	epPort := 8080
   366  
   367  	eps := &discovery.EndpointSlice{
   368  		ObjectMeta: metav1.ObjectMeta{
   369  			Name:      "ep",
   370  			Namespace: "namespace",
   371  		},
   372  		AddressType: discovery.AddressTypeIPv4,
   373  		Endpoints:   []discovery.Endpoint{},
   374  		Ports: []discovery.EndpointPort{{
   375  			Name:     ptr.To(fmt.Sprintf("%d", epPort)),
   376  			Port:     ptr.To(int32(epPort)),
   377  			Protocol: ptr.To(v1.ProtocolTCP),
   378  		}},
   379  	}
   380  
   381  	for j := 0; j < nEndpoints; j++ {
   382  		ipEp := netutils.AddIPOffset(baseEp, j)
   383  		eps.Endpoints = append(eps.Endpoints, discovery.Endpoint{
   384  			Addresses: []string{ipEp.String()},
   385  		})
   386  	}
   387  
   388  	if epsFunc != nil {
   389  		epsFunc(eps)
   390  	}
   391  
   392  	// generate a base service object
   393  	svc := &v1.Service{
   394  		ObjectMeta: metav1.ObjectMeta{
   395  			Name:      "svc",
   396  			Namespace: "namespace",
   397  		},
   398  		Spec: v1.ServiceSpec{
   399  			Type: v1.ServiceTypeClusterIP,
   400  		},
   401  	}
   402  
   403  	if svcFunc != nil {
   404  		svcFunc(svc)
   405  	}
   406  
   407  	// Create the Services and associate and endpoint slice object to each one
   408  	for i := 0; i < nServices; i++ {
   409  		ip := netutils.AddIPOffset(base, i)
   410  		services[i] = svc.DeepCopy()
   411  		services[i].Name = fmt.Sprintf("svc%d", i)
   412  		services[i].Spec.ClusterIP = ip.String()
   413  		services[i].Spec.Ports = []v1.ServicePort{
   414  			{
   415  				Name:       fmt.Sprintf("%d", epPort),
   416  				Protocol:   v1.ProtocolTCP,
   417  				Port:       int32(basePort + i),
   418  				TargetPort: intstr.FromInt32(int32(epPort)),
   419  			},
   420  		}
   421  
   422  		if svc.Spec.Type == v1.ServiceTypeNodePort || svc.Spec.Type == v1.ServiceTypeLoadBalancer {
   423  			services[i].Spec.Ports[0].NodePort = int32(30000 + i)
   424  
   425  		}
   426  		if svc.Spec.Type == v1.ServiceTypeLoadBalancer {
   427  			services[i].Spec.HealthCheckNodePort = int32(32000 + nServices + i)
   428  		}
   429  
   430  		endpointSlices[i] = eps.DeepCopy()
   431  		endpointSlices[i].Name = services[i].Name
   432  		endpointSlices[i].ObjectMeta.Labels = map[string]string{
   433  			discovery.LabelServiceName: services[i].Name,
   434  		}
   435  
   436  	}
   437  
   438  	return services, endpointSlices
   439  }
   440
View as plain text