server_test.go

Documentation: k8s.io/kubernetes/cmd/kube-proxy/app

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package app
    18  
    19  import (
    20  	"context"
    21  	"errors"
    22  	"fmt"
    23  	"io/ioutil"
    24  	"net"
    25  	"path"
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/google/go-cmp/cmp"
    30  	"github.com/spf13/pflag"
    31  	"github.com/stretchr/testify/assert"
    32  	"github.com/stretchr/testify/require"
    33  
    34  	v1 "k8s.io/api/core/v1"
    35  	"k8s.io/apimachinery/pkg/api/resource"
    36  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    37  	clientsetfake "k8s.io/client-go/kubernetes/fake"
    38  	componentbaseconfig "k8s.io/component-base/config"
    39  	logsapi "k8s.io/component-base/logs/api/v1"
    40  	kubeproxyconfig "k8s.io/kubernetes/pkg/proxy/apis/config"
    41  	"k8s.io/kubernetes/test/utils/ktesting"
    42  	netutils "k8s.io/utils/net"
    43  	"k8s.io/utils/ptr"
    44  )
    45  
    46  // TestLoadConfig tests proper operation of loadConfig()
    47  func TestLoadConfig(t *testing.T) {
    48  
    49  	yamlTemplate := `apiVersion: kubeproxy.config.k8s.io/v1alpha1
    50  bindAddress: %s
    51  clientConnection:
    52    acceptContentTypes: "abc"
    53    burst: 100
    54    contentType: content-type
    55    kubeconfig: "/path/to/kubeconfig"
    56    qps: 7
    57  clusterCIDR: "%s"
    58  configSyncPeriod: 15s
    59  conntrack:
    60    maxPerCore: 2
    61    min: 1
    62    tcpCloseWaitTimeout: 10s
    63    tcpEstablishedTimeout: 20s
    64  healthzBindAddress: "%s"
    65  hostnameOverride: "foo"
    66  iptables:
    67    masqueradeAll: true
    68    masqueradeBit: 17
    69    minSyncPeriod: 10s
    70    syncPeriod: 60s
    71    localhostNodePorts: true
    72  ipvs:
    73    minSyncPeriod: 10s
    74    syncPeriod: 60s
    75    excludeCIDRs:
    76      - "10.20.30.40/16"
    77      - "fd00:1::0/64"
    78  nftables:
    79    masqueradeAll: true
    80    masqueradeBit: 18
    81    minSyncPeriod: 10s
    82    syncPeriod: 60s
    83  kind: KubeProxyConfiguration
    84  metricsBindAddress: "%s"
    85  mode: "%s"
    86  oomScoreAdj: 17
    87  portRange: "2-7"
    88  detectLocalMode: "ClusterCIDR"
    89  detectLocal:
    90    bridgeInterface: "cbr0"
    91    interfaceNamePrefix: "veth"
    92  nodePortAddresses:
    93    - "10.20.30.40/16"
    94    - "fd00:1::0/64"
    95  `
    96  
    97  	testCases := []struct {
    98  		name               string
    99  		mode               string
   100  		bindAddress        string
   101  		clusterCIDR        string
   102  		healthzBindAddress string
   103  		metricsBindAddress string
   104  		extraConfig        string
   105  	}{
   106  		{
   107  			name:               "iptables mode, IPv4 all-zeros bind address",
   108  			mode:               "iptables",
   109  			bindAddress:        "0.0.0.0",
   110  			clusterCIDR:        "1.2.3.0/24",
   111  			healthzBindAddress: "1.2.3.4:12345",
   112  			metricsBindAddress: "2.3.4.5:23456",
   113  		},
   114  		{
   115  			name:               "iptables mode, non-zeros IPv4 config",
   116  			mode:               "iptables",
   117  			bindAddress:        "9.8.7.6",
   118  			clusterCIDR:        "1.2.3.0/24",
   119  			healthzBindAddress: "1.2.3.4:12345",
   120  			metricsBindAddress: "2.3.4.5:23456",
   121  		},
   122  		{
   123  			// Test for 'bindAddress: "::"' (IPv6 all-zeros) in kube-proxy
   124  			// config file. The user will need to put quotes around '::' since
   125  			// 'bindAddress: ::' is invalid yaml syntax.
   126  			name:               "iptables mode, IPv6 \"::\" bind address",
   127  			mode:               "iptables",
   128  			bindAddress:        "\"::\"",
   129  			clusterCIDR:        "fd00:1::0/64",
   130  			healthzBindAddress: "[fd00:1::5]:12345",
   131  			metricsBindAddress: "[fd00:2::5]:23456",
   132  		},
   133  		{
   134  			// Test for 'bindAddress: "[::]"' (IPv6 all-zeros in brackets)
   135  			// in kube-proxy config file. The user will need to use
   136  			// surrounding quotes here since 'bindAddress: [::]' is invalid
   137  			// yaml syntax.
   138  			name:               "iptables mode, IPv6 \"[::]\" bind address",
   139  			mode:               "iptables",
   140  			bindAddress:        "\"[::]\"",
   141  			clusterCIDR:        "fd00:1::0/64",
   142  			healthzBindAddress: "[fd00:1::5]:12345",
   143  			metricsBindAddress: "[fd00:2::5]:23456",
   144  		},
   145  		{
   146  			// Test for 'bindAddress: ::0' (another form of IPv6 all-zeros).
   147  			// No surrounding quotes are required around '::0'.
   148  			name:               "iptables mode, IPv6 ::0 bind address",
   149  			mode:               "iptables",
   150  			bindAddress:        "::0",
   151  			clusterCIDR:        "fd00:1::0/64",
   152  			healthzBindAddress: "[fd00:1::5]:12345",
   153  			metricsBindAddress: "[fd00:2::5]:23456",
   154  		},
   155  		{
   156  			name:               "ipvs mode, IPv6 config",
   157  			mode:               "ipvs",
   158  			bindAddress:        "2001:db8::1",
   159  			clusterCIDR:        "fd00:1::0/64",
   160  			healthzBindAddress: "[fd00:1::5]:12345",
   161  			metricsBindAddress: "[fd00:2::5]:23456",
   162  		},
   163  		{
   164  			// Test for unknown field within config.
   165  			// For v1alpha1 a lenient path is implemented and will throw a
   166  			// strict decoding warning instead of failing to load
   167  			name:               "unknown field",
   168  			mode:               "iptables",
   169  			bindAddress:        "9.8.7.6",
   170  			clusterCIDR:        "1.2.3.0/24",
   171  			healthzBindAddress: "1.2.3.4:12345",
   172  			metricsBindAddress: "2.3.4.5:23456",
   173  			extraConfig:        "foo: bar",
   174  		},
   175  		{
   176  			// Test for duplicate field within config.
   177  			// For v1alpha1 a lenient path is implemented and will throw a
   178  			// strict decoding warning instead of failing to load
   179  			name:               "duplicate field",
   180  			mode:               "iptables",
   181  			bindAddress:        "9.8.7.6",
   182  			clusterCIDR:        "1.2.3.0/24",
   183  			healthzBindAddress: "1.2.3.4:12345",
   184  			metricsBindAddress: "2.3.4.5:23456",
   185  			extraConfig:        "bindAddress: 9.8.7.6",
   186  		},
   187  	}
   188  
   189  	for _, tc := range testCases {
   190  		expBindAddr := tc.bindAddress
   191  		if tc.bindAddress[0] == '"' {
   192  			// Surrounding double quotes will get stripped by the yaml parser.
   193  			expBindAddr = expBindAddr[1 : len(tc.bindAddress)-1]
   194  		}
   195  		expected := &kubeproxyconfig.KubeProxyConfiguration{
   196  			BindAddress: expBindAddr,
   197  			ClientConnection: componentbaseconfig.ClientConnectionConfiguration{
   198  				AcceptContentTypes: "abc",
   199  				Burst:              100,
   200  				ContentType:        "content-type",
   201  				Kubeconfig:         "/path/to/kubeconfig",
   202  				QPS:                7,
   203  			},
   204  			ClusterCIDR:      tc.clusterCIDR,
   205  			ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
   206  			Conntrack: kubeproxyconfig.KubeProxyConntrackConfiguration{
   207  				MaxPerCore:            ptr.To[int32](2),
   208  				Min:                   ptr.To[int32](1),
   209  				TCPCloseWaitTimeout:   &metav1.Duration{Duration: 10 * time.Second},
   210  				TCPEstablishedTimeout: &metav1.Duration{Duration: 20 * time.Second},
   211  			},
   212  			FeatureGates:       map[string]bool{},
   213  			HealthzBindAddress: tc.healthzBindAddress,
   214  			HostnameOverride:   "foo",
   215  			IPTables: kubeproxyconfig.KubeProxyIPTablesConfiguration{
   216  				MasqueradeAll:      true,
   217  				MasqueradeBit:      ptr.To[int32](17),
   218  				LocalhostNodePorts: ptr.To(true),
   219  				MinSyncPeriod:      metav1.Duration{Duration: 10 * time.Second},
   220  				SyncPeriod:         metav1.Duration{Duration: 60 * time.Second},
   221  			},
   222  			IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
   223  				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
   224  				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
   225  				ExcludeCIDRs:  []string{"10.20.30.40/16", "fd00:1::0/64"},
   226  			},
   227  			NFTables: kubeproxyconfig.KubeProxyNFTablesConfiguration{
   228  				MasqueradeAll: true,
   229  				MasqueradeBit: ptr.To[int32](18),
   230  				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
   231  				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
   232  			},
   233  			MetricsBindAddress: tc.metricsBindAddress,
   234  			Mode:               kubeproxyconfig.ProxyMode(tc.mode),
   235  			OOMScoreAdj:        ptr.To[int32](17),
   236  			PortRange:          "2-7",
   237  			NodePortAddresses:  []string{"10.20.30.40/16", "fd00:1::0/64"},
   238  			DetectLocalMode:    kubeproxyconfig.LocalModeClusterCIDR,
   239  			DetectLocal: kubeproxyconfig.DetectLocalConfiguration{
   240  				BridgeInterface:     string("cbr0"),
   241  				InterfaceNamePrefix: string("veth"),
   242  			},
   243  			Logging: logsapi.LoggingConfiguration{
   244  				Format:         "text",
   245  				FlushFrequency: logsapi.TimeOrMetaDuration{Duration: metav1.Duration{Duration: 5 * time.Second}, SerializeAsString: true},
   246  			},
   247  		}
   248  
   249  		options := NewOptions()
   250  
   251  		baseYAML := fmt.Sprintf(
   252  			yamlTemplate, tc.bindAddress, tc.clusterCIDR,
   253  			tc.healthzBindAddress, tc.metricsBindAddress, tc.mode)
   254  
   255  		// Append additional configuration to the base yaml template
   256  		yaml := fmt.Sprintf("%s\n%s", baseYAML, tc.extraConfig)
   257  
   258  		config, err := options.loadConfig([]byte(yaml))
   259  
   260  		assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)
   261  
   262  		if diff := cmp.Diff(config, expected); diff != "" {
   263  			t.Fatalf("unexpected config for %s, diff = %s", tc.name, diff)
   264  		}
   265  	}
   266  }
   267  
   268  // TestLoadConfigFailures tests failure modes for loadConfig()
   269  func TestLoadConfigFailures(t *testing.T) {
   270  	// TODO(phenixblue): Uncomment below template when v1alpha2+ of kube-proxy config is
   271  	// released with strict decoding. These associated tests will fail with
   272  	// the lenient codec and only one config API version.
   273  	/*
   274  			yamlTemplate := `bindAddress: 0.0.0.0
   275  		clusterCIDR: "1.2.3.0/24"
   276  		configSyncPeriod: 15s
   277  		kind: KubeProxyConfiguration`
   278  	*/
   279  
   280  	testCases := []struct {
   281  		name    string
   282  		config  string
   283  		expErr  string
   284  		checkFn func(err error) bool
   285  	}{
   286  		{
   287  			name:   "Decode error test",
   288  			config: "Twas bryllyg, and ye slythy toves",
   289  			expErr: "could not find expected ':'",
   290  		},
   291  		{
   292  			name:   "Bad config type test",
   293  			config: "kind: KubeSchedulerConfiguration",
   294  			expErr: "no kind",
   295  		},
   296  		{
   297  			name:   "Missing quotes around :: bindAddress",
   298  			config: "bindAddress: ::",
   299  			expErr: "mapping values are not allowed in this context",
   300  		},
   301  		// TODO(phenixblue): Uncomment below tests when v1alpha2+ of kube-proxy config is
   302  		// released with strict decoding. These tests will fail with the
   303  		// lenient codec and only one config API version.
   304  		/*
   305  			{
   306  				name:    "Duplicate fields",
   307  				config:  fmt.Sprintf("%s\nbindAddress: 1.2.3.4", yamlTemplate),
   308  				checkFn: kuberuntime.IsStrictDecodingError,
   309  			},
   310  			{
   311  				name:    "Unknown field",
   312  				config:  fmt.Sprintf("%s\nfoo: bar", yamlTemplate),
   313  				checkFn: kuberuntime.IsStrictDecodingError,
   314  			},
   315  		*/
   316  	}
   317  
   318  	version := "apiVersion: kubeproxy.config.k8s.io/v1alpha1"
   319  	for _, tc := range testCases {
   320  		t.Run(tc.name, func(t *testing.T) {
   321  			options := NewOptions()
   322  			config := fmt.Sprintf("%s\n%s", version, tc.config)
   323  			_, err := options.loadConfig([]byte(config))
   324  
   325  			if assert.Error(t, err, tc.name) {
   326  				if tc.expErr != "" {
   327  					assert.Contains(t, err.Error(), tc.expErr)
   328  				}
   329  				if tc.checkFn != nil {
   330  					assert.True(t, tc.checkFn(err), tc.name)
   331  				}
   332  			}
   333  		})
   334  	}
   335  }
   336  
   337  // TestProcessHostnameOverrideFlag tests processing hostname-override arg
   338  func TestProcessHostnameOverrideFlag(t *testing.T) {
   339  	testCases := []struct {
   340  		name                 string
   341  		hostnameOverrideFlag string
   342  		expectedHostname     string
   343  		expectError          bool
   344  	}{
   345  		{
   346  			name:                 "Hostname from config file",
   347  			hostnameOverrideFlag: "",
   348  			expectedHostname:     "foo",
   349  			expectError:          false,
   350  		},
   351  		{
   352  			name:                 "Hostname from flag",
   353  			hostnameOverrideFlag: "  bar ",
   354  			expectedHostname:     "bar",
   355  			expectError:          false,
   356  		},
   357  		{
   358  			name:                 "Hostname is space",
   359  			hostnameOverrideFlag: "   ",
   360  			expectError:          true,
   361  		},
   362  	}
   363  	for _, tc := range testCases {
   364  		t.Run(tc.name, func(t *testing.T) {
   365  			options := NewOptions()
   366  			options.config = &kubeproxyconfig.KubeProxyConfiguration{
   367  				HostnameOverride: "foo",
   368  			}
   369  
   370  			options.hostnameOverride = tc.hostnameOverrideFlag
   371  
   372  			err := options.processHostnameOverrideFlag()
   373  			if tc.expectError {
   374  				if err == nil {
   375  					t.Fatalf("should error for this case %s", tc.name)
   376  				}
   377  			} else {
   378  				assert.NoError(t, err, "unexpected error %v", err)
   379  				if tc.expectedHostname != options.config.HostnameOverride {
   380  					t.Fatalf("expected hostname: %s, but got: %s", tc.expectedHostname, options.config.HostnameOverride)
   381  				}
   382  			}
   383  		})
   384  	}
   385  }
   386  
   387  // TestOptionsComplete checks that command line flags are combined with a
   388  // config properly.
   389  func TestOptionsComplete(t *testing.T) {
   390  	header := `apiVersion: kubeproxy.config.k8s.io/v1alpha1
   391  kind: KubeProxyConfiguration
   392  `
   393  
   394  	// Determine default config (depends on platform defaults).
   395  	o := NewOptions()
   396  	require.NoError(t, o.Complete(new(pflag.FlagSet)))
   397  	expected := o.config
   398  
   399  	config := header + `logging:
   400    format: json
   401    flushFrequency: 1s
   402    verbosity: 10
   403    vmodule:
   404    - filePattern: foo.go
   405      verbosity: 6
   406    - filePattern: bar.go
   407      verbosity: 8
   408  `
   409  	expectedLoggingConfig := logsapi.LoggingConfiguration{
   410  		Format:         "json",
   411  		FlushFrequency: logsapi.TimeOrMetaDuration{Duration: metav1.Duration{Duration: time.Second}, SerializeAsString: true},
   412  		Verbosity:      10,
   413  		VModule: []logsapi.VModuleItem{
   414  			{
   415  				FilePattern: "foo.go",
   416  				Verbosity:   6,
   417  			},
   418  			{
   419  				FilePattern: "bar.go",
   420  				Verbosity:   8,
   421  			},
   422  		},
   423  		Options: logsapi.FormatOptions{
   424  			JSON: logsapi.JSONOptions{
   425  				OutputRoutingOptions: logsapi.OutputRoutingOptions{
   426  					InfoBufferSize: resource.QuantityValue{Quantity: resource.MustParse("0")},
   427  				},
   428  			},
   429  			Text: logsapi.TextOptions{
   430  				OutputRoutingOptions: logsapi.OutputRoutingOptions{
   431  					InfoBufferSize: resource.QuantityValue{Quantity: resource.MustParse("0")},
   432  				},
   433  			},
   434  		},
   435  	}
   436  
   437  	for name, tc := range map[string]struct {
   438  		config   string
   439  		flags    []string
   440  		expected *kubeproxyconfig.KubeProxyConfiguration
   441  	}{
   442  		"empty": {
   443  			expected: expected,
   444  		},
   445  		"empty-config": {
   446  			config:   header,
   447  			expected: expected,
   448  		},
   449  		"logging-config": {
   450  			config: config,
   451  			expected: func() *kubeproxyconfig.KubeProxyConfiguration {
   452  				c := expected.DeepCopy()
   453  				c.Logging = *expectedLoggingConfig.DeepCopy()
   454  				return c
   455  			}(),
   456  		},
   457  		"flags": {
   458  			flags: []string{
   459  				"-v=7",
   460  				"--vmodule", "goo.go=8",
   461  			},
   462  			expected: func() *kubeproxyconfig.KubeProxyConfiguration {
   463  				c := expected.DeepCopy()
   464  				c.Logging.Verbosity = 7
   465  				c.Logging.VModule = append(c.Logging.VModule, logsapi.VModuleItem{
   466  					FilePattern: "goo.go",
   467  					Verbosity:   8,
   468  				})
   469  				return c
   470  			}(),
   471  		},
   472  		"both": {
   473  			config: config,
   474  			flags: []string{
   475  				"-v=7",
   476  				"--vmodule", "goo.go=8",
   477  				"--ipvs-scheduler", "some-scheduler", // Overwritten by config.
   478  			},
   479  			expected: func() *kubeproxyconfig.KubeProxyConfiguration {
   480  				c := expected.DeepCopy()
   481  				c.Logging = *expectedLoggingConfig.DeepCopy()
   482  				// Flag wins.
   483  				c.Logging.Verbosity = 7
   484  				// Flag and config get merged with command line flags first.
   485  				c.Logging.VModule = append([]logsapi.VModuleItem{
   486  					{
   487  						FilePattern: "goo.go",
   488  						Verbosity:   8,
   489  					},
   490  				}, c.Logging.VModule...)
   491  				return c
   492  			}(),
   493  		},
   494  	} {
   495  		t.Run(name, func(t *testing.T) {
   496  			options := NewOptions()
   497  			fs := new(pflag.FlagSet)
   498  			options.AddFlags(fs)
   499  			flags := tc.flags
   500  			if len(tc.config) > 0 {
   501  				tmp := t.TempDir()
   502  				configFile := path.Join(tmp, "kube-proxy.conf")
   503  				require.NoError(t, ioutil.WriteFile(configFile, []byte(tc.config), 0666))
   504  				flags = append(flags, "--config", configFile)
   505  			}
   506  			require.NoError(t, fs.Parse(flags))
   507  			require.NoError(t, options.Complete(fs))
   508  			assert.Equal(t, tc.expected, options.config)
   509  		})
   510  	}
   511  }
   512  
   513  type fakeProxyServerLongRun struct{}
   514  
   515  // Run runs the specified ProxyServer.
   516  func (s *fakeProxyServerLongRun) Run() error {
   517  	for {
   518  		time.Sleep(2 * time.Second)
   519  	}
   520  }
   521  
   522  // CleanupAndExit runs in the specified ProxyServer.
   523  func (s *fakeProxyServerLongRun) CleanupAndExit() error {
   524  	return nil
   525  }
   526  
   527  type fakeProxyServerError struct{}
   528  
   529  // Run runs the specified ProxyServer.
   530  func (s *fakeProxyServerError) Run() error {
   531  	for {
   532  		time.Sleep(2 * time.Second)
   533  		return fmt.Errorf("mocking error from ProxyServer.Run()")
   534  	}
   535  }
   536  
   537  // CleanupAndExit runs in the specified ProxyServer.
   538  func (s *fakeProxyServerError) CleanupAndExit() error {
   539  	return errors.New("mocking error from ProxyServer.CleanupAndExit()")
   540  }
   541  
   542  func TestAddressFromDeprecatedFlags(t *testing.T) {
   543  	testCases := []struct {
   544  		name               string
   545  		healthzPort        int32
   546  		healthzBindAddress string
   547  		metricsPort        int32
   548  		metricsBindAddress string
   549  		expHealthz         string
   550  		expMetrics         string
   551  	}{
   552  		{
   553  			name:               "IPv4 bind address",
   554  			healthzBindAddress: "1.2.3.4",
   555  			healthzPort:        12345,
   556  			metricsBindAddress: "2.3.4.5",
   557  			metricsPort:        23456,
   558  			expHealthz:         "1.2.3.4:12345",
   559  			expMetrics:         "2.3.4.5:23456",
   560  		},
   561  		{
   562  			name:               "IPv4 bind address has port",
   563  			healthzBindAddress: "1.2.3.4:12345",
   564  			healthzPort:        23456,
   565  			metricsBindAddress: "2.3.4.5:12345",
   566  			metricsPort:        23456,
   567  			expHealthz:         "1.2.3.4:12345",
   568  			expMetrics:         "2.3.4.5:12345",
   569  		},
   570  		{
   571  			name:               "IPv6 bind address",
   572  			healthzBindAddress: "fd00:1::5",
   573  			healthzPort:        12345,
   574  			metricsBindAddress: "fd00:1::6",
   575  			metricsPort:        23456,
   576  			expHealthz:         "[fd00:1::5]:12345",
   577  			expMetrics:         "[fd00:1::6]:23456",
   578  		},
   579  		{
   580  			name:               "IPv6 bind address has port",
   581  			healthzBindAddress: "[fd00:1::5]:12345",
   582  			healthzPort:        56789,
   583  			metricsBindAddress: "[fd00:1::6]:56789",
   584  			metricsPort:        12345,
   585  			expHealthz:         "[fd00:1::5]:12345",
   586  			expMetrics:         "[fd00:1::6]:56789",
   587  		},
   588  		{
   589  			name:               "Invalid IPv6 Config",
   590  			healthzBindAddress: "[fd00:1::5]",
   591  			healthzPort:        12345,
   592  			metricsBindAddress: "[fd00:1::6]",
   593  			metricsPort:        56789,
   594  			expHealthz:         "[fd00:1::5]",
   595  			expMetrics:         "[fd00:1::6]",
   596  		},
   597  	}
   598  
   599  	for i := range testCases {
   600  		gotHealthz := addressFromDeprecatedFlags(testCases[i].healthzBindAddress, testCases[i].healthzPort)
   601  		gotMetrics := addressFromDeprecatedFlags(testCases[i].metricsBindAddress, testCases[i].metricsPort)
   602  
   603  		errFn := func(name, except, got string) {
   604  			t.Errorf("case %s: expected %v, got %v", name, except, got)
   605  		}
   606  
   607  		if gotHealthz != testCases[i].expHealthz {
   608  			errFn(testCases[i].name, testCases[i].expHealthz, gotHealthz)
   609  		}
   610  
   611  		if gotMetrics != testCases[i].expMetrics {
   612  			errFn(testCases[i].name, testCases[i].expMetrics, gotMetrics)
   613  		}
   614  
   615  	}
   616  }
   617  
   618  func makeNodeWithAddress(name, primaryIP string) *v1.Node {
   619  	node := &v1.Node{
   620  		ObjectMeta: metav1.ObjectMeta{
   621  			Name: name,
   622  		},
   623  		Status: v1.NodeStatus{
   624  			Addresses: []v1.NodeAddress{},
   625  		},
   626  	}
   627  
   628  	if primaryIP != "" {
   629  		node.Status.Addresses = append(node.Status.Addresses,
   630  			v1.NodeAddress{Type: v1.NodeInternalIP, Address: primaryIP},
   631  		)
   632  	}
   633  
   634  	return node
   635  }
   636  
   637  // Test that getNodeIPs retries on failure
   638  func Test_getNodeIPs(t *testing.T) {
   639  	var chans [3]chan error
   640  
   641  	client := clientsetfake.NewSimpleClientset(
   642  		// node1 initially has no IP address.
   643  		makeNodeWithAddress("node1", ""),
   644  
   645  		// node2 initially has an invalid IP address.
   646  		makeNodeWithAddress("node2", "invalid-ip"),
   647  
   648  		// node3 initially does not exist.
   649  	)
   650  
   651  	for i := range chans {
   652  		chans[i] = make(chan error)
   653  		ch := chans[i]
   654  		nodeName := fmt.Sprintf("node%d", i+1)
   655  		expectIP := fmt.Sprintf("192.168.0.%d", i+1)
   656  		go func() {
   657  			logger, _ := ktesting.NewTestContext(t)
   658  			ips := getNodeIPs(logger, client, nodeName)
   659  			if len(ips) == 0 {
   660  				ch <- fmt.Errorf("expected IP %s for %s but got nil", expectIP, nodeName)
   661  			} else if ips[0].String() != expectIP {
   662  				ch <- fmt.Errorf("expected IP %s for %s but got %s", expectIP, nodeName, ips[0].String())
   663  			} else if len(ips) != 1 {
   664  				ch <- fmt.Errorf("expected IP %s for %s but got multiple IPs", expectIP, nodeName)
   665  			}
   666  			close(ch)
   667  		}()
   668  	}
   669  
   670  	// Give the goroutines time to fetch the bad/non-existent nodes, then fix them.
   671  	time.Sleep(1200 * time.Millisecond)
   672  
   673  	_, _ = client.CoreV1().Nodes().UpdateStatus(context.TODO(),
   674  		makeNodeWithAddress("node1", "192.168.0.1"),
   675  		metav1.UpdateOptions{},
   676  	)
   677  	_, _ = client.CoreV1().Nodes().UpdateStatus(context.TODO(),
   678  		makeNodeWithAddress("node2", "192.168.0.2"),
   679  		metav1.UpdateOptions{},
   680  	)
   681  	_, _ = client.CoreV1().Nodes().Create(context.TODO(),
   682  		makeNodeWithAddress("node3", "192.168.0.3"),
   683  		metav1.CreateOptions{},
   684  	)
   685  
   686  	// Ensure each getNodeIP completed as expected
   687  	for i := range chans {
   688  		err := <-chans[i]
   689  		if err != nil {
   690  			t.Error(err.Error())
   691  		}
   692  	}
   693  }
   694  
   695  func Test_detectNodeIPs(t *testing.T) {
   696  	cases := []struct {
   697  		name           string
   698  		rawNodeIPs     []net.IP
   699  		bindAddress    string
   700  		expectedFamily v1.IPFamily
   701  		expectedIPv4   string
   702  		expectedIPv6   string
   703  	}{
   704  		{
   705  			name:           "Bind address IPv4 unicast address and no Node object",
   706  			rawNodeIPs:     nil,
   707  			bindAddress:    "10.0.0.1",
   708  			expectedFamily: v1.IPv4Protocol,
   709  			expectedIPv4:   "10.0.0.1",
   710  			expectedIPv6:   "::1",
   711  		},
   712  		{
   713  			name:           "Bind address IPv6 unicast address and no Node object",
   714  			rawNodeIPs:     nil,
   715  			bindAddress:    "fd00:4321::2",
   716  			expectedFamily: v1.IPv6Protocol,
   717  			expectedIPv4:   "127.0.0.1",
   718  			expectedIPv6:   "fd00:4321::2",
   719  		},
   720  		{
   721  			name:           "No Valid IP found and no bind address",
   722  			rawNodeIPs:     nil,
   723  			bindAddress:    "",
   724  			expectedFamily: v1.IPv4Protocol,
   725  			expectedIPv4:   "127.0.0.1",
   726  			expectedIPv6:   "::1",
   727  		},
   728  		{
   729  			name:           "No Valid IP found and unspecified bind address",
   730  			rawNodeIPs:     nil,
   731  			bindAddress:    "0.0.0.0",
   732  			expectedFamily: v1.IPv4Protocol,
   733  			expectedIPv4:   "127.0.0.1",
   734  			expectedIPv6:   "::1",
   735  		},
   736  		{
   737  			name:           "Bind address 0.0.0.0 and node with IPv4 InternalIP set",
   738  			rawNodeIPs:     []net.IP{netutils.ParseIPSloppy("192.168.1.1")},
   739  			bindAddress:    "0.0.0.0",
   740  			expectedFamily: v1.IPv4Protocol,
   741  			expectedIPv4:   "192.168.1.1",
   742  			expectedIPv6:   "::1",
   743  		},
   744  		{
   745  			name:           "Bind address :: and node with IPv4 InternalIP set",
   746  			rawNodeIPs:     []net.IP{netutils.ParseIPSloppy("192.168.1.1")},
   747  			bindAddress:    "::",
   748  			expectedFamily: v1.IPv4Protocol,
   749  			expectedIPv4:   "192.168.1.1",
   750  			expectedIPv6:   "::1",
   751  		},
   752  		{
   753  			name:           "Bind address 0.0.0.0 and node with IPv6 InternalIP set",
   754  			rawNodeIPs:     []net.IP{netutils.ParseIPSloppy("fd00:1234::1")},
   755  			bindAddress:    "0.0.0.0",
   756  			expectedFamily: v1.IPv6Protocol,
   757  			expectedIPv4:   "127.0.0.1",
   758  			expectedIPv6:   "fd00:1234::1",
   759  		},
   760  		{
   761  			name:           "Bind address :: and node with IPv6 InternalIP set",
   762  			rawNodeIPs:     []net.IP{netutils.ParseIPSloppy("fd00:1234::1")},
   763  			bindAddress:    "::",
   764  			expectedFamily: v1.IPv6Protocol,
   765  			expectedIPv4:   "127.0.0.1",
   766  			expectedIPv6:   "fd00:1234::1",
   767  		},
   768  		{
   769  			name: "Dual stack, primary IPv4",
   770  			rawNodeIPs: []net.IP{
   771  				netutils.ParseIPSloppy("90.90.90.90"),
   772  				netutils.ParseIPSloppy("2001:db8::2"),
   773  			},
   774  			bindAddress:    "::",
   775  			expectedFamily: v1.IPv4Protocol,
   776  			expectedIPv4:   "90.90.90.90",
   777  			expectedIPv6:   "2001:db8::2",
   778  		},
   779  		{
   780  			name: "Dual stack, primary IPv6",
   781  			rawNodeIPs: []net.IP{
   782  				netutils.ParseIPSloppy("2001:db8::2"),
   783  				netutils.ParseIPSloppy("90.90.90.90"),
   784  			},
   785  			bindAddress:    "0.0.0.0",
   786  			expectedFamily: v1.IPv6Protocol,
   787  			expectedIPv4:   "90.90.90.90",
   788  			expectedIPv6:   "2001:db8::2",
   789  		},
   790  		{
   791  			name: "Dual stack, override IPv4",
   792  			rawNodeIPs: []net.IP{
   793  				netutils.ParseIPSloppy("2001:db8::2"),
   794  				netutils.ParseIPSloppy("90.90.90.90"),
   795  			},
   796  			bindAddress:    "80.80.80.80",
   797  			expectedFamily: v1.IPv4Protocol,
   798  			expectedIPv4:   "80.80.80.80",
   799  			expectedIPv6:   "2001:db8::2",
   800  		},
   801  		{
   802  			name: "Dual stack, override IPv6",
   803  			rawNodeIPs: []net.IP{
   804  				netutils.ParseIPSloppy("90.90.90.90"),
   805  				netutils.ParseIPSloppy("2001:db8::2"),
   806  			},
   807  			bindAddress:    "2001:db8::555",
   808  			expectedFamily: v1.IPv6Protocol,
   809  			expectedIPv4:   "90.90.90.90",
   810  			expectedIPv6:   "2001:db8::555",
   811  		},
   812  		{
   813  			name: "Dual stack, override primary family, IPv4",
   814  			rawNodeIPs: []net.IP{
   815  				netutils.ParseIPSloppy("2001:db8::2"),
   816  				netutils.ParseIPSloppy("90.90.90.90"),
   817  			},
   818  			bindAddress:    "127.0.0.1",
   819  			expectedFamily: v1.IPv4Protocol,
   820  			expectedIPv4:   "127.0.0.1",
   821  			expectedIPv6:   "2001:db8::2",
   822  		},
   823  		{
   824  			name: "Dual stack, override primary family, IPv6",
   825  			rawNodeIPs: []net.IP{
   826  				netutils.ParseIPSloppy("90.90.90.90"),
   827  				netutils.ParseIPSloppy("2001:db8::2"),
   828  			},
   829  			bindAddress:    "::1",
   830  			expectedFamily: v1.IPv6Protocol,
   831  			expectedIPv4:   "90.90.90.90",
   832  			expectedIPv6:   "::1",
   833  		},
   834  	}
   835  	for _, c := range cases {
   836  		t.Run(c.name, func(t *testing.T) {
   837  			logger, _ := ktesting.NewTestContext(t)
   838  			primaryFamily, ips := detectNodeIPs(logger, c.rawNodeIPs, c.bindAddress)
   839  			if primaryFamily != c.expectedFamily {
   840  				t.Errorf("Expected family %q got %q", c.expectedFamily, primaryFamily)
   841  			}
   842  			if ips[v1.IPv4Protocol].String() != c.expectedIPv4 {
   843  				t.Errorf("Expected IPv4 %q got %q", c.expectedIPv4, ips[v1.IPv4Protocol].String())
   844  			}
   845  			if ips[v1.IPv6Protocol].String() != c.expectedIPv6 {
   846  				t.Errorf("Expected IPv6 %q got %q", c.expectedIPv6, ips[v1.IPv6Protocol].String())
   847  			}
   848  		})
   849  	}
   850  }
   851  
   852  func Test_checkIPConfig(t *testing.T) {
   853  	cases := []struct {
   854  		name    string
   855  		proxy   *ProxyServer
   856  		ssErr   bool
   857  		ssFatal bool
   858  		dsErr   bool
   859  		dsFatal bool
   860  	}{
   861  		{
   862  			name: "empty config",
   863  			proxy: &ProxyServer{
   864  				Config:          &kubeproxyconfig.KubeProxyConfiguration{},
   865  				PrimaryIPFamily: v1.IPv4Protocol,
   866  			},
   867  			ssErr: false,
   868  			dsErr: false,
   869  		},
   870  
   871  		{
   872  			name: "ok single-stack clusterCIDR",
   873  			proxy: &ProxyServer{
   874  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   875  					ClusterCIDR: "10.0.0.0/8",
   876  				},
   877  				PrimaryIPFamily: v1.IPv4Protocol,
   878  			},
   879  			ssErr: false,
   880  			dsErr: false,
   881  		},
   882  		{
   883  			name: "ok dual-stack clusterCIDR",
   884  			proxy: &ProxyServer{
   885  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   886  					ClusterCIDR: "10.0.0.0/8,fd01:2345::/64",
   887  				},
   888  				PrimaryIPFamily: v1.IPv4Protocol,
   889  			},
   890  			ssErr: false,
   891  			dsErr: false,
   892  		},
   893  		{
   894  			name: "ok reversed dual-stack clusterCIDR",
   895  			proxy: &ProxyServer{
   896  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   897  					ClusterCIDR: "fd01:2345::/64,10.0.0.0/8",
   898  				},
   899  				PrimaryIPFamily: v1.IPv4Protocol,
   900  			},
   901  			ssErr: false,
   902  			dsErr: false,
   903  		},
   904  		{
   905  			name: "wrong-family clusterCIDR",
   906  			proxy: &ProxyServer{
   907  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   908  					ClusterCIDR: "fd01:2345::/64",
   909  				},
   910  				PrimaryIPFamily: v1.IPv4Protocol,
   911  			},
   912  			ssErr:   true,
   913  			ssFatal: false,
   914  			dsErr:   true,
   915  			dsFatal: false,
   916  		},
   917  		{
   918  			name: "wrong-family clusterCIDR when using ClusterCIDR LocalDetector",
   919  			proxy: &ProxyServer{
   920  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   921  					ClusterCIDR:     "fd01:2345::/64",
   922  					DetectLocalMode: kubeproxyconfig.LocalModeClusterCIDR,
   923  				},
   924  				PrimaryIPFamily: v1.IPv4Protocol,
   925  			},
   926  			ssErr:   true,
   927  			ssFatal: true,
   928  			dsErr:   true,
   929  			dsFatal: false,
   930  		},
   931  
   932  		{
   933  			name: "ok single-stack nodePortAddresses",
   934  			proxy: &ProxyServer{
   935  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   936  					NodePortAddresses: []string{"10.0.0.0/8", "192.168.0.0/24"},
   937  				},
   938  				PrimaryIPFamily: v1.IPv4Protocol,
   939  			},
   940  			ssErr: false,
   941  			dsErr: false,
   942  		},
   943  		{
   944  			name: "ok dual-stack nodePortAddresses",
   945  			proxy: &ProxyServer{
   946  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   947  					NodePortAddresses: []string{"10.0.0.0/8", "fd01:2345::/64", "fd01:abcd::/64"},
   948  				},
   949  				PrimaryIPFamily: v1.IPv4Protocol,
   950  			},
   951  			ssErr: false,
   952  			dsErr: false,
   953  		},
   954  		{
   955  			name: "ok reversed dual-stack nodePortAddresses",
   956  			proxy: &ProxyServer{
   957  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   958  					NodePortAddresses: []string{"fd01:2345::/64", "fd01:abcd::/64", "10.0.0.0/8"},
   959  				},
   960  				PrimaryIPFamily: v1.IPv4Protocol,
   961  			},
   962  			ssErr: false,
   963  			dsErr: false,
   964  		},
   965  		{
   966  			name: "wrong-family nodePortAddresses",
   967  			proxy: &ProxyServer{
   968  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   969  					NodePortAddresses: []string{"10.0.0.0/8"},
   970  				},
   971  				PrimaryIPFamily: v1.IPv6Protocol,
   972  			},
   973  			ssErr:   true,
   974  			ssFatal: false,
   975  			dsErr:   true,
   976  			dsFatal: false,
   977  		},
   978  
   979  		{
   980  			name: "ok single-stack node.spec.podCIDRs",
   981  			proxy: &ProxyServer{
   982  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   983  					DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
   984  				},
   985  				PrimaryIPFamily: v1.IPv4Protocol,
   986  				podCIDRs:        []string{"10.0.0.0/8"},
   987  			},
   988  			ssErr: false,
   989  			dsErr: false,
   990  		},
   991  		{
   992  			name: "ok dual-stack node.spec.podCIDRs",
   993  			proxy: &ProxyServer{
   994  				Config: &kubeproxyconfig.KubeProxyConfiguration{
   995  					DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
   996  				},
   997  				PrimaryIPFamily: v1.IPv4Protocol,
   998  				podCIDRs:        []string{"10.0.0.0/8", "fd01:2345::/64"},
   999  			},
  1000  			ssErr: false,
  1001  			dsErr: false,
  1002  		},
  1003  		{
  1004  			name: "ok reversed dual-stack node.spec.podCIDRs",
  1005  			proxy: &ProxyServer{
  1006  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1007  					DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
  1008  				},
  1009  				PrimaryIPFamily: v1.IPv4Protocol,
  1010  				podCIDRs:        []string{"fd01:2345::/64", "10.0.0.0/8"},
  1011  			},
  1012  			ssErr: false,
  1013  			dsErr: false,
  1014  		},
  1015  		{
  1016  			name: "wrong-family node.spec.podCIDRs",
  1017  			proxy: &ProxyServer{
  1018  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1019  					DetectLocalMode: kubeproxyconfig.LocalModeNodeCIDR,
  1020  				},
  1021  				PrimaryIPFamily: v1.IPv4Protocol,
  1022  				podCIDRs:        []string{"fd01:2345::/64"},
  1023  			},
  1024  			ssErr:   true,
  1025  			ssFatal: true,
  1026  			dsErr:   true,
  1027  			dsFatal: true,
  1028  		},
  1029  
  1030  		{
  1031  			name: "ok winkernel.sourceVip",
  1032  			proxy: &ProxyServer{
  1033  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1034  					Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{
  1035  						SourceVip: "10.0.0.1",
  1036  					},
  1037  				},
  1038  				PrimaryIPFamily: v1.IPv4Protocol,
  1039  			},
  1040  			ssErr: false,
  1041  			dsErr: false,
  1042  		},
  1043  		{
  1044  			name: "wrong family winkernel.sourceVip",
  1045  			proxy: &ProxyServer{
  1046  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1047  					Winkernel: kubeproxyconfig.KubeProxyWinkernelConfiguration{
  1048  						SourceVip: "fd01:2345::1",
  1049  					},
  1050  				},
  1051  				PrimaryIPFamily: v1.IPv4Protocol,
  1052  			},
  1053  			ssErr:   true,
  1054  			ssFatal: false,
  1055  			dsErr:   true,
  1056  			dsFatal: false,
  1057  		},
  1058  
  1059  		{
  1060  			name: "ok IPv4 metricsBindAddress",
  1061  			proxy: &ProxyServer{
  1062  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1063  					MetricsBindAddress: "10.0.0.1:9999",
  1064  				},
  1065  				PrimaryIPFamily: v1.IPv4Protocol,
  1066  			},
  1067  			ssErr: false,
  1068  			dsErr: false,
  1069  		},
  1070  		{
  1071  			name: "ok IPv6 metricsBindAddress",
  1072  			proxy: &ProxyServer{
  1073  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1074  					MetricsBindAddress: "[fd01:2345::1]:9999",
  1075  				},
  1076  				PrimaryIPFamily: v1.IPv6Protocol,
  1077  			},
  1078  			ssErr: false,
  1079  			dsErr: false,
  1080  		},
  1081  		{
  1082  			name: "ok unspecified wrong-family metricsBindAddress",
  1083  			proxy: &ProxyServer{
  1084  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1085  					MetricsBindAddress: "0.0.0.0:9999",
  1086  				},
  1087  				PrimaryIPFamily: v1.IPv6Protocol,
  1088  			},
  1089  			ssErr: false,
  1090  			dsErr: false,
  1091  		},
  1092  		{
  1093  			name: "wrong family metricsBindAddress",
  1094  			proxy: &ProxyServer{
  1095  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1096  					MetricsBindAddress: "10.0.0.1:9999",
  1097  				},
  1098  				PrimaryIPFamily: v1.IPv6Protocol,
  1099  			},
  1100  			ssErr:   true,
  1101  			ssFatal: false,
  1102  			dsErr:   false,
  1103  		},
  1104  
  1105  		{
  1106  			name: "ok ipvs.excludeCIDRs",
  1107  			proxy: &ProxyServer{
  1108  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1109  					IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
  1110  						ExcludeCIDRs: []string{"10.0.0.0/8"},
  1111  					},
  1112  				},
  1113  				PrimaryIPFamily: v1.IPv4Protocol,
  1114  			},
  1115  			ssErr: false,
  1116  			dsErr: false,
  1117  		},
  1118  		{
  1119  			name: "wrong family ipvs.excludeCIDRs",
  1120  			proxy: &ProxyServer{
  1121  				Config: &kubeproxyconfig.KubeProxyConfiguration{
  1122  					IPVS: kubeproxyconfig.KubeProxyIPVSConfiguration{
  1123  						ExcludeCIDRs: []string{"10.0.0.0/8", "192.168.0.0/24"},
  1124  					},
  1125  				},
  1126  				PrimaryIPFamily: v1.IPv6Protocol,
  1127  			},
  1128  			ssErr:   true,
  1129  			ssFatal: false,
  1130  			dsErr:   false,
  1131  		},
  1132  	}
  1133  
  1134  	for _, c := range cases {
  1135  		t.Run(c.name, func(t *testing.T) {
  1136  			err, fatal := checkIPConfig(c.proxy, false)
  1137  			if err != nil && !c.ssErr {
  1138  				t.Errorf("unexpected error in single-stack case: %v", err)
  1139  			} else if err == nil && c.ssErr {
  1140  				t.Errorf("unexpected lack of error in single-stack case")
  1141  			} else if fatal != c.ssFatal {
  1142  				t.Errorf("expected fatal=%v, got %v", c.ssFatal, fatal)
  1143  			}
  1144  
  1145  			err, fatal = checkIPConfig(c.proxy, true)
  1146  			if err != nil && !c.dsErr {
  1147  				t.Errorf("unexpected error in dual-stack case: %v", err)
  1148  			} else if err == nil && c.dsErr {
  1149  				t.Errorf("unexpected lack of error in dual-stack case")
  1150  			} else if fatal != c.dsFatal {
  1151  				t.Errorf("expected fatal=%v, got %v", c.dsFatal, fatal)
  1152  			}
  1153  		})
  1154  	}
  1155  }
  1156
View as plain text