Source file src/k8s.io/kubernetes/test/integration/dualstack/dualstack_test.go

Documentation: k8s.io/kubernetes/test/integration/dualstack

     1  /*
     2  Copyright 2020 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package dualstack
    18  
    19  import (
    20  	"encoding/json"
    21  	"fmt"
    22  	"reflect"
    23  	"strings"
    24  	"testing"
    25  	"time"
    26  
    27  	jsonpatch "github.com/evanphx/json-patch"
    28  
    29  	v1 "k8s.io/api/core/v1"
    30  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    31  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    32  	"k8s.io/apimachinery/pkg/types"
    33  
    34  	"k8s.io/apimachinery/pkg/util/intstr"
    35  	"k8s.io/apimachinery/pkg/util/wait"
    36  
    37  	"k8s.io/kubernetes/cmd/kube-apiserver/app/options"
    38  	"k8s.io/kubernetes/test/integration/framework"
    39  	"k8s.io/kubernetes/test/utils/ktesting"
    40  	netutils "k8s.io/utils/net"
    41  )
    42  
    43  // TestCreateServiceSingleStackIPv4 test the Service dualstackness in an IPv4 SingleStack cluster
    44  func TestCreateServiceSingleStackIPv4(t *testing.T) {
    45  	// Create an IPv4 single stack control-plane
    46  	serviceCIDR := "10.0.0.0/16"
    47  
    48  	tCtx := ktesting.Init(t)
    49  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
    50  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
    51  			opts.ServiceClusterIPRanges = serviceCIDR
    52  		},
    53  	})
    54  	defer tearDownFn()
    55  
    56  	// Wait until the default "kubernetes" service is created.
    57  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
    58  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
    59  		if err != nil && !apierrors.IsNotFound(err) {
    60  			return false, err
    61  		}
    62  		return !apierrors.IsNotFound(err), nil
    63  	}); err != nil {
    64  		t.Fatalf("creating kubernetes service timed out")
    65  	}
    66  
    67  	var testcases = []struct {
    68  		name               string
    69  		serviceType        v1.ServiceType
    70  		clusterIPs         []string
    71  		ipFamilies         []v1.IPFamily
    72  		ipFamilyPolicy     v1.IPFamilyPolicy
    73  		expectedIPFamilies []v1.IPFamily
    74  		expectError        bool
    75  	}{
    76  		{
    77  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack",
    78  			serviceType:        v1.ServiceTypeClusterIP,
    79  			clusterIPs:         nil,
    80  			ipFamilies:         nil,
    81  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
    82  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
    83  			expectError:        false,
    84  		},
    85  		{
    86  			name:               "Type ClusterIP - Client Allocated IP - Default IP Family - Policy Single Stack",
    87  			serviceType:        v1.ServiceTypeClusterIP,
    88  			clusterIPs:         []string{"10.0.0.16"},
    89  			ipFamilies:         nil,
    90  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
    91  			expectError:        false,
    92  		},
    93  		{
    94  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack",
    95  			serviceType:        v1.ServiceTypeClusterIP,
    96  			clusterIPs:         []string{},
    97  			ipFamilies:         nil,
    98  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
    99  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   100  			expectError:        false,
   101  		},
   102  		{
   103  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack",
   104  			serviceType:        v1.ServiceTypeClusterIP,
   105  			clusterIPs:         nil,
   106  			ipFamilies:         nil,
   107  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   108  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   109  			expectError:        true,
   110  		},
   111  		{
   112  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack",
   113  			serviceType:        v1.ServiceTypeClusterIP,
   114  			clusterIPs:         nil,
   115  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   116  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   117  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   118  			expectError:        false,
   119  		},
   120  		{
   121  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack",
   122  			serviceType:        v1.ServiceTypeClusterIP,
   123  			clusterIPs:         nil,
   124  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   125  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   126  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   127  			expectError:        false,
   128  		},
   129  		{
   130  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack",
   131  			serviceType:        v1.ServiceTypeClusterIP,
   132  			clusterIPs:         nil,
   133  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   134  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   135  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   136  			expectError:        true,
   137  		},
   138  		{
   139  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack",
   140  			serviceType:        v1.ServiceTypeClusterIP,
   141  			clusterIPs:         nil,
   142  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   143  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   144  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   145  			expectError:        true,
   146  		},
   147  		{
   148  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack",
   149  			serviceType:        v1.ServiceTypeClusterIP,
   150  			clusterIPs:         nil,
   151  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   152  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   153  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   154  			expectError:        true,
   155  		},
   156  		{
   157  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack",
   158  			serviceType:        v1.ServiceTypeClusterIP,
   159  			clusterIPs:         nil,
   160  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   161  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   162  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   163  			expectError:        true,
   164  		},
   165  		{
   166  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack",
   167  			serviceType:        v1.ServiceTypeClusterIP,
   168  			clusterIPs:         nil,
   169  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   170  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   171  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   172  			expectError:        true,
   173  		},
   174  		{
   175  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack",
   176  			serviceType:        v1.ServiceTypeClusterIP,
   177  			clusterIPs:         nil,
   178  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   179  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   180  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   181  			expectError:        true,
   182  		},
   183  		{
   184  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack",
   185  			serviceType:        v1.ServiceTypeClusterIP,
   186  			clusterIPs:         nil,
   187  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   188  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   189  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   190  			expectError:        true,
   191  		},
   192  		{
   193  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack",
   194  			serviceType:        v1.ServiceTypeClusterIP,
   195  			clusterIPs:         nil,
   196  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   197  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   198  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   199  			expectError:        true,
   200  		},
   201  		{
   202  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack",
   203  			serviceType:        v1.ServiceTypeClusterIP,
   204  			clusterIPs:         nil,
   205  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   206  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   207  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   208  			expectError:        true,
   209  		},
   210  		{
   211  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack",
   212  			serviceType:        v1.ServiceTypeClusterIP,
   213  			clusterIPs:         nil,
   214  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   215  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   216  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   217  			expectError:        true,
   218  		},
   219  	}
   220  
   221  	for i, tc := range testcases {
   222  		tc := tc
   223  		t.Run(tc.name, func(t *testing.T) {
   224  
   225  			svc := &v1.Service{
   226  				ObjectMeta: metav1.ObjectMeta{
   227  					Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test
   228  				},
   229  				Spec: v1.ServiceSpec{
   230  					Type:       tc.serviceType,
   231  					ClusterIPs: tc.clusterIPs,
   232  					IPFamilies: tc.ipFamilies,
   233  					Ports: []v1.ServicePort{
   234  						{
   235  							Port:       443,
   236  							TargetPort: intstr.FromInt32(443),
   237  						},
   238  					},
   239  				},
   240  			}
   241  
   242  			if len(tc.ipFamilyPolicy) > 0 {
   243  				svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy
   244  			}
   245  
   246  			if len(tc.clusterIPs) > 0 {
   247  				svc.Spec.ClusterIP = tc.clusterIPs[0]
   248  			}
   249  
   250  			// create the service
   251  			_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
   252  			if (err != nil) != tc.expectError {
   253  				t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err)
   254  			}
   255  			// if no error was expected validate the service otherwise return
   256  			if err != nil {
   257  				return
   258  			}
   259  			// validate the service was created correctly if it was not expected to fail
   260  			svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
   261  			if err != nil {
   262  				t.Errorf("Unexpected error to get the service %s %v", svc.Name, err)
   263  			}
   264  			if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil {
   265  				t.Errorf("Unexpected error validating the service %s\n%+v\n%v", svc.Name, svc, err)
   266  			}
   267  		})
   268  	}
   269  }
   270  
   271  // TestCreateServiceDualStackIPv6 test the Service dualstackness in an IPv6 only DualStack cluster
   272  func TestCreateServiceDualStackIPv6(t *testing.T) {
   273  	// Create an IPv6 only dual stack control-plane
   274  	serviceCIDR := "2001:db8:1::/112"
   275  
   276  	tCtx := ktesting.Init(t)
   277  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   278  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   279  			opts.ServiceClusterIPRanges = serviceCIDR
   280  			opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10")
   281  		},
   282  	})
   283  	defer tearDownFn()
   284  
   285  	// Wait until the default "kubernetes" service is created.
   286  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
   287  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
   288  		if err != nil && !apierrors.IsNotFound(err) {
   289  			return false, err
   290  		}
   291  		return !apierrors.IsNotFound(err), nil
   292  	}); err != nil {
   293  		t.Fatalf("creating kubernetes service timed out")
   294  	}
   295  
   296  	var testcases = []struct {
   297  		name               string
   298  		serviceType        v1.ServiceType
   299  		clusterIPs         []string
   300  		ipFamilies         []v1.IPFamily
   301  		expectedIPFamilies []v1.IPFamily
   302  		ipFamilyPolicy     v1.IPFamilyPolicy
   303  		expectError        bool
   304  	}{
   305  		{
   306  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack",
   307  			serviceType:        v1.ServiceTypeClusterIP,
   308  			clusterIPs:         []string{},
   309  			ipFamilies:         nil,
   310  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol},
   311  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   312  			expectError:        false,
   313  		},
   314  		{
   315  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack",
   316  			serviceType:        v1.ServiceTypeClusterIP,
   317  			clusterIPs:         []string{},
   318  			ipFamilies:         nil,
   319  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol},
   320  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   321  			expectError:        false,
   322  		},
   323  		{
   324  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack",
   325  			serviceType:        v1.ServiceTypeClusterIP,
   326  			clusterIPs:         []string{},
   327  			ipFamilies:         nil,
   328  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   329  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   330  			expectError:        true,
   331  		},
   332  		{
   333  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack",
   334  			serviceType:        v1.ServiceTypeClusterIP,
   335  			clusterIPs:         []string{},
   336  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   337  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   338  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   339  			expectError:        true,
   340  		},
   341  		{
   342  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack",
   343  			serviceType:        v1.ServiceTypeClusterIP,
   344  			clusterIPs:         []string{},
   345  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   346  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   347  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   348  			expectError:        true,
   349  		},
   350  		{
   351  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack",
   352  			serviceType:        v1.ServiceTypeClusterIP,
   353  			clusterIPs:         []string{},
   354  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   355  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   356  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   357  			expectError:        true,
   358  		},
   359  		{
   360  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack",
   361  			serviceType:        v1.ServiceTypeClusterIP,
   362  			clusterIPs:         []string{},
   363  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   364  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol},
   365  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   366  			expectError:        false,
   367  		},
   368  		{
   369  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack",
   370  			serviceType:        v1.ServiceTypeClusterIP,
   371  			clusterIPs:         []string{},
   372  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   373  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol},
   374  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   375  			expectError:        false,
   376  		},
   377  		{
   378  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack",
   379  			serviceType:        v1.ServiceTypeClusterIP,
   380  			clusterIPs:         []string{},
   381  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   382  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   383  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   384  			expectError:        true,
   385  		},
   386  		{
   387  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack",
   388  			serviceType:        v1.ServiceTypeClusterIP,
   389  			clusterIPs:         []string{},
   390  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   391  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   392  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   393  			expectError:        true,
   394  		},
   395  		{
   396  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack",
   397  			serviceType:        v1.ServiceTypeClusterIP,
   398  			clusterIPs:         []string{},
   399  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   400  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   401  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   402  			expectError:        true,
   403  		},
   404  		{
   405  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack",
   406  			serviceType:        v1.ServiceTypeClusterIP,
   407  			clusterIPs:         []string{},
   408  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   409  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   410  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   411  			expectError:        true,
   412  		},
   413  		{
   414  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack",
   415  			serviceType:        v1.ServiceTypeClusterIP,
   416  			clusterIPs:         []string{},
   417  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   418  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   419  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   420  			expectError:        true,
   421  		},
   422  		{
   423  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack",
   424  			serviceType:        v1.ServiceTypeClusterIP,
   425  			clusterIPs:         []string{},
   426  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   427  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   428  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   429  			expectError:        true,
   430  		},
   431  		{
   432  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack",
   433  			serviceType:        v1.ServiceTypeClusterIP,
   434  			clusterIPs:         []string{},
   435  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   436  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   437  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   438  			expectError:        true,
   439  		},
   440  	}
   441  
   442  	for i, tc := range testcases {
   443  		tc := tc
   444  		t.Run(tc.name, func(t *testing.T) {
   445  
   446  			svc := &v1.Service{
   447  				ObjectMeta: metav1.ObjectMeta{
   448  					Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test
   449  				},
   450  				Spec: v1.ServiceSpec{
   451  					Type:           tc.serviceType,
   452  					ClusterIPs:     tc.clusterIPs,
   453  					IPFamilies:     tc.ipFamilies,
   454  					IPFamilyPolicy: &tc.ipFamilyPolicy,
   455  					Ports: []v1.ServicePort{
   456  						{
   457  							Name:       fmt.Sprintf("port-test-%d", i),
   458  							Port:       443,
   459  							TargetPort: intstr.IntOrString{IntVal: 443},
   460  							Protocol:   "TCP",
   461  						},
   462  					},
   463  				},
   464  			}
   465  
   466  			// create the service
   467  			_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
   468  			if (err != nil) != tc.expectError {
   469  				t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err)
   470  			}
   471  			// if no error was expected validate the service otherwise return
   472  			if err != nil {
   473  				return
   474  			}
   475  			// validate the service was created correctly if it was not expected to fail
   476  			svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
   477  			if err != nil {
   478  				t.Errorf("Unexpected error to get the service %s %v", svc.Name, err)
   479  			}
   480  			if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil {
   481  				t.Errorf("Unexpected error validating the service %s %v", svc.Name, err)
   482  			}
   483  		})
   484  	}
   485  }
   486  
   487  // TestCreateServiceDualStackIPv4IPv6 test the Service dualstackness in a IPv4IPv6 DualStack cluster
   488  func TestCreateServiceDualStackIPv4IPv6(t *testing.T) {
   489  	// Create an IPv4IPv6 dual stack control-plane
   490  	serviceCIDR := "10.0.0.0/16"
   491  	secondaryServiceCIDR := "2001:db8:1::/112"
   492  
   493  	tCtx := ktesting.Init(t)
   494  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   495  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   496  			opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR)
   497  		},
   498  	})
   499  	defer tearDownFn()
   500  
   501  	// Wait until the default "kubernetes" service is created.
   502  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
   503  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
   504  		if err != nil && !apierrors.IsNotFound(err) {
   505  			return false, err
   506  		}
   507  		return !apierrors.IsNotFound(err), nil
   508  	}); err != nil {
   509  		t.Fatalf("creating kubernetes service timed out")
   510  	}
   511  
   512  	var testcases = []struct {
   513  		name               string
   514  		serviceType        v1.ServiceType
   515  		clusterIPs         []string
   516  		ipFamilies         []v1.IPFamily
   517  		expectedIPFamilies []v1.IPFamily
   518  		ipFamilyPolicy     v1.IPFamilyPolicy
   519  		expectError        bool
   520  	}{
   521  		{
   522  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack",
   523  			serviceType:        v1.ServiceTypeClusterIP,
   524  			clusterIPs:         []string{},
   525  			ipFamilies:         nil,
   526  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   527  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   528  			expectError:        false,
   529  		},
   530  		{
   531  			name:               "Type ClusterIP - Client Allocated IP - IPv4 Family",
   532  			serviceType:        v1.ServiceTypeClusterIP,
   533  			clusterIPs:         []string{"10.0.0.16"},
   534  			ipFamilies:         nil,
   535  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   536  			expectError:        false,
   537  		},
   538  		{
   539  			name:               "Type ClusterIP - Client Allocated IP - IPv6 Family",
   540  			serviceType:        v1.ServiceTypeClusterIP,
   541  			clusterIPs:         []string{"2001:db8:1::16"},
   542  			ipFamilies:         nil,
   543  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol},
   544  			expectError:        false,
   545  		},
   546  		{
   547  			name:               "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ",
   548  			serviceType:        v1.ServiceTypeClusterIP,
   549  			clusterIPs:         []string{"10.0.0.17", "2001:db8:1::17"},
   550  			ipFamilies:         nil,
   551  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   552  			expectError:        true,
   553  		},
   554  		{
   555  			name:               "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ",
   556  			serviceType:        v1.ServiceTypeClusterIP,
   557  			clusterIPs:         []string{"10.0.0.17", "2001:db8:1::17"},
   558  			ipFamilies:         nil,
   559  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   560  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   561  			expectError:        false,
   562  		},
   563  		{
   564  			name:               "Type ClusterIP - Client Allocated IP - IPv4 IPv6 Family ",
   565  			serviceType:        v1.ServiceTypeClusterIP,
   566  			clusterIPs:         []string{"10.0.0.18", "2001:db8:1::18"},
   567  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   568  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   569  			expectError:        true,
   570  		},
   571  		{
   572  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack",
   573  			serviceType:        v1.ServiceTypeClusterIP,
   574  			clusterIPs:         []string{},
   575  			ipFamilies:         nil,
   576  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   577  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   578  			expectError:        false,
   579  		},
   580  		{
   581  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack",
   582  			serviceType:        v1.ServiceTypeClusterIP,
   583  			clusterIPs:         []string{},
   584  			ipFamilies:         nil,
   585  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   586  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   587  			expectError:        false,
   588  		},
   589  		{
   590  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack",
   591  			serviceType:        v1.ServiceTypeClusterIP,
   592  			clusterIPs:         []string{},
   593  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   594  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol},
   595  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   596  			expectError:        false,
   597  		},
   598  		{
   599  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack",
   600  			serviceType:        v1.ServiceTypeClusterIP,
   601  			clusterIPs:         []string{},
   602  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   603  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   604  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   605  			expectError:        false,
   606  		},
   607  		{
   608  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack",
   609  			serviceType:        v1.ServiceTypeClusterIP,
   610  			clusterIPs:         []string{},
   611  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   612  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   613  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   614  			expectError:        false,
   615  		},
   616  		{
   617  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack",
   618  			serviceType:        v1.ServiceTypeClusterIP,
   619  			clusterIPs:         []string{},
   620  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   621  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol},
   622  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   623  			expectError:        false,
   624  		},
   625  		{
   626  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack",
   627  			serviceType:        v1.ServiceTypeClusterIP,
   628  			clusterIPs:         []string{},
   629  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   630  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   631  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   632  			expectError:        false,
   633  		},
   634  		{
   635  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack",
   636  			serviceType:        v1.ServiceTypeClusterIP,
   637  			clusterIPs:         []string{},
   638  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   639  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   640  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   641  			expectError:        false,
   642  		},
   643  
   644  		{
   645  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Single Stack",
   646  			serviceType:        v1.ServiceTypeClusterIP,
   647  			clusterIPs:         []string{},
   648  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   649  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   650  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   651  			expectError:        true,
   652  		},
   653  		{
   654  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack",
   655  			serviceType:        v1.ServiceTypeClusterIP,
   656  			clusterIPs:         []string{},
   657  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   658  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   659  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   660  			expectError:        false,
   661  		},
   662  		{
   663  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack",
   664  			serviceType:        v1.ServiceTypeClusterIP,
   665  			clusterIPs:         []string{},
   666  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   667  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   668  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   669  			expectError:        false,
   670  		},
   671  		{
   672  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack",
   673  			serviceType:        v1.ServiceTypeClusterIP,
   674  			clusterIPs:         []string{},
   675  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   676  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   677  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   678  			expectError:        true,
   679  		},
   680  		{
   681  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack",
   682  			serviceType:        v1.ServiceTypeClusterIP,
   683  			clusterIPs:         []string{},
   684  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   685  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   686  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   687  			expectError:        false,
   688  		},
   689  		{
   690  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack",
   691  			serviceType:        v1.ServiceTypeClusterIP,
   692  			clusterIPs:         []string{},
   693  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   694  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   695  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   696  			expectError:        false,
   697  		},
   698  	}
   699  
   700  	for i, tc := range testcases {
   701  		tc := tc
   702  		t.Run(tc.name, func(t *testing.T) {
   703  
   704  			svc := &v1.Service{
   705  				ObjectMeta: metav1.ObjectMeta{
   706  					Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test
   707  				},
   708  				Spec: v1.ServiceSpec{
   709  					Type:       tc.serviceType,
   710  					ClusterIPs: tc.clusterIPs,
   711  					IPFamilies: tc.ipFamilies,
   712  					Ports: []v1.ServicePort{
   713  						{
   714  							Port:       443,
   715  							TargetPort: intstr.FromInt32(443),
   716  						},
   717  					},
   718  				},
   719  			}
   720  
   721  			if len(tc.ipFamilyPolicy) > 0 {
   722  				svc.Spec.IPFamilyPolicy = &tc.ipFamilyPolicy
   723  			}
   724  
   725  			if len(tc.clusterIPs) > 0 {
   726  				svc.Spec.ClusterIP = tc.clusterIPs[0]
   727  			}
   728  
   729  			// create a service
   730  			_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
   731  			if (err != nil) != tc.expectError {
   732  				t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err)
   733  			}
   734  			// if no error was expected validate the service otherwise return
   735  			if err != nil {
   736  				return
   737  			}
   738  			// validate the service was created correctly if it was not expected to fail
   739  			svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
   740  			if err != nil {
   741  				t.Errorf("Unexpected error to get the service %s %v", svc.Name, err)
   742  			}
   743  
   744  			if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil {
   745  				t.Errorf("Unexpected error validating the service %s %v", svc.Name, err)
   746  			}
   747  		})
   748  	}
   749  }
   750  
   751  // TestCreateServiceDualStackIPv6IPv4 test the Service dualstackness in a IPv6IPv4 DualStack cluster
   752  func TestCreateServiceDualStackIPv6IPv4(t *testing.T) {
   753  	// Create an IPv6IPv4 dual stack control-plane
   754  	serviceCIDR := "2001:db8:1::/112"
   755  	secondaryServiceCIDR := "10.0.0.0/16"
   756  
   757  	tCtx := ktesting.Init(t)
   758  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   759  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   760  			opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR)
   761  			opts.GenericServerRunOptions.AdvertiseAddress = netutils.ParseIPSloppy("2001:db8::10")
   762  		},
   763  	})
   764  	defer tearDownFn()
   765  
   766  	// Wait until the default "kubernetes" service is created.
   767  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
   768  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
   769  		if err != nil && !apierrors.IsNotFound(err) {
   770  			return false, err
   771  		}
   772  		return !apierrors.IsNotFound(err), nil
   773  	}); err != nil {
   774  		t.Fatalf("creating kubernetes service timed out")
   775  	}
   776  
   777  	// verify client is working
   778  	if err := wait.PollImmediate(5*time.Second, 2*time.Minute, func() (bool, error) {
   779  		_, err := client.CoreV1().Endpoints("default").Get(tCtx, "kubernetes", metav1.GetOptions{})
   780  		if err != nil {
   781  			t.Logf("error fetching endpoints: %v", err)
   782  			return false, nil
   783  		}
   784  		return true, nil
   785  	}); err != nil {
   786  		t.Errorf("server without enabled endpoints failed to register: %v", err)
   787  	}
   788  
   789  	var testcases = []struct {
   790  		name               string
   791  		serviceType        v1.ServiceType
   792  		clusterIPs         []string
   793  		ipFamilies         []v1.IPFamily
   794  		expectedIPFamilies []v1.IPFamily
   795  		ipFamilyPolicy     v1.IPFamilyPolicy
   796  		expectError        bool
   797  	}{
   798  		{
   799  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Single Stack",
   800  			serviceType:        v1.ServiceTypeClusterIP,
   801  			clusterIPs:         []string{},
   802  			ipFamilies:         nil,
   803  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol},
   804  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   805  			expectError:        false,
   806  		},
   807  		{
   808  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Prefer Dual Stack",
   809  			serviceType:        v1.ServiceTypeClusterIP,
   810  			clusterIPs:         []string{},
   811  			ipFamilies:         nil,
   812  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   813  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   814  			expectError:        false,
   815  		},
   816  		{
   817  			name:               "Type ClusterIP - Server Allocated IP - Default IP Family - Policy Required Dual Stack",
   818  			serviceType:        v1.ServiceTypeClusterIP,
   819  			clusterIPs:         []string{},
   820  			ipFamilies:         nil,
   821  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   822  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   823  			expectError:        false,
   824  		},
   825  		{
   826  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Single Stack",
   827  			serviceType:        v1.ServiceTypeClusterIP,
   828  			clusterIPs:         []string{},
   829  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   830  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   831  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   832  			expectError:        true,
   833  		},
   834  		{
   835  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Prefer Dual Stack",
   836  			serviceType:        v1.ServiceTypeClusterIP,
   837  			clusterIPs:         []string{},
   838  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   839  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   840  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   841  			expectError:        false,
   842  		},
   843  		{
   844  			name:               "Type ClusterIP - Server Allocated IP - IPv4 Family - Policy Required Dual Stack",
   845  			serviceType:        v1.ServiceTypeClusterIP,
   846  			clusterIPs:         []string{},
   847  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol},
   848  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   849  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   850  			expectError:        false,
   851  		},
   852  		{
   853  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Single Stack",
   854  			serviceType:        v1.ServiceTypeClusterIP,
   855  			clusterIPs:         []string{},
   856  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   857  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   858  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   859  			expectError:        true,
   860  		},
   861  		{
   862  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Prefer Dual Stack",
   863  			serviceType:        v1.ServiceTypeClusterIP,
   864  			clusterIPs:         []string{},
   865  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   866  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   867  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   868  			expectError:        false,
   869  		},
   870  		{
   871  			name:               "Type ClusterIP - Server Allocated IP - IPv6 Family - Policy Required Dual Stack",
   872  			serviceType:        v1.ServiceTypeClusterIP,
   873  			clusterIPs:         []string{},
   874  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol},
   875  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   876  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   877  			expectError:        false,
   878  		},
   879  
   880  		{
   881  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Prefer Dual Stack",
   882  			serviceType:        v1.ServiceTypeClusterIP,
   883  			clusterIPs:         []string{},
   884  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   885  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   886  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   887  			expectError:        false,
   888  		},
   889  		{
   890  			name:               "Type ClusterIP - Server Allocated IP - IPv4 IPv6 Family - Policy Required Dual Stack",
   891  			serviceType:        v1.ServiceTypeClusterIP,
   892  			clusterIPs:         []string{},
   893  			ipFamilies:         []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   894  			expectedIPFamilies: []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol},
   895  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   896  			expectError:        false,
   897  		},
   898  		{
   899  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Single Stack",
   900  			serviceType:        v1.ServiceTypeClusterIP,
   901  			clusterIPs:         []string{},
   902  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   903  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   904  			ipFamilyPolicy:     v1.IPFamilyPolicySingleStack,
   905  			expectError:        true,
   906  		},
   907  		{
   908  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Prefer Dual Stack",
   909  			serviceType:        v1.ServiceTypeClusterIP,
   910  			clusterIPs:         []string{},
   911  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   912  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   913  			ipFamilyPolicy:     v1.IPFamilyPolicyPreferDualStack,
   914  			expectError:        false,
   915  		},
   916  		{
   917  			name:               "Type ClusterIP - Server Allocated IP - IPv6 IPv4 Family - Policy Required Dual Stack",
   918  			serviceType:        v1.ServiceTypeClusterIP,
   919  			clusterIPs:         []string{},
   920  			ipFamilies:         []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   921  			expectedIPFamilies: []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol},
   922  			ipFamilyPolicy:     v1.IPFamilyPolicyRequireDualStack,
   923  			expectError:        false,
   924  		},
   925  	}
   926  
   927  	for i, tc := range testcases {
   928  		tc := tc
   929  		t.Run(tc.name, func(t *testing.T) {
   930  
   931  			svc := &v1.Service{
   932  				ObjectMeta: metav1.ObjectMeta{
   933  					Name: fmt.Sprintf("svc-test-%d", i), // use different services for each test
   934  				},
   935  				Spec: v1.ServiceSpec{
   936  					Type:           tc.serviceType,
   937  					ClusterIPs:     tc.clusterIPs,
   938  					IPFamilies:     tc.ipFamilies,
   939  					IPFamilyPolicy: &tc.ipFamilyPolicy,
   940  					Ports: []v1.ServicePort{
   941  						{
   942  							Port:       443,
   943  							TargetPort: intstr.FromInt32(443),
   944  						},
   945  					},
   946  				},
   947  			}
   948  
   949  			// create a service
   950  			_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
   951  			if (err != nil) != tc.expectError {
   952  				t.Errorf("Test failed expected result: %v received %v ", tc.expectError, err)
   953  			}
   954  			// if no error was expected validate the service otherwise return
   955  			if err != nil {
   956  				return
   957  			}
   958  			// validate the service was created correctly if it was not expected to fail
   959  			svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
   960  			if err != nil {
   961  				t.Errorf("Unexpected error to get the service %s %v", svc.Name, err)
   962  			}
   963  
   964  			if err := validateServiceAndClusterIPFamily(svc, tc.expectedIPFamilies); err != nil {
   965  				t.Errorf("Unexpected error validating the service %s %v", svc.Name, err)
   966  			}
   967  		})
   968  	}
   969  }
   970  
   971  // TestUpgradeDowngrade tests upgrading and downgrading a service from/to dual-stack
   972  func TestUpgradeDowngrade(t *testing.T) {
   973  	// Create an IPv4IPv6 dual stack control-plane
   974  	serviceCIDR := "10.0.0.0/16"
   975  	secondaryServiceCIDR := "2001:db8:1::/112"
   976  
   977  	tCtx := ktesting.Init(t)
   978  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
   979  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
   980  			opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR)
   981  		},
   982  	})
   983  	defer tearDownFn()
   984  
   985  	// Wait until the default "kubernetes" service is created.
   986  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
   987  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
   988  		if err != nil && !apierrors.IsNotFound(err) {
   989  			return false, err
   990  		}
   991  		return !apierrors.IsNotFound(err), nil
   992  	}); err != nil {
   993  		t.Fatalf("creating kubernetes service timed out")
   994  	}
   995  
   996  	upgradeServiceName := "svc-upgrade"
   997  
   998  	svc := &v1.Service{
   999  		ObjectMeta: metav1.ObjectMeta{
  1000  			Name: upgradeServiceName,
  1001  		},
  1002  		Spec: v1.ServiceSpec{
  1003  			Type: v1.ServiceTypeClusterIP,
  1004  			Ports: []v1.ServicePort{
  1005  				{
  1006  					Port:       443,
  1007  					TargetPort: intstr.FromInt32(443),
  1008  				},
  1009  			},
  1010  		},
  1011  	}
  1012  
  1013  	// create a service
  1014  	_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
  1015  	if err != nil {
  1016  		t.Fatalf("unexpected error while creating service:%v", err)
  1017  	}
  1018  	// validate the service was created correctly if it was not expected to fail
  1019  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1020  	if err != nil {
  1021  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1022  	}
  1023  
  1024  	if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol} /* default cluster config */); err != nil {
  1025  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1026  	}
  1027  
  1028  	// upgrade it
  1029  	requireDualStack := v1.IPFamilyPolicyRequireDualStack
  1030  	svc.Spec.IPFamilyPolicy = &requireDualStack
  1031  	upgraded, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, svc, metav1.UpdateOptions{})
  1032  	if err != nil {
  1033  		t.Fatalf("unexpected error upgrading service to dual stack. %v", err)
  1034  	}
  1035  	if err := validateServiceAndClusterIPFamily(upgraded, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} /* +1 family */); err != nil {
  1036  		t.Fatalf("Unexpected error validating the service(after upgrade) %s %v", svc.Name, err)
  1037  	}
  1038  
  1039  	// downgrade it
  1040  	singleStack := v1.IPFamilyPolicySingleStack
  1041  	upgraded.Spec.IPFamilyPolicy = &singleStack
  1042  	upgraded.Spec.ClusterIPs = upgraded.Spec.ClusterIPs[0:1]
  1043  	upgraded.Spec.IPFamilies = upgraded.Spec.IPFamilies[0:1]
  1044  	downgraded, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, upgraded, metav1.UpdateOptions{})
  1045  	if err != nil {
  1046  		t.Fatalf("unexpected error downgrading service to single stack. %v", err)
  1047  	}
  1048  	if err := validateServiceAndClusterIPFamily(downgraded, []v1.IPFamily{v1.IPv4Protocol} /* -1 family */); err != nil {
  1049  		t.Fatalf("unexpected error validating the service(after downgrade) %s %v", svc.Name, err)
  1050  	}
  1051  
  1052  	// run test again this time without removing secondary IPFamily or ClusterIP
  1053  	downgraded.Spec.IPFamilyPolicy = &requireDualStack
  1054  	upgradedAgain, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, downgraded, metav1.UpdateOptions{})
  1055  	if err != nil {
  1056  		t.Fatalf("unexpected error upgrading service to dual stack. %v", err)
  1057  	}
  1058  	if err := validateServiceAndClusterIPFamily(upgradedAgain, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol} /* +1 family */); err != nil {
  1059  		t.Fatalf("Unexpected error validating the service(after upgrade) %s %v", svc.Name, err)
  1060  	}
  1061  
  1062  	upgradedAgain.Spec.IPFamilyPolicy = &singleStack
  1063  	// api-server automatically  removes the secondary ClusterIP and IPFamily
  1064  	// when a servie is downgraded.
  1065  	downgradedAgain, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, upgradedAgain, metav1.UpdateOptions{})
  1066  	if err != nil {
  1067  		t.Fatalf("unexpected error downgrading service to single stack. %v", err)
  1068  	}
  1069  	if err := validateServiceAndClusterIPFamily(downgradedAgain, []v1.IPFamily{v1.IPv4Protocol} /* -1 family */); err != nil {
  1070  		t.Fatalf("unexpected error validating the service(after downgrade) %s %v", svc.Name, err)
  1071  	}
  1072  }
  1073  
  1074  // TestConvertToFromExternalName tests the compatibility with old clients that
  1075  // may not clear ClusterIPs
  1076  func TestConvertToFromExternalName(t *testing.T) {
  1077  	// Create an IPv4IPv6 dual stack control-plane
  1078  	serviceCIDR := "10.0.0.0/16"
  1079  	secondaryServiceCIDR := "2001:db8:1::/112"
  1080  
  1081  	tCtx := ktesting.Init(t)
  1082  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
  1083  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
  1084  			opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR)
  1085  		},
  1086  	})
  1087  	defer tearDownFn()
  1088  
  1089  	// Wait until the default "kubernetes" service is created.
  1090  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
  1091  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
  1092  		if err != nil && !apierrors.IsNotFound(err) {
  1093  			return false, err
  1094  		}
  1095  		return !apierrors.IsNotFound(err), nil
  1096  	}); err != nil {
  1097  		t.Fatalf("creating kubernetes service timed out")
  1098  	}
  1099  
  1100  	serviceName := "svc-ext-name"
  1101  	svc := &v1.Service{
  1102  		ObjectMeta: metav1.ObjectMeta{
  1103  			Name: serviceName,
  1104  		},
  1105  		Spec: v1.ServiceSpec{
  1106  			Type: v1.ServiceTypeClusterIP,
  1107  			Ports: []v1.ServicePort{
  1108  				{
  1109  					Port:       443,
  1110  					TargetPort: intstr.FromInt32(443),
  1111  				},
  1112  			},
  1113  		},
  1114  	}
  1115  
  1116  	// create a service
  1117  	_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
  1118  	if err != nil {
  1119  		t.Fatalf("unexpected error while creating service:%v", err)
  1120  	}
  1121  	// validate the service was created correctly if it was not expected to fail
  1122  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1123  	if err != nil {
  1124  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1125  	}
  1126  
  1127  	if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol}); err != nil {
  1128  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1129  	}
  1130  
  1131  	// convert to ExternalName
  1132  	svc.Spec.Type = v1.ServiceTypeExternalName
  1133  	svc.Spec.ClusterIP = "" // not clearing ClusterIPs
  1134  	svc.Spec.ExternalName = "something.somewhere"
  1135  
  1136  	externalNameSvc, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, svc, metav1.UpdateOptions{})
  1137  	if err != nil {
  1138  		t.Fatalf("unexpected error converting service to external name. %v", err)
  1139  	}
  1140  
  1141  	if len(externalNameSvc.Spec.ClusterIPs) > 0 || len(externalNameSvc.Spec.ClusterIP) > 0 || len(externalNameSvc.Spec.IPFamilies) > 0 {
  1142  		t.Fatalf("unpexpected externalname service with ClusterIPs %v or ClusterIP %v or IPFamilies %v", externalNameSvc.Spec.ClusterIPs, externalNameSvc.Spec.ClusterIP, externalNameSvc.Spec.IPFamilies)
  1143  	}
  1144  
  1145  	// convert to a ClusterIP service
  1146  	externalNameSvc.Spec.Type = v1.ServiceTypeClusterIP
  1147  	externalNameSvc.Spec.ExternalName = ""
  1148  	clusterIPSvc, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, externalNameSvc, metav1.UpdateOptions{})
  1149  	if err != nil {
  1150  		t.Fatalf("unexpected error converting service to ClusterIP. %v", err)
  1151  	}
  1152  	if err := validateServiceAndClusterIPFamily(clusterIPSvc, []v1.IPFamily{v1.IPv4Protocol}); err != nil {
  1153  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1154  	}
  1155  }
  1156  
  1157  // TestPreferDualStack preferDualstack on create and update
  1158  func TestPreferDualStack(t *testing.T) {
  1159  	// Create an IPv4IPv6 dual stack control-plane
  1160  	serviceCIDR := "10.0.0.0/16"
  1161  	secondaryServiceCIDR := "2001:db8:1::/112"
  1162  
  1163  	tCtx := ktesting.Init(t)
  1164  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
  1165  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
  1166  			opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR)
  1167  		},
  1168  	})
  1169  	defer tearDownFn()
  1170  
  1171  	// Wait until the default "kubernetes" service is created.
  1172  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
  1173  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
  1174  		if err != nil && !apierrors.IsNotFound(err) {
  1175  			return false, err
  1176  		}
  1177  		return !apierrors.IsNotFound(err), nil
  1178  	}); err != nil {
  1179  		t.Fatalf("creating kubernetes service timed out")
  1180  	}
  1181  
  1182  	preferDualStack := v1.IPFamilyPolicyPreferDualStack
  1183  
  1184  	serviceName := "svc-upgrade"
  1185  
  1186  	svc := &v1.Service{
  1187  		ObjectMeta: metav1.ObjectMeta{
  1188  			Name: serviceName,
  1189  		},
  1190  		Spec: v1.ServiceSpec{
  1191  			Type:           v1.ServiceTypeClusterIP,
  1192  			IPFamilyPolicy: &preferDualStack,
  1193  			Ports: []v1.ServicePort{
  1194  				{
  1195  					Port:       443,
  1196  					TargetPort: intstr.FromInt32(443),
  1197  				},
  1198  			},
  1199  		},
  1200  	}
  1201  
  1202  	// create a service
  1203  	_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
  1204  	if err != nil {
  1205  		t.Fatalf("unexpected error while creating service:%v", err)
  1206  	}
  1207  	// validate the service was created correctly if it was not expected to fail
  1208  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1209  	if err != nil {
  1210  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1211  	}
  1212  
  1213  	if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil {
  1214  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1215  	}
  1216  
  1217  	// update it
  1218  	svc.Spec.Selector = map[string]string{"foo": "bar"}
  1219  	upgraded, err := client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, svc, metav1.UpdateOptions{})
  1220  	if err != nil {
  1221  		t.Fatalf("unexpected error upgrading service to dual stack. %v", err)
  1222  	}
  1223  	if err := validateServiceAndClusterIPFamily(upgraded, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil {
  1224  		t.Fatalf("Unexpected error validating the service(after upgrade) %s %v", svc.Name, err)
  1225  	}
  1226  }
  1227  
  1228  type labelsForMergePatch struct {
  1229  	Labels map[string]string `json:"lables,omitempty"`
  1230  }
  1231  
  1232  // tests an update service while dualstack flag is off
  1233  func TestServiceUpdate(t *testing.T) {
  1234  	// Create an IPv4 single stack control-plane
  1235  	serviceCIDR := "10.0.0.0/16"
  1236  
  1237  	tCtx := ktesting.Init(t)
  1238  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
  1239  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
  1240  			opts.ServiceClusterIPRanges = serviceCIDR
  1241  		},
  1242  	})
  1243  	defer tearDownFn()
  1244  
  1245  	// Wait until the default "kubernetes" service is created.
  1246  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
  1247  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
  1248  		if err != nil && !apierrors.IsNotFound(err) {
  1249  			return false, err
  1250  		}
  1251  		return !apierrors.IsNotFound(err), nil
  1252  	}); err != nil {
  1253  		t.Fatalf("creating kubernetes service timed out")
  1254  	}
  1255  
  1256  	serviceName := "test-service"
  1257  	svc := &v1.Service{
  1258  		ObjectMeta: metav1.ObjectMeta{
  1259  			Name: serviceName,
  1260  		},
  1261  		Spec: v1.ServiceSpec{
  1262  			Type: v1.ServiceTypeClusterIP,
  1263  			Ports: []v1.ServicePort{
  1264  				{
  1265  					Port:       443,
  1266  					TargetPort: intstr.FromInt32(443),
  1267  				},
  1268  			},
  1269  		},
  1270  	}
  1271  
  1272  	// create the service
  1273  	_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
  1274  	// if no error was expected validate the service otherwise return
  1275  	if err != nil {
  1276  		t.Errorf("unexpected error creating service:%v", err)
  1277  		return
  1278  	}
  1279  
  1280  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1281  	if err != nil {
  1282  		t.Errorf("Unexpected error to get the service %s %v", svc.Name, err)
  1283  	}
  1284  
  1285  	// update using put
  1286  	svc.Labels = map[string]string{"x": "y"}
  1287  	_, err = client.CoreV1().Services(metav1.NamespaceDefault).Update(tCtx, svc, metav1.UpdateOptions{})
  1288  	if err != nil {
  1289  		t.Errorf("Unexpected error updating the service %s %v", svc.Name, err)
  1290  	}
  1291  
  1292  	_, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1293  	if err != nil {
  1294  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1295  	}
  1296  
  1297  	// update using StrategicMergePatchType
  1298  	labels := labelsForMergePatch{
  1299  		Labels: map[string]string{"foo": "bar"},
  1300  	}
  1301  
  1302  	patchBytes, err := json.Marshal(&labels)
  1303  	if err != nil {
  1304  		t.Fatalf("failed to json.Marshal labels: %v", err)
  1305  	}
  1306  
  1307  	_, err = client.CoreV1().Services(metav1.NamespaceDefault).Patch(tCtx, svc.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
  1308  	if err != nil {
  1309  		t.Fatalf("unexpected error patching service using strategic merge patch. %v", err)
  1310  	}
  1311  
  1312  	current, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1313  	if err != nil {
  1314  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1315  	}
  1316  
  1317  	// update using json patch
  1318  	toUpdate := current.DeepCopy()
  1319  	currentJSON, err := json.Marshal(current)
  1320  	if err != nil {
  1321  		t.Fatalf("unexpected error marshal current service. %v", err)
  1322  	}
  1323  	toUpdate.Labels = map[string]string{"alpha": "bravo"}
  1324  	toUpdateJSON, err := json.Marshal(toUpdate)
  1325  	if err != nil {
  1326  		t.Fatalf("unexpected error marshal toupdate service. %v", err)
  1327  	}
  1328  
  1329  	patchBytes, err = jsonpatch.CreateMergePatch(currentJSON, toUpdateJSON)
  1330  	if err != nil {
  1331  		t.Fatalf("unexpected error creating json patch. %v", err)
  1332  	}
  1333  
  1334  	_, err = client.CoreV1().Services(metav1.NamespaceDefault).Patch(tCtx, svc.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
  1335  	if err != nil {
  1336  		t.Fatalf("unexpected error patching service using merge patch. %v", err)
  1337  	}
  1338  
  1339  	// validate the service was created correctly if it was not expected to fail
  1340  	_, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1341  	if err != nil {
  1342  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1343  	}
  1344  }
  1345  
  1346  // validateServiceAndClusterIPFamily checks that the service has the expected IPFamilies
  1347  func validateServiceAndClusterIPFamily(svc *v1.Service, expectedIPFamilies []v1.IPFamily) error {
  1348  	// create a slice for the errors
  1349  	var errstrings []string
  1350  
  1351  	if svc.Spec.IPFamilies == nil {
  1352  		return fmt.Errorf("service ip family nil for service %s/%s", svc.Namespace, svc.Name)
  1353  	}
  1354  	if !reflect.DeepEqual(svc.Spec.IPFamilies, expectedIPFamilies) {
  1355  		return fmt.Errorf("ip families mismatch for service: %s/%s, expected: %s, actual: %s", svc.Namespace, svc.Name, expectedIPFamilies, svc.Spec.IPFamilies)
  1356  	}
  1357  
  1358  	if len(svc.Spec.ClusterIPs) == 0 {
  1359  		return fmt.Errorf("svc %s is invalid it does not have ClusterIP", svc.Name)
  1360  	}
  1361  
  1362  	// not headless
  1363  	if len(svc.Spec.ClusterIPs) > 0 && svc.Spec.ClusterIPs[0] != v1.ClusterIPNone {
  1364  		if len(svc.Spec.ClusterIPs) != len(svc.Spec.IPFamilies) {
  1365  			return fmt.Errorf("svc %v is invalid len(ClusterIPs:%v) != len(IPFamilies:%v)", svc.Name, svc.Spec.ClusterIPs, svc.Spec.IPFamilies)
  1366  		}
  1367  	}
  1368  
  1369  	for j, ip := range svc.Spec.ClusterIPs {
  1370  		// we should never be here
  1371  		if ip == v1.ClusterIPNone && len(svc.Spec.ClusterIPs) > 1 {
  1372  			errstrings = append(errstrings, fmt.Sprintf("Error validating Service: %s, None is used with +1 clusterIPs (%v)", svc.Name, svc.Spec.ClusterIPs))
  1373  		}
  1374  
  1375  		if ip == v1.ClusterIPNone {
  1376  			break // the service is headless. the rest of family check is pointless
  1377  		}
  1378  
  1379  		// the clusterIP assigned should have the same IPFamily requested
  1380  		if netutils.IsIPv6String(ip) != (expectedIPFamilies[j] == v1.IPv6Protocol) {
  1381  			errstrings = append(errstrings, fmt.Sprintf("got unexpected service ip %s, should belong to %s ip family", ip, expectedIPFamilies[j]))
  1382  		}
  1383  	}
  1384  
  1385  	if len(errstrings) > 0 {
  1386  		errstrings = append(errstrings, fmt.Sprintf("Error validating Service: %s, ClusterIPs: %v Expected IPFamilies %v", svc.Name, svc.Spec.ClusterIPs, expectedIPFamilies))
  1387  		return fmt.Errorf(strings.Join(errstrings, "\n"))
  1388  	}
  1389  
  1390  	return nil
  1391  }
  1392  
  1393  func TestUpgradeServicePreferToDualStack(t *testing.T) {
  1394  	sharedEtcd := framework.SharedEtcd()
  1395  
  1396  	tCtx := ktesting.Init(t)
  1397  
  1398  	// Create an IPv4 only dual stack control-plane
  1399  	serviceCIDR := "192.168.0.0/24"
  1400  
  1401  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
  1402  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
  1403  			opts.Etcd.StorageConfig = *sharedEtcd
  1404  			opts.ServiceClusterIPRanges = serviceCIDR
  1405  		},
  1406  	})
  1407  
  1408  	// Wait until the default "kubernetes" service is created.
  1409  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
  1410  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
  1411  		if err != nil && !apierrors.IsNotFound(err) {
  1412  			return false, err
  1413  		}
  1414  		return !apierrors.IsNotFound(err), nil
  1415  	}); err != nil {
  1416  		t.Fatalf("creating kubernetes service timed out")
  1417  	}
  1418  
  1419  	preferDualStack := v1.IPFamilyPolicyPreferDualStack
  1420  	svc := &v1.Service{
  1421  		ObjectMeta: metav1.ObjectMeta{
  1422  			Name: "svc-prefer-dual",
  1423  		},
  1424  		Spec: v1.ServiceSpec{
  1425  			Type:           v1.ServiceTypeClusterIP,
  1426  			ClusterIPs:     nil,
  1427  			IPFamilies:     nil,
  1428  			IPFamilyPolicy: &preferDualStack,
  1429  			Ports: []v1.ServicePort{
  1430  				{
  1431  					Name:       "svc-port-1",
  1432  					Port:       443,
  1433  					TargetPort: intstr.IntOrString{IntVal: 443},
  1434  					Protocol:   "TCP",
  1435  				},
  1436  			},
  1437  		},
  1438  	}
  1439  
  1440  	// create the service
  1441  	_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
  1442  	if err != nil {
  1443  		t.Fatalf("Unexpected error: %v", err)
  1444  	}
  1445  	// validate the service was created correctly if it was not expected to fail
  1446  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1447  	if err != nil {
  1448  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1449  	}
  1450  	if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol}); err != nil {
  1451  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1452  	}
  1453  
  1454  	// reconfigure the apiserver to be dual-stack
  1455  	tearDownFn()
  1456  
  1457  	secondaryServiceCIDR := "2001:db8:1::/112"
  1458  
  1459  	client, _, tearDownFn = framework.StartTestServer(tCtx, t, framework.TestServerSetup{
  1460  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
  1461  			opts.Etcd.StorageConfig = *sharedEtcd
  1462  			opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR)
  1463  		},
  1464  	})
  1465  	defer tearDownFn()
  1466  
  1467  	// Wait until the default "kubernetes" service is created.
  1468  	if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
  1469  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
  1470  		if err != nil && !apierrors.IsNotFound(err) {
  1471  			return false, err
  1472  		}
  1473  		return !apierrors.IsNotFound(err), nil
  1474  	}); err != nil {
  1475  		t.Fatalf("creating kubernetes service timed out")
  1476  	}
  1477  	// validate the service was created correctly if it was not expected to fail
  1478  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1479  	if err != nil {
  1480  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1481  	}
  1482  	// service should remain single stack
  1483  	if err = validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol}); err != nil {
  1484  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1485  	}
  1486  }
  1487  
  1488  func TestDowngradeServicePreferToDualStack(t *testing.T) {
  1489  	sharedEtcd := framework.SharedEtcd()
  1490  
  1491  	tCtx := ktesting.Init(t)
  1492  
  1493  	// Create a dual stack control-plane
  1494  	serviceCIDR := "192.168.0.0/24"
  1495  	secondaryServiceCIDR := "2001:db8:1::/112"
  1496  
  1497  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{
  1498  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
  1499  			opts.Etcd.StorageConfig = *sharedEtcd
  1500  			opts.ServiceClusterIPRanges = fmt.Sprintf("%s,%s", serviceCIDR, secondaryServiceCIDR)
  1501  		},
  1502  	})
  1503  
  1504  	// Wait until the default "kubernetes" service is created.
  1505  	if err := wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
  1506  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
  1507  		if err != nil && !apierrors.IsNotFound(err) {
  1508  			return false, err
  1509  		}
  1510  		return !apierrors.IsNotFound(err), nil
  1511  	}); err != nil {
  1512  		t.Fatalf("creating kubernetes service timed out")
  1513  	}
  1514  	preferDualStack := v1.IPFamilyPolicyPreferDualStack
  1515  	svc := &v1.Service{
  1516  		ObjectMeta: metav1.ObjectMeta{
  1517  			Name: "svc-prefer-dual01",
  1518  		},
  1519  		Spec: v1.ServiceSpec{
  1520  			Type:           v1.ServiceTypeClusterIP,
  1521  			ClusterIPs:     nil,
  1522  			IPFamilies:     nil,
  1523  			IPFamilyPolicy: &preferDualStack,
  1524  			Ports: []v1.ServicePort{
  1525  				{
  1526  					Name:       "svc-port-1",
  1527  					Port:       443,
  1528  					TargetPort: intstr.IntOrString{IntVal: 443},
  1529  					Protocol:   "TCP",
  1530  				},
  1531  			},
  1532  		},
  1533  	}
  1534  	// create the service
  1535  	_, err := client.CoreV1().Services(metav1.NamespaceDefault).Create(tCtx, svc, metav1.CreateOptions{})
  1536  	if err != nil {
  1537  		t.Fatalf("Unexpected error: %v", err)
  1538  	}
  1539  	// validate the service was created correctly if it was not expected to fail
  1540  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1541  	if err != nil {
  1542  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1543  	}
  1544  	if err := validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil {
  1545  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1546  	}
  1547  	// reconfigure the apiserver to be sinlge stack
  1548  	tearDownFn()
  1549  
  1550  	// reset secondary
  1551  	client, _, tearDownFn = framework.StartTestServer(tCtx, t, framework.TestServerSetup{
  1552  		ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
  1553  			opts.Etcd.StorageConfig = *sharedEtcd
  1554  			opts.ServiceClusterIPRanges = serviceCIDR
  1555  		},
  1556  	})
  1557  	defer tearDownFn()
  1558  
  1559  	// Wait until the default "kubernetes" service is created.
  1560  	if err = wait.Poll(250*time.Millisecond, time.Minute, func() (bool, error) {
  1561  		_, err := client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, "kubernetes", metav1.GetOptions{})
  1562  		if err != nil && !apierrors.IsNotFound(err) {
  1563  			return false, err
  1564  		}
  1565  		return !apierrors.IsNotFound(err), nil
  1566  	}); err != nil {
  1567  		t.Fatalf("creating kubernetes service timed out")
  1568  	}
  1569  	// validate the service is still there.
  1570  	svc, err = client.CoreV1().Services(metav1.NamespaceDefault).Get(tCtx, svc.Name, metav1.GetOptions{})
  1571  	if err != nil {
  1572  		t.Fatalf("Unexpected error to get the service %s %v", svc.Name, err)
  1573  	}
  1574  	// service should remain dual stack
  1575  	if err = validateServiceAndClusterIPFamily(svc, []v1.IPFamily{v1.IPv4Protocol, v1.IPv6Protocol}); err != nil {
  1576  		t.Fatalf("Unexpected error validating the service %s %v", svc.Name, err)
  1577  	}
  1578  }
  1579  
  1580  type serviceMergePatch struct {
  1581  	Spec specMergePatch `json:"spec,omitempty"`
  1582  }
  1583  type specMergePatch struct {
  1584  	Type         v1.ServiceType `json:"type,omitempty"`
  1585  	ExternalName string         `json:"externalName,omitempty"`
  1586  }
  1587  
  1588  // tests success when converting ClusterIP:Headless service to ExternalName
  1589  func Test_ServiceChangeTypeHeadlessToExternalNameWithPatch(t *testing.T) {
  1590  	tCtx := ktesting.Init(t)
  1591  	client, _, tearDownFn := framework.StartTestServer(tCtx, t, framework.TestServerSetup{})
  1592  	defer tearDownFn()
  1593  
  1594  	ns := framework.CreateNamespaceOrDie(client, "test-service-allocate-node-ports", t)
  1595  	defer framework.DeleteNamespaceOrDie(client, ns, t)
  1596  
  1597  	service := &v1.Service{
  1598  		ObjectMeta: metav1.ObjectMeta{
  1599  			Name: "test-123",
  1600  		},
  1601  		Spec: v1.ServiceSpec{
  1602  			Type:      v1.ServiceTypeClusterIP,
  1603  			ClusterIP: "None",
  1604  			Selector:  map[string]string{"foo": "bar"},
  1605  		},
  1606  	}
  1607  
  1608  	var err error
  1609  	service, err = client.CoreV1().Services(ns.Name).Create(tCtx, service, metav1.CreateOptions{})
  1610  	if err != nil {
  1611  		t.Fatalf("Error creating test service: %v", err)
  1612  	}
  1613  
  1614  	serviceMergePatch := serviceMergePatch{
  1615  		Spec: specMergePatch{
  1616  			Type:         v1.ServiceTypeExternalName,
  1617  			ExternalName: "foo.bar",
  1618  		},
  1619  	}
  1620  	patchBytes, err := json.Marshal(&serviceMergePatch)
  1621  	if err != nil {
  1622  		t.Fatalf("failed to json.Marshal ports: %v", err)
  1623  	}
  1624  
  1625  	_, err = client.CoreV1().Services(ns.Name).Patch(tCtx, service.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
  1626  	if err != nil {
  1627  		t.Fatalf("unexpected error patching service using strategic merge patch. %v", err)
  1628  	}
  1629  }
  1630  

View as plain text