ccm_test.go

Documentation: k8s.io/kubernetes/test/integration/cloudprovider

     1  /*
     2  Copyright 2024 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package cloudprovider
    18  
    19  import (
    20  	"context"
    21  	"io"
    22  	"os"
    23  	"testing"
    24  	"time"
    25  
    26  	v1 "k8s.io/api/core/v1"
    27  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    28  	"k8s.io/apimachinery/pkg/types"
    29  	"k8s.io/apimachinery/pkg/util/wait"
    30  	clientset "k8s.io/client-go/kubernetes"
    31  	"k8s.io/client-go/rest"
    32  	"k8s.io/client-go/tools/clientcmd"
    33  	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
    34  	cloudprovider "k8s.io/cloud-provider"
    35  	cloudproviderapi "k8s.io/cloud-provider/api"
    36  	ccmservertesting "k8s.io/cloud-provider/app/testing"
    37  	fakecloud "k8s.io/cloud-provider/fake"
    38  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    39  	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
    40  	"k8s.io/kubernetes/test/integration/framework"
    41  )
    42  
    43  func Test_RemoveExternalCloudProviderTaint(t *testing.T) {
    44  	ctx, cancel := context.WithCancel(context.Background())
    45  	defer cancel()
    46  
    47  	// Disable ServiceAccount admission plugin as we don't have serviceaccount controller running.
    48  	server := kubeapiservertesting.StartTestServerOrDie(t, nil, []string{"--disable-admission-plugins=ServiceAccount"}, framework.SharedEtcd())
    49  	defer server.TearDownFn()
    50  
    51  	client := clientset.NewForConfigOrDie(server.ClientConfig)
    52  
    53  	ns := framework.CreateNamespaceOrDie(client, "config-map", t)
    54  	defer framework.DeleteNamespaceOrDie(client, ns, t)
    55  
    56  	// Create fake node
    57  	_, err := client.CoreV1().Nodes().Create(ctx, makeNode("node0"), metav1.CreateOptions{})
    58  	if err != nil {
    59  		t.Fatalf("Failed to create Node %v", err)
    60  	}
    61  
    62  	// start cloud-controller-manager
    63  	kubeconfig := createKubeconfigFileForRestConfig(server.ClientConfig)
    64  	// nolint:errcheck // Ignore the error trying to delete the kubeconfig file used for the test
    65  	defer os.Remove(kubeconfig)
    66  	args := []string{
    67  		"--kubeconfig=" + kubeconfig,
    68  		"--cloud-provider=fakeCloud",
    69  		"--cidr-allocator-type=" + string(ipam.RangeAllocatorType),
    70  		"--configure-cloud-routes=false",
    71  	}
    72  
    73  	fakeCloud := &fakecloud.Cloud{
    74  		Zone: cloudprovider.Zone{
    75  			FailureDomain: "zone-0",
    76  			Region:        "region-1",
    77  		},
    78  		EnableInstancesV2:  true,
    79  		ExistsByProviderID: true,
    80  		ProviderID: map[types.NodeName]string{
    81  			types.NodeName("node0"): "12345",
    82  		},
    83  		InstanceTypes: map[types.NodeName]string{
    84  			types.NodeName("node0"): "t1.micro",
    85  		},
    86  		ExtID: map[types.NodeName]string{
    87  			types.NodeName("node0"): "12345",
    88  		},
    89  		Addresses: []v1.NodeAddress{
    90  			{
    91  				Type:    v1.NodeHostName,
    92  				Address: "node0.cloud.internal",
    93  			},
    94  			{
    95  				Type:    v1.NodeInternalIP,
    96  				Address: "10.0.0.1",
    97  			},
    98  			{
    99  				Type:    v1.NodeExternalIP,
   100  				Address: "132.143.154.163",
   101  			},
   102  		},
   103  		ErrByProviderID: nil,
   104  		Err:             nil,
   105  	}
   106  
   107  	// register fake GCE cloud provider
   108  	cloudprovider.RegisterCloudProvider(
   109  		"fakeCloud",
   110  		func(config io.Reader) (cloudprovider.Interface, error) {
   111  			return fakeCloud, nil
   112  		})
   113  
   114  	ccm := ccmservertesting.StartTestServerOrDie(ctx, args)
   115  	defer ccm.TearDownFn()
   116  
   117  	// There should be only the taint TaintNodeNotReady, added by the admission plugin TaintNodesByCondition
   118  	err = wait.PollUntilContextTimeout(ctx, 1*time.Second, 50*time.Second, true, func(ctx context.Context) (done bool, err error) {
   119  		n, err := client.CoreV1().Nodes().Get(ctx, "node0", metav1.GetOptions{})
   120  		if err != nil {
   121  			return false, err
   122  		}
   123  		if len(n.Spec.Taints) != 1 {
   124  			return false, nil
   125  		}
   126  		if n.Spec.Taints[0].Key != v1.TaintNodeNotReady {
   127  			return false, nil
   128  		}
   129  		return true, nil
   130  	})
   131  	if err != nil {
   132  		t.Logf("Fake Cloud Provider calls: %v", fakeCloud.Calls)
   133  		t.Fatalf("expected node to not have Taint: %v", err)
   134  	}
   135  }
   136  
   137  // sigs.k8s.io/controller-runtime/pkg/envtest
   138  func createKubeconfigFileForRestConfig(restConfig *rest.Config) string {
   139  	clusters := make(map[string]*clientcmdapi.Cluster)
   140  	clusters["default-cluster"] = &clientcmdapi.Cluster{
   141  		Server:                   restConfig.Host,
   142  		TLSServerName:            restConfig.ServerName,
   143  		CertificateAuthorityData: restConfig.CAData,
   144  	}
   145  	contexts := make(map[string]*clientcmdapi.Context)
   146  	contexts["default-context"] = &clientcmdapi.Context{
   147  		Cluster:  "default-cluster",
   148  		AuthInfo: "default-user",
   149  	}
   150  	authinfos := make(map[string]*clientcmdapi.AuthInfo)
   151  	authinfos["default-user"] = &clientcmdapi.AuthInfo{
   152  		ClientCertificateData: restConfig.CertData,
   153  		ClientKeyData:         restConfig.KeyData,
   154  		Token:                 restConfig.BearerToken,
   155  	}
   156  	clientConfig := clientcmdapi.Config{
   157  		Kind:           "Config",
   158  		APIVersion:     "v1",
   159  		Clusters:       clusters,
   160  		Contexts:       contexts,
   161  		CurrentContext: "default-context",
   162  		AuthInfos:      authinfos,
   163  	}
   164  	kubeConfigFile, _ := os.CreateTemp("", "kubeconfig")
   165  	_ = clientcmd.WriteToFile(clientConfig, kubeConfigFile.Name())
   166  	return kubeConfigFile.Name()
   167  }
   168  
   169  func makeNode(name string) *v1.Node {
   170  	return &v1.Node{
   171  		ObjectMeta: metav1.ObjectMeta{
   172  			Name: name,
   173  		},
   174  		Spec: v1.NodeSpec{
   175  			Taints: []v1.Taint{{
   176  				Key:    cloudproviderapi.TaintExternalCloudProvider,
   177  				Value:  "true",
   178  				Effect: v1.TaintEffectNoSchedule,
   179  			}},
   180  			Unschedulable: false,
   181  		},
   182  		Status: v1.NodeStatus{
   183  			Conditions: []v1.NodeCondition{
   184  				{
   185  					Type:              v1.NodeReady,
   186  					Status:            v1.ConditionUnknown,
   187  					LastHeartbeatTime: metav1.Time{Time: time.Now()},
   188  				},
   189  			},
   190  		},
   191  	}
   192  }
   193
View as plain text