hollow_node.go

Documentation: k8s.io/kubernetes/cmd/kubemark/app

     1  /*
     2  Copyright 2023 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package app
    18  
    19  import (
    20  	"context"
    21  	goflag "flag"
    22  	"fmt"
    23  	"time"
    24  
    25  	"github.com/spf13/cobra"
    26  	"github.com/spf13/pflag"
    27  	oteltrace "go.opentelemetry.io/otel/trace"
    28  	internalapi "k8s.io/cri-api/pkg/apis"
    29  	"k8s.io/klog/v2"
    30  
    31  	v1 "k8s.io/api/core/v1"
    32  	"k8s.io/apimachinery/pkg/api/resource"
    33  	"k8s.io/apimachinery/pkg/types"
    34  	"k8s.io/apimachinery/pkg/util/sets"
    35  	clientset "k8s.io/client-go/kubernetes"
    36  	restclient "k8s.io/client-go/rest"
    37  	"k8s.io/client-go/tools/clientcmd"
    38  	"k8s.io/client-go/tools/events"
    39  	cliflag "k8s.io/component-base/cli/flag"
    40  	_ "k8s.io/component-base/metrics/prometheus/restclient" // for client metric registration
    41  	_ "k8s.io/component-base/metrics/prometheus/version"    // for version metric registration
    42  	"k8s.io/component-base/version"
    43  	"k8s.io/component-base/version/verflag"
    44  	"k8s.io/kubernetes/pkg/api/legacyscheme"
    45  	"k8s.io/kubernetes/pkg/cluster/ports"
    46  	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
    47  	"k8s.io/kubernetes/pkg/kubelet/certificate/bootstrap"
    48  	"k8s.io/kubernetes/pkg/kubelet/cm"
    49  	"k8s.io/kubernetes/pkg/kubelet/cri/remote"
    50  	fakeremote "k8s.io/kubernetes/pkg/kubelet/cri/remote/fake"
    51  	"k8s.io/kubernetes/pkg/kubemark"
    52  	kubemarkproxy "k8s.io/kubernetes/pkg/proxy/kubemark"
    53  	utilflag "k8s.io/kubernetes/pkg/util/flag"
    54  )
    55  
    56  type hollowNodeConfig struct {
    57  	KubeconfigPath          string
    58  	BootstrapKubeconfigPath string
    59  	CertDirectory           string
    60  	KubeletPort             int
    61  	KubeletReadOnlyPort     int
    62  	Morph                   string
    63  	NodeName                string
    64  	ServerPort              int
    65  	ContentType             string
    66  	NodeLabels              map[string]string
    67  	RegisterWithTaints      []v1.Taint
    68  	MaxPods                 int
    69  	ExtendedResources       map[string]string
    70  	UseHostImageService     bool
    71  
    72  	// Deprecated config; remove these with the corresponding flags
    73  	UseRealProxier       bool
    74  	ProxierSyncPeriod    time.Duration
    75  	ProxierMinSyncPeriod time.Duration
    76  }
    77  
    78  const (
    79  	maxPods     = 110
    80  	podsPerCore = 0
    81  )
    82  
    83  // TODO(#45650): Refactor hollow-node into hollow-kubelet and hollow-proxy
    84  // and make the config driven.
    85  var knownMorphs = sets.NewString("kubelet", "proxy")
    86  
    87  func (c *hollowNodeConfig) addFlags(fs *pflag.FlagSet) {
    88  	fs.StringVar(&c.KubeconfigPath, "kubeconfig", "/kubeconfig/kubeconfig", "Path to kubeconfig file.")
    89  	fs.StringVar(&c.BootstrapKubeconfigPath, "bootstrap-kubeconfig", "", "Path to bootstrap kubeconfig file.")
    90  	fs.StringVar(&c.CertDirectory, "cert-dir", "/etc/srv/", "Path to cert directory for bootstraping.")
    91  	fs.IntVar(&c.KubeletPort, "kubelet-port", ports.KubeletPort, "Port on which HollowKubelet should be listening.")
    92  	fs.IntVar(&c.KubeletReadOnlyPort, "kubelet-read-only-port", ports.KubeletReadOnlyPort, "Read-only port on which Kubelet is listening.")
    93  	fs.StringVar(&c.NodeName, "name", "fake-node", "Name of this Hollow Node.")
    94  	fs.IntVar(&c.ServerPort, "api-server-port", 443, "Port on which API server is listening.")
    95  	fs.StringVar(&c.Morph, "morph", "", fmt.Sprintf("Specifies into which Hollow component this binary should morph. Allowed values: %v", knownMorphs.List()))
    96  	fs.StringVar(&c.ContentType, "kube-api-content-type", "application/vnd.kubernetes.protobuf", "ContentType of requests sent to apiserver.")
    97  	bindableNodeLabels := cliflag.ConfigurationMap(c.NodeLabels)
    98  	fs.Var(&bindableNodeLabels, "node-labels", "Additional node labels")
    99  	fs.Var(utilflag.RegisterWithTaintsVar{Value: &c.RegisterWithTaints}, "register-with-taints", "Register the node with the given list of taints (comma separated \"<key>=<value>:<effect>\"). No-op if register-node is false.")
   100  	fs.IntVar(&c.MaxPods, "max-pods", maxPods, "Number of pods that can run on this Kubelet.")
   101  	bindableExtendedResources := cliflag.ConfigurationMap(c.ExtendedResources)
   102  	fs.Var(&bindableExtendedResources, "extended-resources", "Register the node with extended resources (comma separated \"<name>=<quantity>\")")
   103  	fs.BoolVar(&c.UseHostImageService, "use-host-image-service", true, "Set to true if the hollow-kubelet should use the host image service. If set to false the fake image service will be used")
   104  
   105  	fs.BoolVar(&c.UseRealProxier, "use-real-proxier", true, "Has no effect.")
   106  	_ = fs.MarkDeprecated("use-real-proxier", "This flag is deprecated and will be removed in a future release.")
   107  	fs.DurationVar(&c.ProxierSyncPeriod, "proxier-sync-period", 30*time.Second, "Has no effect.")
   108  	_ = fs.MarkDeprecated("proxier-sync-period", "This flag is deprecated and will be removed in a future release.")
   109  	fs.DurationVar(&c.ProxierMinSyncPeriod, "proxier-min-sync-period", 0, "Has no effect.")
   110  	_ = fs.MarkDeprecated("proxier-min-sync-period", "This flag is deprecated and will be removed in a future release.")
   111  }
   112  
   113  func (c *hollowNodeConfig) createClientConfigFromFile() (*restclient.Config, error) {
   114  	clientConfig, err := clientcmd.LoadFromFile(c.KubeconfigPath)
   115  	if err != nil {
   116  		return nil, fmt.Errorf("error while loading kubeconfig from file %v: %v", c.KubeconfigPath, err)
   117  	}
   118  	config, err := clientcmd.NewDefaultClientConfig(*clientConfig, &clientcmd.ConfigOverrides{}).ClientConfig()
   119  	if err != nil {
   120  		return nil, fmt.Errorf("error while creating kubeconfig: %v", err)
   121  	}
   122  	config.ContentType = c.ContentType
   123  	config.QPS = 10
   124  	config.Burst = 20
   125  	return config, nil
   126  }
   127  
   128  func (c *hollowNodeConfig) bootstrapClientConfig() error {
   129  	if c.BootstrapKubeconfigPath != "" {
   130  		return bootstrap.LoadClientCert(context.TODO(), c.KubeconfigPath, c.BootstrapKubeconfigPath, c.CertDirectory, types.NodeName(c.NodeName))
   131  	}
   132  	return nil
   133  }
   134  
   135  func (c *hollowNodeConfig) createHollowKubeletOptions() *kubemark.HollowKubeletOptions {
   136  	return &kubemark.HollowKubeletOptions{
   137  		NodeName:            c.NodeName,
   138  		KubeletPort:         c.KubeletPort,
   139  		KubeletReadOnlyPort: c.KubeletReadOnlyPort,
   140  		MaxPods:             c.MaxPods,
   141  		PodsPerCore:         podsPerCore,
   142  		NodeLabels:          c.NodeLabels,
   143  		RegisterWithTaints:  c.RegisterWithTaints,
   144  	}
   145  }
   146  
   147  // NewHollowNodeCommand creates a *cobra.Command object with default parameters
   148  func NewHollowNodeCommand() *cobra.Command {
   149  	s := &hollowNodeConfig{
   150  		NodeLabels:        make(map[string]string),
   151  		ExtendedResources: make(map[string]string),
   152  	}
   153  
   154  	cmd := &cobra.Command{
   155  		Use:  "kubemark",
   156  		Long: "kubemark",
   157  		RunE: func(cmd *cobra.Command, args []string) error {
   158  			verflag.PrintAndExitIfRequested()
   159  			cliflag.PrintFlags(cmd.Flags())
   160  			return run(s)
   161  		},
   162  		Args: func(cmd *cobra.Command, args []string) error {
   163  			for _, arg := range args {
   164  				if len(arg) > 0 {
   165  					return fmt.Errorf("%q does not take any arguments, got %q", cmd.CommandPath(), args)
   166  				}
   167  			}
   168  			return nil
   169  		},
   170  	}
   171  
   172  	fs := cmd.Flags()
   173  	fs.AddGoFlagSet(goflag.CommandLine) // for flags like --docker-only
   174  	s.addFlags(fs)
   175  
   176  	return cmd
   177  }
   178  
   179  func run(config *hollowNodeConfig) error {
   180  	// To help debugging, immediately log version and print flags.
   181  	klog.Infof("Version: %+v", version.Get())
   182  
   183  	if !knownMorphs.Has(config.Morph) {
   184  		return fmt.Errorf("Unknown morph: %v. allowed values: %v", config.Morph, knownMorphs.List())
   185  	}
   186  
   187  	// create a client to communicate with API server.
   188  	err := config.bootstrapClientConfig()
   189  	if err != nil {
   190  		return fmt.Errorf("Failed to bootstrap, error: %w. Exiting", err)
   191  	}
   192  	clientConfig, err := config.createClientConfigFromFile()
   193  	if err != nil {
   194  		return fmt.Errorf("Failed to create a ClientConfig, error: %w. Exiting", err)
   195  	}
   196  
   197  	if config.Morph == "kubelet" {
   198  		clientConfig.UserAgent = "hollow-kubelet"
   199  		client, err := clientset.NewForConfig(clientConfig)
   200  		if err != nil {
   201  			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
   202  		}
   203  
   204  		f, c := kubemark.GetHollowKubeletConfig(config.createHollowKubeletOptions())
   205  
   206  		heartbeatClientConfig := *clientConfig
   207  		heartbeatClientConfig.Timeout = c.NodeStatusUpdateFrequency.Duration
   208  		// The timeout is the minimum of the lease duration and status update frequency
   209  		leaseTimeout := time.Duration(c.NodeLeaseDurationSeconds) * time.Second
   210  		if heartbeatClientConfig.Timeout > leaseTimeout {
   211  			heartbeatClientConfig.Timeout = leaseTimeout
   212  		}
   213  
   214  		heartbeatClientConfig.QPS = float32(-1)
   215  		heartbeatClient, err := clientset.NewForConfig(&heartbeatClientConfig)
   216  		if err != nil {
   217  			return fmt.Errorf("Failed to create a ClientSet, error: %w. Exiting", err)
   218  		}
   219  
   220  		cadvisorInterface := &cadvisortest.Fake{
   221  			NodeName: config.NodeName,
   222  		}
   223  
   224  		var containerManager cm.ContainerManager
   225  		if config.ExtendedResources != nil {
   226  			extendedResources := v1.ResourceList{}
   227  			for k, v := range config.ExtendedResources {
   228  				extendedResources[v1.ResourceName(k)] = resource.MustParse(v)
   229  			}
   230  
   231  			containerManager = cm.NewStubContainerManagerWithDevicePluginResource(extendedResources)
   232  		} else {
   233  			containerManager = cm.NewStubContainerManager()
   234  		}
   235  
   236  		endpoint, err := fakeremote.GenerateEndpoint()
   237  		if err != nil {
   238  			return fmt.Errorf("Failed to generate fake endpoint, error: %w", err)
   239  		}
   240  		fakeRemoteRuntime := fakeremote.NewFakeRemoteRuntime()
   241  		if err = fakeRemoteRuntime.Start(endpoint); err != nil {
   242  			return fmt.Errorf("Failed to start fake runtime, error: %w", err)
   243  		}
   244  		defer fakeRemoteRuntime.Stop()
   245  		runtimeService, err := remote.NewRemoteRuntimeService(endpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
   246  		if err != nil {
   247  			return fmt.Errorf("Failed to init runtime service, error: %w", err)
   248  		}
   249  
   250  		var imageService internalapi.ImageManagerService = fakeRemoteRuntime.ImageService
   251  		if config.UseHostImageService {
   252  			imageService, err = remote.NewRemoteImageService(c.ImageServiceEndpoint, 15*time.Second, oteltrace.NewNoopTracerProvider())
   253  			if err != nil {
   254  				return fmt.Errorf("Failed to init image service, error: %w", err)
   255  			}
   256  		}
   257  
   258  		hollowKubelet := kubemark.NewHollowKubelet(
   259  			f, c,
   260  			client,
   261  			heartbeatClient,
   262  			cadvisorInterface,
   263  			imageService,
   264  			runtimeService,
   265  			containerManager,
   266  		)
   267  		hollowKubelet.Run()
   268  	}
   269  
   270  	if config.Morph == "proxy" {
   271  		clientConfig.UserAgent = "hollow-proxy"
   272  
   273  		client, err := clientset.NewForConfig(clientConfig)
   274  		if err != nil {
   275  			return fmt.Errorf("Failed to create API Server client, error: %w", err)
   276  		}
   277  		eventBroadcaster := events.NewBroadcaster(&events.EventSinkImpl{Interface: client.EventsV1()})
   278  		recorder := eventBroadcaster.NewRecorder(legacyscheme.Scheme, "kube-proxy")
   279  
   280  		hollowProxy := kubemarkproxy.NewHollowProxy(
   281  			config.NodeName,
   282  			client,
   283  			client.CoreV1(),
   284  			eventBroadcaster,
   285  			recorder,
   286  		)
   287  		return hollowProxy.Run()
   288  	}
   289  
   290  	return nil
   291  }
   292
View as plain text