hollow_kubelet.go

Documentation: k8s.io/kubernetes/pkg/kubemark

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package kubemark
    18  
    19  import (
    20  	"fmt"
    21  	"time"
    22  
    23  	"go.opentelemetry.io/otel/trace"
    24  	v1 "k8s.io/api/core/v1"
    25  	"k8s.io/klog/v2"
    26  	"k8s.io/mount-utils"
    27  
    28  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    29  	clientset "k8s.io/client-go/kubernetes"
    30  	"k8s.io/client-go/tools/record"
    31  	internalapi "k8s.io/cri-api/pkg/apis"
    32  	kubeletapp "k8s.io/kubernetes/cmd/kubelet/app"
    33  	"k8s.io/kubernetes/cmd/kubelet/app/options"
    34  	"k8s.io/kubernetes/pkg/kubelet"
    35  	kubeletconfig "k8s.io/kubernetes/pkg/kubelet/apis/config"
    36  	"k8s.io/kubernetes/pkg/kubelet/cadvisor"
    37  	"k8s.io/kubernetes/pkg/kubelet/cm"
    38  	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
    39  	probetest "k8s.io/kubernetes/pkg/kubelet/prober/testing"
    40  	kubetypes "k8s.io/kubernetes/pkg/kubelet/types"
    41  	kubeletutil "k8s.io/kubernetes/pkg/kubelet/util"
    42  	"k8s.io/kubernetes/pkg/util/oom"
    43  	"k8s.io/kubernetes/pkg/volume"
    44  	"k8s.io/kubernetes/pkg/volume/cephfs"
    45  	"k8s.io/kubernetes/pkg/volume/configmap"
    46  	"k8s.io/kubernetes/pkg/volume/csi"
    47  	"k8s.io/kubernetes/pkg/volume/downwardapi"
    48  	"k8s.io/kubernetes/pkg/volume/emptydir"
    49  	"k8s.io/kubernetes/pkg/volume/fc"
    50  	"k8s.io/kubernetes/pkg/volume/git_repo"
    51  	"k8s.io/kubernetes/pkg/volume/hostpath"
    52  	"k8s.io/kubernetes/pkg/volume/iscsi"
    53  	"k8s.io/kubernetes/pkg/volume/local"
    54  	"k8s.io/kubernetes/pkg/volume/nfs"
    55  	"k8s.io/kubernetes/pkg/volume/portworx"
    56  	"k8s.io/kubernetes/pkg/volume/projected"
    57  	"k8s.io/kubernetes/pkg/volume/rbd"
    58  	"k8s.io/kubernetes/pkg/volume/secret"
    59  	"k8s.io/kubernetes/pkg/volume/util/hostutil"
    60  	"k8s.io/kubernetes/pkg/volume/util/subpath"
    61  	"k8s.io/kubernetes/test/utils"
    62  )
    63  
    64  type HollowKubelet struct {
    65  	KubeletFlags         *options.KubeletFlags
    66  	KubeletConfiguration *kubeletconfig.KubeletConfiguration
    67  	KubeletDeps          *kubelet.Dependencies
    68  }
    69  
    70  func volumePlugins() []volume.VolumePlugin {
    71  	allPlugins := []volume.VolumePlugin{}
    72  	allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
    73  	allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
    74  	allPlugins = append(allPlugins, hostpath.FakeProbeVolumePlugins(volume.VolumeConfig{})...)
    75  	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
    76  	allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
    77  	allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
    78  	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
    79  	allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...)
    80  	allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
    81  	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
    82  	allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
    83  	allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
    84  	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
    85  	allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
    86  	allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
    87  	return allPlugins
    88  }
    89  
    90  func NewHollowKubelet(
    91  	flags *options.KubeletFlags,
    92  	config *kubeletconfig.KubeletConfiguration,
    93  	client *clientset.Clientset,
    94  	heartbeatClient *clientset.Clientset,
    95  	cadvisorInterface cadvisor.Interface,
    96  	imageService internalapi.ImageManagerService,
    97  	runtimeService internalapi.RuntimeService,
    98  	containerManager cm.ContainerManager) *HollowKubelet {
    99  	d := &kubelet.Dependencies{
   100  		KubeClient:                client,
   101  		HeartbeatClient:           heartbeatClient,
   102  		ProbeManager:              probetest.FakeManager{},
   103  		RemoteRuntimeService:      runtimeService,
   104  		RemoteImageService:        imageService,
   105  		CAdvisorInterface:         cadvisorInterface,
   106  		Cloud:                     nil,
   107  		OSInterface:               &containertest.FakeOS{},
   108  		ContainerManager:          containerManager,
   109  		VolumePlugins:             volumePlugins(),
   110  		TLSOptions:                nil,
   111  		OOMAdjuster:               oom.NewFakeOOMAdjuster(),
   112  		Mounter:                   &mount.FakeMounter{},
   113  		Subpather:                 &subpath.FakeSubpath{},
   114  		HostUtil:                  hostutil.NewFakeHostUtil(nil),
   115  		PodStartupLatencyTracker:  kubeletutil.NewPodStartupLatencyTracker(),
   116  		NodeStartupLatencyTracker: kubeletutil.NewNodeStartupLatencyTracker(),
   117  		TracerProvider:            trace.NewNoopTracerProvider(),
   118  		Recorder:                  &record.FakeRecorder{}, // With real recorder we attempt to read /dev/kmsg.
   119  	}
   120  
   121  	return &HollowKubelet{
   122  		KubeletFlags:         flags,
   123  		KubeletConfiguration: config,
   124  		KubeletDeps:          d,
   125  	}
   126  }
   127  
   128  // Starts this HollowKubelet and blocks.
   129  func (hk *HollowKubelet) Run() {
   130  	if err := kubeletapp.RunKubelet(&options.KubeletServer{
   131  		KubeletFlags:         *hk.KubeletFlags,
   132  		KubeletConfiguration: *hk.KubeletConfiguration,
   133  	}, hk.KubeletDeps, false); err != nil {
   134  		klog.Fatalf("Failed to run HollowKubelet: %v. Exiting.", err)
   135  	}
   136  	select {}
   137  }
   138  
   139  // HollowKubeletOptions contains settable parameters for hollow kubelet.
   140  type HollowKubeletOptions struct {
   141  	NodeName            string
   142  	KubeletPort         int
   143  	KubeletReadOnlyPort int
   144  	MaxPods             int
   145  	PodsPerCore         int
   146  	NodeLabels          map[string]string
   147  	RegisterWithTaints  []v1.Taint
   148  }
   149  
   150  // Builds a KubeletConfiguration for the HollowKubelet, ensuring that the
   151  // usual defaults are applied for fields we do not override.
   152  func GetHollowKubeletConfig(opt *HollowKubeletOptions) (*options.KubeletFlags, *kubeletconfig.KubeletConfiguration) {
   153  	testRootDir := utils.MakeTempDirOrDie("hollow-kubelet.", "")
   154  	podFilePath := utils.MakeTempDirOrDie("static-pods", testRootDir)
   155  	podLogsPath := utils.MakeTempDirOrDie("pod-logs", testRootDir)
   156  	klog.Infof("Using %s as root dir for hollow-kubelet", testRootDir)
   157  
   158  	// Flags struct
   159  	f := options.NewKubeletFlags()
   160  	f.RootDirectory = testRootDir
   161  	f.HostnameOverride = opt.NodeName
   162  	f.MinimumGCAge = metav1.Duration{Duration: 1 * time.Minute}
   163  	f.MaxContainerCount = 100
   164  	f.MaxPerPodContainerCount = 2
   165  	f.NodeLabels = opt.NodeLabels
   166  	f.RegisterSchedulable = true
   167  
   168  	// Config struct
   169  	c, err := options.NewKubeletConfiguration()
   170  	if err != nil {
   171  		panic(err)
   172  	}
   173  
   174  	c.ImageServiceEndpoint = "unix:///run/containerd/containerd.sock"
   175  	c.StaticPodURL = ""
   176  	c.EnableServer = true
   177  	c.Address = "0.0.0.0" /* bind address */
   178  	c.Port = int32(opt.KubeletPort)
   179  	c.ReadOnlyPort = int32(opt.KubeletReadOnlyPort)
   180  	c.StaticPodPath = podFilePath
   181  	c.FileCheckFrequency.Duration = 20 * time.Second
   182  	c.HTTPCheckFrequency.Duration = 20 * time.Second
   183  	c.NodeStatusUpdateFrequency.Duration = 10 * time.Second
   184  	c.NodeStatusReportFrequency.Duration = 5 * time.Minute
   185  	c.SyncFrequency.Duration = 10 * time.Second
   186  	c.EvictionPressureTransitionPeriod.Duration = 5 * time.Minute
   187  	c.MaxPods = int32(opt.MaxPods)
   188  	c.PodsPerCore = int32(opt.PodsPerCore)
   189  	c.ClusterDNS = []string{}
   190  	c.ImageGCHighThresholdPercent = 90
   191  	c.ImageGCLowThresholdPercent = 80
   192  	c.ProviderID = fmt.Sprintf("kubemark://%v", opt.NodeName)
   193  	c.VolumeStatsAggPeriod.Duration = time.Minute
   194  	c.CgroupRoot = ""
   195  	c.CPUCFSQuota = true
   196  	c.EnableControllerAttachDetach = false
   197  	c.EnableDebuggingHandlers = true
   198  	c.CgroupsPerQOS = false
   199  	// hairpin-veth is used to allow hairpin packets. Note that this deviates from
   200  	// what the "real" kubelet currently does, because there's no way to
   201  	// set promiscuous mode on docker0.
   202  	c.HairpinMode = kubeletconfig.HairpinVeth
   203  	c.MaxOpenFiles = 1024
   204  	c.RegistryBurst = 10
   205  	c.RegistryPullQPS = 5.0
   206  	c.ResolverConfig = kubetypes.ResolvConfDefault
   207  	c.KubeletCgroups = "/kubelet"
   208  	c.SerializeImagePulls = true
   209  	c.SystemCgroups = ""
   210  	c.ProtectKernelDefaults = false
   211  	c.RegisterWithTaints = opt.RegisterWithTaints
   212  	c.RegisterNode = true
   213  	c.LocalStorageCapacityIsolation = true
   214  	c.PodLogsDir = podLogsPath
   215  
   216  	return f, c
   217  }
   218
View as plain text