manifests.go

Documentation: edge-infra.dev/pkg/sds/lanoutage/detector

     1  package detector
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"path/filepath"
     8  	"strings"
     9  
    10  	schedulercfg "edge-infra.dev/config/pallets/sds/lanoutage/scheduler"
    11  	"edge-infra.dev/pkg/edge/component/build"
    12  	"edge-infra.dev/pkg/lib/fog"
    13  	"edge-infra.dev/pkg/sds/lanoutage/detector/internal/config"
    14  	"edge-infra.dev/pkg/sds/lanoutage/detector/internal/constants"
    15  	"edge-infra.dev/pkg/sds/lanoutage/detector/internal/enter"
    16  	"edge-infra.dev/pkg/sds/lanoutage/detector/internal/leave"
    17  	"edge-infra.dev/pkg/sds/lib/dbus/systemd"
    18  	"edge-infra.dev/pkg/sds/lib/k8s/manifest"
    19  
    20  	"github.com/spf13/afero"
    21  	corev1 "k8s.io/api/core/v1"
    22  	"k8s.io/apimachinery/pkg/runtime"
    23  	clientcmdv1 "k8s.io/client-go/tools/clientcmd/api/v1"
    24  	bootstraptokenv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1"
    25  	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
    26  	kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme"
    27  	kubeadmapiv1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
    28  	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
    29  	"k8s.io/kubernetes/cmd/kubeadm/app/phases/controlplane"
    30  )
    31  
    32  // removeKubeManifests removes the kubernetes kube-apiserver, controller manager,
    33  // and scheduler static pod manifests from the node's filesystem.
    34  func RemoveKubeManifests(ctx context.Context, cfg leave.Conf) error {
    35  	log := fog.FromContext(ctx)
    36  	log.Info("removing kubernetes manifests")
    37  
    38  	err := cfg.ReconcilerConfig.Fs.RemoveAll(constants.KubeAPIServerManifestFilepath)
    39  	if err != nil {
    40  		return fmt.Errorf("there was an error removing the kube-apiserver manifest: %w", err)
    41  	}
    42  
    43  	err = cfg.ReconcilerConfig.Fs.RemoveAll(constants.ControllerManagerManifestFilepath)
    44  	if err != nil {
    45  		return fmt.Errorf("there was an error removing the kube-controller-manager manifest: %w", err)
    46  	}
    47  
    48  	return cfg.ReconcilerConfig.Fs.RemoveAll(constants.SchedulerManifestFilepath)
    49  }
    50  
    51  func createKubeadmConfig(vipAddress, registrySuffix, kubeVersion, etcdImageTag string) (*kubeadmapi.InitConfiguration, error) {
    52  	versionedInitCfg := &kubeadmapiv1.InitConfiguration{
    53  		LocalAPIEndpoint: kubeadmapiv1.APIEndpoint{AdvertiseAddress: vipAddress},
    54  		BootstrapTokens:  []bootstraptokenv1.BootstrapToken{PlaceholderToken},
    55  		NodeRegistration: kubeadmapiv1.NodeRegistrationOptions{
    56  			CRISocket: kubeadmconstants.DefaultCRISocket, // avoid CRI detection
    57  			Name:      "node",
    58  		},
    59  	}
    60  
    61  	versionedClusterCfg := &kubeadmapiv1.ClusterConfiguration{
    62  		KubernetesVersion: kubeVersion, // avoid going to the Internet for the current Kubernetes version
    63  		ImageRepository:   filepath.Join(build.DefaultPublicContainerRegistry, registrySuffix),
    64  	}
    65  
    66  	internalcfg := &kubeadmapi.InitConfiguration{}
    67  
    68  	kubeadmscheme.Scheme.Default(versionedInitCfg)
    69  	err := kubeadmscheme.Scheme.Convert(versionedInitCfg, internalcfg, nil)
    70  	if err != nil {
    71  		return nil, err
    72  	}
    73  
    74  	kubeadmscheme.Scheme.Default(versionedClusterCfg)
    75  	if err := kubeadmscheme.Scheme.Convert(versionedClusterCfg, &internalcfg.ClusterConfiguration, nil); err != nil {
    76  		return nil, err
    77  	}
    78  
    79  	// replace the etcd image tag with the one found in container-versions.yaml
    80  	internalcfg.ClusterConfiguration.Etcd.Local.ImageMeta.ImageTag = etcdImageTag
    81  	return internalcfg, err
    82  }
    83  
    84  func getEtcdImageTag(fs afero.Fs) (string, error) {
    85  	containerVersionsCfg := config.ContainerVersionsConfig{}
    86  	err := containerVersionsCfg.Load(fs)
    87  	if err != nil {
    88  		return "", fmt.Errorf("failed to retrieve container versions: %w", err)
    89  	}
    90  	etcdImage := containerVersionsCfg.Containers.Etcd
    91  	imageParts := strings.Split(etcdImage, ":")
    92  	imageTag := imageParts[len(imageParts)-1]
    93  
    94  	return imageTag, nil
    95  }
    96  
    97  func getRegistrySuffix(fs afero.Fs) (string, error) {
    98  	containerVersionsCfg := config.ContainerVersionsConfig{}
    99  	err := containerVersionsCfg.Load(fs)
   100  	if err != nil {
   101  		return "", fmt.Errorf("failed to retrieve container versions: %w", err)
   102  	}
   103  	etcdImage := containerVersionsCfg.Containers.Etcd
   104  	imageParts := strings.Split(etcdImage, "/")
   105  	prefix := imageParts[0]
   106  
   107  	return prefix, nil
   108  }
   109  
   110  // createManifests() creates manifests for kube-apiserver, controller manager,
   111  // and the scheduler
   112  func createManifests(cfg enter.Conf, components []string) error {
   113  	manifestDir := "/etc/kubernetes/manifests"
   114  	err := controlplane.CreateStaticPodFiles(manifestDir, "", &cfg.InitConf.ClusterConfiguration, &cfg.InitConf.LocalAPIEndpoint, false, components...)
   115  	if err != nil {
   116  		return fmt.Errorf("an error occurred creating the yaml manifests: %w", err)
   117  	}
   118  
   119  	return createSchedulerManifest(cfg.ReconcilerConfig)
   120  }
   121  
   122  // createSchedulerManifest creates the static manifest for the LAN outage
   123  // scheduler
   124  func createSchedulerManifest(cfg config.Config) error {
   125  	m := manifest.New(cfg.Fs, constants.SchedulerManifestFilepath, &corev1.Pod{}, 0600)
   126  	if err := m.Load(schedulercfg.SchedulerManifest); err != nil {
   127  		return err
   128  	}
   129  
   130  	return m.WithCreate(func(obj runtime.Object) error {
   131  		return addSchedulerArgs(cfg, obj)
   132  	})
   133  }
   134  
   135  func addSchedulerArgs(cfg config.Config, obj runtime.Object) error {
   136  	pod, ok := obj.(*corev1.Pod)
   137  	if !ok {
   138  		return errors.New("invalid LAN Outage Scheduler manifest")
   139  	}
   140  
   141  	pod.Spec.Containers[0].Image = cfg.SchedulerImage
   142  	nodeNameArg := strings.Join([]string{"--NODE_NAME", cfg.NodeName}, "=")
   143  	pod.Spec.Containers[0].Args = []string{"run", nodeNameArg}
   144  	return nil
   145  }
   146  
   147  func configureControllerManagerConfig(cfg enter.Conf) error {
   148  	m := manifest.New(cfg.ReconcilerConfig.Fs, constants.ControllerManagerConfFilepath, &clientcmdv1.Config{}, 0)
   149  	return m.WithUpdate(func(_ runtime.Object) error {
   150  		config, ok := m.Content().(*clientcmdv1.Config)
   151  		if !ok {
   152  			return fmt.Errorf("current content of the manifest is not a valid pod")
   153  		}
   154  		return setVIPAsClusterServer(cfg.Vip, config)
   155  	})
   156  }
   157  
   158  // setupKubeConfig copies the kubelet kubeconfig file to the zylevel0 kubeconfig file.
   159  // It sets the client cert and key fields to point to the zylevel0 pem file,
   160  // and copies the kubelet pem file there.
   161  func setupKubeConfig(fs afero.Fs) error {
   162  	kubeletManifest := manifest.New(fs, constants.KubeletConfFilepath, &clientcmdv1.Config{}, 0)
   163  	if err := kubeletManifest.Read(true); err != nil {
   164  		return err
   165  	}
   166  	config, ok := kubeletManifest.Content().(*clientcmdv1.Config)
   167  	if !ok {
   168  		return fmt.Errorf("current content of the kubelet.conf manifest is not a valid Config file")
   169  	}
   170  
   171  	for i, ai := range config.AuthInfos {
   172  		if ai.Name == "default-auth" {
   173  			config.AuthInfos[i].AuthInfo.ClientCertificate = constants.Zylevel0PEMFilepath
   174  			config.AuthInfos[i].AuthInfo.ClientKey = constants.Zylevel0PEMFilepath
   175  		}
   176  	}
   177  	zylevel0Manifest := manifest.New(fs, constants.Zylevel0ConfFilepath, config, 0755)
   178  	if err := zylevel0Manifest.Write(); err != nil {
   179  		return fmt.Errorf("an error occurred writing the zylevel0 config file: %w", err)
   180  	}
   181  
   182  	configPemBytes, err := afero.ReadFile(fs, constants.KubeletPEMFilepath)
   183  	if err != nil {
   184  		return fmt.Errorf("an error occurred reading the kubelet config pem file: %w", err)
   185  	}
   186  	err = afero.WriteFile(fs, constants.Zylevel0PEMFilepath, configPemBytes, 0600)
   187  	if err != nil {
   188  		return fmt.Errorf("an error occurred writing the zylevel0 config pem file: %w", err)
   189  	}
   190  
   191  	// user 1000 is always zylevel0 on the IEN
   192  	err = fs.Chown(constants.Zylevel0ConfFilepath, 1000, 1000)
   193  	if err != nil {
   194  		return err
   195  	}
   196  	return fs.Chown(constants.Zylevel0PEMFilepath, 1000, 1000)
   197  }
   198  
   199  // restartKubelet restarts the kubelet systemd service.
   200  func restartKubelet(ctx context.Context) (err error) {
   201  	conn, err := systemd.NewConnection(ctx)
   202  	if err != nil {
   203  		return err
   204  	}
   205  	defer conn.Close()
   206  
   207  	return conn.Restart(ctx, "kubelet.service", systemd.Replace, true)
   208  }
   209
View as plain text