package testutil import ( "fmt" "strconv" "strings" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "github.com/linkerd/linkerd2/pkg/k8s" ) const enabled = "true" // InjectValidator is used as a helper to generate // correct injector flags and annotations and verify // injected pods type InjectValidator struct { NoInitContainer bool AutoInject bool AdminPort int ControlPort int EnableDebug bool EnableExternalProfiles bool ImagePullPolicy string InboundPort int InitImage string InitImageVersion string OutboundPort int CPULimit string EphemeralStorageLimit string CPURequest string MemoryLimit string MemoryRequest string EphemeralStorageRequest string Image string LogLevel string LogFormat string UID int GID int Version string RequireIdentityOnPorts string SkipOutboundPorts string OpaquePorts string SkipInboundPorts string OutboundConnectTimeout string InboundConnectTimeout string WaitBeforeExitSeconds int SkipSubnets string ShutdownGracePeriod string } func (iv *InjectValidator) getContainer(pod *v1.PodSpec, name string, isInit bool) *v1.Container { containers := pod.Containers if isInit { containers = pod.InitContainers } for _, container := range containers { if container.Name == name { return &container } } return nil } func (iv *InjectValidator) validateEnvVar(container *v1.Container, envName, expectedValue string) error { for _, env := range container.Env { if env.Name == envName { if env.Value == expectedValue { return nil } return fmt.Errorf("env: %s, expected: %s, actual %s", envName, expectedValue, env.Value) } } return fmt.Errorf("cannot find env: %s", envName) } func (iv *InjectValidator) validatePort(container *v1.Container, portName string, expectedValue int) error { for _, port := range container.Ports { if port.Name == portName { if port.ContainerPort == int32(expectedValue) { return nil } return fmt.Errorf("port: %s, expected: %d, actual %d", portName, expectedValue, port.ContainerPort) } } return fmt.Errorf("cannot find port: %s", portName) } func (iv *InjectValidator) validateDebugContainer(pod *v1.PodSpec) error { if iv.EnableDebug { proxyContainer := iv.getContainer(pod, k8s.DebugContainerName, false) if proxyContainer == nil { return fmt.Errorf("container %s missing", k8s.DebugContainerName) } } return nil } func (iv *InjectValidator) validateProxyContainer(pod *v1.PodSpec) error { proxyContainer := iv.getContainer(pod, k8s.ProxyContainerName, false) if proxyContainer == nil { return fmt.Errorf("container %s missing", k8s.ProxyContainerName) } if iv.AdminPort != 0 { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_ADMIN_LISTEN_ADDR", fmt.Sprintf("[::]:%d", iv.AdminPort)); err != nil { return err } if proxyContainer.LivenessProbe.HTTPGet.Port.IntVal != int32(iv.AdminPort) { return fmt.Errorf("livenessProbe: expected: %d, actual %d", iv.AdminPort, proxyContainer.LivenessProbe.HTTPGet.Port.IntVal) } if proxyContainer.ReadinessProbe.HTTPGet.Port.IntVal != int32(iv.AdminPort) { return fmt.Errorf("readinessProbe: expected: %d, actual %d", iv.AdminPort, proxyContainer.LivenessProbe.HTTPGet.Port.IntVal) } if err := iv.validatePort(proxyContainer, k8s.ProxyAdminPortName, iv.AdminPort); err != nil { return err } } if iv.ControlPort != 0 { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_CONTROL_LISTEN_ADDR", fmt.Sprintf("[::]:%d", iv.ControlPort)); err != nil { return err } } if iv.EnableExternalProfiles { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_DESTINATION_PROFILE_SUFFIXES", "."); err != nil { return err } } if iv.ImagePullPolicy != "" { if string(proxyContainer.ImagePullPolicy) != iv.ImagePullPolicy { return fmt.Errorf("pullPolicy: expected: %s, actual %s", iv.ImagePullPolicy, string(proxyContainer.ImagePullPolicy)) } } if iv.InboundPort != 0 { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_INBOUND_LISTEN_ADDR", fmt.Sprintf("[::]:%d", iv.InboundPort)); err != nil { return err } if proxyContainer.LivenessProbe.HTTPGet.Port.IntVal != int32(iv.AdminPort) { return fmt.Errorf("livenessProbe: expected: %d, actual %d", iv.AdminPort, proxyContainer.LivenessProbe.HTTPGet.Port.IntVal) } if proxyContainer.ReadinessProbe.HTTPGet.Port.IntVal != int32(iv.AdminPort) { return fmt.Errorf("readinessProbe: expected: %d, actual %d", iv.AdminPort, proxyContainer.LivenessProbe.HTTPGet.Port.IntVal) } if err := iv.validatePort(proxyContainer, k8s.ProxyPortName, iv.InboundPort); err != nil { return err } } if iv.OutboundPort != 0 { if err := iv.validateEnvVar( proxyContainer, "LINKERD2_PROXY_OUTBOUND_LISTEN_ADDRS", fmt.Sprintf("127.0.0.1:%d", iv.OutboundPort), ); err != nil { return err } } if iv.CPULimit != "" { limit := resource.MustParse(iv.CPULimit) if proxyContainer.Resources.Limits.Cpu() != nil { if !proxyContainer.Resources.Limits.Cpu().Equal(limit) { return fmt.Errorf("CpuLimit: expected %v, actual %v", &limit, proxyContainer.Resources.Limits.Cpu()) } } else { return fmt.Errorf("CpuLimit: expected %v, but none", &limit) } } if iv.CPURequest != "" { request := resource.MustParse(iv.CPURequest) if proxyContainer.Resources.Requests.Cpu() != nil { if !proxyContainer.Resources.Requests.Cpu().Equal(request) { return fmt.Errorf("CpuRequest: expected %v, actual %v", &request, proxyContainer.Resources.Requests.Cpu()) } } else { return fmt.Errorf("CpuRequest: expected %v, but none", &request) } } if iv.MemoryLimit != "" { limit := resource.MustParse(iv.MemoryLimit) if proxyContainer.Resources.Limits.Memory() != nil { if !proxyContainer.Resources.Limits.Memory().Equal(limit) { return fmt.Errorf("MemLimit: expected %v, actual %v", &limit, proxyContainer.Resources.Limits.Memory()) } } else { return fmt.Errorf("MemLimit: expected %v, but none", &limit) } } if iv.MemoryRequest != "" { request := resource.MustParse(iv.MemoryRequest) if proxyContainer.Resources.Requests.Memory() != nil { if !proxyContainer.Resources.Requests.Memory().Equal(request) { return fmt.Errorf("MemRequest: expected %v, actual %v", &request, proxyContainer.Resources.Requests.Memory()) } } else { return fmt.Errorf("MemRequest: expected %v, but none", &request) } } if iv.EphemeralStorageLimit != "" { limit := resource.MustParse(iv.EphemeralStorageLimit) if proxyContainer.Resources.Limits.StorageEphemeral() != nil { if !proxyContainer.Resources.Limits.StorageEphemeral().Equal(limit) { return fmt.Errorf("EphemeralStorageLimit: expected %v, actual %v", &limit, proxyContainer.Resources.Limits.StorageEphemeral()) } } else { return fmt.Errorf("EphemeralStorageLimit: expected %v, but none", &limit) } } if iv.EphemeralStorageRequest != "" { request := resource.MustParse(iv.EphemeralStorageRequest) if proxyContainer.Resources.Requests.StorageEphemeral() != nil { if !proxyContainer.Resources.Requests.StorageEphemeral().Equal(request) { return fmt.Errorf("EphemeralStorageRequest: expected %v, actual %v", &request, proxyContainer.Resources.Requests.StorageEphemeral()) } } else { return fmt.Errorf("EphemeralStorageRequest: expected %v, but none", &request) } } if iv.Image != "" || iv.Version != "" { image := strings.Split(proxyContainer.Image, ":") if len(image) != 2 { return fmt.Errorf("invalid proxy container image string: %s", proxyContainer.Image) } if iv.Image != "" { if image[0] != iv.Image { return fmt.Errorf("proxyImage: expected %s, actual %s", iv.Image, image[0]) } } if iv.Version != "" { if image[1] != iv.Version { return fmt.Errorf("proxyImageVersion: expected %s, actual %s", iv.Version, image[1]) } } } if iv.LogLevel != "" { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_LOG", iv.LogLevel); err != nil { return err } } if iv.LogFormat != "" { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_LOG_FORMAT", iv.LogFormat); err != nil { return err } } if iv.UID != 0 { if proxyContainer.SecurityContext.RunAsUser == nil { return fmt.Errorf("no RunAsUser specified") } if *proxyContainer.SecurityContext.RunAsUser != int64(iv.UID) { return fmt.Errorf("runAsUser: expected %d, actual %d", iv.UID, *proxyContainer.SecurityContext.RunAsUser) } } if iv.GID != 0 { if proxyContainer.SecurityContext.RunAsGroup == nil { return fmt.Errorf("no RunAsGroup specified") } if *proxyContainer.SecurityContext.RunAsGroup != int64(iv.GID) { return fmt.Errorf("runAsGroup: expected %d, actual %d", iv.GID, *proxyContainer.SecurityContext.RunAsGroup) } } if iv.RequireIdentityOnPorts != "" { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_INBOUND_PORTS_REQUIRE_IDENTITY", iv.RequireIdentityOnPorts); err != nil { return err } } if iv.OpaquePorts != "" { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_INBOUND_PORTS_DISABLE_PROTOCOL_DETECTION", iv.OpaquePorts); err != nil { return err } } if iv.OutboundConnectTimeout != "" { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_OUTBOUND_CONNECT_TIMEOUT", iv.OutboundConnectTimeout); err != nil { return err } } if iv.OutboundConnectTimeout != "" { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_INBOUND_CONNECT_TIMEOUT", iv.InboundConnectTimeout); err != nil { return err } } if iv.WaitBeforeExitSeconds != 0 { expectedCmd := fmt.Sprintf("/bin/sleep,%d", iv.WaitBeforeExitSeconds) actual := strings.Join(proxyContainer.Lifecycle.PreStop.Exec.Command, ",") if expectedCmd != strings.Join(proxyContainer.Lifecycle.PreStop.Exec.Command, ",") { return fmt.Errorf("preStopHook: expected %s, actual %s", expectedCmd, actual) } } if iv.ShutdownGracePeriod != "" { if err := iv.validateEnvVar(proxyContainer, "LINKERD2_PROXY_SHUTDOWN_GRACE_PERIOD", iv.ShutdownGracePeriod); err != nil { return err } } return nil } func (iv *InjectValidator) validateInitContainer(pod *v1.PodSpec) error { if iv.NoInitContainer { return nil } initContainer := iv.getContainer(pod, k8s.InitContainerName, true) if initContainer == nil { return fmt.Errorf("container %s missing", k8s.InitContainerName) } if iv.InitImage != "" || iv.InitImageVersion != "" { image := strings.Split(initContainer.Image, ":") if len(image) != 2 { return fmt.Errorf("invalid proxy init image string: %s", initContainer.Image) } if iv.InitImage != "" { if image[0] != iv.InitImage { return fmt.Errorf("proxyInitImage: expected %s, actual %s", iv.InitImage, image[0]) } } if iv.InitImageVersion != "" { if image[1] != iv.InitImageVersion { return fmt.Errorf("proxyInitImageVersion: expected %s, actual %s", iv.InitImageVersion, image[1]) } } } if iv.InboundPort != 0 { if err := iv.validateArg(initContainer, "--incoming-proxy-port", strconv.Itoa(iv.InboundPort)); err != nil { return err } } if iv.OutboundPort != 0 { if err := iv.validateArg(initContainer, "--proxy-uid", strconv.Itoa(iv.UID)); err != nil { return err } } if iv.UID != 0 { if err := iv.validateArg(initContainer, "--outgoing-proxy-port", strconv.Itoa(iv.OutboundPort)); err != nil { return err } } if iv.SkipInboundPorts != "" { expectedPorts := fmt.Sprintf("%d,%d,%s", iv.ControlPort, iv.AdminPort, iv.SkipInboundPorts) if err := iv.validateArg(initContainer, "--inbound-ports-to-ignore", expectedPorts); err != nil { return err } } if iv.SkipOutboundPorts != "" { if err := iv.validateArg(initContainer, "--outbound-ports-to-ignore", iv.SkipOutboundPorts); err != nil { return err } } if iv.SkipSubnets != "" { if err := iv.validateArg(initContainer, "--skip-subnets", iv.SkipSubnets); err != nil { return err } } return nil } func (iv *InjectValidator) validateArg(container *v1.Container, argName, expectedValue string) error { for i, arg := range container.Args { if arg == argName { if len(container.Args) < i+2 { return fmt.Errorf("No value for arg %s", argName) } if container.Args[i+1] != expectedValue { return fmt.Errorf("container arg %s expected: %s, actual %s", argName, expectedValue, container.Args[i+1]) } return nil } } return fmt.Errorf("Could not find arg: %s", argName) } // ValidatePod validates that the pod had been configured // according by the injector correctly func (iv *InjectValidator) ValidatePod(pod *v1.PodSpec) error { if err := iv.validateProxyContainer(pod); err != nil { return err } if err := iv.validateInitContainer(pod); err != nil { return err } if err := iv.validateDebugContainer(pod); err != nil { return err } return nil } // GetFlagsAndAnnotations retrieves the injector config flags and annotations // based on the options provided func (iv *InjectValidator) GetFlagsAndAnnotations() ([]string, map[string]string) { annotations := make(map[string]string) var flags []string if iv.AutoInject { annotations[k8s.ProxyInjectAnnotation] = k8s.ProxyInjectEnabled } if iv.AdminPort != 0 { annotations[k8s.ProxyAdminPortAnnotation] = strconv.Itoa(iv.AdminPort) flags = append(flags, fmt.Sprintf("--admin-port=%s", strconv.Itoa(iv.AdminPort))) } if iv.ControlPort != 0 { annotations[k8s.ProxyControlPortAnnotation] = strconv.Itoa(iv.ControlPort) flags = append(flags, fmt.Sprintf("--control-port=%s", strconv.Itoa(iv.ControlPort))) } if iv.EnableDebug { annotations[k8s.ProxyEnableDebugAnnotation] = enabled flags = append(flags, "--enable-debug-sidecar") } if iv.EnableExternalProfiles { annotations[k8s.ProxyEnableExternalProfilesAnnotation] = enabled flags = append(flags, "--enable-external-profiles") } if iv.ImagePullPolicy != "" { annotations[k8s.ProxyImagePullPolicyAnnotation] = iv.ImagePullPolicy flags = append(flags, fmt.Sprintf("--image-pull-policy=%s", iv.ImagePullPolicy)) } if iv.InboundPort != 0 { annotations[k8s.ProxyInboundPortAnnotation] = strconv.Itoa(iv.InboundPort) flags = append(flags, fmt.Sprintf("--inbound-port=%s", strconv.Itoa(iv.InboundPort))) } if iv.InitImage != "" { annotations[k8s.ProxyInitImageAnnotation] = iv.InitImage flags = append(flags, fmt.Sprintf("--init-image=%s", iv.InitImage)) } if iv.InitImageVersion != "" { annotations[k8s.ProxyInitImageVersionAnnotation] = iv.InitImageVersion flags = append(flags, fmt.Sprintf("--init-image-version=%s", iv.InitImageVersion)) } if iv.OutboundPort != 0 { annotations[k8s.ProxyOutboundPortAnnotation] = strconv.Itoa(iv.OutboundPort) flags = append(flags, fmt.Sprintf("--outbound-port=%s", strconv.Itoa(iv.OutboundPort))) } if iv.CPULimit != "" { annotations[k8s.ProxyCPULimitAnnotation] = iv.CPULimit flags = append(flags, fmt.Sprintf("--proxy-cpu-limit=%s", iv.CPULimit)) } if iv.CPURequest != "" { annotations[k8s.ProxyCPURequestAnnotation] = iv.CPURequest flags = append(flags, fmt.Sprintf("--proxy-cpu-request=%s", iv.CPURequest)) } if iv.MemoryLimit != "" { annotations[k8s.ProxyMemoryLimitAnnotation] = iv.MemoryLimit flags = append(flags, fmt.Sprintf("--proxy-memory-limit=%s", iv.MemoryLimit)) } if iv.EphemeralStorageLimit != "" { annotations[k8s.ProxyEphemeralStorageLimitAnnotation] = iv.EphemeralStorageLimit // TODO - `inject` does not support the `--set` flag, so we can't add this // flags = append(flags, "--set", fmt.Sprintf("proxy.resources.ephemeral-storage.limit=%s", iv.EphemeralStorageLimit)) } if iv.MemoryRequest != "" { annotations[k8s.ProxyMemoryRequestAnnotation] = iv.MemoryRequest flags = append(flags, fmt.Sprintf("--proxy-memory-request=%s", iv.MemoryRequest)) } if iv.EphemeralStorageRequest != "" { annotations[k8s.ProxyEphemeralStorageRequestAnnotation] = iv.EphemeralStorageRequest // TODO - `inject` does not support the `--set` flag, so we can't add this // flags = append(flags, "--set", fmt.Sprintf("proxy.resources.ephemeral-storage.request=%s", iv.EphemeralStorageRequest)) } if iv.Image != "" { annotations[k8s.ProxyImageAnnotation] = iv.Image flags = append(flags, fmt.Sprintf("--proxy-image=%s", iv.Image)) } if iv.LogLevel != "" { annotations[k8s.ProxyLogLevelAnnotation] = iv.LogLevel flags = append(flags, fmt.Sprintf("--proxy-log-level=%s", iv.LogLevel)) } if iv.LogFormat != "" { annotations[k8s.ProxyLogFormatAnnotation] = iv.LogFormat } if iv.UID != 0 { annotations[k8s.ProxyUIDAnnotation] = strconv.Itoa(iv.UID) flags = append(flags, fmt.Sprintf("--proxy-uid=%s", strconv.Itoa(iv.UID))) } if iv.GID != 0 { annotations[k8s.ProxyGIDAnnotation] = strconv.Itoa(iv.GID) flags = append(flags, fmt.Sprintf("--proxy-gid=%s", strconv.Itoa(iv.GID))) } if iv.Version != "" { annotations[k8s.ProxyVersionOverrideAnnotation] = iv.Version flags = append(flags, fmt.Sprintf("--proxy-version=%s", iv.Version)) } if iv.RequireIdentityOnPorts != "" { annotations[k8s.ProxyRequireIdentityOnInboundPortsAnnotation] = iv.RequireIdentityOnPorts flags = append(flags, fmt.Sprintf("--require-identity-on-inbound-ports =%s", iv.RequireIdentityOnPorts)) } if iv.SkipInboundPorts != "" { annotations[k8s.ProxyIgnoreInboundPortsAnnotation] = iv.SkipInboundPorts flags = append(flags, fmt.Sprintf("--skip-inbound-ports=%s", iv.SkipInboundPorts)) } if iv.OpaquePorts != "" { annotations[k8s.ProxyOpaquePortsAnnotation] = iv.OpaquePorts } if iv.SkipOutboundPorts != "" { annotations[k8s.ProxyIgnoreOutboundPortsAnnotation] = iv.SkipOutboundPorts flags = append(flags, fmt.Sprintf("--skip-outbound-ports=%s", iv.SkipOutboundPorts)) } if iv.OutboundConnectTimeout != "" { annotations[k8s.ProxyOutboundConnectTimeout] = iv.OutboundConnectTimeout } if iv.InboundConnectTimeout != "" { annotations[k8s.ProxyInboundConnectTimeout] = iv.InboundConnectTimeout } if iv.WaitBeforeExitSeconds != 0 { annotations[k8s.ProxyWaitBeforeExitSecondsAnnotation] = strconv.Itoa(iv.WaitBeforeExitSeconds) flags = append(flags, fmt.Sprintf("--wait-before-exit-secondst=%s", strconv.Itoa(iv.WaitBeforeExitSeconds))) } if iv.SkipSubnets != "" { annotations[k8s.ProxySkipSubnetsAnnotation] = iv.SkipSubnets flags = append(flags, fmt.Sprintf("--skip-subnets=%s", iv.SkipSubnets)) } if iv.ShutdownGracePeriod != "" { annotations[k8s.ProxyShutdownGracePeriodAnnotation] = iv.ShutdownGracePeriod flags = append(flags, fmt.Sprintf("--shutdown-grace-period=%s", iv.ShutdownGracePeriod)) } return flags, annotations }