firewall.go

Documentation: k8s.io/kubernetes/test/e2e/network

     1  //go:build !providerless
     2  // +build !providerless
     3  
     4  /*
     5  Copyright 2016 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package network
    21  
    22  import (
    23  	"context"
    24  	"fmt"
    25  	"strings"
    26  	"time"
    27  
    28  	v1 "k8s.io/api/core/v1"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/util/sets"
    31  	"k8s.io/apimachinery/pkg/util/wait"
    32  	clientset "k8s.io/client-go/kubernetes"
    33  	cloudprovider "k8s.io/cloud-provider"
    34  	"k8s.io/kubernetes/pkg/cluster/ports"
    35  	kubeschedulerconfig "k8s.io/kubernetes/pkg/scheduler/apis/config"
    36  	"k8s.io/kubernetes/test/e2e/framework"
    37  	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
    38  	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
    39  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    40  	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
    41  	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
    42  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
    43  	"k8s.io/kubernetes/test/e2e/network/common"
    44  	gcecloud "k8s.io/legacy-cloud-providers/gce"
    45  	admissionapi "k8s.io/pod-security-admission/api"
    46  
    47  	"github.com/onsi/ginkgo/v2"
    48  )
    49  
    50  const (
    51  	firewallTestTCPTimeout = time.Duration(1 * time.Second)
    52  	// Set ports outside of 30000-32767, 80 and 8080 to avoid being allowlisted by the e2e cluster
    53  	firewallTestHTTPPort = int32(29999)
    54  	firewallTestUDPPort  = int32(29998)
    55  )
    56  
    57  var _ = common.SIGDescribe("Firewall rule", func() {
    58  	var firewallTestName = "firewall-test"
    59  	f := framework.NewDefaultFramework(firewallTestName)
    60  	f.NamespacePodSecurityLevel = admissionapi.LevelPrivileged
    61  
    62  	var cs clientset.Interface
    63  	var cloudConfig framework.CloudConfig
    64  	var gceCloud *gcecloud.Cloud
    65  
    66  	ginkgo.BeforeEach(func() {
    67  		e2eskipper.SkipUnlessProviderIs("gce")
    68  
    69  		var err error
    70  		cs = f.ClientSet
    71  		cloudConfig = framework.TestContext.CloudConfig
    72  		gceCloud, err = gce.GetGCECloud()
    73  		framework.ExpectNoError(err)
    74  	})
    75  
    76  	// This test takes around 6 minutes to run
    77  	f.It(f.WithSlow(), f.WithSerial(), "should create valid firewall rules for LoadBalancer type service", func(ctx context.Context) {
    78  		ns := f.Namespace.Name
    79  		// This source ranges is just used to examine we have exact same things on LB firewall rules
    80  		firewallTestSourceRanges := []string{"0.0.0.0/1", "128.0.0.0/1"}
    81  		serviceName := "firewall-test-loadbalancer"
    82  
    83  		jig := e2eservice.NewTestJig(cs, ns, serviceName)
    84  		nodeList, err := e2enode.GetBoundedReadySchedulableNodes(ctx, cs, e2eservice.MaxNodesForEndpointsTests)
    85  		framework.ExpectNoError(err)
    86  
    87  		nodesNames := []string{}
    88  		for _, node := range nodeList.Items {
    89  			nodesNames = append(nodesNames, node.Name)
    90  		}
    91  		nodesSet := sets.NewString(nodesNames...)
    92  
    93  		ginkgo.By("Creating a LoadBalancer type service with ExternalTrafficPolicy=Global")
    94  		svc, err := jig.CreateLoadBalancerService(ctx, e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs), func(svc *v1.Service) {
    95  			svc.Spec.Ports = []v1.ServicePort{{Protocol: v1.ProtocolTCP, Port: firewallTestHTTPPort}}
    96  			svc.Spec.LoadBalancerSourceRanges = firewallTestSourceRanges
    97  		})
    98  		framework.ExpectNoError(err)
    99  
   100  		// This configmap is guaranteed to exist after a Loadbalancer type service is created
   101  		ginkgo.By("Getting cluster ID")
   102  		clusterID, err := gce.GetClusterID(ctx, cs)
   103  		framework.ExpectNoError(err)
   104  		framework.Logf("Got cluster ID: %v", clusterID)
   105  
   106  		defer func() {
   107  			_, err = jig.UpdateService(ctx, func(svc *v1.Service) {
   108  				svc.Spec.Type = v1.ServiceTypeNodePort
   109  				svc.Spec.LoadBalancerSourceRanges = nil
   110  			})
   111  			framework.ExpectNoError(err)
   112  			err = cs.CoreV1().Services(svc.Namespace).Delete(ctx, svc.Name, metav1.DeleteOptions{})
   113  			framework.ExpectNoError(err)
   114  			ginkgo.By("Waiting for the local traffic health check firewall rule to be deleted")
   115  			localHCFwName := gce.MakeHealthCheckFirewallNameForLBService(clusterID, cloudprovider.DefaultLoadBalancerName(svc), false)
   116  			_, err := gce.WaitForFirewallRule(ctx, gceCloud, localHCFwName, false, e2eservice.LoadBalancerCleanupTimeout)
   117  			framework.ExpectNoError(err)
   118  		}()
   119  		svcExternalIP := svc.Status.LoadBalancer.Ingress[0].IP
   120  
   121  		ginkgo.By("Checking if service's firewall rule is correct")
   122  		lbFw := gce.ConstructFirewallForLBService(svc, cloudConfig.NodeTag)
   123  		fw, err := gceCloud.GetFirewall(lbFw.Name)
   124  		framework.ExpectNoError(err)
   125  		err = gce.VerifyFirewallRule(fw, lbFw, cloudConfig.Network, false)
   126  		framework.ExpectNoError(err)
   127  
   128  		ginkgo.By("Checking if service's nodes health check firewall rule is correct")
   129  		nodesHCFw := gce.ConstructHealthCheckFirewallForLBService(clusterID, svc, cloudConfig.NodeTag, true)
   130  		fw, err = gceCloud.GetFirewall(nodesHCFw.Name)
   131  		framework.ExpectNoError(err)
   132  		err = gce.VerifyFirewallRule(fw, nodesHCFw, cloudConfig.Network, false)
   133  		framework.ExpectNoError(err)
   134  
   135  		// OnlyLocal service is needed to examine which exact nodes the requests are being forwarded to by the Load Balancer on GCE
   136  		ginkgo.By("Updating LoadBalancer service to ExternalTrafficPolicy=Local")
   137  		svc, err = jig.UpdateService(ctx, func(svc *v1.Service) {
   138  			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyLocal
   139  		})
   140  		framework.ExpectNoError(err)
   141  
   142  		ginkgo.By("Waiting for the nodes health check firewall rule to be deleted")
   143  		_, err = gce.WaitForFirewallRule(ctx, gceCloud, nodesHCFw.Name, false, e2eservice.LoadBalancerCleanupTimeout)
   144  		framework.ExpectNoError(err)
   145  
   146  		ginkgo.By("Waiting for the correct local traffic health check firewall rule to be created")
   147  		localHCFw := gce.ConstructHealthCheckFirewallForLBService(clusterID, svc, cloudConfig.NodeTag, false)
   148  		fw, err = gce.WaitForFirewallRule(ctx, gceCloud, localHCFw.Name, true, e2eservice.GetServiceLoadBalancerCreationTimeout(ctx, cs))
   149  		framework.ExpectNoError(err)
   150  		err = gce.VerifyFirewallRule(fw, localHCFw, cloudConfig.Network, false)
   151  		framework.ExpectNoError(err)
   152  
   153  		ginkgo.By(fmt.Sprintf("Creating netexec pods on at most %v nodes", e2eservice.MaxNodesForEndpointsTests))
   154  		for i, nodeName := range nodesNames {
   155  			podName := fmt.Sprintf("netexec%v", i)
   156  
   157  			framework.Logf("Creating netexec pod %q on node %v in namespace %q", podName, nodeName, ns)
   158  			pod := e2epod.NewAgnhostPod(ns, podName, nil, nil, nil,
   159  				"netexec",
   160  				fmt.Sprintf("--http-port=%d", firewallTestHTTPPort),
   161  				fmt.Sprintf("--udp-port=%d", firewallTestUDPPort))
   162  			pod.ObjectMeta.Labels = jig.Labels
   163  			nodeSelection := e2epod.NodeSelection{Name: nodeName}
   164  			e2epod.SetNodeSelection(&pod.Spec, nodeSelection)
   165  			pod.Spec.HostNetwork = true
   166  			_, err := cs.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{})
   167  			framework.ExpectNoError(err)
   168  			framework.ExpectNoError(e2epod.WaitTimeoutForPodReadyInNamespace(ctx, f.ClientSet, podName, f.Namespace.Name, framework.PodStartTimeout))
   169  			framework.Logf("Netexec pod %q in namespace %q running", podName, ns)
   170  
   171  			defer func() {
   172  				framework.Logf("Cleaning up the netexec pod: %v", podName)
   173  				err = cs.CoreV1().Pods(ns).Delete(ctx, podName, metav1.DeleteOptions{})
   174  				framework.ExpectNoError(err)
   175  			}()
   176  		}
   177  
   178  		// Send requests from outside of the cluster because internal traffic is allowlisted
   179  		ginkgo.By("Accessing the external service ip from outside, all non-master nodes should be reached")
   180  		err = testHitNodesFromOutside(svcExternalIP, firewallTestHTTPPort, e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, cs), nodesSet)
   181  		framework.ExpectNoError(err)
   182  
   183  		// Check if there are overlapping tags on the firewall that extend beyond just the vms in our cluster
   184  		// by removing the tag on one vm and make sure it doesn't get any traffic. This is an imperfect
   185  		// simulation, we really want to check that traffic doesn't reach a vm outside the GKE cluster, but
   186  		// that's much harder to do in the current e2e framework.
   187  		ginkgo.By(fmt.Sprintf("Removing tags from one of the nodes: %v", nodesNames[0]))
   188  		nodesSet.Delete(nodesNames[0])
   189  		// Instance could run in a different zone in multi-zone test. Figure out which zone
   190  		// it is in before proceeding.
   191  		zone := cloudConfig.Zone
   192  		if zoneInLabel, ok := nodeList.Items[0].Labels[v1.LabelFailureDomainBetaZone]; ok {
   193  			zone = zoneInLabel
   194  		} else if zoneInLabel, ok := nodeList.Items[0].Labels[v1.LabelTopologyZone]; ok {
   195  			zone = zoneInLabel
   196  		}
   197  		removedTags := gce.SetInstanceTags(cloudConfig, nodesNames[0], zone, []string{})
   198  		defer func() {
   199  			ginkgo.By("Adding tags back to the node and wait till the traffic is recovered")
   200  			nodesSet.Insert(nodesNames[0])
   201  			gce.SetInstanceTags(cloudConfig, nodesNames[0], zone, removedTags)
   202  			// Make sure traffic is recovered before exit
   203  			err = testHitNodesFromOutside(svcExternalIP, firewallTestHTTPPort, e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, cs), nodesSet)
   204  			framework.ExpectNoError(err)
   205  		}()
   206  
   207  		ginkgo.By("Accessing service through the external ip and examine got no response from the node without tags")
   208  		err = testHitNodesFromOutsideWithCount(svcExternalIP, firewallTestHTTPPort, e2eservice.GetServiceLoadBalancerPropagationTimeout(ctx, cs), nodesSet, 15)
   209  		framework.ExpectNoError(err)
   210  	})
   211  
   212  	ginkgo.It("control plane should not expose well-known ports", func(ctx context.Context) {
   213  		nodes, err := e2enode.GetReadySchedulableNodes(ctx, cs)
   214  		framework.ExpectNoError(err)
   215  
   216  		ginkgo.By("Checking well known ports on master and nodes are not exposed externally")
   217  		nodeAddr := e2enode.FirstAddress(nodes, v1.NodeExternalIP)
   218  		if nodeAddr != "" {
   219  			assertNotReachableHTTPTimeout(nodeAddr, "/", ports.KubeletPort, firewallTestTCPTimeout, false)
   220  			assertNotReachableHTTPTimeout(nodeAddr, "/", ports.KubeletReadOnlyPort, firewallTestTCPTimeout, false)
   221  			assertNotReachableHTTPTimeout(nodeAddr, "/", ports.ProxyStatusPort, firewallTestTCPTimeout, false)
   222  		}
   223  
   224  		controlPlaneAddresses := framework.GetControlPlaneAddresses(ctx, cs)
   225  		for _, instanceAddress := range controlPlaneAddresses {
   226  			assertNotReachableHTTPTimeout(instanceAddress, "/healthz", ports.KubeControllerManagerPort, firewallTestTCPTimeout, true)
   227  			assertNotReachableHTTPTimeout(instanceAddress, "/healthz", kubeschedulerconfig.DefaultKubeSchedulerPort, firewallTestTCPTimeout, true)
   228  		}
   229  	})
   230  })
   231  
   232  func assertNotReachableHTTPTimeout(ip, path string, port int, timeout time.Duration, enableHTTPS bool) {
   233  	result := e2enetwork.PokeHTTP(ip, port, path, &e2enetwork.HTTPPokeParams{Timeout: timeout, EnableHTTPS: enableHTTPS})
   234  	if result.Status == e2enetwork.HTTPError {
   235  		framework.Failf("Unexpected error checking for reachability of %s:%d: %v", ip, port, result.Error)
   236  	}
   237  	if result.Code != 0 {
   238  		framework.Failf("Was unexpectedly able to reach %s:%d", ip, port)
   239  	}
   240  }
   241  
   242  // testHitNodesFromOutside checks HTTP connectivity from outside.
   243  func testHitNodesFromOutside(externalIP string, httpPort int32, timeout time.Duration, expectedHosts sets.String) error {
   244  	return testHitNodesFromOutsideWithCount(externalIP, httpPort, timeout, expectedHosts, 1)
   245  }
   246  
   247  // testHitNodesFromOutsideWithCount checks HTTP connectivity from outside with count.
   248  func testHitNodesFromOutsideWithCount(externalIP string, httpPort int32, timeout time.Duration, expectedHosts sets.String,
   249  	countToSucceed int) error {
   250  	framework.Logf("Waiting up to %v for satisfying expectedHosts for %v times", timeout, countToSucceed)
   251  	hittedHosts := sets.NewString()
   252  	count := 0
   253  	condition := func() (bool, error) {
   254  		result := e2enetwork.PokeHTTP(externalIP, int(httpPort), "/hostname", &e2enetwork.HTTPPokeParams{Timeout: 1 * time.Second})
   255  		if result.Status != e2enetwork.HTTPSuccess {
   256  			return false, nil
   257  		}
   258  
   259  		hittedHost := strings.TrimSpace(string(result.Body))
   260  		if !expectedHosts.Has(hittedHost) {
   261  			framework.Logf("Error hitting unexpected host: %v, reset counter: %v", hittedHost, count)
   262  			count = 0
   263  			return false, nil
   264  		}
   265  		if !hittedHosts.Has(hittedHost) {
   266  			hittedHosts.Insert(hittedHost)
   267  			framework.Logf("Missing %+v, got %+v", expectedHosts.Difference(hittedHosts), hittedHosts)
   268  		}
   269  		if hittedHosts.Equal(expectedHosts) {
   270  			count++
   271  			if count >= countToSucceed {
   272  				return true, nil
   273  			}
   274  		}
   275  		return false, nil
   276  	}
   277  
   278  	if err := wait.Poll(time.Second, timeout, condition); err != nil {
   279  		return fmt.Errorf("error waiting for expectedHosts: %v, hittedHosts: %v, count: %v, expected count: %v",
   280  			expectedHosts, hittedHosts, count, countToSucceed)
   281  	}
   282  	return nil
   283  }
   284
View as plain text