generate.go

Documentation: edge-infra.dev/pkg/sds/ien/k8s/controllers/nodeagent/plugins/networking/netplan

     1  package netplan
     2  
     3  import (
     4  	"context"
     5  	"errors"
     6  	"fmt"
     7  	"net"
     8  	"sort"
     9  
    10  	v1core "k8s.io/api/core/v1"
    11  	"sigs.k8s.io/controller-runtime/pkg/client"
    12  
    13  	calico "edge-infra.dev/pkg/k8s/net/calico"
    14  	"edge-infra.dev/pkg/lib/kernel/netlink/ip"
    15  	"edge-infra.dev/pkg/lib/kernel/netlink/link"
    16  	v1ien "edge-infra.dev/pkg/sds/ien/k8s/apis/v1"
    17  	"edge-infra.dev/pkg/sds/ien/k8s/controllers/nodeagent/plugins/networking/netplan/internal"
    18  	nodemeta "edge-infra.dev/pkg/sds/ien/node"
    19  	"edge-infra.dev/pkg/sds/lib/networking"
    20  	"edge-infra.dev/pkg/sds/lib/networking/routing"
    21  )
    22  
    23  var (
    24  	ErrNoEthernetInterfaces = errors.New("no ethernet interfaces have been configured")
    25  )
    26  
    27  func GenerateTargetNetplanConfig(ienode *v1ien.IENode, nodes []v1core.Node, apiConfigured, gatewayEnabled bool, tunnelNetwork *net.IPNet) (*Config, error) {
    28  	netplanConfig := Template()
    29  	if err := configureEthernetInterfaces(netplanConfig, ienode, apiConfigured, gatewayEnabled); err != nil {
    30  		return nil, err
    31  	}
    32  
    33  	if apiConfigured {
    34  		return netplanConfig, nil
    35  	}
    36  
    37  	nodes = sortK8sNodesByCreationTimestamp(nodes)
    38  	gatewayNodeIndexes, nonGatewayNodeIndexes, err := filterGatewayNodeIndexes(nodes)
    39  	if err != nil {
    40  		return nil, err
    41  	}
    42  
    43  	subnets, err := internal.PairTunnelIPs(tunnelNetwork)
    44  	if err != nil {
    45  		return nil, err
    46  	}
    47  	if !gatewayEnabled { // no need for tunnels if egress gateway is disabled
    48  		return netplanConfig, nil
    49  	}
    50  	if ienode.Spec.IsGatewayNode() {
    51  		return configureGatewayNodeTunnels(netplanConfig, ienode, nodes, nonGatewayNodeIndexes, subnets)
    52  	}
    53  	return configureNonGatewayNodeTunnels(netplanConfig, ienode, nodes, gatewayNodeIndexes, subnets)
    54  }
    55  
    56  // Configures all of the the Edge O/S' ethernet interfaces and assigns them to netplan configuration.
    57  func configureEthernetInterfaces(netplanConfig *Config, ienode *v1ien.IENode, apiConfigured, gatewayEnabled bool) error {
    58  	// retrieve the default mtu
    59  	if err := setDefaultMTU(ienode, apiConfigured); err != nil {
    60  		return err
    61  	}
    62  
    63  	var mtu = ienode.Status.Network.DefaultMTU
    64  
    65  	// reduce mtu on non-gateway nodes to allow room for ipip header
    66  	if !ienode.Spec.IsGatewayNode() && gatewayEnabled && mtu != 0 {
    67  		mtu = mtu - ip.IPIPHeaderSize
    68  	}
    69  
    70  	// API netplan generation can only return API defined interfaces
    71  	if apiConfigured {
    72  		return configureAPIManagedEthernet(netplanConfig, ienode, mtu)
    73  	}
    74  
    75  	netlink := link.NetLink{}
    76  
    77  	// Retrieve all backbone interfaces (e*) from netlink
    78  	links, err := netlink.GetAllNetworkDeviceLinks()
    79  	if err != nil {
    80  		return err
    81  	}
    82  
    83  	for _, link := range links {
    84  		if iface, isPrimary, apiManaged := isInterfaceAPIManaged(ienode.Spec.Network, link.Attrs().HardwareAddr.String(), ienode.Spec.PrimaryInterface); apiManaged {
    85  			eth, err := createEthernet(ienode, iface, isPrimary, mtu)
    86  			if err != nil {
    87  				return err
    88  			}
    89  			netplanConfig.AddEthernet(link.Attrs().Name, eth)
    90  		} else {
    91  			// add non api defined backbone interfaces with activation mode set to off
    92  			netplanConfig.AddEthernet(link.Attrs().Name, Ethernet{DHCP4: true, ActivationMode: "off"})
    93  		}
    94  	}
    95  	if len(netplanConfig.Ethernets) == 0 {
    96  		return ErrNoEthernetInterfaces
    97  	}
    98  
    99  	return nil
   100  }
   101  
   102  // Determines if the macaddress for interface is listed and managed by the API.
   103  // If true, it will return the managed interface and whether it is the primary interface.
   104  func isInterfaceAPIManaged(managedInterfaces []v1ien.Network, macaddress string, primaryInterface *v1ien.PrimaryInterface) (v1ien.Network, bool, bool) {
   105  	for idx, iface := range managedInterfaces {
   106  		if iface.MacAddress == macaddress {
   107  			isPrimary := false
   108  			if primaryInterface == nil || len(primaryInterface.MacAddresses) == 0 {
   109  				isPrimary = idx == 0
   110  			} else {
   111  				for _, mac := range primaryInterface.MacAddresses {
   112  					if iface.MacAddress == mac {
   113  						isPrimary = true
   114  						break
   115  					}
   116  				}
   117  			}
   118  			return iface, isPrimary, true
   119  		}
   120  	}
   121  
   122  	return v1ien.Network{}, false, false
   123  }
   124  
   125  // Configures an API managed interface with basic netplan configuration.
   126  // This excludes any egress gateway configuration
   127  func configureAPIManagedEthernet(netplanConfig *Config, ienode *v1ien.IENode, mtu int) error {
   128  	for idx, iface := range ienode.Spec.Network {
   129  		isPrimary := false
   130  		if ienode.Spec.PrimaryInterface == nil || len(ienode.Spec.PrimaryInterface.MacAddresses) == 0 {
   131  			isPrimary = idx == 0
   132  		} else {
   133  			for _, mac := range ienode.Spec.PrimaryInterface.MacAddresses {
   134  				if iface.MacAddress == mac {
   135  					isPrimary = true
   136  					break
   137  				}
   138  			}
   139  		}
   140  		eth, err := createEthernet(ienode, iface, isPrimary, mtu)
   141  		if err != nil {
   142  			return err
   143  		}
   144  		tempName := fmt.Sprintf("eth%d", idx)
   145  		netplanConfig.AddEthernet(tempName, eth)
   146  	}
   147  	return nil
   148  }
   149  
   150  func configureGatewayNodeTunnels(netplanConfig *Config, localNode *v1ien.IENode, nodes []v1core.Node, nonGatewayNodeIndexes []int, subnets []*net.IPNet) (*Config, error) {
   151  	localNodeIP, err := localNodeIP(localNode.ObjectMeta.Name, nodes)
   152  	if err != nil {
   153  		return nil, err
   154  	}
   155  	// for gateway nodes we want list of all tunnels to the non-gateway nodes
   156  	for _, idx := range nonGatewayNodeIndexes {
   157  		remoteIPCIDR, err := getNodeIP(nodes[idx])
   158  		if err != nil {
   159  			return nil, err
   160  		}
   161  
   162  		remoteIP, _, err := net.ParseCIDR(remoteIPCIDR)
   163  		if err != nil {
   164  			return nil, err
   165  		}
   166  
   167  		subnet := subnets[idx]
   168  
   169  		// use first ip for gateway nodes
   170  		tunnelIP, err := networking.GetAddressFromSubnet(subnet, 0)
   171  		if err != nil {
   172  			return nil, err
   173  		}
   174  
   175  		tunnel := createEgressTunnel(localNodeIP, remoteIP.String(), tunnelIP)
   176  		netplanConfig.AddTunnel(fmt.Sprintf("tun%d", idx), tunnel)
   177  	}
   178  	return netplanConfig, nil
   179  }
   180  
   181  func configureNonGatewayNodeTunnels(netplanConfig *Config, localNode *v1ien.IENode, nodes []v1core.Node, gatewayNodeIndexes []int, subnets []*net.IPNet) (*Config, error) {
   182  	localNodeIP, err := localNodeIP(localNode.ObjectMeta.Name, nodes)
   183  	if err != nil {
   184  		return nil, err
   185  	}
   186  	// create tunnel for non-gateway node to gateway node
   187  	localNodeIdx, err := localNodeIdx(localNode.ObjectMeta.Name, nodes)
   188  	if err != nil {
   189  		return nil, err
   190  	}
   191  
   192  	// for non-gateway tunnels we want list of all gateway nodes
   193  	for _, idx := range gatewayNodeIndexes {
   194  		remoteIPCIDR, err := getNodeIP(nodes[idx])
   195  		if err != nil {
   196  			return nil, err
   197  		}
   198  
   199  		remoteIP, _, err := net.ParseCIDR(remoteIPCIDR)
   200  		if err != nil {
   201  			return nil, err
   202  		}
   203  
   204  		subnet := subnets[localNodeIdx]
   205  
   206  		// use second ip for non-gateway nodes
   207  		tunnelIP, err := networking.GetAddressFromSubnet(subnet, 1)
   208  		if err != nil {
   209  			return nil, err
   210  		}
   211  
   212  		// use first ip for gateway nodes
   213  		gatewayTunnelCIDR, err := networking.GetAddressFromSubnet(subnet, 0)
   214  		if err != nil {
   215  			return nil, err
   216  		}
   217  
   218  		gatewayTunnelIP, _, err := net.ParseCIDR(gatewayTunnelCIDR)
   219  		if err != nil {
   220  			return nil, err
   221  		}
   222  
   223  		tunnel := createEgressTunnel(localNodeIP, remoteIP.String(), tunnelIP)
   224  		tunnel.AddRoute(Route{
   225  			To:    "default",
   226  			Table: routing.EgressGatewayTable,
   227  			Via:   gatewayTunnelIP.String(),
   228  		})
   229  		tunnel.AddRoutingPolicy(RoutingPolicy{
   230  			Mark:     512,
   231  			To:       "0.0.0.0/0",
   232  			Table:    routing.EgressGatewayTable,
   233  			Priority: 32765,
   234  		})
   235  		netplanConfig.AddTunnel(fmt.Sprintf("tun%d", localNodeIdx), tunnel)
   236  	}
   237  	return netplanConfig, nil
   238  }
   239  
   240  func sortK8sNodesByCreationTimestamp(nodes []v1core.Node) []v1core.Node {
   241  	sort.Slice(nodes, func(i, j int) bool {
   242  		return nodes[i].ObjectMeta.CreationTimestamp.Before(&nodes[j].ObjectMeta.CreationTimestamp)
   243  	})
   244  	return nodes
   245  }
   246  
   247  // returns list of indexes for gateway and non-gateway nodes
   248  func filterGatewayNodeIndexes(nodes []v1core.Node) ([]int, []int, error) {
   249  	var gatewayNodeIndexList, nonGatewayNodeIndexList []int
   250  	for idx, node := range nodes {
   251  		role, ok := nodes[idx].ObjectMeta.Labels[nodemeta.RoleLabel]
   252  		if !ok {
   253  			return nil, nil, fmt.Errorf("role label %s for node %s", nodemeta.RoleLabel, node.ObjectMeta.Name)
   254  		}
   255  		if role == string(v1ien.ControlPlane) {
   256  			gatewayNodeIndexList = append(gatewayNodeIndexList, idx)
   257  		}
   258  		if role == string(v1ien.Worker) {
   259  			nonGatewayNodeIndexList = append(nonGatewayNodeIndexList, idx)
   260  		}
   261  	}
   262  	return gatewayNodeIndexList, nonGatewayNodeIndexList, nil
   263  }
   264  
   265  func localNodeIdx(localNodeName string, nodes []v1core.Node) (int, error) {
   266  	localNodeIdx := -1
   267  	for idx, k8sNode := range nodes {
   268  		if k8sNode.ObjectMeta.Name == localNodeName {
   269  			localNodeIdx = idx
   270  		}
   271  	}
   272  
   273  	if localNodeIdx == -1 {
   274  		return -1, errors.New("could not find owning k8s node")
   275  	}
   276  
   277  	return localNodeIdx, nil
   278  }
   279  
   280  func localNodeIP(localNodeName string, nodes []v1core.Node) (string, error) {
   281  	localNodeIdx, err := localNodeIdx(localNodeName, nodes)
   282  	if err != nil {
   283  		return "", err
   284  	}
   285  
   286  	localNodeIP, err := getNodeIP(nodes[localNodeIdx])
   287  	if err != nil {
   288  		return "", err
   289  	}
   290  
   291  	ip, _, err := net.ParseCIDR(localNodeIP)
   292  	if err != nil {
   293  		return "", err
   294  	}
   295  
   296  	return ip.String(), nil
   297  }
   298  
   299  func getNodeIP(node v1core.Node) (string, error) {
   300  	nodeIP, ok := node.ObjectMeta.Annotations[calico.IPAnnotation]
   301  	if !ok {
   302  		return "", fmt.Errorf("could not find node ip address via %s annotation for node %s", calico.IPAnnotation, node.ObjectMeta.Name)
   303  	}
   304  	return nodeIP, nil
   305  }
   306  
   307  func createEthernet(ienode *v1ien.IENode, iface v1ien.Network, isPrimary bool, mtu int) (Ethernet, error) {
   308  	if !isPrimary {
   309  		return NewEthernet(
   310  			iface.Addresses,
   311  			iface.DHCP4,
   312  			[]Route{},
   313  			[]RoutingPolicy{},
   314  			ienode.Spec.DNSServers,
   315  			false,
   316  			0,
   317  			Match{
   318  				MacAddress: iface.MacAddress,
   319  			},
   320  		), nil
   321  	}
   322  	routeList := []Route{}
   323  	if !iface.DHCP4 && iface.Gateway4 != "" {
   324  		staticRoutes, err := staticRoutes(iface)
   325  		if err != nil {
   326  			return Ethernet{}, err
   327  		}
   328  		routeList = append(routeList, staticRoutes...)
   329  	}
   330  	return NewEthernet(
   331  		iface.Addresses,
   332  		iface.DHCP4,
   333  		routeList,
   334  		[]RoutingPolicy{},
   335  		ienode.Spec.DNSServers,
   336  		true,
   337  		mtu,
   338  		Match{
   339  			MacAddress: iface.MacAddress,
   340  		},
   341  	), nil
   342  }
   343  
   344  func createEgressTunnel(local, remote, address string) Tunnel {
   345  	return Tunnel{
   346  		Mode:      "ipip",
   347  		Local:     local,
   348  		Remote:    remote,
   349  		Addresses: []string{address},
   350  	}
   351  }
   352  
   353  func setDefaultMTU(ienode *v1ien.IENode, apiConfigured bool) error {
   354  	if ienode.Status == nil {
   355  		ienode.Status = &v1ien.IENodeStatus{Network: v1ien.NetworkStatus{}}
   356  	}
   357  
   358  	if apiConfigured { // do not configure the mtu if set from API
   359  		return nil
   360  	}
   361  
   362  	// return if mtu is already set
   363  	if ienode.Status.Network.DefaultMTU != 0 {
   364  		return nil
   365  	}
   366  
   367  	hardwareAddress, err := ienode.Spec.DefaultMacAddress()
   368  	if err != nil {
   369  		return err
   370  	}
   371  
   372  	netlink := link.NetLink{}
   373  
   374  	defaultLink, err := netlink.GetFromHardwareAddr(hardwareAddress)
   375  	if err != nil {
   376  		return err
   377  	}
   378  
   379  	ienode.Status.Network.DefaultMTU = defaultLink.Attrs().MTU
   380  	ienode.Status.Network.DefaultInterfaceName = defaultLink.Attrs().Name
   381  	return nil
   382  }
   383  
   384  // patch the ienode with network information
   385  func patchIENode(ctx context.Context, k8sClient client.Client, ienode *v1ien.IENode) error {
   386  	currentIENode := &v1ien.IENode{}
   387  	if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(ienode), currentIENode); err != nil {
   388  		return err
   389  	}
   390  	patch := client.MergeFrom(currentIENode.DeepCopy())
   391  	currentIENode.Status = ienode.Status
   392  	return k8sClient.Status().Patch(ctx, currentIENode, patch)
   393  }
   394
View as plain text