hns.go

Documentation: k8s.io/kubernetes/pkg/proxy/winkernel

     1  //go:build windows
     2  // +build windows
     3  
     4  /*
     5  Copyright 2018 The Kubernetes Authors.
     6  
     7  Licensed under the Apache License, Version 2.0 (the "License");
     8  you may not use this file except in compliance with the License.
     9  You may obtain a copy of the License at
    10  
    11      http://www.apache.org/licenses/LICENSE-2.0
    12  
    13  Unless required by applicable law or agreed to in writing, software
    14  distributed under the License is distributed on an "AS IS" BASIS,
    15  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    16  See the License for the specific language governing permissions and
    17  limitations under the License.
    18  */
    19  
    20  package winkernel
    21  
    22  import (
    23  	"crypto/sha1"
    24  	"encoding/json"
    25  	"fmt"
    26  
    27  	"github.com/Microsoft/hcsshim/hcn"
    28  	"k8s.io/klog/v2"
    29  
    30  	"strings"
    31  )
    32  
    33  type HostNetworkService interface {
    34  	getNetworkByName(name string) (*hnsNetworkInfo, error)
    35  	getAllEndpointsByNetwork(networkName string) (map[string]*endpointInfo, error)
    36  	getEndpointByID(id string) (*endpointInfo, error)
    37  	getEndpointByIpAddress(ip string, networkName string) (*endpointInfo, error)
    38  	getEndpointByName(id string) (*endpointInfo, error)
    39  	createEndpoint(ep *endpointInfo, networkName string) (*endpointInfo, error)
    40  	deleteEndpoint(hnsID string) error
    41  	getLoadBalancer(endpoints []endpointInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16, previousLoadBalancers map[loadBalancerIdentifier]*loadBalancerInfo) (*loadBalancerInfo, error)
    42  	getAllLoadBalancers() (map[loadBalancerIdentifier]*loadBalancerInfo, error)
    43  	deleteLoadBalancer(hnsID string) error
    44  }
    45  
    46  type hns struct {
    47  	hcn HcnService
    48  }
    49  
    50  var (
    51  	// LoadBalancerFlagsIPv6 enables IPV6.
    52  	LoadBalancerFlagsIPv6 hcn.LoadBalancerFlags = 2
    53  	// LoadBalancerPortMappingFlagsVipExternalIP enables VipExternalIP.
    54  	LoadBalancerPortMappingFlagsVipExternalIP hcn.LoadBalancerPortMappingFlags = 16
    55  )
    56  
    57  func (hns hns) getNetworkByName(name string) (*hnsNetworkInfo, error) {
    58  	hnsnetwork, err := hns.hcn.GetNetworkByName(name)
    59  	if err != nil {
    60  		klog.ErrorS(err, "Error getting network by name")
    61  		return nil, err
    62  	}
    63  
    64  	var remoteSubnets []*remoteSubnetInfo
    65  	for _, policy := range hnsnetwork.Policies {
    66  		if policy.Type == hcn.RemoteSubnetRoute {
    67  			policySettings := hcn.RemoteSubnetRoutePolicySetting{}
    68  			err = json.Unmarshal(policy.Settings, &policySettings)
    69  			if err != nil {
    70  				return nil, fmt.Errorf("failed to unmarshal Remote Subnet policy settings")
    71  			}
    72  			rs := &remoteSubnetInfo{
    73  				destinationPrefix: policySettings.DestinationPrefix,
    74  				isolationID:       policySettings.IsolationId,
    75  				providerAddress:   policySettings.ProviderAddress,
    76  				drMacAddress:      policySettings.DistributedRouterMacAddress,
    77  			}
    78  			remoteSubnets = append(remoteSubnets, rs)
    79  		}
    80  	}
    81  
    82  	return &hnsNetworkInfo{
    83  		id:            hnsnetwork.Id,
    84  		name:          hnsnetwork.Name,
    85  		networkType:   string(hnsnetwork.Type),
    86  		remoteSubnets: remoteSubnets,
    87  	}, nil
    88  }
    89  
    90  func (hns hns) getAllEndpointsByNetwork(networkName string) (map[string]*(endpointInfo), error) {
    91  	hcnnetwork, err := hns.hcn.GetNetworkByName(networkName)
    92  	if err != nil {
    93  		klog.ErrorS(err, "failed to get HNS network by name", "name", networkName)
    94  		return nil, err
    95  	}
    96  	endpoints, err := hns.hcn.ListEndpointsOfNetwork(hcnnetwork.Id)
    97  	if err != nil {
    98  		return nil, fmt.Errorf("failed to list endpoints: %w", err)
    99  	}
   100  	endpointInfos := make(map[string]*(endpointInfo))
   101  	for _, ep := range endpoints {
   102  
   103  		if len(ep.IpConfigurations) == 0 {
   104  			klog.V(3).InfoS("No IpConfigurations found in endpoint info of queried endpoints", "endpoint", ep)
   105  			continue
   106  		}
   107  
   108  		// Add to map with key endpoint ID or IP address
   109  		// Storing this is expensive in terms of memory, however there is a bug in Windows Server 2019 that can cause two endpoints to be created with the same IP address.
   110  		// TODO: Store by IP only and remove any lookups by endpoint ID.
   111  		endpointInfos[ep.Id] = &endpointInfo{
   112  			ip:         ep.IpConfigurations[0].IpAddress,
   113  			isLocal:    uint32(ep.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0,
   114  			macAddress: ep.MacAddress,
   115  			hnsID:      ep.Id,
   116  			hns:        hns,
   117  			// only ready and not terminating endpoints were added to HNS
   118  			ready:       true,
   119  			serving:     true,
   120  			terminating: false,
   121  		}
   122  		endpointInfos[ep.IpConfigurations[0].IpAddress] = endpointInfos[ep.Id]
   123  
   124  		if len(ep.IpConfigurations) == 1 {
   125  			continue
   126  		}
   127  
   128  		// If ipFamilyPolicy is RequireDualStack or PreferDualStack, then there will be 2 IPS (iPV4 and IPV6)
   129  		// in the endpoint list
   130  		endpointDualstack := &endpointInfo{
   131  			ip:         ep.IpConfigurations[1].IpAddress,
   132  			isLocal:    uint32(ep.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0,
   133  			macAddress: ep.MacAddress,
   134  			hnsID:      ep.Id,
   135  			hns:        hns,
   136  			// only ready and not terminating endpoints were added to HNS
   137  			ready:       true,
   138  			serving:     true,
   139  			terminating: false,
   140  		}
   141  		endpointInfos[ep.IpConfigurations[1].IpAddress] = endpointDualstack
   142  	}
   143  	klog.V(3).InfoS("Queried endpoints from network", "network", networkName)
   144  	klog.V(5).InfoS("Queried endpoints details", "network", networkName, "endpointInfos", endpointInfos)
   145  	return endpointInfos, nil
   146  }
   147  
   148  func (hns hns) getEndpointByID(id string) (*endpointInfo, error) {
   149  	hnsendpoint, err := hns.hcn.GetEndpointByID(id)
   150  	if err != nil {
   151  		return nil, err
   152  	}
   153  	return &endpointInfo{ //TODO: fill out PA
   154  		ip:         hnsendpoint.IpConfigurations[0].IpAddress,
   155  		isLocal:    uint32(hnsendpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, //TODO: Change isLocal to isRemote
   156  		macAddress: hnsendpoint.MacAddress,
   157  		hnsID:      hnsendpoint.Id,
   158  		hns:        hns,
   159  	}, nil
   160  }
   161  func (hns hns) getEndpointByIpAddress(ip string, networkName string) (*endpointInfo, error) {
   162  	hnsnetwork, err := hns.hcn.GetNetworkByName(networkName)
   163  	if err != nil {
   164  		klog.ErrorS(err, "Error getting network by name")
   165  		return nil, err
   166  	}
   167  
   168  	endpoints, err := hns.hcn.ListEndpoints()
   169  	if err != nil {
   170  		return nil, fmt.Errorf("failed to list endpoints: %w", err)
   171  	}
   172  	for _, endpoint := range endpoints {
   173  		equal := false
   174  		if endpoint.IpConfigurations != nil && len(endpoint.IpConfigurations) > 0 {
   175  			equal = endpoint.IpConfigurations[0].IpAddress == ip
   176  
   177  			if !equal && len(endpoint.IpConfigurations) > 1 {
   178  				equal = endpoint.IpConfigurations[1].IpAddress == ip
   179  			}
   180  		}
   181  		if equal && strings.EqualFold(endpoint.HostComputeNetwork, hnsnetwork.Id) {
   182  			return &endpointInfo{
   183  				ip:         ip,
   184  				isLocal:    uint32(endpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, //TODO: Change isLocal to isRemote
   185  				macAddress: endpoint.MacAddress,
   186  				hnsID:      endpoint.Id,
   187  				hns:        hns,
   188  			}, nil
   189  		}
   190  	}
   191  	return nil, fmt.Errorf("Endpoint %v not found on network %s", ip, networkName)
   192  }
   193  func (hns hns) getEndpointByName(name string) (*endpointInfo, error) {
   194  	hnsendpoint, err := hns.hcn.GetEndpointByName(name)
   195  	if err != nil {
   196  		return nil, err
   197  	}
   198  	return &endpointInfo{ //TODO: fill out PA
   199  		ip:         hnsendpoint.IpConfigurations[0].IpAddress,
   200  		isLocal:    uint32(hnsendpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0, //TODO: Change isLocal to isRemote
   201  		macAddress: hnsendpoint.MacAddress,
   202  		hnsID:      hnsendpoint.Id,
   203  		hns:        hns,
   204  	}, nil
   205  }
   206  func (hns hns) createEndpoint(ep *endpointInfo, networkName string) (*endpointInfo, error) {
   207  	hnsNetwork, err := hns.hcn.GetNetworkByName(networkName)
   208  	if err != nil {
   209  		return nil, err
   210  	}
   211  	var flags hcn.EndpointFlags
   212  	if !ep.isLocal {
   213  		flags |= hcn.EndpointFlagsRemoteEndpoint
   214  	}
   215  	ipConfig := &hcn.IpConfig{
   216  		IpAddress: ep.ip,
   217  	}
   218  	hnsEndpoint := &hcn.HostComputeEndpoint{
   219  		IpConfigurations: []hcn.IpConfig{*ipConfig},
   220  		MacAddress:       ep.macAddress,
   221  		Flags:            flags,
   222  		SchemaVersion: hcn.SchemaVersion{
   223  			Major: 2,
   224  			Minor: 0,
   225  		},
   226  	}
   227  
   228  	var createdEndpoint *hcn.HostComputeEndpoint
   229  	if !ep.isLocal {
   230  		if len(ep.providerAddress) != 0 {
   231  			policySettings := hcn.ProviderAddressEndpointPolicySetting{
   232  				ProviderAddress: ep.providerAddress,
   233  			}
   234  			policySettingsJson, err := json.Marshal(policySettings)
   235  			if err != nil {
   236  				return nil, fmt.Errorf("PA Policy creation failed: %v", err)
   237  			}
   238  			paPolicy := hcn.EndpointPolicy{
   239  				Type:     hcn.NetworkProviderAddress,
   240  				Settings: policySettingsJson,
   241  			}
   242  			hnsEndpoint.Policies = append(hnsEndpoint.Policies, paPolicy)
   243  		}
   244  		createdEndpoint, err = hns.hcn.CreateRemoteEndpoint(hnsNetwork, hnsEndpoint)
   245  		if err != nil {
   246  			return nil, err
   247  		}
   248  	} else {
   249  		createdEndpoint, err = hns.hcn.CreateEndpoint(hnsNetwork, hnsEndpoint)
   250  		if err != nil {
   251  			return nil, err
   252  		}
   253  	}
   254  	return &endpointInfo{
   255  		ip:              createdEndpoint.IpConfigurations[0].IpAddress,
   256  		isLocal:         uint32(createdEndpoint.Flags&hcn.EndpointFlagsRemoteEndpoint) == 0,
   257  		macAddress:      createdEndpoint.MacAddress,
   258  		hnsID:           createdEndpoint.Id,
   259  		providerAddress: ep.providerAddress, //TODO get from createdEndpoint
   260  		hns:             hns,
   261  	}, nil
   262  }
   263  func (hns hns) deleteEndpoint(hnsID string) error {
   264  	hnsendpoint, err := hns.hcn.GetEndpointByID(hnsID)
   265  	if err != nil {
   266  		return err
   267  	}
   268  	err = hns.hcn.DeleteEndpoint(hnsendpoint)
   269  	if err == nil {
   270  		klog.V(3).InfoS("Remote endpoint resource deleted", "hnsID", hnsID)
   271  	}
   272  	return err
   273  }
   274  
   275  // findLoadBalancerID will construct a id from the provided loadbalancer fields
   276  func findLoadBalancerID(endpoints []endpointInfo, vip string, protocol, internalPort, externalPort uint16) (loadBalancerIdentifier, error) {
   277  	// Compute hash from backends (endpoint IDs)
   278  	hash, err := hashEndpoints(endpoints)
   279  	if err != nil {
   280  		klog.V(2).ErrorS(err, "Error hashing endpoints", "endpoints", endpoints)
   281  		return loadBalancerIdentifier{}, err
   282  	}
   283  	if len(vip) > 0 {
   284  		return loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, vip: vip, endpointsHash: hash}, nil
   285  	}
   286  	return loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, endpointsHash: hash}, nil
   287  }
   288  
   289  func (hns hns) getAllLoadBalancers() (map[loadBalancerIdentifier]*loadBalancerInfo, error) {
   290  	lbs, err := hns.hcn.ListLoadBalancers()
   291  	var id loadBalancerIdentifier
   292  	if err != nil {
   293  		return nil, err
   294  	}
   295  	loadBalancers := make(map[loadBalancerIdentifier]*(loadBalancerInfo))
   296  	for _, lb := range lbs {
   297  		portMap := lb.PortMappings[0]
   298  		// Compute hash from backends (endpoint IDs)
   299  		hash, err := hashEndpoints(lb.HostComputeEndpoints)
   300  		if err != nil {
   301  			klog.V(2).ErrorS(err, "Error hashing endpoints", "policy", lb)
   302  			return nil, err
   303  		}
   304  		if len(lb.FrontendVIPs) == 0 {
   305  			// Leave VIP uninitialized
   306  			id = loadBalancerIdentifier{protocol: uint16(portMap.Protocol), internalPort: portMap.InternalPort, externalPort: portMap.ExternalPort, endpointsHash: hash}
   307  		} else {
   308  			id = loadBalancerIdentifier{protocol: uint16(portMap.Protocol), internalPort: portMap.InternalPort, externalPort: portMap.ExternalPort, vip: lb.FrontendVIPs[0], endpointsHash: hash}
   309  		}
   310  		loadBalancers[id] = &loadBalancerInfo{
   311  			hnsID: lb.Id,
   312  		}
   313  	}
   314  	klog.V(3).InfoS("Queried load balancers", "count", len(lbs))
   315  	return loadBalancers, nil
   316  }
   317  
   318  func (hns hns) getLoadBalancer(endpoints []endpointInfo, flags loadBalancerFlags, sourceVip string, vip string, protocol uint16, internalPort uint16, externalPort uint16, previousLoadBalancers map[loadBalancerIdentifier]*loadBalancerInfo) (*loadBalancerInfo, error) {
   319  	var id loadBalancerIdentifier
   320  	vips := []string{}
   321  	// Compute hash from backends (endpoint IDs)
   322  	hash, err := hashEndpoints(endpoints)
   323  	if err != nil {
   324  		klog.V(2).ErrorS(err, "Error hashing endpoints", "endpoints", endpoints)
   325  		return nil, err
   326  	}
   327  	if len(vip) > 0 {
   328  		id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, vip: vip, endpointsHash: hash}
   329  		vips = append(vips, vip)
   330  	} else {
   331  		id = loadBalancerIdentifier{protocol: protocol, internalPort: internalPort, externalPort: externalPort, endpointsHash: hash}
   332  	}
   333  
   334  	if lb, found := previousLoadBalancers[id]; found {
   335  		klog.V(1).InfoS("Found cached Hns loadbalancer policy resource", "policies", lb)
   336  		return lb, nil
   337  	}
   338  
   339  	lbPortMappingFlags := hcn.LoadBalancerPortMappingFlagsNone
   340  	if flags.isILB {
   341  		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsILB
   342  	}
   343  	if flags.useMUX {
   344  		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsUseMux
   345  	}
   346  	if flags.preserveDIP {
   347  		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsPreserveDIP
   348  	}
   349  	if flags.localRoutedVIP {
   350  		lbPortMappingFlags |= hcn.LoadBalancerPortMappingFlagsLocalRoutedVIP
   351  	}
   352  	if flags.isVipExternalIP {
   353  		lbPortMappingFlags |= LoadBalancerPortMappingFlagsVipExternalIP
   354  	}
   355  
   356  	lbFlags := hcn.LoadBalancerFlagsNone
   357  	if flags.isDSR {
   358  		lbFlags |= hcn.LoadBalancerFlagsDSR
   359  	}
   360  
   361  	if flags.isIPv6 {
   362  		lbFlags |= LoadBalancerFlagsIPv6
   363  	}
   364  
   365  	lbDistributionType := hcn.LoadBalancerDistributionNone
   366  
   367  	if flags.sessionAffinity {
   368  		lbDistributionType = hcn.LoadBalancerDistributionSourceIP
   369  	}
   370  
   371  	loadBalancer := &hcn.HostComputeLoadBalancer{
   372  		SourceVIP: sourceVip,
   373  		PortMappings: []hcn.LoadBalancerPortMapping{
   374  			{
   375  				Protocol:         uint32(protocol),
   376  				InternalPort:     internalPort,
   377  				ExternalPort:     externalPort,
   378  				DistributionType: lbDistributionType,
   379  				Flags:            lbPortMappingFlags,
   380  			},
   381  		},
   382  		FrontendVIPs: vips,
   383  		SchemaVersion: hcn.SchemaVersion{
   384  			Major: 2,
   385  			Minor: 0,
   386  		},
   387  		Flags: lbFlags,
   388  	}
   389  
   390  	for _, ep := range endpoints {
   391  		loadBalancer.HostComputeEndpoints = append(loadBalancer.HostComputeEndpoints, ep.hnsID)
   392  	}
   393  
   394  	lb, err := hns.hcn.CreateLoadBalancer(loadBalancer)
   395  
   396  	if err != nil {
   397  		return nil, err
   398  	}
   399  
   400  	klog.V(1).InfoS("Created Hns loadbalancer policy resource", "loadBalancer", lb)
   401  	lbInfo := &loadBalancerInfo{
   402  		hnsID: lb.Id,
   403  	}
   404  	// Add to map of load balancers
   405  	previousLoadBalancers[id] = lbInfo
   406  	return lbInfo, err
   407  }
   408  
   409  func (hns hns) deleteLoadBalancer(hnsID string) error {
   410  	lb, err := hns.hcn.GetLoadBalancerByID(hnsID)
   411  	if err != nil {
   412  		// Return silently
   413  		return nil
   414  	}
   415  
   416  	err = hns.hcn.DeleteLoadBalancer(lb)
   417  	if err != nil {
   418  		// There is a bug in Windows Server 2019, that can cause the delete call to fail sometimes. We retry one more time.
   419  		// TODO: The logic in syncProxyRules  should be rewritten in the future to better stage and handle a call like this failing using the policyApplied fields.
   420  		klog.V(1).ErrorS(err, "Error deleting Hns loadbalancer policy resource. Attempting one more time...", "loadBalancer", lb)
   421  		return hns.hcn.DeleteLoadBalancer(lb)
   422  	}
   423  	return err
   424  }
   425  
   426  // Calculates a hash from the given endpoint IDs.
   427  func hashEndpoints[T string | endpointInfo](endpoints []T) (hash [20]byte, err error) {
   428  	var id string
   429  	// Recover in case something goes wrong. Return error and null byte array.
   430  	defer func() {
   431  		if r := recover(); r != nil {
   432  			err = r.(error)
   433  			hash = [20]byte{}
   434  		}
   435  	}()
   436  
   437  	// Iterate over endpoints, compute hash
   438  	for _, ep := range endpoints {
   439  		switch x := any(ep).(type) {
   440  		case endpointInfo:
   441  			id = strings.ToUpper(x.hnsID)
   442  		case string:
   443  			id = x
   444  		}
   445  		if len(id) > 0 {
   446  			// We XOR the hashes of endpoints, since they are an unordered set.
   447  			// This can cause collisions, but is sufficient since we are using other keys to identify the load balancer.
   448  			hash = xor(hash, sha1.Sum(([]byte(id))))
   449  		}
   450  	}
   451  	return
   452  }
   453  
   454  func xor(b1 [20]byte, b2 [20]byte) (xorbytes [20]byte) {
   455  	for i := 0; i < 20; i++ {
   456  		xorbytes[i] = b1[i] ^ b2[i]
   457  	}
   458  	return xorbytes
   459  }
   460
View as plain text