trafficcontrol.go

Documentation: edge-infra.dev/pkg/sds/ien/k8s/controllers/nodeagent/plugins/networking/trafficshaping

     1  //go:build linux
     2  
     3  package trafficshaping
     4  
     5  import (
     6  	"net"
     7  
     8  	"github.com/vishvananda/netlink"
     9  	"golang.org/x/sys/unix"
    10  
    11  	"edge-infra.dev/pkg/lib/kernel/netlink/ip"
    12  	"edge-infra.dev/pkg/lib/kernel/netlink/tc"
    13  )
    14  
    15  var (
    16  	egressMajorClass    uint16 = 0x3    // 3
    17  	ingressMajorClass   uint16 = 0x4    // 4
    18  	defaultMinorClass   uint16 = 0x30   // 30
    19  	rateLimitMinorClass uint16 = 0x40   // 40
    20  	minorFilterHandle   uint16 = 0x800  // 2047
    21  	majorFilterHandle   uint16 = 0x8000 // 32768 bug: https://github.com/vishvananda/netlink/issues/901
    22  	ingressParentMajor  uint16 = 0xffff // 65535
    23  )
    24  
    25  var (
    26  	rate2Quantum uint32 = 10
    27  	priority     uint32 = 49152
    28  	divisor      uint32 = 1
    29  	filterRate   uint64 = 1e+10 // filter rate is 10gbit/s
    30  )
    31  
    32  var (
    33  	srcPorts = []uint32{
    34  		0x00160000, // port 22 (ssh)
    35  		0x00430000, // port 67 (dhcp)
    36  		0x00440000, // port 68 (dhcp)
    37  		0x758F0000, // port 30095 (pxe)
    38  	}
    39  	dstPorts = []uint32{
    40  		0x16,   // port 22 (ssh)
    41  		0x43,   // port 67 (dhcp)
    42  		0x44,   // port 68 (dhcp)
    43  		0x758F, // port 30095 (pxe)
    44  	}
    45  	ipCatchAll = net.ParseIP("0.0.0.0")
    46  )
    47  
    48  func reconcileTrafficControl(defaultLinkIndex, ifbLinkIndex int, egressRate, ingressRate uint64, excludeNetworks []*net.IPNet) error {
    49  	if err := createQdiscs(defaultLinkIndex, ifbLinkIndex); err != nil {
    50  		return err
    51  	}
    52  
    53  	if err := createClasses(defaultLinkIndex, ifbLinkIndex, egressRate, ingressRate); err != nil {
    54  		return err
    55  	}
    56  
    57  	if err := createMajorFilterHandles(defaultLinkIndex, ifbLinkIndex); err != nil {
    58  		return err
    59  	}
    60  
    61  	return createMinorFilters(defaultLinkIndex, ifbLinkIndex, excludeNetworks)
    62  }
    63  
    64  func createQdiscs(defaultLinkIndex, ifbLinkIndex int) error {
    65  	// tc adder closure function
    66  	add := tc.Adder()
    67  
    68  	return add(
    69  		// create default link qdiscs
    70  		defaultHtbQdisc(defaultLinkIndex),
    71  		defaultIngressQdisc(defaultLinkIndex),
    72  
    73  		// create ifb link qdiscs
    74  		ifbHtbQdisc(ifbLinkIndex),
    75  	)
    76  }
    77  
    78  func createClasses(defaultLinkIndex, ifbLinkIndex int, egressRate, ingressRate uint64) error {
    79  	// tc replacer closure function
    80  	replace := tc.Replacer()
    81  
    82  	return replace(
    83  		// create egress rate limit class
    84  		defaultLinkClass(defaultLinkIndex, egressRate),
    85  
    86  		// create ingress rate limit class
    87  		ifbLinkClass(ifbLinkIndex, ingressRate),
    88  
    89  		// create filter class for backbone link
    90  		defaultLinkFilterClass(defaultLinkIndex, filterRate),
    91  
    92  		// create filter class for ifb link
    93  		ifbLinkFilterClass(ifbLinkIndex, filterRate),
    94  	)
    95  }
    96  
    97  func createMajorFilterHandles(defaultLinkIndex, ifbLinkIndex int) error {
    98  	// tc adder and replacer closure functions
    99  	add := tc.Adder()
   100  
   101  	return add(
   102  		// redirect ingress traffic to ifb link
   103  		createU32HandleFilter(defaultLinkIndex, ingressParentMajor, majorFilterHandle),
   104  
   105  		// create filter table handles
   106  		createU32HandleFilter(defaultLinkIndex, egressMajorClass, majorFilterHandle),
   107  		createU32HandleFilter(ifbLinkIndex, ingressMajorClass, majorFilterHandle),
   108  	)
   109  }
   110  
   111  func createMinorFilters(defaultLinkIndex, ifbLinkIndex int, excludeNetworks []*net.IPNet) error {
   112  	// minor handle index to increment each filter
   113  	var minorHandleIdx = minorFilterHandle
   114  
   115  	// tc replacer closure functions
   116  	replace := tc.Replacer()
   117  
   118  	// redirect ingress traffic to ifb link
   119  	filters := []tc.Tc{createRedirectFilter(defaultLinkIndex, ifbLinkIndex, majorFilterHandle, minorFilterHandle)}
   120  
   121  	dynamicFilters := dynamicMinorFilters(defaultLinkIndex, ifbLinkIndex, minorHandleIdx, excludeNetworks)
   122  	filters = append(filters, dynamicFilters...)
   123  
   124  	return replace(filters...)
   125  }
   126  
   127  // create dynamic filters i.e. ignore intra-cluster subnets from rate limiting
   128  func dynamicMinorFilters(defaultLinkIndex, ifbLinkIndex int, minorHandleIdx uint16, excludeNetworks []*net.IPNet) []tc.Tc {
   129  	filters := []tc.Tc{}
   130  
   131  	// add dynamic ip filters on backbone and ifb links
   132  	minorHandleIdx, ipFilters := dynamicIPFilters(defaultLinkIndex, ifbLinkIndex, minorHandleIdx, excludeNetworks)
   133  	filters = append(filters, ipFilters...)
   134  
   135  	// add source port filters on backbone link
   136  	minorHandleIdx, srcPortFilters := dynamicSrcPortFilters(defaultLinkIndex, minorHandleIdx)
   137  	filters = append(filters, srcPortFilters...)
   138  
   139  	// add destination port filters on ifb link
   140  	_, dstPortFilters := dynamicDstPortFilters(ifbLinkIndex, minorHandleIdx)
   141  	filters = append(filters, dstPortFilters...)
   142  
   143  	return filters
   144  }
   145  
   146  // exclude ignored networks
   147  func dynamicIPFilters(defaultLinkIndex, ifbLinkIndex int, minorHandleIdx uint16, excludeNetworks []*net.IPNet) (newMinorHandleIdx uint16, filters []tc.Tc) {
   148  	newMinorHandleIdx = minorHandleIdx
   149  	for _, ignoreAddr := range excludeNetworks {
   150  		ipAddr := ignoreAddr.IP
   151  		mask := net.IP(ignoreAddr.Mask)
   152  		matchSrcIP := tc.NewMatchIP().WithIP(&ipAddr).WithMask(&mask).WithIPv4HeaderOffset(ip.SrcAddressBitOffset)
   153  		matchDstIP := tc.NewMatchIP().WithIP(&ipAddr).WithMask(&mask).WithIPv4HeaderOffset(ip.DstAddressBitOffset)
   154  		filters = append(filters,
   155  			createU32MatchFilter(defaultLinkIndex, egressMajorClass, defaultMinorClass, majorFilterHandle, newMinorHandleIdx).WithMatchIP(matchDstIP),
   156  			createU32MatchFilter(ifbLinkIndex, ingressMajorClass, defaultMinorClass, majorFilterHandle, newMinorHandleIdx).WithMatchIP(matchSrcIP),
   157  		)
   158  		newMinorHandleIdx++
   159  	}
   160  	return newMinorHandleIdx, filters
   161  }
   162  
   163  // exclude source ports
   164  func dynamicSrcPortFilters(linkIndex int, minorHandleIdx uint16) (newMinorHandleIdx uint16, filters []tc.Tc) {
   165  	newMinorHandleIdx = minorHandleIdx
   166  	for _, port := range srcPorts {
   167  		filters = append(filters,
   168  			createU32MatchFilter(linkIndex, egressMajorClass, defaultMinorClass, majorFilterHandle, newMinorHandleIdx).WithMatchPort(tc.NewMatchPort().WithSrcPort(port)),
   169  		)
   170  		newMinorHandleIdx++
   171  	}
   172  	return newMinorHandleIdx, filters
   173  }
   174  
   175  // exclude destination ports
   176  func dynamicDstPortFilters(linkIndex int, minorHandleIdx uint16) (newMinorHandleIdx uint16, filters []tc.Tc) {
   177  	newMinorHandleIdx = minorHandleIdx
   178  	for _, port := range dstPorts {
   179  		filters = append(filters,
   180  			createU32MatchFilter(linkIndex, ingressMajorClass, defaultMinorClass, majorFilterHandle, newMinorHandleIdx).WithMatchPort(tc.NewMatchPort().WithDstPort(port)),
   181  		)
   182  		newMinorHandleIdx++
   183  	}
   184  	return newMinorHandleIdx, filters
   185  }
   186  
   187  // equivalent: "tc qdisc add dev ens4 root handle 0x1: htb default 0x10 r2q 10"
   188  func defaultHtbQdisc(defaultLinkIndex int) tc.Tc {
   189  	htbAttrs := tc.NewQdiscAttrs().WithMakeHandle(egressMajorClass, 0x0).WithLinkIndex(defaultLinkIndex).WithParent(netlink.HANDLE_ROOT).Build()
   190  	return tc.NewHtb().WithAttrs(htbAttrs).WithDefaultClass(uint32(rateLimitMinorClass)).WithRate2Quantum(rate2Quantum)
   191  }
   192  
   193  // equivalent: "tc qdisc replace dev ens4 handle ffff: ingress"
   194  func defaultIngressQdisc(defaultLinkIndex int) tc.Tc {
   195  	ingressAttrs := tc.NewQdiscAttrs().WithMakeHandle(0xffff, 0x0).WithLinkIndex(defaultLinkIndex).WithParent(netlink.HANDLE_INGRESS).Build()
   196  	return tc.NewIngressQdisc().WithAttrs(ingressAttrs)
   197  }
   198  
   199  // equivalent: "tc qdisc add dev ifbens4 root handle 0x2: htb default 0x10 r2q 10"
   200  func ifbHtbQdisc(ifbLinkIndex int) tc.Tc {
   201  	ifbQdiscAttrs := tc.NewQdiscAttrs().WithMakeHandle(ingressMajorClass, 0x0).WithLinkIndex(ifbLinkIndex).WithParent(netlink.HANDLE_ROOT).Build()
   202  	return tc.NewHtb().WithAttrs(ifbQdiscAttrs).WithDefaultClass(uint32(rateLimitMinorClass)).WithRate2Quantum(rate2Quantum)
   203  }
   204  
   205  // equivalent: "tc class replace dev ens4 parent 0x1:0x1 classid 0x1:0x20 htb rate 10Mbit ceil 10Mbit"
   206  func defaultLinkClass(defaultLinkIndex int, rate uint64) *tc.Class {
   207  	classAttrs := tc.NewClassAttrs().WithLinkIndex(defaultLinkIndex).WithMakeHandle(egressMajorClass, rateLimitMinorClass).WithParent(netlink.HANDLE_ROOT).Build()
   208  	return tc.NewClass().WithAttrs(classAttrs).WithRateLimit(rate).WithPriority(priority)
   209  }
   210  
   211  // equivalent: "tc class replace dev ifbens4 parent 0x2:0x1 classid 0x2:0x20 htb rate 10Mbit ceil 10Mbit"
   212  func ifbLinkClass(ifbLinkIndex int, rate uint64) *tc.Class {
   213  	ifbClassAttrs := tc.NewClassAttrs().WithLinkIndex(ifbLinkIndex).WithMakeHandle(ingressMajorClass, rateLimitMinorClass).WithParent(netlink.HANDLE_ROOT).Build()
   214  	return tc.NewClass().WithAttrs(ifbClassAttrs).WithRateLimit(rate).WithPriority(priority)
   215  }
   216  
   217  // equivalent: "tc class replace dev ens4 parent 0x1:0x1 classid 0x1:0x20 htb rate 10Gbit ceil 10Gbit"
   218  func defaultLinkFilterClass(defaultLinkIndex int, rate uint64) *tc.Class {
   219  	classAttrs := tc.NewClassAttrs().WithLinkIndex(defaultLinkIndex).WithMakeHandle(egressMajorClass, defaultMinorClass).WithParent(netlink.HANDLE_ROOT).Build()
   220  	return tc.NewClass().WithAttrs(classAttrs).WithRateLimit(rate).WithPriority(priority)
   221  }
   222  
   223  // equivalent: "tc class replace dev ifbens4 parent 0x2:0x1 classid 0x2:0x20 htb rate 10Gbit ceil 10Gbit"
   224  func ifbLinkFilterClass(ifbLinkIndex int, rate uint64) *tc.Class {
   225  	ifbClassAttrs := tc.NewClassAttrs().WithLinkIndex(ifbLinkIndex).WithMakeHandle(ingressMajorClass, defaultMinorClass).WithParent(netlink.HANDLE_ROOT).Build()
   226  	return tc.NewClass().WithAttrs(ifbClassAttrs).WithRateLimit(rate).WithPriority(priority)
   227  }
   228  
   229  // equivalent: "tc filter add dev ens4 parent ffff: protocol ip u32 match u32 0 0 action mirred egress redirect dev ifbens4"
   230  func createRedirectFilter(defaultLinkIndex, ifbLinkIndex int, majorHandle, minorHandle uint16) *tc.U32Filter {
   231  	filterAttrs := tc.NewFilterAttrs().WithLinkIndex(defaultLinkIndex).WithMakeParent(0xffff, 0x0).WithProtocol(unix.ETH_P_IP).WithPriority(uint16(priority)).WithMakeHandle(majorHandle, minorHandle).Build() /* #nosec G115 */
   232  	actionAttrs := tc.NewActionAttrs().WithAction(netlink.TC_ACT_STOLEN).Build()
   233  	mirredAction := tc.NewMirredAction().WithAttrs(actionAttrs).WithMirredAction(netlink.TCA_EGRESS_REDIR).WithRedirectIfIndex(ifbLinkIndex).Build()
   234  	matchIP := tc.NewMatchIP().WithIPv4HeaderOffset(0).WithIP(&ipCatchAll).WithMask(&ipCatchAll)
   235  	hashTable := netlink.MakeHandle(majorHandle, 0x0)
   236  	return tc.NewU32Filter().WithAttrs(filterAttrs).WithAction(&mirredAction).WithHashTable(hashTable).WithMatchIP(matchIP)
   237  }
   238  
   239  // equivalent: "tc filter add dev ens4 protocol ip parent 1: prio 1 handle 800: u32 divisor 1"
   240  func createU32HandleFilter(linkIndex int, major, handleMajor uint16) *tc.U32Filter {
   241  	filterAttrs := tc.NewFilterAttrs().WithLinkIndex(linkIndex).WithMakeParent(major, 0x0).WithProtocol(unix.ETH_P_IP).WithPriority(uint16(priority)).WithMakeHandle(handleMajor, 0x0).Build() /* #nosec G115 */
   242  	return tc.NewU32Filter().WithAttrs(filterAttrs).WithDivisor(divisor)
   243  }
   244  
   245  // equivalent: "tc filter add dev ens4 protocol ip parent 1: prio 1 handle 800::801 u32 match ... flowid 1:20"
   246  func createU32MatchFilter(linkIndex int, major, minor, majorHandle, minorHandle uint16) *tc.U32Filter {
   247  	filterAttrs := tc.NewFilterAttrs().WithLinkIndex(linkIndex).WithMakeParent(major, 0x0).WithProtocol(unix.ETH_P_IP).WithPriority(uint16(priority)).WithMakeHandle(majorHandle, minorHandle).Build() /* #nosec G115 */
   248  	hashTable := netlink.MakeHandle(majorHandle, 0x0)
   249  	return tc.NewU32Filter().WithAttrs(filterAttrs).WithMakeClassID(major, minor).WithHashTable(hashTable)
   250  }
   251
View as plain text