1
2
3 package main
4
5 import (
6 "context"
7 "flag"
8 "fmt"
9 "math/rand"
10 "net"
11 "os"
12 "os/signal"
13 "strconv"
14 "strings"
15 "syscall"
16
17 "github.com/Microsoft/go-winio/pkg/guid"
18 "github.com/Microsoft/hcsshim/hcn"
19 "github.com/Microsoft/hcsshim/internal/log"
20 ncproxygrpc "github.com/Microsoft/hcsshim/pkg/ncproxy/ncproxygrpc/v1"
21 nodenetsvcV0 "github.com/Microsoft/hcsshim/pkg/ncproxy/nodenetsvc/v0"
22 nodenetsvc "github.com/Microsoft/hcsshim/pkg/ncproxy/nodenetsvc/v1"
23 "github.com/sirupsen/logrus"
24 "google.golang.org/grpc"
25 )
26
27
28
29
30 var configPath = flag.String("config", "", "Path to JSON configuration file.")
31
32 const (
33 prefixLength uint32 = 24
34 ipVersion = "4"
35 )
36
37 func generateMAC() (string, error) {
38 buf := make([]byte, 6)
39
40 _, err := rand.Read(buf)
41 if err != nil {
42 return "", err
43 }
44
45
46 buf[0] = 0
47 mac := net.HardwareAddr(buf)
48 macString := strings.ToUpper(mac.String())
49 macString = strings.Replace(macString, ":", "-", -1)
50
51 return macString, nil
52 }
53
54 func generateIPs(prefixLength string) (string, string, string) {
55 buf := []byte{192, 168, 50}
56
57
58 ipPrefixBytes := append(buf, 0)
59 ipPrefix := net.IP(ipPrefixBytes)
60 ipPrefixString := ipPrefix.String() + "/" + prefixLength
61
62
63 ipGatewayBytes := append(buf, 1)
64 ipGateway := net.IP(ipGatewayBytes)
65 ipGatewayString := ipGateway.String()
66
67
68 last := byte(rand.Intn(255-2) + 2)
69 ipBytes := append(buf, last)
70 ip := net.IP(ipBytes)
71 ipString := ip.String()
72
73 return ipPrefixString, ipGatewayString, ipString
74 }
75
76 func (s *service) configureHCNNetworkingHelper(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) {
77 prefixIP, gatewayIP, midIP := generateIPs(strconv.Itoa(int(prefixLength)))
78
79 mode := ncproxygrpc.HostComputeNetworkSettings_NAT
80 if s.conf.NetworkingSettings.HNSSettings.IOVSettings != nil {
81 mode = ncproxygrpc.HostComputeNetworkSettings_Transparent
82 }
83 addNetworkReq := &ncproxygrpc.CreateNetworkRequest{
84 Network: &ncproxygrpc.Network{
85 Settings: &ncproxygrpc.Network_HcnNetwork{
86 HcnNetwork: &ncproxygrpc.HostComputeNetworkSettings{
87 Name: req.ContainerID + "_network_hcn",
88 Mode: mode,
89 SwitchName: s.conf.NetworkingSettings.HNSSettings.SwitchName,
90 IpamType: ncproxygrpc.HostComputeNetworkSettings_Static,
91 SubnetIpaddressPrefix: []string{prefixIP},
92 DefaultGateway: gatewayIP,
93 },
94 },
95 },
96 }
97
98 networkResp, err := s.client.CreateNetwork(ctx, addNetworkReq)
99 if err != nil {
100 return nil, err
101 }
102
103 network, err := hcn.GetNetworkByID(networkResp.ID)
104 if err != nil {
105 return nil, err
106 }
107 s.containerToNetwork[req.ContainerID] = append(s.containerToNetwork[req.ContainerID], network.Name)
108
109 mac, err := generateMAC()
110 if err != nil {
111 return nil, err
112 }
113
114 policies := &ncproxygrpc.HcnEndpointPolicies{}
115 if s.conf.NetworkingSettings.HNSSettings.IOVSettings != nil {
116 policies.IovPolicySettings = s.conf.NetworkingSettings.HNSSettings.IOVSettings
117 }
118
119 name := req.ContainerID + "_endpoint_hcn"
120 endpointCreateReq := &ncproxygrpc.CreateEndpointRequest{
121 EndpointSettings: &ncproxygrpc.EndpointSettings{
122 Settings: &ncproxygrpc.EndpointSettings_HcnEndpoint{
123 HcnEndpoint: &ncproxygrpc.HcnEndpointSettings{
124 Name: name,
125 Macaddress: mac,
126 Ipaddress: midIP,
127 IpaddressPrefixlength: prefixLength,
128 NetworkName: network.Name,
129 Policies: policies,
130 },
131 },
132 },
133 }
134
135 endpt, err := s.client.CreateEndpoint(ctx, endpointCreateReq)
136 if err != nil {
137 return nil, err
138 }
139
140 log.G(ctx).WithField("endpt", endpt).Info("ConfigureContainerNetworking created endpoint")
141
142 addEndpointReq := &ncproxygrpc.AddEndpointRequest{
143 Name: name,
144 NamespaceID: req.NetworkNamespaceID,
145 }
146 _, err = s.client.AddEndpoint(ctx, addEndpointReq)
147 if err != nil {
148 return nil, err
149 }
150 s.containerToNamespace[req.ContainerID] = req.NetworkNamespaceID
151
152 resultIPAddr := &nodenetsvc.ContainerIPAddress{
153 Version: ipVersion,
154 Ip: midIP,
155 PrefixLength: strconv.Itoa(int(prefixLength)),
156 DefaultGateway: gatewayIP,
157 }
158 netInterface := &nodenetsvc.ContainerNetworkInterface{
159 Name: network.Name,
160 MacAddress: mac,
161 NetworkNamespaceID: req.NetworkNamespaceID,
162 Ipaddresses: []*nodenetsvc.ContainerIPAddress{resultIPAddr},
163 }
164
165 return &nodenetsvc.ConfigureContainerNetworkingResponse{
166 Interfaces: []*nodenetsvc.ContainerNetworkInterface{netInterface},
167 }, nil
168 }
169
170 func (s *service) configureNCProxyNetworkingHelper(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) {
171 _, gatewayIP, midIP := generateIPs(strconv.Itoa(int(prefixLength)))
172 networkName := req.ContainerID + "_network_ncproxy"
173 addNetworkReq := &ncproxygrpc.CreateNetworkRequest{
174 Network: &ncproxygrpc.Network{
175 Settings: &ncproxygrpc.Network_NcproxyNetwork{
176 NcproxyNetwork: &ncproxygrpc.NCProxyNetworkSettings{
177 Name: networkName,
178 },
179 },
180 },
181 }
182
183 _, err = s.client.CreateNetwork(ctx, addNetworkReq)
184 if err != nil {
185 return nil, err
186 }
187 s.containerToNetwork[req.ContainerID] = append(s.containerToNetwork[req.ContainerID], networkName)
188
189 mac, err := generateMAC()
190 if err != nil {
191 return nil, err
192 }
193
194 name := req.ContainerID + "_endpoint_ncproxy"
195 endpointCreateReq := &ncproxygrpc.CreateEndpointRequest{
196 EndpointSettings: &ncproxygrpc.EndpointSettings{
197 Settings: &ncproxygrpc.EndpointSettings_NcproxyEndpoint{
198 NcproxyEndpoint: &ncproxygrpc.NCProxyEndpointSettings{
199 Name: name,
200 Macaddress: mac,
201 Ipaddress: midIP,
202 IpaddressPrefixlength: prefixLength,
203 NetworkName: networkName,
204 DefaultGateway: gatewayIP,
205 DeviceDetails: &ncproxygrpc.NCProxyEndpointSettings_PciDeviceDetails{
206 PciDeviceDetails: &ncproxygrpc.PCIDeviceDetails{
207 DeviceID: s.conf.NetworkingSettings.NCProxyNetworkingSettings.DeviceID,
208 VirtualFunctionIndex: s.conf.NetworkingSettings.NCProxyNetworkingSettings.VirtualFunctionIndex,
209 },
210 },
211 },
212 },
213 },
214 }
215
216 endpt, err := s.client.CreateEndpoint(ctx, endpointCreateReq)
217 if err != nil {
218 return nil, err
219 }
220
221 log.G(ctx).WithField("endpt", endpt).Info("ConfigureContainerNetworking created endpoint")
222
223 addEndpointReq := &ncproxygrpc.AddEndpointRequest{
224 Name: name,
225 NamespaceID: req.NetworkNamespaceID,
226 }
227 _, err = s.client.AddEndpoint(ctx, addEndpointReq)
228 if err != nil {
229 return nil, err
230 }
231 s.containerToNamespace[req.ContainerID] = req.NetworkNamespaceID
232
233 resultIPAddr := &nodenetsvc.ContainerIPAddress{
234 Version: ipVersion,
235 Ip: midIP,
236 PrefixLength: strconv.Itoa(int(prefixLength)),
237 DefaultGateway: gatewayIP,
238 }
239 netInterface := &nodenetsvc.ContainerNetworkInterface{
240 Name: networkName,
241 MacAddress: mac,
242 NetworkNamespaceID: req.NetworkNamespaceID,
243 Ipaddresses: []*nodenetsvc.ContainerIPAddress{resultIPAddr},
244 }
245
246 return &nodenetsvc.ConfigureContainerNetworkingResponse{
247 Interfaces: []*nodenetsvc.ContainerNetworkInterface{netInterface},
248 }, nil
249 }
250
251 func (s *service) teardownConfigureContainerNetworking(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) {
252 eReq := &ncproxygrpc.GetEndpointsRequest{}
253 resp, err := s.client.GetEndpoints(ctx, eReq)
254 if err != nil {
255 return nil, err
256 }
257
258 for _, endpoint := range resp.Endpoints {
259 if endpoint == nil {
260 log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint to delete")
261 continue
262 }
263 if endpoint.Endpoint == nil || endpoint.Endpoint.Settings == nil {
264 log.G(ctx).WithField("name", endpoint.ID).Warn("failed to get endpoint settings")
265 continue
266 }
267 if endpoint.Namespace == req.NetworkNamespaceID {
268 endpointName := ""
269 switch ep := endpoint.Endpoint.GetSettings().(type) {
270 case *ncproxygrpc.EndpointSettings_NcproxyEndpoint:
271 endpointName = ep.NcproxyEndpoint.Name
272 case *ncproxygrpc.EndpointSettings_HcnEndpoint:
273 endpointName = ep.HcnEndpoint.Name
274 default:
275 log.G(ctx).WithField("name", endpoint.ID).Warn("invalid endpoint settings type")
276 continue
277 }
278 deleteEndptReq := &ncproxygrpc.DeleteEndpointRequest{
279 Name: endpointName,
280 }
281 if _, err := s.client.DeleteEndpoint(ctx, deleteEndptReq); err != nil {
282 log.G(ctx).WithField("name", endpointName).Warn("failed to delete endpoint")
283 }
284 }
285 }
286
287 if networks, ok := s.containerToNetwork[req.ContainerID]; ok {
288 for _, networkName := range networks {
289 deleteReq := &ncproxygrpc.DeleteNetworkRequest{
290 Name: networkName,
291 }
292 if _, err := s.client.DeleteNetwork(ctx, deleteReq); err != nil {
293 log.G(ctx).WithField("name", networkName).Warn("failed to delete network")
294 }
295 }
296 delete(s.containerToNetwork, req.ContainerID)
297 }
298
299 return &nodenetsvc.ConfigureContainerNetworkingResponse{}, nil
300 }
301
302 func (s *service) ConfigureContainerNetworking(ctx context.Context, req *nodenetsvc.ConfigureContainerNetworkingRequest) (_ *nodenetsvc.ConfigureContainerNetworkingResponse, err error) {
303
304 log.G(ctx).WithField("req", req).Info("ConfigureContainerNetworking request")
305
306 if req.RequestType == nodenetsvc.RequestType_Setup {
307 interfaces := []*nodenetsvc.ContainerNetworkInterface{}
308 if s.conf.NetworkingSettings != nil && s.conf.NetworkingSettings.HNSSettings != nil {
309 result, err := s.configureHCNNetworkingHelper(ctx, req)
310 if err != nil {
311 return nil, err
312 }
313 interfaces = append(interfaces, result.Interfaces...)
314 }
315 if s.conf.NetworkingSettings != nil && s.conf.NetworkingSettings.NCProxyNetworkingSettings != nil {
316 result, err := s.configureNCProxyNetworkingHelper(ctx, req)
317 if err != nil {
318 return nil, err
319 }
320 interfaces = append(interfaces, result.Interfaces...)
321 }
322 return &nodenetsvc.ConfigureContainerNetworkingResponse{
323 Interfaces: interfaces,
324 }, nil
325 } else if req.RequestType == nodenetsvc.RequestType_Teardown {
326 return s.teardownConfigureContainerNetworking(ctx, req)
327 }
328 return nil, fmt.Errorf("invalid request type %v", req.RequestType)
329 }
330
331 func (s *service) addHelper(ctx context.Context, req *nodenetsvc.ConfigureNetworkingRequest, containerNamespaceID string) (_ *nodenetsvc.ConfigureNetworkingResponse, err error) {
332 eReq := &ncproxygrpc.GetEndpointsRequest{}
333 resp, err := s.client.GetEndpoints(ctx, eReq)
334 if err != nil {
335 return nil, err
336 }
337 log.G(ctx).WithField("endpts", resp.Endpoints).Info("ConfigureNetworking addrequest")
338
339 for _, endpoint := range resp.Endpoints {
340 if endpoint == nil {
341 log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint")
342 continue
343 }
344 if endpoint.Endpoint == nil || endpoint.Endpoint.Settings == nil {
345 log.G(ctx).WithField("name", endpoint.ID).Warn("failed to get endpoint settings")
346 continue
347 }
348 if endpoint.Namespace == containerNamespaceID {
349
350 nicID, err := guid.NewV4()
351 if err != nil {
352 return nil, fmt.Errorf("failed to create nic GUID: %s", err)
353 }
354 endpointName := ""
355 switch ep := endpoint.Endpoint.GetSettings().(type) {
356 case *ncproxygrpc.EndpointSettings_NcproxyEndpoint:
357 endpointName = ep.NcproxyEndpoint.Name
358 case *ncproxygrpc.EndpointSettings_HcnEndpoint:
359 endpointName = ep.HcnEndpoint.Name
360 default:
361 log.G(ctx).WithField("name", endpoint.ID).Warn("invalid endpoint settings type")
362 continue
363 }
364 nsReq := &ncproxygrpc.AddNICRequest{
365 ContainerID: req.ContainerID,
366 NicID: nicID.String(),
367 EndpointName: endpointName,
368 }
369 if _, err := s.client.AddNIC(ctx, nsReq); err != nil {
370 return nil, err
371 }
372 s.endpointToNicID[endpointName] = nicID.String()
373 }
374 }
375
376 defer func() {
377 if err != nil {
378 _, _ = s.teardownHelper(ctx, req, containerNamespaceID)
379 }
380 }()
381
382 return &nodenetsvc.ConfigureNetworkingResponse{}, nil
383 }
384
385 func (s *service) teardownHelper(ctx context.Context, req *nodenetsvc.ConfigureNetworkingRequest, containerNamespaceID string) (*nodenetsvc.ConfigureNetworkingResponse, error) {
386 eReq := &ncproxygrpc.GetEndpointsRequest{}
387 resp, err := s.client.GetEndpoints(ctx, eReq)
388 if err != nil {
389 return nil, err
390 }
391
392 for _, endpoint := range resp.Endpoints {
393 if endpoint == nil {
394 log.G(ctx).WithField("name", endpoint.ID).Warn("failed to find endpoint to delete")
395 continue
396 }
397 if endpoint.Endpoint == nil || endpoint.Endpoint.Settings == nil {
398 log.G(ctx).WithField("name", endpoint.ID).Warn("failed to get endpoint settings")
399 continue
400 }
401
402 if endpoint.Namespace == containerNamespaceID {
403 endpointName := ""
404 switch ep := endpoint.Endpoint.GetSettings().(type) {
405 case *ncproxygrpc.EndpointSettings_NcproxyEndpoint:
406 endpointName = ep.NcproxyEndpoint.Name
407 case *ncproxygrpc.EndpointSettings_HcnEndpoint:
408 endpointName = ep.HcnEndpoint.Name
409 default:
410 log.G(ctx).WithField("name", endpoint.ID).Warn("invalid endpoint settings type")
411 continue
412 }
413 nicID, ok := s.endpointToNicID[endpointName]
414 if !ok {
415 log.G(ctx).WithField("name", endpointName).Warn("endpoint was not assigned a NIC ID previously")
416 continue
417 }
418
419 nsReq := &ncproxygrpc.DeleteNICRequest{
420 ContainerID: req.ContainerID,
421 NicID: nicID,
422 EndpointName: endpointName,
423 }
424 if _, err := s.client.DeleteNIC(ctx, nsReq); err != nil {
425 log.G(ctx).WithField("name", endpointName).Warn("failed to delete endpoint nic")
426 }
427 delete(s.endpointToNicID, endpointName)
428 }
429 }
430 return &nodenetsvc.ConfigureNetworkingResponse{}, nil
431 }
432
433 func (s *service) ConfigureNetworking(ctx context.Context, req *nodenetsvc.ConfigureNetworkingRequest) (*nodenetsvc.ConfigureNetworkingResponse, error) {
434 log.G(ctx).WithField("req", req).Info("ConfigureNetworking request")
435
436 containerNamespaceID, ok := s.containerToNamespace[req.ContainerID]
437 if !ok {
438 return nil, fmt.Errorf("no namespace was previously created for containerID %s", req.ContainerID)
439 }
440
441 if req.RequestType == nodenetsvc.RequestType_Setup {
442 return s.addHelper(ctx, req, containerNamespaceID)
443 }
444 return s.teardownHelper(ctx, req, containerNamespaceID)
445 }
446
447
448 func (s *service) GetHostLocalIpAddress(ctx context.Context, req *nodenetsvc.GetHostLocalIpAddressRequest) (*nodenetsvc.GetHostLocalIpAddressResponse, error) {
449 return &nodenetsvc.GetHostLocalIpAddressResponse{IpAddr: ""}, nil
450 }
451
452 func (s *service) PingNodeNetworkService(ctx context.Context, req *nodenetsvc.PingNodeNetworkServiceRequest) (*nodenetsvc.PingNodeNetworkServiceResponse, error) {
453 return &nodenetsvc.PingNodeNetworkServiceResponse{}, nil
454 }
455
456 func main() {
457 var err error
458 ctx := context.Background()
459
460 flag.Parse()
461 conf, err := readConfig(*configPath)
462 if err != nil {
463 log.G(ctx).WithError(err).Fatalf("failed to read network agent's config file at %s", *configPath)
464 }
465 log.G(ctx).WithFields(logrus.Fields{
466 "config path": *configPath,
467 "conf": conf,
468 }).Info("network agent configuration")
469
470 sigChan := make(chan os.Signal, 1)
471 serveErr := make(chan error, 1)
472 defer close(serveErr)
473 signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
474 defer signal.Stop(sigChan)
475
476 grpcClient, err := grpc.Dial(
477 conf.GRPCAddr,
478 grpc.WithInsecure(),
479 )
480 if err != nil {
481 log.G(ctx).WithError(err).Fatalf("failed to connect to ncproxy at %s", conf.GRPCAddr)
482 }
483 defer grpcClient.Close()
484
485 log.G(ctx).WithField("addr", conf.GRPCAddr).Info("connected to ncproxy")
486 ncproxyClient := ncproxygrpc.NewNetworkConfigProxyClient(grpcClient)
487 service := &service{
488 conf: conf,
489 client: ncproxyClient,
490 containerToNamespace: make(map[string]string),
491 endpointToNicID: make(map[string]string),
492 containerToNetwork: make(map[string][]string),
493 }
494 v0Service := &v0ServiceWrapper{
495 s: service,
496 }
497 server := grpc.NewServer()
498 nodenetsvc.RegisterNodeNetworkServiceServer(server, service)
499 nodenetsvcV0.RegisterNodeNetworkServiceServer(server, v0Service)
500
501 grpcListener, err := net.Listen("tcp", conf.NodeNetSvcAddr)
502 if err != nil {
503 log.G(ctx).WithError(err).Fatalf("failed to listen on %s", grpcListener.Addr().String())
504 }
505
506 go func() {
507 defer grpcListener.Close()
508 if err := server.Serve(grpcListener); err != nil {
509 if strings.Contains(err.Error(), "use of closed network connection") {
510 serveErr <- nil
511 }
512 serveErr <- err
513 }
514 }()
515
516 log.G(ctx).WithField("addr", conf.NodeNetSvcAddr).Info("serving network service agent")
517
518
519 select {
520 case <-sigChan:
521 log.G(ctx).Info("Received interrupt. Closing")
522 case err := <-serveErr:
523 if err != nil {
524 log.G(ctx).WithError(err).Fatal("grpc service failure")
525 }
526 }
527
528
529 server.GracefulStop()
530 }
531
View as plain text