1
16
17 package stats
18
19 import (
20 "context"
21 "math/rand"
22 "os"
23 "path/filepath"
24 "runtime"
25 "strings"
26 "testing"
27 "time"
28
29 "github.com/golang/mock/gomock"
30 cadvisorfs "github.com/google/cadvisor/fs"
31 cadvisorapiv2 "github.com/google/cadvisor/info/v2"
32 "github.com/stretchr/testify/assert"
33 "k8s.io/apimachinery/pkg/api/resource"
34 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
35 "k8s.io/apimachinery/pkg/types"
36 "k8s.io/apimachinery/pkg/util/uuid"
37 runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
38 critest "k8s.io/cri-api/pkg/apis/testing"
39 statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
40 kubelettypes "k8s.io/kubelet/pkg/types"
41 cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
42 "k8s.io/kubernetes/pkg/kubelet/cm"
43 kubecontainertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
44 "k8s.io/kubernetes/pkg/kubelet/kuberuntime"
45 kubepodtest "k8s.io/kubernetes/pkg/kubelet/pod/testing"
46 serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
47 "k8s.io/kubernetes/pkg/volume"
48 )
49
50 const (
51 offsetInodeUsage = iota
52 offsetUsage
53 )
54
55 const (
56
57 offsetCRI = 1000
58 )
59
60 const (
61 seedRoot = 0
62 seedKubelet = 200
63 seedMisc = 300
64 seedSandbox0 = 1000
65 seedContainer0 = 2000
66 seedSandbox1 = 3000
67 seedContainer1 = 4000
68 seedContainer2 = 5000
69 seedSandbox2 = 6000
70 seedContainer3 = 7000
71 seedSandbox3 = 8000
72 )
73
74 const (
75 pName0 = "pod0"
76 pName1 = "pod1"
77 pName2 = "pod2"
78 )
79
80 const (
81 cName0 = "container0-name"
82 cName1 = "container1-name"
83 cName2 = "container2-name"
84 cName3 = "container3-name"
85 cName5 = "container5-name"
86 cName6 = "container6-name"
87 cName7 = "container7-name"
88 cName8 = "container8-name"
89 cName9 = "container9-name"
90 )
91
92 const testPodLogDirectory = "/var/log/kube/pods/"
93
94 func TestCRIListPodStats(t *testing.T) {
95 ctx := context.Background()
96 var (
97 imageFsMountpoint = "/test/mount/point"
98 unknownMountpoint = "/unknown/mount/point"
99 imageFsInfo = getTestFsInfo(2000)
100 rootFsInfo = getTestFsInfo(1000)
101
102 sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns", false)
103 sandbox0Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox0.PodSandboxStatus.Metadata.Uid))
104 container0 = makeFakeContainer(sandbox0, cName0, 0, false)
105 containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
106 containerLogStats0 = makeFakeLogStats(1000)
107 container1 = makeFakeContainer(sandbox0, cName1, 0, false)
108 containerStats1 = makeFakeContainerStats(container1, unknownMountpoint)
109 containerLogStats1 = makeFakeLogStats(2000)
110
111 sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns", false)
112 sandbox1Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox1.PodSandboxStatus.Metadata.Uid))
113 container2 = makeFakeContainer(sandbox1, cName2, 0, false)
114 containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
115 containerLogStats2 = makeFakeLogStats(3000)
116
117 sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns", false)
118 sandbox2Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid))
119 container3 = makeFakeContainer(sandbox2, cName3, 0, true)
120 containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
121 container4 = makeFakeContainer(sandbox2, cName3, 1, false)
122 containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
123 containerLogStats4 = makeFakeLogStats(4000)
124
125
126 sandbox3 = makeFakePodSandbox("sandbox3-name", "sandbox3-uid", "sandbox3-ns", false)
127 sandbox3Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox3.PodSandboxStatus.Metadata.Uid))
128 container5 = makeFakeContainer(sandbox3, cName5, 0, true)
129 containerStats5 = makeFakeContainerStats(container5, imageFsMountpoint)
130 containerLogStats5 = makeFakeLogStats(5000)
131 container8 = makeFakeContainer(sandbox3, cName8, 0, false)
132 containerStats8 = makeFakeContainerStats(container8, imageFsMountpoint)
133 containerLogStats8 = makeFakeLogStats(6000)
134
135
136 sandbox4 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns", true)
137 container6 = makeFakeContainer(sandbox4, cName6, 0, true)
138 containerStats6 = makeFakeContainerStats(container6, imageFsMountpoint)
139
140
141 sandbox5 = makeFakePodSandbox("sandbox1-name", "sandbox5-uid", "sandbox1-ns", true)
142 container7 = makeFakeContainer(sandbox5, cName7, 0, true)
143 containerStats7 = makeFakeContainerStats(container7, imageFsMountpoint)
144
145 podLogName0 = "pod-log-0"
146 podLogName1 = "pod-log-1"
147 podLogStats0 = makeFakeLogStats(5000)
148 podLogStats1 = makeFakeLogStats(6000)
149 )
150
151 mockCtrl := gomock.NewController(t)
152 defer mockCtrl.Finish()
153
154 var (
155 mockCadvisor = cadvisortest.NewMockInterface(mockCtrl)
156 mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
157 mockPodManager = new(kubepodtest.MockManager)
158 resourceAnalyzer = new(fakeResourceAnalyzer)
159 fakeRuntimeService = critest.NewFakeRuntimeService()
160 fakeImageService = critest.NewFakeImageService()
161 )
162
163 infos := map[string]cadvisorapiv2.ContainerInfo{
164 "/": getTestContainerInfo(seedRoot, "", "", ""),
165 "/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
166 "/system": getTestContainerInfo(seedMisc, "", "", ""),
167 sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, kubelettypes.PodInfraContainerName),
168 sandbox0Cgroup: getTestContainerInfo(seedSandbox0, "", "", ""),
169 container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
170 container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
171 sandbox1.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, kubelettypes.PodInfraContainerName),
172 sandbox1Cgroup: getTestContainerInfo(seedSandbox1, "", "", ""),
173 container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2),
174 sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, kubelettypes.PodInfraContainerName),
175 sandbox2Cgroup: getTestContainerInfo(seedSandbox2, "", "", ""),
176 container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
177 sandbox3Cgroup: getTestContainerInfo(seedSandbox3, "", "", ""),
178 }
179
180 options := cadvisorapiv2.RequestOptions{
181 IdType: cadvisorapiv2.TypeName,
182 Count: 2,
183 Recursive: true,
184 }
185
186 mockCadvisor.EXPECT().ContainerInfoV2("/", options).Return(infos, nil)
187 mockCadvisor.EXPECT().RootFsInfo().Return(rootFsInfo, nil)
188 mockCadvisor.EXPECT().GetDirFsInfo(imageFsMountpoint).Return(imageFsInfo, nil)
189 mockCadvisor.EXPECT().GetDirFsInfo(unknownMountpoint).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice)
190
191 fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
192 sandbox0, sandbox1, sandbox2, sandbox3, sandbox4, sandbox5,
193 })
194 fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
195 container0, container1, container2, container3, container4, container5, container6, container7, container8,
196 })
197 fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
198 containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, containerStats6, containerStats7, containerStats8,
199 })
200
201 ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
202 persistentVolumes := makeFakeVolumeStats([]string{"persisVolume1, persisVolumes2"})
203 resourceAnalyzer.podVolumeStats = serverstats.PodVolumeStats{
204 EphemeralVolumes: ephemeralVolumes,
205 PersistentVolumes: persistentVolumes,
206 }
207
208 fakeStats := map[string]*volume.Metrics{
209 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName0): containerLogStats0,
210 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName1): containerLogStats1,
211 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid"), cName2): containerLogStats2,
212 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox2-ns", "sandbox2-name", types.UID("sandbox2-uid"), cName3): containerLogStats4,
213 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox3-ns", "sandbox3-name", types.UID("sandbox3-uid"), cName5): containerLogStats5,
214 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox3-ns", "sandbox3-name", types.UID("sandbox3-uid"), cName8): containerLogStats8,
215 filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")), podLogName0): podLogStats0,
216 filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")), podLogName1): podLogStats1,
217 }
218
219 ctrl := gomock.NewController(t)
220 defer ctrl.Finish()
221
222 fakeOS := &kubecontainertest.FakeOS{}
223 fakeOS.ReadDirFn = func(path string) ([]os.DirEntry, error) {
224 var dirEntries []os.DirEntry
225 mockDE := kubecontainertest.NewMockDirEntry(ctrl)
226 switch path {
227 case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")):
228 mockDE.EXPECT().Name().Return(podLogName0)
229 case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")):
230 mockDE.EXPECT().Name().Return(podLogName1)
231 default:
232 return nil, nil
233 }
234 mockDE.EXPECT().IsDir().Return(false)
235 dirEntries = append(dirEntries, mockDE)
236 return dirEntries, nil
237 }
238
239 provider := NewCRIStatsProvider(
240 mockCadvisor,
241 resourceAnalyzer,
242 mockPodManager,
243 mockRuntimeCache,
244 fakeRuntimeService,
245 fakeImageService,
246 NewFakeHostStatsProviderWithData(fakeStats, fakeOS),
247 false,
248 )
249
250 stats, err := provider.ListPodStats(ctx)
251 assert := assert.New(t)
252 assert.NoError(err)
253 assert.Equal(4, len(stats))
254
255 podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
256 for _, s := range stats {
257 podStatsMap[s.PodRef] = s
258 }
259
260 p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}]
261 assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
262 assert.Equal(2, len(p0.Containers))
263
264 checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1},
265 []*volume.Metrics{containerLogStats0, containerLogStats1}, podLogStats0)
266
267 containerStatsMap := make(map[string]statsapi.ContainerStats)
268 for _, s := range p0.Containers {
269 containerStatsMap[s.Name] = s
270 }
271
272 c0 := containerStatsMap[cName0]
273 assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
274 checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
275 assert.Nil(c0.Accelerators)
276 checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
277 checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0)
278
279 c1 := containerStatsMap[cName1]
280 assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
281 checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
282 assert.Nil(c0.Accelerators)
283 checkCRIRootfsStats(assert, c1, containerStats1, nil)
284 checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
285 checkCRINetworkStats(assert, p0.Network, infos[sandbox0.PodSandboxStatus.Id].Stats[0].Network)
286 checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
287 checkCRIPodSwapStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
288
289 p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
290 assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
291 assert.Equal(1, len(p1.Containers))
292
293 checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2},
294 []*volume.Metrics{containerLogStats2}, podLogStats1)
295 c2 := p1.Containers[0]
296 assert.Equal(cName2, c2.Name)
297 assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
298 checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
299 assert.Nil(c0.Accelerators)
300 checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
301 checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
302 checkCRINetworkStats(assert, p1.Network, infos[sandbox1.PodSandboxStatus.Id].Stats[0].Network)
303 checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
304 checkCRIPodSwapStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
305
306 p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
307 assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
308 assert.Equal(1, len(p2.Containers))
309
310 checkEphemeralStorageStats(assert, p2, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats4},
311 []*volume.Metrics{containerLogStats4}, nil)
312
313 c3 := p2.Containers[0]
314 assert.Equal(cName3, c3.Name)
315 assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
316 checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
317 assert.Nil(c0.Accelerators)
318 checkCRIRootfsStats(assert, c3, containerStats4, &imageFsInfo)
319
320 checkCRILogsStats(assert, c3, &rootFsInfo, containerLogStats4)
321 checkCRINetworkStats(assert, p2.Network, infos[sandbox2.PodSandboxStatus.Id].Stats[0].Network)
322 checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
323 checkCRIPodSwapStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
324
325 p3 := podStatsMap[statsapi.PodReference{Name: "sandbox3-name", UID: "sandbox3-uid", Namespace: "sandbox3-ns"}]
326 assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano())
327 assert.Equal(1, len(p3.Containers))
328
329 c8 := p3.Containers[0]
330 assert.Equal(cName8, c8.Name)
331 assert.Equal(container8.CreatedAt, c8.StartTime.UnixNano())
332 assert.NotNil(c8.CPU.Time)
333 assert.NotNil(c8.Memory.Time)
334 checkCRIPodCPUAndMemoryStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
335 checkCRIPodSwapStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
336 }
337
338 func TestListPodStatsStrictlyFromCRI(t *testing.T) {
339 ctx := context.Background()
340 var (
341 imageFsMountpoint = "/test/mount/point"
342 unknownMountpoint = "/unknown/mount/point"
343 imageFsInfo = getTestFsInfo(2000)
344 rootFsInfo = getTestFsInfo(1000)
345
346
347
348 sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns", false)
349 sandbox0Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox0.PodSandboxStatus.Metadata.Uid))
350 container0 = makeFakeContainer(sandbox0, cName0, 0, false)
351 containerStats0 = makeFakeContainerStatsStrictlyFromCRI(seedContainer0, container0, imageFsMountpoint)
352 containerLogStats0 = makeFakeLogStats(1000)
353 container1 = makeFakeContainer(sandbox0, cName1, 0, false)
354 containerStats1 = makeFakeContainerStatsStrictlyFromCRI(seedContainer1, container1, unknownMountpoint)
355 containerLogStats1 = makeFakeLogStats(2000)
356 sandboxPodStats0 = makeFakePodSandboxStatsStrictlyFromCRI(seedSandbox0, sandbox0, containerStats0, containerStats1)
357
358
359
360 sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns", false)
361 sandbox1Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox1.PodSandboxStatus.Metadata.Uid))
362 container2 = makeFakeContainer(sandbox1, cName2, 0, false)
363 containerStats2 = makeFakeContainerStatsStrictlyFromCRI(seedContainer2, container2, imageFsMountpoint)
364 containerLogStats2 = makeFakeLogStats(3000)
365 sandboxPodStats1 = makeFakePodSandboxStatsStrictlyFromCRI(seedSandbox1, sandbox1, containerStats2)
366
367 podLogName0 = "pod-log-0"
368 podLogName1 = "pod-log-1"
369 podLogStats0 = makeFakeLogStats(5000)
370 podLogStats1 = makeFakeLogStats(6000)
371 )
372 mockCtrl := gomock.NewController(t)
373 defer mockCtrl.Finish()
374 var (
375 mockCadvisor = cadvisortest.NewMockInterface(mockCtrl)
376 mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
377 mockPodManager = new(kubepodtest.MockManager)
378 resourceAnalyzer = new(fakeResourceAnalyzer)
379 fakeRuntimeService = critest.NewFakeRuntimeService()
380 fakeImageService = critest.NewFakeImageService()
381 )
382 infos := map[string]cadvisorapiv2.ContainerInfo{
383 "/": getTestContainerInfo(seedRoot, "", "", ""),
384 "/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
385 "/system": getTestContainerInfo(seedMisc, "", "", ""),
386 sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, kubelettypes.PodInfraContainerName),
387 sandbox0Cgroup: getTestContainerInfo(seedSandbox0, "", "", ""),
388 container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
389 container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
390 }
391
392 exceptedContainerStatsMap := map[string]statsapi.ContainerStats{
393 cName0: getCRIContainerStatsStrictlyFromCRI(seedContainer0, cName0),
394 cName1: getCRIContainerStatsStrictlyFromCRI(seedContainer1, cName1),
395 cName2: getCRIContainerStatsStrictlyFromCRI(seedContainer2, cName2),
396 }
397
398 prf0 := statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}
399 prf1 := statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}
400
401 exceptedPodStatsMap := map[statsapi.PodReference]statsapi.PodStats{
402 prf0: getPodSandboxStatsStrictlyFromCRI(seedSandbox0, sandbox0),
403 prf1: getPodSandboxStatsStrictlyFromCRI(seedSandbox1, sandbox1),
404 }
405
406 options := cadvisorapiv2.RequestOptions{
407 IdType: cadvisorapiv2.TypeName,
408 Count: 2,
409 Recursive: true,
410 }
411 mockCadvisor.EXPECT().ContainerInfoV2("/", options).Return(infos, nil)
412 mockCadvisor.EXPECT().RootFsInfo().Return(rootFsInfo, nil)
413 mockCadvisor.EXPECT().GetDirFsInfo(imageFsMountpoint).Return(imageFsInfo, nil)
414 mockCadvisor.EXPECT().GetDirFsInfo(unknownMountpoint).Return(cadvisorapiv2.FsInfo{}, cadvisorfs.ErrNoSuchDevice)
415 fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
416 sandbox0, sandbox1,
417 })
418 fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
419 container0, container1, container2,
420 })
421 fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
422 containerStats0, containerStats1, containerStats2,
423 })
424
425 fakeRuntimeService.SetFakePodSandboxStats([]*runtimeapi.PodSandboxStats{
426 sandboxPodStats0, sandboxPodStats1,
427 })
428
429 ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
430 persistentVolumes := makeFakeVolumeStats([]string{"persisVolume1, persisVolumes2"})
431 resourceAnalyzer.podVolumeStats = serverstats.PodVolumeStats{
432 EphemeralVolumes: ephemeralVolumes,
433 PersistentVolumes: persistentVolumes,
434 }
435 fakeStats := map[string]*volume.Metrics{
436 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName0): containerLogStats0,
437 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid"), cName1): containerLogStats1,
438 kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid"), cName2): containerLogStats2,
439 filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")), podLogName0): podLogStats0,
440 filepath.Join(kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")), podLogName1): podLogStats1,
441 }
442 ctrl := gomock.NewController(t)
443 defer ctrl.Finish()
444 fakeOS := &kubecontainertest.FakeOS{}
445 fakeOS.ReadDirFn = func(path string) ([]os.DirEntry, error) {
446 var dirEntries []os.DirEntry
447 mockDE := kubecontainertest.NewMockDirEntry(ctrl)
448 switch path {
449 case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox0-ns", "sandbox0-name", types.UID("sandbox0-uid")):
450 mockDE.EXPECT().Name().Return(podLogName0)
451 case kuberuntime.BuildPodLogsDirectory(testPodLogDirectory, "sandbox1-ns", "sandbox1-name", types.UID("sandbox1-uid")):
452 mockDE.EXPECT().Name().Return(podLogName1)
453 default:
454 return nil, nil
455 }
456 mockDE.EXPECT().IsDir().Return(false)
457 dirEntries = append(dirEntries, mockDE)
458 return dirEntries, nil
459 }
460 provider := NewCRIStatsProvider(
461 mockCadvisor,
462 resourceAnalyzer,
463 mockPodManager,
464 mockRuntimeCache,
465 fakeRuntimeService,
466 fakeImageService,
467 NewFakeHostStatsProviderWithData(fakeStats, fakeOS),
468 true,
469 )
470
471 cadvisorInfos, err := getCadvisorContainerInfo(mockCadvisor)
472 if err != nil {
473 t.Errorf("failed to get container info from cadvisor: %v", err)
474 }
475 stats, err := provider.ListPodStats(ctx)
476 assert := assert.New(t)
477 assert.NoError(err)
478 assert.Equal(2, len(stats))
479 podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
480 for _, s := range stats {
481 podStatsMap[s.PodRef] = s
482 }
483 p0 := podStatsMap[prf0]
484 assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
485 assert.Equal(2, len(p0.Containers))
486
487 checkEphemeralStorageStats(assert, p0, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats0, containerStats1},
488 []*volume.Metrics{containerLogStats0, containerLogStats1}, podLogStats0)
489
490 containerStatsMap := make(map[string]statsapi.ContainerStats)
491 for _, s := range p0.Containers {
492 containerStatsMap[s.Name] = s
493 }
494
495 c0 := containerStatsMap[cName0]
496 assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
497 checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c0, exceptedContainerStatsMap[cName0])
498 assert.Nil(c0.Accelerators)
499 checkCRIRootfsStats(assert, c0, containerStats0, &imageFsInfo)
500 checkCRILogsStats(assert, c0, &rootFsInfo, containerLogStats0)
501
502 c1 := containerStatsMap[cName1]
503 assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
504 checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c1, exceptedContainerStatsMap[cName1])
505 assert.Nil(c0.Accelerators)
506 checkCRIRootfsStats(assert, c1, containerStats1, nil)
507 checkCRILogsStats(assert, c1, &rootFsInfo, containerLogStats1)
508 checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert, p0, exceptedPodStatsMap[prf0])
509 assert.NotNil(cadvisorInfos[sandbox0Cgroup].Stats[0].Cpu)
510 assert.NotNil(cadvisorInfos[sandbox0Cgroup].Stats[0].Memory)
511
512 p1 := podStatsMap[prf1]
513 assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
514 assert.Equal(1, len(p1.Containers))
515
516 checkEphemeralStorageStats(assert, p1, ephemeralVolumes, []*runtimeapi.ContainerStats{containerStats2},
517 []*volume.Metrics{containerLogStats2}, podLogStats1)
518 c2 := p1.Containers[0]
519 assert.Equal(cName2, c2.Name)
520 assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
521 checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert, c2, exceptedContainerStatsMap[cName2])
522 assert.Nil(c0.Accelerators)
523 checkCRIRootfsStats(assert, c2, containerStats2, &imageFsInfo)
524 checkCRILogsStats(assert, c2, &rootFsInfo, containerLogStats2)
525 checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert, p1, exceptedPodStatsMap[prf1])
526
527 if runtime.GOOS == "linux" {
528 if _, ok := cadvisorInfos[sandbox1Cgroup]; ok {
529 t.Errorf("expect no cadvisor stats for pod %v", prf1)
530 }
531 }
532 }
533 func TestCRIListPodCPUAndMemoryStats(t *testing.T) {
534 ctx := context.Background()
535
536 var (
537 imageFsMountpoint = "/test/mount/point"
538 unknownMountpoint = "/unknown/mount/point"
539
540 sandbox0 = makeFakePodSandbox("sandbox0-name", "sandbox0-uid", "sandbox0-ns", false)
541 sandbox0Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox0.PodSandboxStatus.Metadata.Uid))
542 container0 = makeFakeContainer(sandbox0, cName0, 0, false)
543 containerStats0 = makeFakeContainerStats(container0, imageFsMountpoint)
544 container1 = makeFakeContainer(sandbox0, cName1, 0, false)
545 containerStats1 = makeFakeContainerStats(container1, unknownMountpoint)
546
547 sandbox1 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns", false)
548 sandbox1Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox1.PodSandboxStatus.Metadata.Uid))
549 container2 = makeFakeContainer(sandbox1, cName2, 0, false)
550 containerStats2 = makeFakeContainerStats(container2, imageFsMountpoint)
551
552 sandbox2 = makeFakePodSandbox("sandbox2-name", "sandbox2-uid", "sandbox2-ns", false)
553 sandbox2Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox2.PodSandboxStatus.Metadata.Uid))
554 container3 = makeFakeContainer(sandbox2, cName3, 0, true)
555 containerStats3 = makeFakeContainerStats(container3, imageFsMountpoint)
556 container4 = makeFakeContainer(sandbox2, cName3, 1, false)
557 containerStats4 = makeFakeContainerStats(container4, imageFsMountpoint)
558
559
560 sandbox3 = makeFakePodSandbox("sandbox3-name", "sandbox3-uid", "sandbox3-ns", false)
561 sandbox3Cgroup = "/" + cm.GetPodCgroupNameSuffix(types.UID(sandbox3.PodSandboxStatus.Metadata.Uid))
562 container5 = makeFakeContainer(sandbox3, cName5, 0, true)
563 containerStats5 = makeFakeContainerStats(container5, imageFsMountpoint)
564 container8 = makeFakeContainer(sandbox3, cName8, 0, false)
565 containerStats8 = makeFakeContainerStats(container8, imageFsMountpoint)
566
567
568 sandbox4 = makeFakePodSandbox("sandbox1-name", "sandbox1-uid", "sandbox1-ns", true)
569 container6 = makeFakeContainer(sandbox4, cName6, 0, true)
570 containerStats6 = makeFakeContainerStats(container6, imageFsMountpoint)
571
572
573 sandbox5 = makeFakePodSandbox("sandbox1-name", "sandbox5-uid", "sandbox1-ns", true)
574 container7 = makeFakeContainer(sandbox5, cName7, 0, true)
575 containerStats7 = makeFakeContainerStats(container7, imageFsMountpoint)
576
577
578 sandbox6 = makeFakePodSandbox("sandbox6-name", "sandbox6-uid", "sandbox6-ns", false)
579 container9 = makeFakeContainer(sandbox6, cName9, 0, false)
580 containerStats9 = makeFakeContainerStats(container9, imageFsMountpoint)
581 )
582
583 mockCtrl := gomock.NewController(t)
584 defer mockCtrl.Finish()
585
586 var (
587 mockCadvisor = cadvisortest.NewMockInterface(mockCtrl)
588 mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
589 mockPodManager = new(kubepodtest.MockManager)
590 resourceAnalyzer = new(fakeResourceAnalyzer)
591 fakeRuntimeService = critest.NewFakeRuntimeService()
592 )
593
594 infos := map[string]cadvisorapiv2.ContainerInfo{
595 "/": getTestContainerInfo(seedRoot, "", "", ""),
596 "/kubelet": getTestContainerInfo(seedKubelet, "", "", ""),
597 "/system": getTestContainerInfo(seedMisc, "", "", ""),
598 sandbox0.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, kubelettypes.PodInfraContainerName),
599 sandbox0Cgroup: getTestContainerInfo(seedSandbox0, "", "", ""),
600 container0.ContainerStatus.Id: getTestContainerInfo(seedContainer0, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName0),
601 container1.ContainerStatus.Id: getTestContainerInfo(seedContainer1, pName0, sandbox0.PodSandboxStatus.Metadata.Namespace, cName1),
602 sandbox1.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox1, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, kubelettypes.PodInfraContainerName),
603 sandbox1Cgroup: getTestContainerInfo(seedSandbox1, "", "", ""),
604 container2.ContainerStatus.Id: getTestContainerInfo(seedContainer2, pName1, sandbox1.PodSandboxStatus.Metadata.Namespace, cName2),
605 sandbox2.PodSandboxStatus.Id: getTestContainerInfo(seedSandbox2, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, kubelettypes.PodInfraContainerName),
606 sandbox2Cgroup: getTestContainerInfo(seedSandbox2, "", "", ""),
607 container4.ContainerStatus.Id: getTestContainerInfo(seedContainer3, pName2, sandbox2.PodSandboxStatus.Metadata.Namespace, cName3),
608 sandbox3Cgroup: getTestContainerInfo(seedSandbox3, "", "", ""),
609 }
610
611 options := cadvisorapiv2.RequestOptions{
612 IdType: cadvisorapiv2.TypeName,
613 Count: 2,
614 Recursive: true,
615 }
616
617 mockCadvisor.EXPECT().ContainerInfoV2("/", options).Return(infos, nil)
618
619 fakeRuntimeService.SetFakeSandboxes([]*critest.FakePodSandbox{
620 sandbox0, sandbox1, sandbox2, sandbox3, sandbox4, sandbox5, sandbox6,
621 })
622 fakeRuntimeService.SetFakeContainers([]*critest.FakeContainer{
623 container0, container1, container2, container3, container4, container5, container6, container7, container8, container9,
624 })
625 fakeRuntimeService.SetFakeContainerStats([]*runtimeapi.ContainerStats{
626 containerStats0, containerStats1, containerStats2, containerStats3, containerStats4, containerStats5, containerStats6, containerStats7, containerStats8, containerStats9,
627 })
628
629 ephemeralVolumes := makeFakeVolumeStats([]string{"ephVolume1, ephVolumes2"})
630 persistentVolumes := makeFakeVolumeStats([]string{"persisVolume1, persisVolumes2"})
631 resourceAnalyzer.podVolumeStats = serverstats.PodVolumeStats{
632 EphemeralVolumes: ephemeralVolumes,
633 PersistentVolumes: persistentVolumes,
634 }
635
636 provider := NewCRIStatsProvider(
637 mockCadvisor,
638 resourceAnalyzer,
639 mockPodManager,
640 mockRuntimeCache,
641 fakeRuntimeService,
642 nil,
643 NewFakeHostStatsProvider(),
644 false,
645 )
646
647 stats, err := provider.ListPodCPUAndMemoryStats(ctx)
648 assert := assert.New(t)
649 assert.NoError(err)
650 assert.Equal(5, len(stats))
651
652 podStatsMap := make(map[statsapi.PodReference]statsapi.PodStats)
653 for _, s := range stats {
654 podStatsMap[s.PodRef] = s
655 }
656
657 p0 := podStatsMap[statsapi.PodReference{Name: "sandbox0-name", UID: "sandbox0-uid", Namespace: "sandbox0-ns"}]
658 assert.Equal(sandbox0.CreatedAt, p0.StartTime.UnixNano())
659 assert.Equal(2, len(p0.Containers))
660 assert.Nil(p0.EphemeralStorage)
661 assert.Nil(p0.VolumeStats)
662 assert.Nil(p0.Network)
663 checkCRIPodCPUAndMemoryStats(assert, p0, infos[sandbox0Cgroup].Stats[0])
664
665 containerStatsMap := make(map[string]statsapi.ContainerStats)
666 for _, s := range p0.Containers {
667 containerStatsMap[s.Name] = s
668 }
669
670 c0 := containerStatsMap[cName0]
671 assert.Equal(container0.CreatedAt, c0.StartTime.UnixNano())
672 checkCRICPUAndMemoryStats(assert, c0, infos[container0.ContainerStatus.Id].Stats[0])
673 assert.Nil(c0.Rootfs)
674 assert.Nil(c0.Logs)
675 assert.Nil(c0.Accelerators)
676 assert.Nil(c0.UserDefinedMetrics)
677 c1 := containerStatsMap[cName1]
678 assert.Equal(container1.CreatedAt, c1.StartTime.UnixNano())
679 checkCRICPUAndMemoryStats(assert, c1, infos[container1.ContainerStatus.Id].Stats[0])
680 assert.Nil(c1.Rootfs)
681 assert.Nil(c1.Logs)
682 assert.Nil(c1.Accelerators)
683 assert.Nil(c1.UserDefinedMetrics)
684
685 p1 := podStatsMap[statsapi.PodReference{Name: "sandbox1-name", UID: "sandbox1-uid", Namespace: "sandbox1-ns"}]
686 assert.Equal(sandbox1.CreatedAt, p1.StartTime.UnixNano())
687 assert.Equal(1, len(p1.Containers))
688 assert.Nil(p1.EphemeralStorage)
689 assert.Nil(p1.VolumeStats)
690 assert.Nil(p1.Network)
691 checkCRIPodCPUAndMemoryStats(assert, p1, infos[sandbox1Cgroup].Stats[0])
692
693 c2 := p1.Containers[0]
694 assert.Equal(cName2, c2.Name)
695 assert.Equal(container2.CreatedAt, c2.StartTime.UnixNano())
696 checkCRICPUAndMemoryStats(assert, c2, infos[container2.ContainerStatus.Id].Stats[0])
697 assert.Nil(c2.Rootfs)
698 assert.Nil(c2.Logs)
699 assert.Nil(c2.Accelerators)
700 assert.Nil(c2.UserDefinedMetrics)
701
702 p2 := podStatsMap[statsapi.PodReference{Name: "sandbox2-name", UID: "sandbox2-uid", Namespace: "sandbox2-ns"}]
703 assert.Equal(sandbox2.CreatedAt, p2.StartTime.UnixNano())
704 assert.Equal(1, len(p2.Containers))
705 assert.Nil(p2.EphemeralStorage)
706 assert.Nil(p2.VolumeStats)
707 assert.Nil(p2.Network)
708 checkCRIPodCPUAndMemoryStats(assert, p2, infos[sandbox2Cgroup].Stats[0])
709
710 c3 := p2.Containers[0]
711 assert.Equal(cName3, c3.Name)
712 assert.Equal(container4.CreatedAt, c3.StartTime.UnixNano())
713 checkCRICPUAndMemoryStats(assert, c3, infos[container4.ContainerStatus.Id].Stats[0])
714 assert.Nil(c2.Rootfs)
715 assert.Nil(c2.Logs)
716 assert.Nil(c2.Accelerators)
717 assert.Nil(c2.UserDefinedMetrics)
718
719 p3 := podStatsMap[statsapi.PodReference{Name: "sandbox3-name", UID: "sandbox3-uid", Namespace: "sandbox3-ns"}]
720 assert.Equal(sandbox3.CreatedAt, p3.StartTime.UnixNano())
721 assert.Equal(1, len(p3.Containers))
722
723 c8 := p3.Containers[0]
724 assert.Equal(cName8, c8.Name)
725 assert.Equal(container8.CreatedAt, c8.StartTime.UnixNano())
726 assert.NotNil(c8.CPU.Time)
727 assert.NotNil(c8.Memory.Time)
728 checkCRIPodCPUAndMemoryStats(assert, p3, infos[sandbox3Cgroup].Stats[0])
729
730 p6 := podStatsMap[statsapi.PodReference{Name: "sandbox6-name", UID: "sandbox6-uid", Namespace: "sandbox6-ns"}]
731 assert.Equal(sandbox6.CreatedAt, p6.StartTime.UnixNano())
732 assert.Equal(1, len(p6.Containers))
733
734 c9 := p6.Containers[0]
735 assert.Equal(cName9, c9.Name)
736 assert.Equal(container9.CreatedAt, c9.StartTime.UnixNano())
737 assert.NotNil(c9.CPU.Time)
738 assert.Equal(containerStats9.Cpu.Timestamp, p6.CPU.Time.UnixNano())
739 assert.NotNil(c9.Memory.Time)
740 assert.Equal(containerStats9.Memory.Timestamp, p6.Memory.Time.UnixNano())
741 }
742
743 func TestCRIImagesFsStats(t *testing.T) {
744 ctx := context.Background()
745 var (
746 imageFsMountpoint = "/test/mount/point"
747 imageFsInfo = getTestFsInfo(2000)
748 imageFsUsage = makeFakeImageFsUsage(imageFsMountpoint)
749 )
750
751 mockCtrl := gomock.NewController(t)
752 defer mockCtrl.Finish()
753
754 var (
755 mockCadvisor = cadvisortest.NewMockInterface(mockCtrl)
756 mockRuntimeCache = new(kubecontainertest.MockRuntimeCache)
757 mockPodManager = new(kubepodtest.MockManager)
758 resourceAnalyzer = new(fakeResourceAnalyzer)
759 fakeRuntimeService = critest.NewFakeRuntimeService()
760 fakeImageService = critest.NewFakeImageService()
761 )
762 mockCadvisor.EXPECT().GetDirFsInfo(imageFsMountpoint).Return(imageFsInfo, nil)
763 fakeImageService.SetFakeFilesystemUsage([]*runtimeapi.FilesystemUsage{
764 imageFsUsage,
765 })
766
767 provider := NewCRIStatsProvider(
768 mockCadvisor,
769 resourceAnalyzer,
770 mockPodManager,
771 mockRuntimeCache,
772 fakeRuntimeService,
773 fakeImageService,
774 NewFakeHostStatsProvider(),
775 false,
776 )
777
778 stats, containerStats, err := provider.ImageFsStats(ctx)
779 assert := assert.New(t)
780 assert.NoError(err)
781
782 assert.Equal(imageFsUsage.Timestamp, stats.Time.UnixNano())
783 assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
784 assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
785 assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
786 assert.Equal(imageFsInfo.Inodes, stats.Inodes)
787 assert.Equal(imageFsUsage.UsedBytes.Value, *stats.UsedBytes)
788 assert.Equal(imageFsUsage.InodesUsed.Value, *stats.InodesUsed)
789
790 assert.Equal(imageFsUsage.Timestamp, containerStats.Time.UnixNano())
791 assert.Equal(imageFsInfo.Available, *containerStats.AvailableBytes)
792 assert.Equal(imageFsInfo.Capacity, *containerStats.CapacityBytes)
793 assert.Equal(imageFsInfo.InodesFree, containerStats.InodesFree)
794 assert.Equal(imageFsInfo.Inodes, containerStats.Inodes)
795 assert.Equal(imageFsUsage.UsedBytes.Value, *containerStats.UsedBytes)
796 assert.Equal(imageFsUsage.InodesUsed.Value, *containerStats.InodesUsed)
797
798 }
799
800 func makeFakePodSandbox(name, uid, namespace string, terminated bool) *critest.FakePodSandbox {
801 p := &critest.FakePodSandbox{
802 PodSandboxStatus: runtimeapi.PodSandboxStatus{
803 Metadata: &runtimeapi.PodSandboxMetadata{
804 Name: name,
805 Uid: uid,
806 Namespace: namespace,
807 },
808 State: runtimeapi.PodSandboxState_SANDBOX_READY,
809 CreatedAt: time.Now().UnixNano(),
810 },
811 }
812 if terminated {
813 p.PodSandboxStatus.State = runtimeapi.PodSandboxState_SANDBOX_NOTREADY
814 }
815 p.PodSandboxStatus.Id = strings.ReplaceAll(string(uuid.NewUUID()), "-", "")
816 return p
817 }
818
819 func makeFakeContainer(sandbox *critest.FakePodSandbox, name string, attempt uint32, terminated bool) *critest.FakeContainer {
820 sandboxID := sandbox.PodSandboxStatus.Id
821 c := &critest.FakeContainer{
822 SandboxID: sandboxID,
823 ContainerStatus: runtimeapi.ContainerStatus{
824 Metadata: &runtimeapi.ContainerMetadata{Name: name, Attempt: attempt},
825 Image: &runtimeapi.ImageSpec{},
826 ImageRef: "fake-image-ref",
827 CreatedAt: time.Now().UnixNano(),
828 },
829 }
830 c.ContainerStatus.Labels = map[string]string{
831 "io.kubernetes.pod.name": sandbox.Metadata.Name,
832 "io.kubernetes.pod.uid": sandbox.Metadata.Uid,
833 "io.kubernetes.pod.namespace": sandbox.Metadata.Namespace,
834 "io.kubernetes.container.name": name,
835 }
836 if terminated {
837 c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_EXITED
838 } else {
839 c.ContainerStatus.State = runtimeapi.ContainerState_CONTAINER_RUNNING
840 }
841 c.ContainerStatus.Id = strings.ReplaceAll(string(uuid.NewUUID()), "-", "")
842 return c
843 }
844
845 func makeFakeContainerStats(container *critest.FakeContainer, imageFsMountpoint string) *runtimeapi.ContainerStats {
846 containerStats := &runtimeapi.ContainerStats{
847 Attributes: &runtimeapi.ContainerAttributes{
848 Id: container.ContainerStatus.Id,
849 Metadata: container.ContainerStatus.Metadata,
850 },
851 WritableLayer: &runtimeapi.FilesystemUsage{
852 Timestamp: time.Now().UnixNano(),
853 FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: imageFsMountpoint},
854 UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64() / 100},
855 InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64() / 100},
856 },
857 }
858 if container.State == runtimeapi.ContainerState_CONTAINER_EXITED {
859 containerStats.Cpu = nil
860 containerStats.Memory = nil
861 } else {
862 containerStats.Cpu = &runtimeapi.CpuUsage{
863 Timestamp: time.Now().UnixNano(),
864 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: rand.Uint64()},
865 }
866 containerStats.Memory = &runtimeapi.MemoryUsage{
867 Timestamp: time.Now().UnixNano(),
868 WorkingSetBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
869 }
870 }
871 return containerStats
872 }
873
874
875 func makeFakeContainerStatsStrictlyFromCRI(seed int, container *critest.FakeContainer, imageFsMountpoint string) *runtimeapi.ContainerStats {
876 containerStats := &runtimeapi.ContainerStats{
877 Attributes: &runtimeapi.ContainerAttributes{
878 Id: container.ContainerStatus.Id,
879 Metadata: container.ContainerStatus.Metadata,
880 },
881 WritableLayer: &runtimeapi.FilesystemUsage{
882 Timestamp: timestamp.UnixNano(),
883 FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: imageFsMountpoint},
884 UsedBytes: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetFsUsage)},
885 InodesUsed: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetFsInodeUsage)},
886 },
887 }
888 if container.State == runtimeapi.ContainerState_CONTAINER_EXITED {
889 containerStats.Cpu = nil
890 containerStats.Memory = nil
891 } else {
892 containerStats.Cpu = &runtimeapi.CpuUsage{
893 Timestamp: timestamp.UnixNano(),
894 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)},
895 }
896 containerStats.Memory = &runtimeapi.MemoryUsage{
897 Timestamp: timestamp.UnixNano(),
898 WorkingSetBytes: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetMemWorkingSetBytes)},
899 }
900 }
901 return containerStats
902 }
903
904 func makeFakePodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePodSandbox, podContainerStats ...*runtimeapi.ContainerStats) *runtimeapi.PodSandboxStats {
905 podSandboxStats := &runtimeapi.PodSandboxStats{
906 Attributes: &runtimeapi.PodSandboxAttributes{
907 Id: podSandbox.Id,
908 Metadata: podSandbox.Metadata,
909 },
910 Linux: &runtimeapi.LinuxPodSandboxStats{},
911 }
912 podSandboxStats.Linux.Containers = append(podSandboxStats.Linux.Containers, podContainerStats...)
913 if podSandbox.State == runtimeapi.PodSandboxState_SANDBOX_NOTREADY {
914 podSandboxStats.Linux.Cpu = nil
915 podSandboxStats.Linux.Memory = nil
916 } else {
917 podSandboxStats.Linux.Cpu = &runtimeapi.CpuUsage{
918 Timestamp: timestamp.UnixNano(),
919 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)},
920 }
921 podSandboxStats.Linux.Memory = &runtimeapi.MemoryUsage{
922 Timestamp: timestamp.UnixNano(),
923 WorkingSetBytes: &runtimeapi.UInt64Value{Value: uint64(seed + offsetCRI + offsetMemWorkingSetBytes)},
924 }
925 }
926 return podSandboxStats
927 }
928 func getPodSandboxStatsStrictlyFromCRI(seed int, podSandbox *critest.FakePodSandbox) statsapi.PodStats {
929 podStats := statsapi.PodStats{
930 PodRef: statsapi.PodReference{
931 Name: podSandbox.Metadata.Name,
932 UID: podSandbox.Metadata.Uid,
933 Namespace: podSandbox.Metadata.Namespace,
934 },
935
936 StartTime: metav1.NewTime(time.Unix(0, podSandbox.CreatedAt)),
937 }
938 if podSandbox.State == runtimeapi.PodSandboxState_SANDBOX_NOTREADY {
939 podStats.CPU = nil
940 podStats.Memory = nil
941 } else {
942 usageCoreNanoSeconds := uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)
943 workingSetBytes := uint64(seed + offsetCRI + offsetMemWorkingSetBytes)
944 podStats.CPU = &statsapi.CPUStats{
945 Time: metav1.NewTime(timestamp),
946 UsageCoreNanoSeconds: &usageCoreNanoSeconds,
947 }
948 podStats.Memory = &statsapi.MemoryStats{
949 Time: metav1.NewTime(timestamp),
950 WorkingSetBytes: &workingSetBytes,
951 }
952 }
953
954 return podStats
955 }
956
957 func makeFakeImageFsUsage(fsMountpoint string) *runtimeapi.FilesystemUsage {
958 return &runtimeapi.FilesystemUsage{
959 Timestamp: time.Now().UnixNano(),
960 FsId: &runtimeapi.FilesystemIdentifier{Mountpoint: fsMountpoint},
961 UsedBytes: &runtimeapi.UInt64Value{Value: rand.Uint64()},
962 InodesUsed: &runtimeapi.UInt64Value{Value: rand.Uint64()},
963 }
964 }
965
966 func makeFakeVolumeStats(volumeNames []string) []statsapi.VolumeStats {
967 volumes := make([]statsapi.VolumeStats, len(volumeNames))
968 availableBytes := rand.Uint64()
969 capacityBytes := rand.Uint64()
970 usedBytes := rand.Uint64() / 100
971 inodes := rand.Uint64()
972 inodesFree := rand.Uint64()
973 inodesUsed := rand.Uint64() / 100
974 for i, name := range volumeNames {
975 fsStats := statsapi.FsStats{
976 Time: metav1.NewTime(time.Now()),
977 AvailableBytes: &availableBytes,
978 CapacityBytes: &capacityBytes,
979 UsedBytes: &usedBytes,
980 Inodes: &inodes,
981 InodesFree: &inodesFree,
982 InodesUsed: &inodesUsed,
983 }
984 volumes[i] = statsapi.VolumeStats{
985 FsStats: fsStats,
986 Name: name,
987 }
988 }
989 return volumes
990 }
991
992 func checkCRICPUAndMemoryStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *cadvisorapiv2.ContainerStats) {
993 assert.Equal(cs.Timestamp.UnixNano(), actual.CPU.Time.UnixNano())
994 assert.Equal(cs.Cpu.Usage.Total, *actual.CPU.UsageCoreNanoSeconds)
995 assert.Equal(cs.CpuInst.Usage.Total, *actual.CPU.UsageNanoCores)
996
997 assert.Equal(cs.Memory.Usage, *actual.Memory.UsageBytes)
998 assert.Equal(cs.Memory.WorkingSet, *actual.Memory.WorkingSetBytes)
999 assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes)
1000 assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults)
1001 assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults)
1002 }
1003
1004 func checkCRICPUAndMemoryStatsForStrictlyFromCRI(assert *assert.Assertions, actual statsapi.ContainerStats, excepted statsapi.ContainerStats) {
1005 assert.Equal(excepted.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano())
1006 assert.Equal(*excepted.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds)
1007 assert.Equal(*excepted.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes)
1008 }
1009
1010 func checkCRIRootfsStats(assert *assert.Assertions, actual statsapi.ContainerStats, cs *runtimeapi.ContainerStats, imageFsInfo *cadvisorapiv2.FsInfo) {
1011 assert.Equal(cs.WritableLayer.Timestamp, actual.Rootfs.Time.UnixNano())
1012 if imageFsInfo != nil {
1013 assert.Equal(imageFsInfo.Available, *actual.Rootfs.AvailableBytes)
1014 assert.Equal(imageFsInfo.Capacity, *actual.Rootfs.CapacityBytes)
1015 assert.Equal(*imageFsInfo.InodesFree, *actual.Rootfs.InodesFree)
1016 assert.Equal(*imageFsInfo.Inodes, *actual.Rootfs.Inodes)
1017 } else {
1018 assert.Nil(actual.Rootfs.AvailableBytes)
1019 assert.Nil(actual.Rootfs.CapacityBytes)
1020 assert.Nil(actual.Rootfs.InodesFree)
1021 assert.Nil(actual.Rootfs.Inodes)
1022 }
1023 assert.Equal(cs.WritableLayer.UsedBytes.Value, *actual.Rootfs.UsedBytes)
1024 assert.Equal(cs.WritableLayer.InodesUsed.Value, *actual.Rootfs.InodesUsed)
1025 }
1026
1027 func checkCRILogsStats(assert *assert.Assertions, actual statsapi.ContainerStats, rootFsInfo *cadvisorapiv2.FsInfo, logStats *volume.Metrics) {
1028 assert.Equal(rootFsInfo.Timestamp, actual.Logs.Time.Time)
1029 assert.Equal(rootFsInfo.Available, *actual.Logs.AvailableBytes)
1030 assert.Equal(rootFsInfo.Capacity, *actual.Logs.CapacityBytes)
1031 assert.Equal(*rootFsInfo.InodesFree, *actual.Logs.InodesFree)
1032 assert.Equal(*rootFsInfo.Inodes, *actual.Logs.Inodes)
1033 assert.Equal(uint64(logStats.Used.Value()), *actual.Logs.UsedBytes)
1034 assert.Equal(uint64(logStats.InodesUsed.Value()), *actual.Logs.InodesUsed)
1035 }
1036
1037 func checkEphemeralStorageStats(assert *assert.Assertions,
1038 actual statsapi.PodStats,
1039 volumes []statsapi.VolumeStats,
1040 containers []*runtimeapi.ContainerStats,
1041 containerLogStats []*volume.Metrics,
1042 podLogStats *volume.Metrics) {
1043 var totalUsed, inodesUsed uint64
1044 for _, container := range containers {
1045 totalUsed = totalUsed + container.WritableLayer.UsedBytes.Value
1046 inodesUsed = inodesUsed + container.WritableLayer.InodesUsed.Value
1047 }
1048
1049 for _, volume := range volumes {
1050 totalUsed = totalUsed + *volume.FsStats.UsedBytes
1051 inodesUsed = inodesUsed + *volume.FsStats.InodesUsed
1052 }
1053
1054 for _, logStats := range containerLogStats {
1055 totalUsed = totalUsed + uint64(logStats.Used.Value())
1056 inodesUsed = inodesUsed + uint64(logStats.InodesUsed.Value())
1057 }
1058
1059 if podLogStats != nil {
1060 totalUsed = totalUsed + uint64(podLogStats.Used.Value())
1061 inodesUsed = inodesUsed + uint64(podLogStats.InodesUsed.Value())
1062 }
1063
1064 assert.Equal(int(totalUsed), int(*actual.EphemeralStorage.UsedBytes))
1065 assert.Equal(int(inodesUsed), int(*actual.EphemeralStorage.InodesUsed))
1066 }
1067
1068 func checkCRINetworkStats(assert *assert.Assertions, actual *statsapi.NetworkStats, expected *cadvisorapiv2.NetworkStats) {
1069 assert.Equal(expected.Interfaces[0].RxBytes, *actual.RxBytes)
1070 assert.Equal(expected.Interfaces[0].RxErrors, *actual.RxErrors)
1071 assert.Equal(expected.Interfaces[0].TxBytes, *actual.TxBytes)
1072 assert.Equal(expected.Interfaces[0].TxErrors, *actual.TxErrors)
1073 }
1074
1075 func checkCRIPodCPUAndMemoryStats(assert *assert.Assertions, actual statsapi.PodStats, cs *cadvisorapiv2.ContainerStats) {
1076 if runtime.GOOS != "linux" {
1077 return
1078 }
1079 assert.Equal(cs.Timestamp.UnixNano(), actual.CPU.Time.UnixNano())
1080 assert.Equal(cs.Cpu.Usage.Total, *actual.CPU.UsageCoreNanoSeconds)
1081 assert.Equal(cs.CpuInst.Usage.Total, *actual.CPU.UsageNanoCores)
1082
1083 assert.Equal(cs.Memory.Usage, *actual.Memory.UsageBytes)
1084 assert.Equal(cs.Memory.WorkingSet, *actual.Memory.WorkingSetBytes)
1085 assert.Equal(cs.Memory.RSS, *actual.Memory.RSSBytes)
1086 assert.Equal(cs.Memory.ContainerData.Pgfault, *actual.Memory.PageFaults)
1087 assert.Equal(cs.Memory.ContainerData.Pgmajfault, *actual.Memory.MajorPageFaults)
1088 }
1089
1090 func checkCRIPodSwapStats(assert *assert.Assertions, actual statsapi.PodStats, cs *cadvisorapiv2.ContainerStats) {
1091 if runtime.GOOS != "linux" {
1092 return
1093 }
1094
1095 assert.Equal(cs.Timestamp.UnixNano(), actual.Swap.Time.UnixNano())
1096 assert.Equal(cs.Memory.Swap, *actual.Swap.SwapUsageBytes)
1097 }
1098
1099 func checkCRIPodCPUAndMemoryStatsStrictlyFromCRI(assert *assert.Assertions, actual statsapi.PodStats, excepted statsapi.PodStats) {
1100 if runtime.GOOS != "linux" {
1101 return
1102 }
1103 assert.Equal(excepted.CPU.Time.UnixNano(), actual.CPU.Time.UnixNano())
1104 assert.Equal(*excepted.CPU.UsageCoreNanoSeconds, *actual.CPU.UsageCoreNanoSeconds)
1105 assert.Equal(*excepted.Memory.WorkingSetBytes, *actual.Memory.WorkingSetBytes)
1106 }
1107
1108 func makeFakeLogStats(seed int) *volume.Metrics {
1109 m := &volume.Metrics{}
1110 m.Used = resource.NewQuantity(int64(seed+offsetUsage), resource.BinarySI)
1111 m.InodesUsed = resource.NewQuantity(int64(seed+offsetInodeUsage), resource.BinarySI)
1112 return m
1113 }
1114
1115 func TestGetContainerUsageNanoCores(t *testing.T) {
1116 var value0 uint64
1117 var value1 uint64 = 10000000000
1118
1119
1120 var value2 uint64 = 188427786383
1121
1122 tests := []struct {
1123 desc string
1124 cpuUsageCache map[string]*cpuUsageRecord
1125 stats *runtimeapi.ContainerStats
1126 expected *uint64
1127 }{
1128 {
1129 desc: "should return nil if stats is nil",
1130 cpuUsageCache: map[string]*cpuUsageRecord{},
1131 },
1132 {
1133 desc: "should return nil if cpu stats is nil",
1134 cpuUsageCache: map[string]*cpuUsageRecord{},
1135 stats: &runtimeapi.ContainerStats{
1136 Attributes: &runtimeapi.ContainerAttributes{
1137 Id: "1",
1138 },
1139 Cpu: nil,
1140 },
1141 },
1142 {
1143 desc: "should return nil if usageCoreNanoSeconds is nil",
1144 cpuUsageCache: map[string]*cpuUsageRecord{},
1145 stats: &runtimeapi.ContainerStats{
1146 Attributes: &runtimeapi.ContainerAttributes{
1147 Id: "1",
1148 },
1149 Cpu: &runtimeapi.CpuUsage{
1150 Timestamp: 1,
1151 UsageCoreNanoSeconds: nil,
1152 },
1153 },
1154 },
1155 {
1156 desc: "should return nil if cpu stats is not cached yet",
1157 cpuUsageCache: map[string]*cpuUsageRecord{},
1158 stats: &runtimeapi.ContainerStats{
1159 Attributes: &runtimeapi.ContainerAttributes{
1160 Id: "1",
1161 },
1162 Cpu: &runtimeapi.CpuUsage{
1163 Timestamp: 1,
1164 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1165 Value: 10000000000,
1166 },
1167 },
1168 },
1169 },
1170 {
1171 desc: "should return zero value if cached cpu stats is equal to current value",
1172 stats: &runtimeapi.ContainerStats{
1173 Attributes: &runtimeapi.ContainerAttributes{
1174 Id: "1",
1175 },
1176 Cpu: &runtimeapi.CpuUsage{
1177 Timestamp: 1,
1178 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1179 Value: 10000000000,
1180 },
1181 },
1182 },
1183 cpuUsageCache: map[string]*cpuUsageRecord{
1184 "1": {
1185 stats: &runtimeapi.CpuUsage{
1186 Timestamp: 0,
1187 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1188 Value: 10000000000,
1189 },
1190 },
1191 },
1192 },
1193 expected: &value0,
1194 },
1195 {
1196 desc: "should return correct value if cached cpu stats is not equal to current value",
1197 stats: &runtimeapi.ContainerStats{
1198 Attributes: &runtimeapi.ContainerAttributes{
1199 Id: "1",
1200 },
1201 Cpu: &runtimeapi.CpuUsage{
1202 Timestamp: int64(time.Second / time.Nanosecond),
1203 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1204 Value: 20000000000,
1205 },
1206 },
1207 },
1208 cpuUsageCache: map[string]*cpuUsageRecord{
1209 "1": {
1210 stats: &runtimeapi.CpuUsage{
1211 Timestamp: 0,
1212 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1213 Value: 10000000000,
1214 },
1215 },
1216 },
1217 },
1218 expected: &value1,
1219 },
1220 {
1221 desc: "should return correct value if elapsed UsageCoreNanoSeconds exceeds 18446744073",
1222 stats: &runtimeapi.ContainerStats{
1223 Attributes: &runtimeapi.ContainerAttributes{
1224 Id: "1",
1225 },
1226 Cpu: &runtimeapi.CpuUsage{
1227 Timestamp: int64(time.Second / time.Nanosecond),
1228 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1229 Value: 68172016162105,
1230 },
1231 },
1232 },
1233 cpuUsageCache: map[string]*cpuUsageRecord{
1234 "1": {
1235 stats: &runtimeapi.CpuUsage{
1236 Timestamp: 0,
1237 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1238 Value: 67983588375722,
1239 },
1240 },
1241 },
1242 },
1243 expected: &value2,
1244 },
1245 {
1246 desc: "should return nil if cpuacct is reset to 0 in a live container",
1247 stats: &runtimeapi.ContainerStats{
1248 Attributes: &runtimeapi.ContainerAttributes{
1249 Id: "1",
1250 },
1251 Cpu: &runtimeapi.CpuUsage{
1252 Timestamp: 2,
1253 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1254 Value: 0,
1255 },
1256 },
1257 },
1258 cpuUsageCache: map[string]*cpuUsageRecord{
1259 "1": {
1260 stats: &runtimeapi.CpuUsage{
1261 Timestamp: 1,
1262 UsageCoreNanoSeconds: &runtimeapi.UInt64Value{
1263 Value: 10000000000,
1264 },
1265 },
1266 },
1267 },
1268 expected: nil,
1269 },
1270 }
1271 for _, test := range tests {
1272 provider := &criStatsProvider{cpuUsageCache: test.cpuUsageCache}
1273
1274 cached := provider.getContainerUsageNanoCores(test.stats)
1275 assert.Nil(t, cached)
1276
1277
1278 real := provider.getAndUpdateContainerUsageNanoCores(test.stats)
1279 assert.Equal(t, test.expected, real, test.desc)
1280
1281
1282 cached = provider.getContainerUsageNanoCores(test.stats)
1283 assert.Equal(t, test.expected, cached, test.desc)
1284 }
1285 }
1286
1287 func TestExtractIDFromCgroupPath(t *testing.T) {
1288 tests := []struct {
1289 cgroupPath string
1290 expected string
1291 }{
1292 {
1293 cgroupPath: "/kubepods/burstable/pod2fc932ce-fdcc-454b-97bd-aadfdeb4c340/9be25294016e2dc0340dd605ce1f57b492039b267a6a618a7ad2a7a58a740f32",
1294 expected: "9be25294016e2dc0340dd605ce1f57b492039b267a6a618a7ad2a7a58a740f32",
1295 },
1296 {
1297 cgroupPath: "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod2fc932ce_fdcc_454b_97bd_aadfdeb4c340.slice/cri-containerd-aaefb9d8feed2d453b543f6d928cede7a4dbefa6a0ae7c9b990dd234c56e93b9.scope",
1298 expected: "aaefb9d8feed2d453b543f6d928cede7a4dbefa6a0ae7c9b990dd234c56e93b9",
1299 },
1300 {
1301 cgroupPath: "/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod2fc932ce_fdcc_454b_97bd_aadfdeb4c340.slice/cri-o-aaefb9d8feed2d453b543f6d928cede7a4dbefa6a0ae7c9b990dd234c56e93b9.scope",
1302 expected: "aaefb9d8feed2d453b543f6d928cede7a4dbefa6a0ae7c9b990dd234c56e93b9",
1303 },
1304 }
1305
1306 for _, test := range tests {
1307 id := extractIDFromCgroupPath(test.cgroupPath)
1308 assert.Equal(t, test.expected, id)
1309 }
1310 }
1311
1312 func getCRIContainerStatsStrictlyFromCRI(seed int, containerName string) statsapi.ContainerStats {
1313 result := statsapi.ContainerStats{
1314 Name: containerName,
1315 StartTime: metav1.NewTime(timestamp),
1316 CPU: &statsapi.CPUStats{},
1317 Memory: &statsapi.MemoryStats{},
1318
1319 Rootfs: &statsapi.FsStats{},
1320 }
1321
1322 result.CPU.Time = metav1.NewTime(timestamp)
1323 usageCoreNanoSeconds := uint64(seed + offsetCRI + offsetCPUUsageCoreSeconds)
1324 result.CPU.UsageCoreNanoSeconds = &usageCoreNanoSeconds
1325
1326 result.Memory.Time = metav1.NewTime(timestamp)
1327 workingSetBytes := uint64(seed + offsetCRI + offsetMemWorkingSetBytes)
1328 result.Memory.WorkingSetBytes = &workingSetBytes
1329
1330 result.Rootfs.Time = metav1.NewTime(timestamp)
1331 usedBytes := uint64(seed + offsetCRI + offsetFsUsage)
1332 result.Rootfs.UsedBytes = &usedBytes
1333
1334 inodesUsed := uint64(seed + offsetCRI + offsetFsInodeUsage)
1335 result.Rootfs.InodesUsed = &inodesUsed
1336
1337 return result
1338 }
1339
View as plain text