cadvisor_stats_provider_test.go

Documentation: k8s.io/kubernetes/pkg/kubelet/stats

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package stats
    18  
    19  import (
    20  	"context"
    21  	"runtime"
    22  	"testing"
    23  
    24  	"github.com/golang/mock/gomock"
    25  	cadvisorapiv2 "github.com/google/cadvisor/info/v2"
    26  	"github.com/stretchr/testify/assert"
    27  
    28  	v1 "k8s.io/api/core/v1"
    29  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    30  	"k8s.io/apimachinery/pkg/types"
    31  	utilfeature "k8s.io/apiserver/pkg/util/feature"
    32  	featuregatetesting "k8s.io/component-base/featuregate/testing"
    33  	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
    34  	statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
    35  	kubelettypes "k8s.io/kubelet/pkg/types"
    36  	"k8s.io/kubernetes/pkg/features"
    37  	cadvisortest "k8s.io/kubernetes/pkg/kubelet/cadvisor/testing"
    38  	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
    39  	containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
    40  	"k8s.io/kubernetes/pkg/kubelet/kuberuntime"
    41  	serverstats "k8s.io/kubernetes/pkg/kubelet/server/stats"
    42  	statustest "k8s.io/kubernetes/pkg/kubelet/status/testing"
    43  	"k8s.io/kubernetes/pkg/volume"
    44  )
    45  
    46  func TestFilterTerminatedContainerInfoAndAssembleByPodCgroupKey(t *testing.T) {
    47  	const (
    48  		seedPastPod0Infra      = 1000
    49  		seedPastPod0Container0 = 2000
    50  		seedPod0Infra          = 3000
    51  		seedPod0Container0     = 4000
    52  	)
    53  	const (
    54  		namespace = "test"
    55  		pName0    = "pod0"
    56  		cName00   = "c0"
    57  		pName1    = "pod1"
    58  		cName11   = "c1"
    59  		pName2    = "pod2"
    60  		cName22   = "c2"
    61  		cName222  = "c222"
    62  	)
    63  	infos := map[string]cadvisorapiv2.ContainerInfo{
    64  		// ContainerInfo with past creation time and no CPU/memory usage for
    65  		// simulating uncleaned cgroups of already terminated containers, which
    66  		// should not be shown in the results.
    67  		"/pod0-i-terminated-1":  getTerminatedContainerInfo(seedPastPod0Infra, pName0, namespace, kubelettypes.PodInfraContainerName),
    68  		"/pod0-c0-terminated-1": getTerminatedContainerInfo(seedPastPod0Container0, pName0, namespace, cName00),
    69  
    70  		// Same as above but uses the same creation time as the latest
    71  		// containers. They are terminated containers, so they should not be in
    72  		// the results.
    73  		"/pod0-i-terminated-2":  getTerminatedContainerInfo(seedPod0Infra, pName0, namespace, kubelettypes.PodInfraContainerName),
    74  		"/pod0-c0-terminated-2": getTerminatedContainerInfo(seedPod0Container0, pName0, namespace, cName00),
    75  
    76  		// The latest containers, which should be in the results.
    77  		"/pod0-i":  getTestContainerInfo(seedPod0Infra, pName0, namespace, kubelettypes.PodInfraContainerName),
    78  		"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace, cName00),
    79  
    80  		"/pod1-i.slice":  getTestContainerInfo(seedPod0Infra, pName1, namespace, kubelettypes.PodInfraContainerName),
    81  		"/pod1-c1.slice": getTestContainerInfo(seedPod0Container0, pName1, namespace, cName11),
    82  
    83  		"/pod2-i-terminated-1": getTerminatedContainerInfo(seedPastPod0Infra, pName2, namespace, kubelettypes.PodInfraContainerName),
    84  		// ContainerInfo with past creation time and no CPU/memory usage for
    85  		// simulating uncleaned cgroups of already terminated containers, which
    86  		// should not be shown in the results.
    87  		"/pod2-c2-terminated-1": getTerminatedContainerInfo(seedPastPod0Container0, pName2, namespace, cName22),
    88  
    89  		//ContainerInfo with no CPU/memory usage but has network usage for uncleaned cgroups, should not be filtered out
    90  		"/pod2-c222-zerocpumem-1": getContainerInfoWithZeroCpuMem(seedPastPod0Container0, pName2, namespace, cName222),
    91  	}
    92  	filteredInfos, allInfos := filterTerminatedContainerInfoAndAssembleByPodCgroupKey(infos)
    93  	assert.Len(t, filteredInfos, 5)
    94  	assert.Len(t, allInfos, 11)
    95  	for _, c := range []string{"/pod0-i", "/pod0-c0"} {
    96  		if _, found := filteredInfos[c]; !found {
    97  			t.Errorf("%q is expected to be in the output\n", c)
    98  		}
    99  	}
   100  
   101  	expectedInfoKeys := []string{"pod0-i-terminated-1", "pod0-c0-terminated-1", "pod0-i-terminated-2", "pod0-c0-terminated-2", "pod0-i", "pod0-c0"}
   102  	// NOTE: on Windows, IsSystemdStyleName will return false, which means that the Container Info will
   103  	// not be assembled by cgroup key.
   104  	if runtime.GOOS != "windows" {
   105  		expectedInfoKeys = append(expectedInfoKeys, "c1")
   106  	} else {
   107  		expectedInfoKeys = append(expectedInfoKeys, "pod1-c1.slice")
   108  	}
   109  	for _, c := range expectedInfoKeys {
   110  		if _, found := allInfos[c]; !found {
   111  			t.Errorf("%q is expected to be in the output\n", c)
   112  		}
   113  	}
   114  }
   115  
   116  func TestCadvisorListPodStats(t *testing.T) {
   117  	ctx := context.Background()
   118  	const (
   119  		namespace0 = "test0"
   120  		namespace2 = "test2"
   121  	)
   122  	const (
   123  		seedRoot              = 0
   124  		seedRuntime           = 100
   125  		seedKubelet           = 200
   126  		seedMisc              = 300
   127  		seedPod0Infra         = 1000
   128  		seedPod0Container0    = 2000
   129  		seedPod0Container1    = 2001
   130  		seedPod1Infra         = 3000
   131  		seedPod1Container     = 4000
   132  		seedPod2Infra         = 5000
   133  		seedPod2Container     = 6000
   134  		seedPod3Infra         = 7000
   135  		seedPod3Container0    = 8000
   136  		seedPod3Container1    = 8001
   137  		seedEphemeralVolume1  = 10000
   138  		seedEphemeralVolume2  = 10001
   139  		seedPersistentVolume1 = 20000
   140  		seedPersistentVolume2 = 20001
   141  	)
   142  	const (
   143  		pName0 = "pod0"
   144  		pName1 = "pod1"
   145  		pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace
   146  		pName3 = "pod3"
   147  	)
   148  	const (
   149  		cName00 = "c0"
   150  		cName01 = "c1"
   151  		cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod
   152  		cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace
   153  		cName30 = "c0-init"
   154  		cName31 = "c1"
   155  	)
   156  	const (
   157  		rootfsCapacity    = uint64(10000000)
   158  		rootfsAvailable   = uint64(5000000)
   159  		rootfsInodesFree  = uint64(1000)
   160  		rootfsInodes      = uint64(2000)
   161  		imagefsCapacity   = uint64(20000000)
   162  		imagefsAvailable  = uint64(8000000)
   163  		imagefsInodesFree = uint64(2000)
   164  		imagefsInodes     = uint64(4000)
   165  	)
   166  
   167  	prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0}
   168  	prf1 := statsapi.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1}
   169  	prf2 := statsapi.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2}
   170  	prf3 := statsapi.PodReference{Name: pName3, Namespace: namespace0, UID: "UID" + pName3}
   171  	infos := map[string]cadvisorapiv2.ContainerInfo{
   172  		"/":              getTestContainerInfo(seedRoot, "", "", ""),
   173  		"/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""),
   174  		"/kubelet":       getTestContainerInfo(seedKubelet, "", "", ""),
   175  		"/system":        getTestContainerInfo(seedMisc, "", "", ""),
   176  		// Pod0 - Namespace0
   177  		"/pod0-i":  getTestContainerInfo(seedPod0Infra, pName0, namespace0, kubelettypes.PodInfraContainerName),
   178  		"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00),
   179  		"/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
   180  		// Pod1 - Namespace0
   181  		"/pod1-i":  getTestContainerInfo(seedPod1Infra, pName1, namespace0, kubelettypes.PodInfraContainerName),
   182  		"/pod1-c0": getTestContainerInfo(seedPod1Container, pName1, namespace0, cName10),
   183  		// Pod2 - Namespace2
   184  		"/pod2-i":                        getTestContainerInfo(seedPod2Infra, pName2, namespace2, kubelettypes.PodInfraContainerName),
   185  		"/pod2-c0":                       getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20),
   186  		"/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, kubelettypes.PodInfraContainerName),
   187  		"/kubepods/podUIDpod1":           getTestContainerInfo(seedPod1Infra, pName1, namespace0, kubelettypes.PodInfraContainerName),
   188  		// Pod3 - Namespace0
   189  		"/pod3-i":       getTestContainerInfo(seedPod3Infra, pName3, namespace0, kubelettypes.PodInfraContainerName),
   190  		"/pod3-c0-init": getTestContainerInfo(seedPod3Container0, pName3, namespace0, cName30),
   191  		"/pod3-c1":      getTestContainerInfo(seedPod3Container1, pName3, namespace0, cName31),
   192  	}
   193  
   194  	freeRootfsInodes := rootfsInodesFree
   195  	totalRootfsInodes := rootfsInodes
   196  	rootfs := cadvisorapiv2.FsInfo{
   197  		Capacity:   rootfsCapacity,
   198  		Available:  rootfsAvailable,
   199  		InodesFree: &freeRootfsInodes,
   200  		Inodes:     &totalRootfsInodes,
   201  	}
   202  
   203  	freeImagefsInodes := imagefsInodesFree
   204  	totalImagefsInodes := imagefsInodes
   205  	imagefs := cadvisorapiv2.FsInfo{
   206  		Capacity:   imagefsCapacity,
   207  		Available:  imagefsAvailable,
   208  		InodesFree: &freeImagefsInodes,
   209  		Inodes:     &totalImagefsInodes,
   210  	}
   211  
   212  	// memory limit overrides for each container (used to test available bytes if a memory limit is known)
   213  	memoryLimitOverrides := map[string]uint64{
   214  		"/":        uint64(1 << 30),
   215  		"/pod2-c0": uint64(1 << 15),
   216  	}
   217  	for name, memoryLimitOverride := range memoryLimitOverrides {
   218  		info, found := infos[name]
   219  		if !found {
   220  			t.Errorf("No container defined with name %v", name)
   221  		}
   222  		info.Spec.Memory.Limit = memoryLimitOverride
   223  		infos[name] = info
   224  	}
   225  	// any container for which cadvisor should return no stats (as might be the case for an exited init container)
   226  	nostatsOverrides := []string{
   227  		"/pod3-c0-init",
   228  	}
   229  	for _, name := range nostatsOverrides {
   230  		info, found := infos[name]
   231  		if !found {
   232  			t.Errorf("No container defined with name %v", name)
   233  		}
   234  		info.Spec.Memory = cadvisorapiv2.MemorySpec{}
   235  		info.Spec.Cpu = cadvisorapiv2.CpuSpec{}
   236  		info.Spec.HasMemory = false
   237  		info.Spec.HasCpu = false
   238  		info.Spec.HasNetwork = false
   239  		infos[name] = info
   240  	}
   241  
   242  	options := cadvisorapiv2.RequestOptions{
   243  		IdType:    cadvisorapiv2.TypeName,
   244  		Count:     2,
   245  		Recursive: true,
   246  	}
   247  
   248  	mockCtrl := gomock.NewController(t)
   249  	defer mockCtrl.Finish()
   250  
   251  	mockCadvisor := cadvisortest.NewMockInterface(mockCtrl)
   252  	mockCadvisor.EXPECT().ContainerInfoV2("/", options).Return(infos, nil)
   253  	mockCadvisor.EXPECT().RootFsInfo().Return(rootfs, nil)
   254  	mockCadvisor.EXPECT().ImagesFsInfo().Return(imagefs, nil)
   255  
   256  	mockRuntime := containertest.NewMockRuntime(mockCtrl)
   257  
   258  	ephemeralVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedEphemeralVolume1, "ephemeralVolume1"),
   259  		getPodVolumeStats(seedEphemeralVolume2, "ephemeralVolume2")}
   260  	persistentVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedPersistentVolume1, "persistentVolume1"),
   261  		getPodVolumeStats(seedPersistentVolume2, "persistentVolume2")}
   262  	volumeStats := serverstats.PodVolumeStats{
   263  		EphemeralVolumes:  ephemeralVolumes,
   264  		PersistentVolumes: persistentVolumes,
   265  	}
   266  	p0Time := metav1.Now()
   267  	p1Time := metav1.Now()
   268  	p2Time := metav1.Now()
   269  	p3Time := metav1.Now()
   270  	mockStatus := statustest.NewMockPodStatusProvider(mockCtrl)
   271  	mockStatus.EXPECT().GetPodStatus(types.UID("UID"+pName0)).Return(v1.PodStatus{StartTime: &p0Time}, true)
   272  	mockStatus.EXPECT().GetPodStatus(types.UID("UID"+pName1)).Return(v1.PodStatus{StartTime: &p1Time}, true)
   273  	mockStatus.EXPECT().GetPodStatus(types.UID("UID"+pName2)).Return(v1.PodStatus{StartTime: &p2Time}, true)
   274  	mockStatus.EXPECT().GetPodStatus(types.UID("UID"+pName3)).Return(v1.PodStatus{StartTime: &p3Time}, true)
   275  
   276  	resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
   277  
   278  	p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime, mockStatus, NewFakeHostStatsProvider())
   279  	pods, err := p.ListPodStats(ctx)
   280  	assert.NoError(t, err)
   281  
   282  	assert.Equal(t, 4, len(pods))
   283  	indexPods := make(map[statsapi.PodReference]statsapi.PodStats, len(pods))
   284  	for _, pod := range pods {
   285  		indexPods[pod.PodRef] = pod
   286  	}
   287  
   288  	// Validate Pod0 Results
   289  	ps, found := indexPods[prf0]
   290  	assert.True(t, found)
   291  	assert.Len(t, ps.Containers, 2)
   292  	indexCon := make(map[string]statsapi.ContainerStats, len(ps.Containers))
   293  	for _, con := range ps.Containers {
   294  		indexCon[con.Name] = con
   295  	}
   296  	con := indexCon[cName00]
   297  	assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix())
   298  	checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU)
   299  	checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory)
   300  	checkSwapStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Swap)
   301  
   302  	con = indexCon[cName01]
   303  	assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
   304  	checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU)
   305  	checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory)
   306  	checkSwapStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Swap)
   307  
   308  	assert.EqualValues(t, p0Time.Unix(), ps.StartTime.Time.Unix())
   309  	checkNetworkStats(t, "Pod0", seedPod0Infra, ps.Network)
   310  	checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, []int{seedEphemeralVolume1, seedEphemeralVolume2}, nil, ps.EphemeralStorage)
   311  	if ps.CPU != nil {
   312  		checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
   313  	}
   314  	if ps.Memory != nil {
   315  		checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory)
   316  	}
   317  	if ps.Swap != nil {
   318  		checkSwapStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Swap)
   319  	}
   320  
   321  	// Validate Pod1 Results
   322  	ps, found = indexPods[prf1]
   323  	assert.True(t, found)
   324  	assert.Len(t, ps.Containers, 1)
   325  	con = ps.Containers[0]
   326  	assert.Equal(t, cName10, con.Name)
   327  	checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU)
   328  	checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory)
   329  	checkSwapStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Swap)
   330  	checkNetworkStats(t, "Pod1", seedPod1Infra, ps.Network)
   331  
   332  	// Validate Pod2 Results
   333  	ps, found = indexPods[prf2]
   334  	assert.True(t, found)
   335  	assert.Len(t, ps.Containers, 1)
   336  	con = ps.Containers[0]
   337  	assert.Equal(t, cName20, con.Name)
   338  	checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
   339  	checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
   340  	checkSwapStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Swap)
   341  	checkNetworkStats(t, "Pod2", seedPod2Infra, ps.Network)
   342  
   343  	// Validate Pod3 Results
   344  
   345  	ps, found = indexPods[prf3]
   346  	assert.True(t, found)
   347  	// /pod3-c0-init has no stats should be filtered
   348  	assert.Len(t, ps.Containers, 1)
   349  	indexCon = make(map[string]statsapi.ContainerStats, len(ps.Containers))
   350  	for _, con := range ps.Containers {
   351  		indexCon[con.Name] = con
   352  	}
   353  	con = indexCon[cName31]
   354  	assert.Equal(t, cName31, con.Name)
   355  	checkCPUStats(t, "Pod3Container1", seedPod3Container1, con.CPU)
   356  	checkMemoryStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Memory)
   357  	checkSwapStats(t, "Pod3Container1", seedPod3Container1, infos["/pod3-c1"], con.Swap)
   358  }
   359  
   360  func TestCadvisorListPodCPUAndMemoryStats(t *testing.T) {
   361  	ctx := context.Background()
   362  	const (
   363  		namespace0 = "test0"
   364  		namespace2 = "test2"
   365  	)
   366  	const (
   367  		seedRoot              = 0
   368  		seedRuntime           = 100
   369  		seedKubelet           = 200
   370  		seedMisc              = 300
   371  		seedPod0Infra         = 1000
   372  		seedPod0Container0    = 2000
   373  		seedPod0Container1    = 2001
   374  		seedPod1Infra         = 3000
   375  		seedPod1Container     = 4000
   376  		seedPod2Infra         = 5000
   377  		seedPod2Container     = 6000
   378  		seedEphemeralVolume1  = 10000
   379  		seedEphemeralVolume2  = 10001
   380  		seedPersistentVolume1 = 20000
   381  		seedPersistentVolume2 = 20001
   382  	)
   383  	const (
   384  		pName0 = "pod0"
   385  		pName1 = "pod1"
   386  		pName2 = "pod0" // ensure pName2 conflicts with pName0, but is in a different namespace
   387  	)
   388  	const (
   389  		cName00 = "c0"
   390  		cName01 = "c1"
   391  		cName10 = "c0" // ensure cName10 conflicts with cName02, but is in a different pod
   392  		cName20 = "c1" // ensure cName20 conflicts with cName01, but is in a different pod + namespace
   393  	)
   394  
   395  	prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0}
   396  	prf1 := statsapi.PodReference{Name: pName1, Namespace: namespace0, UID: "UID" + pName1}
   397  	prf2 := statsapi.PodReference{Name: pName2, Namespace: namespace2, UID: "UID" + pName2}
   398  	infos := map[string]cadvisorapiv2.ContainerInfo{
   399  		"/":              getTestContainerInfo(seedRoot, "", "", ""),
   400  		"/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""),
   401  		"/kubelet":       getTestContainerInfo(seedKubelet, "", "", ""),
   402  		"/system":        getTestContainerInfo(seedMisc, "", "", ""),
   403  		// Pod0 - Namespace0
   404  		"/pod0-i":  getTestContainerInfo(seedPod0Infra, pName0, namespace0, kubelettypes.PodInfraContainerName),
   405  		"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00),
   406  		"/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
   407  		// Pod1 - Namespace0
   408  		"/pod1-i":  getTestContainerInfo(seedPod1Infra, pName1, namespace0, kubelettypes.PodInfraContainerName),
   409  		"/pod1-c0": getTestContainerInfo(seedPod1Container, pName1, namespace0, cName10),
   410  		// Pod2 - Namespace2
   411  		"/pod2-i":                        getTestContainerInfo(seedPod2Infra, pName2, namespace2, kubelettypes.PodInfraContainerName),
   412  		"/pod2-c0":                       getTestContainerInfo(seedPod2Container, pName2, namespace2, cName20),
   413  		"/kubepods/burstable/podUIDpod0": getTestContainerInfo(seedPod0Infra, pName0, namespace0, kubelettypes.PodInfraContainerName),
   414  		"/kubepods/podUIDpod1":           getTestContainerInfo(seedPod1Infra, pName1, namespace0, kubelettypes.PodInfraContainerName),
   415  	}
   416  
   417  	// memory limit overrides for each container (used to test available bytes if a memory limit is known)
   418  	memoryLimitOverrides := map[string]uint64{
   419  		"/":        uint64(1 << 30),
   420  		"/pod2-c0": uint64(1 << 15),
   421  	}
   422  	for name, memoryLimitOverride := range memoryLimitOverrides {
   423  		info, found := infos[name]
   424  		if !found {
   425  			t.Errorf("No container defined with name %v", name)
   426  		}
   427  		info.Spec.Memory.Limit = memoryLimitOverride
   428  		infos[name] = info
   429  	}
   430  
   431  	options := cadvisorapiv2.RequestOptions{
   432  		IdType:    cadvisorapiv2.TypeName,
   433  		Count:     2,
   434  		Recursive: true,
   435  	}
   436  
   437  	mockCtrl := gomock.NewController(t)
   438  	defer mockCtrl.Finish()
   439  
   440  	mockCadvisor := cadvisortest.NewMockInterface(mockCtrl)
   441  	mockCadvisor.EXPECT().ContainerInfoV2("/", options).Return(infos, nil)
   442  
   443  	ephemeralVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedEphemeralVolume1, "ephemeralVolume1"),
   444  		getPodVolumeStats(seedEphemeralVolume2, "ephemeralVolume2")}
   445  	persistentVolumes := []statsapi.VolumeStats{getPodVolumeStats(seedPersistentVolume1, "persistentVolume1"),
   446  		getPodVolumeStats(seedPersistentVolume2, "persistentVolume2")}
   447  	volumeStats := serverstats.PodVolumeStats{
   448  		EphemeralVolumes:  ephemeralVolumes,
   449  		PersistentVolumes: persistentVolumes,
   450  	}
   451  
   452  	resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
   453  
   454  	p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, nil, nil, NewFakeHostStatsProvider())
   455  	pods, err := p.ListPodCPUAndMemoryStats(ctx)
   456  	assert.NoError(t, err)
   457  
   458  	assert.Equal(t, 3, len(pods))
   459  	indexPods := make(map[statsapi.PodReference]statsapi.PodStats, len(pods))
   460  	for _, pod := range pods {
   461  		indexPods[pod.PodRef] = pod
   462  	}
   463  
   464  	// Validate Pod0 Results
   465  	ps, found := indexPods[prf0]
   466  	assert.True(t, found)
   467  	assert.Len(t, ps.Containers, 2)
   468  	indexCon := make(map[string]statsapi.ContainerStats, len(ps.Containers))
   469  	for _, con := range ps.Containers {
   470  		indexCon[con.Name] = con
   471  	}
   472  	con := indexCon[cName00]
   473  	assert.EqualValues(t, testTime(creationTime, seedPod0Container0).Unix(), con.StartTime.Time.Unix())
   474  	checkCPUStats(t, "Pod0Container0", seedPod0Container0, con.CPU)
   475  	checkMemoryStats(t, "Pod0Conainer0", seedPod0Container0, infos["/pod0-c0"], con.Memory)
   476  	assert.Nil(t, con.Rootfs)
   477  	assert.Nil(t, con.Logs)
   478  	assert.Nil(t, con.Accelerators)
   479  	assert.Nil(t, con.UserDefinedMetrics)
   480  
   481  	con = indexCon[cName01]
   482  	assert.EqualValues(t, testTime(creationTime, seedPod0Container1).Unix(), con.StartTime.Time.Unix())
   483  	checkCPUStats(t, "Pod0Container1", seedPod0Container1, con.CPU)
   484  	checkMemoryStats(t, "Pod0Container1", seedPod0Container1, infos["/pod0-c1"], con.Memory)
   485  	assert.Nil(t, con.Rootfs)
   486  	assert.Nil(t, con.Logs)
   487  	assert.Nil(t, con.Accelerators)
   488  	assert.Nil(t, con.UserDefinedMetrics)
   489  
   490  	assert.EqualValues(t, testTime(creationTime, seedPod0Infra).Unix(), ps.StartTime.Time.Unix())
   491  	assert.Nil(t, ps.EphemeralStorage)
   492  	assert.Nil(t, ps.VolumeStats)
   493  	assert.Nil(t, ps.Network)
   494  	if ps.CPU != nil {
   495  		checkCPUStats(t, "Pod0", seedPod0Infra, ps.CPU)
   496  	}
   497  	if ps.Memory != nil {
   498  		checkMemoryStats(t, "Pod0", seedPod0Infra, infos["/pod0-i"], ps.Memory)
   499  	}
   500  
   501  	// Validate Pod1 Results
   502  	ps, found = indexPods[prf1]
   503  	assert.True(t, found)
   504  	assert.Len(t, ps.Containers, 1)
   505  	con = ps.Containers[0]
   506  	assert.Equal(t, cName10, con.Name)
   507  	checkCPUStats(t, "Pod1Container0", seedPod1Container, con.CPU)
   508  	checkMemoryStats(t, "Pod1Container0", seedPod1Container, infos["/pod1-c0"], con.Memory)
   509  	assert.Nil(t, ps.EphemeralStorage)
   510  	assert.Nil(t, ps.VolumeStats)
   511  	assert.Nil(t, ps.Network)
   512  
   513  	// Validate Pod2 Results
   514  	ps, found = indexPods[prf2]
   515  	assert.True(t, found)
   516  	assert.Len(t, ps.Containers, 1)
   517  	con = ps.Containers[0]
   518  	assert.Equal(t, cName20, con.Name)
   519  	checkCPUStats(t, "Pod2Container0", seedPod2Container, con.CPU)
   520  	checkMemoryStats(t, "Pod2Container0", seedPod2Container, infos["/pod2-c0"], con.Memory)
   521  	assert.Nil(t, ps.EphemeralStorage)
   522  	assert.Nil(t, ps.VolumeStats)
   523  	assert.Nil(t, ps.Network)
   524  }
   525  
   526  func TestCadvisorImagesFsStatsKubeletSeparateDiskOff(t *testing.T) {
   527  	ctx := context.Background()
   528  	mockCtrl := gomock.NewController(t)
   529  	defer mockCtrl.Finish()
   530  	var (
   531  		assert       = assert.New(t)
   532  		mockCadvisor = cadvisortest.NewMockInterface(mockCtrl)
   533  		mockRuntime  = containertest.NewMockRuntime(mockCtrl)
   534  
   535  		seed        = 1000
   536  		imageFsInfo = getTestFsInfo(seed)
   537  		imageStats  = &kubecontainer.ImageStats{TotalStorageBytes: 100}
   538  	)
   539  
   540  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletSeparateDiskGC, false)()
   541  
   542  	mockCadvisor.EXPECT().ImagesFsInfo().Return(imageFsInfo, nil)
   543  	mockRuntime.EXPECT().ImageStats(ctx).Return(imageStats, nil)
   544  
   545  	provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil, NewFakeHostStatsProvider())
   546  	stats, _, err := provider.ImageFsStats(ctx)
   547  	assert.NoError(err)
   548  
   549  	assert.Equal(imageFsInfo.Timestamp, stats.Time.Time)
   550  	assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
   551  	assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
   552  	assert.Equal(imageStats.TotalStorageBytes, *stats.UsedBytes)
   553  	assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
   554  	assert.Equal(imageFsInfo.Inodes, stats.Inodes)
   555  	assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed)
   556  }
   557  
   558  func TestCadvisorImagesFsStats(t *testing.T) {
   559  	ctx := context.Background()
   560  	mockCtrl := gomock.NewController(t)
   561  	defer mockCtrl.Finish()
   562  	var (
   563  		assert       = assert.New(t)
   564  		mockCadvisor = cadvisortest.NewMockInterface(mockCtrl)
   565  		mockRuntime  = containertest.NewMockRuntime(mockCtrl)
   566  
   567  		seed        = 1000
   568  		imageFsInfo = getTestFsInfo(seed)
   569  	)
   570  	imageFsInfoCRI := &runtimeapi.FilesystemUsage{
   571  		Timestamp:  imageFsInfo.Timestamp.Unix(),
   572  		FsId:       &runtimeapi.FilesystemIdentifier{Mountpoint: "images"},
   573  		UsedBytes:  &runtimeapi.UInt64Value{Value: imageFsInfo.Usage},
   574  		InodesUsed: &runtimeapi.UInt64Value{Value: *imageFsInfo.Inodes},
   575  	}
   576  	imageFsInfoResponse := &runtimeapi.ImageFsInfoResponse{
   577  		ImageFilesystems:     []*runtimeapi.FilesystemUsage{imageFsInfoCRI},
   578  		ContainerFilesystems: []*runtimeapi.FilesystemUsage{imageFsInfoCRI},
   579  	}
   580  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletSeparateDiskGC, true)()
   581  
   582  	mockCadvisor.EXPECT().ImagesFsInfo().Return(imageFsInfo, nil)
   583  	mockCadvisor.EXPECT().ContainerFsInfo().Return(imageFsInfo, nil)
   584  	mockRuntime.EXPECT().ImageFsInfo(ctx).Return(imageFsInfoResponse, nil)
   585  
   586  	provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil, NewFakeHostStatsProvider())
   587  	stats, containerfs, err := provider.ImageFsStats(ctx)
   588  	assert.NoError(err)
   589  
   590  	assert.Equal(imageFsInfo.Timestamp, stats.Time.Time)
   591  	assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
   592  	assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
   593  	assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
   594  	assert.Equal(imageFsInfo.Inodes, stats.Inodes)
   595  	assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed)
   596  
   597  	assert.Equal(imageFsInfo.Timestamp, containerfs.Time.Time)
   598  	assert.Equal(imageFsInfo.Available, *containerfs.AvailableBytes)
   599  	assert.Equal(imageFsInfo.Capacity, *containerfs.CapacityBytes)
   600  	assert.Equal(imageFsInfo.InodesFree, containerfs.InodesFree)
   601  	assert.Equal(imageFsInfo.Inodes, containerfs.Inodes)
   602  	assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *containerfs.InodesUsed)
   603  
   604  }
   605  
   606  func TestCadvisorSplitImagesFsStats(t *testing.T) {
   607  	ctx := context.Background()
   608  	mockCtrl := gomock.NewController(t)
   609  	defer mockCtrl.Finish()
   610  	var (
   611  		assert       = assert.New(t)
   612  		mockCadvisor = cadvisortest.NewMockInterface(mockCtrl)
   613  		mockRuntime  = containertest.NewMockRuntime(mockCtrl)
   614  
   615  		seed            = 1000
   616  		imageFsInfo     = getTestFsInfo(seed)
   617  		containerSeed   = 1001
   618  		containerFsInfo = getTestFsInfo(containerSeed)
   619  	)
   620  	imageFsInfoCRI := &runtimeapi.FilesystemUsage{
   621  		Timestamp:  imageFsInfo.Timestamp.Unix(),
   622  		FsId:       &runtimeapi.FilesystemIdentifier{Mountpoint: "images"},
   623  		UsedBytes:  &runtimeapi.UInt64Value{Value: imageFsInfo.Usage},
   624  		InodesUsed: &runtimeapi.UInt64Value{Value: *imageFsInfo.Inodes},
   625  	}
   626  	containerFsInfoCRI := &runtimeapi.FilesystemUsage{
   627  		Timestamp:  containerFsInfo.Timestamp.Unix(),
   628  		FsId:       &runtimeapi.FilesystemIdentifier{Mountpoint: "containers"},
   629  		UsedBytes:  &runtimeapi.UInt64Value{Value: containerFsInfo.Usage},
   630  		InodesUsed: &runtimeapi.UInt64Value{Value: *containerFsInfo.Inodes},
   631  	}
   632  	imageFsInfoResponse := &runtimeapi.ImageFsInfoResponse{
   633  		ImageFilesystems:     []*runtimeapi.FilesystemUsage{imageFsInfoCRI},
   634  		ContainerFilesystems: []*runtimeapi.FilesystemUsage{containerFsInfoCRI},
   635  	}
   636  
   637  	mockCadvisor.EXPECT().ImagesFsInfo().Return(imageFsInfo, nil)
   638  	mockCadvisor.EXPECT().ContainerFsInfo().Return(containerFsInfo, nil)
   639  	mockRuntime.EXPECT().ImageFsInfo(ctx).Return(imageFsInfoResponse, nil)
   640  	defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.KubeletSeparateDiskGC, true)()
   641  
   642  	provider := newCadvisorStatsProvider(mockCadvisor, &fakeResourceAnalyzer{}, mockRuntime, nil, NewFakeHostStatsProvider())
   643  	stats, containerfs, err := provider.ImageFsStats(ctx)
   644  	assert.NoError(err)
   645  
   646  	assert.Equal(imageFsInfo.Timestamp, stats.Time.Time)
   647  	assert.Equal(imageFsInfo.Available, *stats.AvailableBytes)
   648  	assert.Equal(imageFsInfo.Capacity, *stats.CapacityBytes)
   649  	assert.Equal(imageFsInfo.InodesFree, stats.InodesFree)
   650  	assert.Equal(imageFsInfo.Inodes, stats.Inodes)
   651  	assert.Equal(*imageFsInfo.Inodes-*imageFsInfo.InodesFree, *stats.InodesUsed)
   652  
   653  	assert.Equal(containerFsInfo.Timestamp, containerfs.Time.Time)
   654  	assert.Equal(containerFsInfo.Available, *containerfs.AvailableBytes)
   655  	assert.Equal(containerFsInfo.Capacity, *containerfs.CapacityBytes)
   656  	assert.Equal(containerFsInfo.InodesFree, containerfs.InodesFree)
   657  	assert.Equal(containerFsInfo.Inodes, containerfs.Inodes)
   658  	assert.Equal(*containerFsInfo.Inodes-*containerFsInfo.InodesFree, *containerfs.InodesUsed)
   659  
   660  }
   661  
   662  func TestCadvisorListPodStatsWhenContainerLogFound(t *testing.T) {
   663  	ctx := context.Background()
   664  	const (
   665  		namespace0 = "test0"
   666  	)
   667  	const (
   668  		seedRoot           = 0
   669  		seedRuntime        = 100
   670  		seedKubelet        = 200
   671  		seedMisc           = 300
   672  		seedPod0Infra      = 1000
   673  		seedPod0Container0 = 0
   674  		seedPod0Container1 = 0
   675  	)
   676  	const (
   677  		pName0 = "pod0"
   678  	)
   679  	const (
   680  		cName00 = "c0"
   681  		cName01 = "c1"
   682  	)
   683  	const (
   684  		rootfsCapacity    = uint64(10000000)
   685  		rootfsAvailable   = uint64(5000000)
   686  		rootfsInodesFree  = uint64(1000)
   687  		rootfsInodes      = uint64(2000)
   688  		imagefsCapacity   = uint64(20000000)
   689  		imagefsAvailable  = uint64(8000000)
   690  		imagefsInodesFree = uint64(2000)
   691  		imagefsInodes     = uint64(4000)
   692  	)
   693  
   694  	prf0 := statsapi.PodReference{Name: pName0, Namespace: namespace0, UID: "UID" + pName0}
   695  	infos := map[string]cadvisorapiv2.ContainerInfo{
   696  		"/":              getTestContainerInfo(seedRoot, "", "", ""),
   697  		"/docker-daemon": getTestContainerInfo(seedRuntime, "", "", ""),
   698  		"/kubelet":       getTestContainerInfo(seedKubelet, "", "", ""),
   699  		"/system":        getTestContainerInfo(seedMisc, "", "", ""),
   700  		// Pod0 - Namespace0
   701  		"/pod0-i":  getTestContainerInfo(seedPod0Infra, pName0, namespace0, kubelettypes.PodInfraContainerName),
   702  		"/pod0-c0": getTestContainerInfo(seedPod0Container0, pName0, namespace0, cName00),
   703  		"/pod0-c1": getTestContainerInfo(seedPod0Container1, pName0, namespace0, cName01),
   704  	}
   705  
   706  	containerLogStats0 := makeFakeLogStats(0)
   707  	containerLogStats1 := makeFakeLogStats(0)
   708  	fakeStats := map[string]*volume.Metrics{
   709  		kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName00): containerLogStats0,
   710  		kuberuntime.BuildContainerLogsDirectory(testPodLogDirectory, prf0.Namespace, prf0.Name, types.UID(prf0.UID), cName01): containerLogStats1,
   711  	}
   712  	fakeStatsSlice := []*volume.Metrics{containerLogStats0, containerLogStats1}
   713  	fakeOS := &containertest.FakeOS{}
   714  
   715  	freeRootfsInodes := rootfsInodesFree
   716  	totalRootfsInodes := rootfsInodes
   717  	rootfs := cadvisorapiv2.FsInfo{
   718  		Capacity:   rootfsCapacity,
   719  		Available:  rootfsAvailable,
   720  		InodesFree: &freeRootfsInodes,
   721  		Inodes:     &totalRootfsInodes,
   722  	}
   723  
   724  	freeImagefsInodes := imagefsInodesFree
   725  	totalImagefsInodes := imagefsInodes
   726  	imagefs := cadvisorapiv2.FsInfo{
   727  		Capacity:   imagefsCapacity,
   728  		Available:  imagefsAvailable,
   729  		InodesFree: &freeImagefsInodes,
   730  		Inodes:     &totalImagefsInodes,
   731  	}
   732  
   733  	options := cadvisorapiv2.RequestOptions{
   734  		IdType:    cadvisorapiv2.TypeName,
   735  		Count:     2,
   736  		Recursive: true,
   737  	}
   738  
   739  	mockCtrl := gomock.NewController(t)
   740  	defer mockCtrl.Finish()
   741  
   742  	mockCadvisor := cadvisortest.NewMockInterface(mockCtrl)
   743  	mockCadvisor.EXPECT().ContainerInfoV2("/", options).Return(infos, nil)
   744  	mockCadvisor.EXPECT().RootFsInfo().Return(rootfs, nil)
   745  	mockCadvisor.EXPECT().ImagesFsInfo().Return(imagefs, nil)
   746  
   747  	mockRuntime := containertest.NewMockRuntime(mockCtrl)
   748  	mockRuntime.EXPECT().ImageStats(ctx).Return(&kubecontainer.ImageStats{TotalStorageBytes: 123}, nil).AnyTimes()
   749  
   750  	volumeStats := serverstats.PodVolumeStats{}
   751  	p0Time := metav1.Now()
   752  	mockStatus := statustest.NewMockPodStatusProvider(mockCtrl)
   753  	mockStatus.EXPECT().GetPodStatus(types.UID("UID"+pName0)).Return(v1.PodStatus{StartTime: &p0Time}, true)
   754  
   755  	resourceAnalyzer := &fakeResourceAnalyzer{podVolumeStats: volumeStats}
   756  
   757  	p := NewCadvisorStatsProvider(mockCadvisor, resourceAnalyzer, nil, nil, mockRuntime, mockStatus, NewFakeHostStatsProviderWithData(fakeStats, fakeOS))
   758  	pods, err := p.ListPodStats(ctx)
   759  	assert.NoError(t, err)
   760  
   761  	assert.Equal(t, 1, len(pods))
   762  	// Validate Pod0 Results
   763  	checkEphemeralStats(t, "Pod0", []int{seedPod0Container0, seedPod0Container1}, nil, fakeStatsSlice, pods[0].EphemeralStorage)
   764  }
   765
View as plain text