runtime_conformance_test.go

Documentation: k8s.io/kubernetes/test/e2e_node

     1  /*
     2  Copyright 2016 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package e2enode
    18  
    19  import (
    20  	"context"
    21  	"fmt"
    22  	"os"
    23  	"path/filepath"
    24  	"time"
    25  
    26  	v1 "k8s.io/api/core/v1"
    27  	"k8s.io/kubernetes/pkg/kubelet/images"
    28  	"k8s.io/kubernetes/test/e2e/common/node"
    29  	"k8s.io/kubernetes/test/e2e/framework"
    30  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    31  	"k8s.io/kubernetes/test/e2e_node/services"
    32  	admissionapi "k8s.io/pod-security-admission/api"
    33  
    34  	"github.com/onsi/ginkgo/v2"
    35  )
    36  
    37  var _ = SIGDescribe("Container Runtime Conformance Test", func() {
    38  	f := framework.NewDefaultFramework("runtime-conformance")
    39  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
    40  
    41  	ginkgo.Describe("container runtime conformance blackbox test", func() {
    42  
    43  		ginkgo.Context("when running a container with a new image", func() {
    44  			// The service account only has pull permission
    45  			auth := `
    46  {
    47  	"auths": {
    48  		"https://gcr.io": {
    49  			"auth": "X2pzb25fa2V5OnsKICAidHlwZSI6ICJzZXJ2aWNlX2FjY291bnQiLAogICJwcm9qZWN0X2lkIjogImF1dGhlbnRpY2F0ZWQtaW1hZ2UtcHVsbGluZyIsCiAgInByaXZhdGVfa2V5X2lkIjogImI5ZjJhNjY0YWE5YjIwNDg0Y2MxNTg2MDYzZmVmZGExOTIyNGFjM2IiLAogICJwcml2YXRlX2tleSI6ICItLS0tLUJFR0lOIFBSSVZBVEUgS0VZLS0tLS1cbk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktjd2dnU2pBZ0VBQW9JQkFRQzdTSG5LVEVFaVlMamZcbkpmQVBHbUozd3JCY2VJNTBKS0xxS21GWE5RL3REWGJRK2g5YVl4aldJTDhEeDBKZTc0bVovS01uV2dYRjVLWlNcbm9BNktuSU85Yi9SY1NlV2VpSXRSekkzL1lYVitPNkNjcmpKSXl4anFWam5mVzJpM3NhMzd0OUE5VEZkbGZycm5cbjR6UkpiOWl4eU1YNGJMdHFGR3ZCMDNOSWl0QTNzVlo1ODhrb1FBZmgzSmhhQmVnTWorWjRSYko0aGVpQlFUMDNcbnZVbzViRWFQZVQ5RE16bHdzZWFQV2dydDZOME9VRGNBRTl4bGNJek11MjUzUG4vSzgySFpydEx4akd2UkhNVXhcbng0ZjhwSnhmQ3h4QlN3Z1NORit3OWpkbXR2b0wwRmE3ZGducFJlODZWRDY2ejNZenJqNHlLRXRqc2hLZHl5VWRcbkl5cVhoN1JSQWdNQkFBRUNnZ0VBT3pzZHdaeENVVlFUeEFka2wvSTVTRFVidi9NazRwaWZxYjJEa2FnbmhFcG9cbjFJajJsNGlWMTByOS9uenJnY2p5VlBBd3pZWk1JeDFBZVF0RDdoUzRHWmFweXZKWUc3NkZpWFpQUm9DVlB6b3VcbmZyOGRDaWFwbDV0enJDOWx2QXNHd29DTTdJWVRjZmNWdDdjRTEyRDNRS3NGNlo3QjJ6ZmdLS251WVBmK0NFNlRcbmNNMHkwaCtYRS9kMERvSERoVy96YU1yWEhqOFRvd2V1eXRrYmJzNGYvOUZqOVBuU2dET1lQd2xhbFZUcitGUWFcbkpSd1ZqVmxYcEZBUW14M0Jyd25rWnQzQ2lXV2lGM2QrSGk5RXRVYnRWclcxYjZnK1JRT0licWFtcis4YlJuZFhcbjZWZ3FCQWtKWjhSVnlkeFVQMGQxMUdqdU9QRHhCbkhCbmM0UW9rSXJFUUtCZ1FEMUNlaWN1ZGhXdGc0K2dTeGJcbnplanh0VjFONDFtZHVjQnpvMmp5b1dHbzNQVDh3ckJPL3lRRTM0cU9WSi9pZCs4SThoWjRvSWh1K0pBMDBzNmdcblRuSXErdi9kL1RFalk4MW5rWmlDa21SUFdiWHhhWXR4UjIxS1BYckxOTlFKS2ttOHRkeVh5UHFsOE1veUdmQ1dcbjJ2aVBKS05iNkhabnY5Q3lqZEo5ZzJMRG5RS0JnUUREcVN2eURtaGViOTIzSW96NGxlZ01SK205Z2xYVWdTS2dcbkVzZlllbVJmbU5XQitDN3ZhSXlVUm1ZNU55TXhmQlZXc3dXRldLYXhjK0krYnFzZmx6elZZdFpwMThNR2pzTURcbmZlZWZBWDZCWk1zVXQ3Qmw3WjlWSjg1bnRFZHFBQ0xwWitaLzN0SVJWdWdDV1pRMWhrbmxHa0dUMDI0SkVFKytcbk55SDFnM2QzUlFLQmdRQ1J2MXdKWkkwbVBsRklva0tGTkh1YTBUcDNLb1JTU1hzTURTVk9NK2xIckcxWHJtRjZcbkMwNGNTKzQ0N0dMUkxHOFVUaEpKbTRxckh0Ti9aK2dZOTYvMm1xYjRIakpORDM3TVhKQnZFYTN5ZUxTOHEvK1JcbjJGOU1LamRRaU5LWnhQcG84VzhOSlREWTVOa1BaZGh4a2pzSHdVNGRTNjZwMVRESUU0MGd0TFpaRFFLQmdGaldcbktyblFpTnEzOS9iNm5QOFJNVGJDUUFKbmR3anhTUU5kQTVmcW1rQTlhRk9HbCtqamsxQ1BWa0tNSWxLSmdEYkpcbk9heDl2OUc2Ui9NSTFIR1hmV3QxWU56VnRocjRIdHNyQTB0U3BsbWhwZ05XRTZWejZuQURqdGZQSnMyZUdqdlhcbmpQUnArdjhjY21MK3dTZzhQTGprM3ZsN2VlNXJsWWxNQndNdUdjUHhBb0dBZWRueGJXMVJMbVZubEFpSEx1L0xcbmxtZkF3RFdtRWlJMFVnK1BMbm9Pdk81dFE1ZDRXMS94RU44bFA0cWtzcGtmZk1Rbk5oNFNZR0VlQlQzMlpxQ1RcbkpSZ2YwWGpveXZ2dXA5eFhqTWtYcnBZL3ljMXpmcVRaQzBNTzkvMVVjMWJSR2RaMmR5M2xSNU5XYXA3T1h5Zk9cblBQcE5Gb1BUWGd2M3FDcW5sTEhyR3pNPVxuLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLVxuIiwKICAiY2xpZW50X2VtYWlsIjogImltYWdlLXB1bGxpbmdAYXV0aGVudGljYXRlZC1pbWFnZS1wdWxsaW5nLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAiY2xpZW50X2lkIjogIjExMzc5NzkxNDUzMDA3MzI3ODcxMiIsCiAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAidG9rZW5fdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi90b2tlbiIsCiAgImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHMiLAogICJjbGllbnRfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9yb2JvdC92MS9tZXRhZGF0YS94NTA5L2ltYWdlLXB1bGxpbmclNDBhdXRoZW50aWNhdGVkLWltYWdlLXB1bGxpbmcuaWFtLmdzZXJ2aWNlYWNjb3VudC5jb20iCn0=",
    50  			"email": "image-pulling@authenticated-image-pulling.iam.gserviceaccount.com"
    51  		}
    52  	}
    53  }`
    54  			// The following images are not added into NodePrePullImageList, because this test is
    55  			// testing image pulling, these images don't need to be prepulled. The ImagePullPolicy
    56  			// is v1.PullAlways, so it won't be blocked by framework image pre-pull list check.
    57  			for _, testCase := range []struct {
    58  				description string
    59  				image       string
    60  				phase       v1.PodPhase
    61  				waiting     bool
    62  			}{
    63  				{
    64  					description: "should be able to pull from private registry with credential provider",
    65  					image:       "gcr.io/authenticated-image-pulling/alpine:3.7",
    66  					phase:       v1.PodRunning,
    67  					waiting:     false,
    68  				},
    69  			} {
    70  				testCase := testCase
    71  				f.It(testCase.description+"", f.WithNodeConformance(), func(ctx context.Context) {
    72  					name := "image-pull-test"
    73  					command := []string{"/bin/sh", "-c", "while true; do sleep 1; done"}
    74  					container := node.ConformanceContainer{
    75  						PodClient: e2epod.NewPodClient(f),
    76  						Container: v1.Container{
    77  							Name:    name,
    78  							Image:   testCase.image,
    79  							Command: command,
    80  							// PullAlways makes sure that the image will always be pulled even if it is present before the test.
    81  							ImagePullPolicy: v1.PullAlways,
    82  						},
    83  						RestartPolicy: v1.RestartPolicyNever,
    84  					}
    85  
    86  					configFile := filepath.Join(services.KubeletRootDirectory, "config.json")
    87  					err := os.WriteFile(configFile, []byte(auth), 0644)
    88  					framework.ExpectNoError(err)
    89  					defer os.Remove(configFile)
    90  
    91  					// checkContainerStatus checks whether the container status matches expectation.
    92  					checkContainerStatus := func(ctx context.Context) error {
    93  						status, err := container.GetStatus(ctx)
    94  						if err != nil {
    95  							return fmt.Errorf("failed to get container status: %w", err)
    96  						}
    97  						// We need to check container state first. The default pod status is pending, If we check
    98  						// pod phase first, and the expected pod phase is Pending, the container status may not
    99  						// even show up when we check it.
   100  						// Check container state
   101  						if !testCase.waiting {
   102  							if status.State.Running == nil {
   103  								return fmt.Errorf("expected container state: Running, got: %q",
   104  									node.GetContainerState(status.State))
   105  							}
   106  						}
   107  						if testCase.waiting {
   108  							if status.State.Waiting == nil {
   109  								return fmt.Errorf("expected container state: Waiting, got: %q",
   110  									node.GetContainerState(status.State))
   111  							}
   112  							reason := status.State.Waiting.Reason
   113  							if reason != images.ErrImagePull.Error() &&
   114  								reason != images.ErrImagePullBackOff.Error() {
   115  								return fmt.Errorf("unexpected waiting reason: %q", reason)
   116  							}
   117  						}
   118  						// Check pod phase
   119  						phase, err := container.GetPhase(ctx)
   120  						if err != nil {
   121  							return fmt.Errorf("failed to get pod phase: %w", err)
   122  						}
   123  						if phase != testCase.phase {
   124  							return fmt.Errorf("expected pod phase: %q, got: %q", testCase.phase, phase)
   125  						}
   126  						return nil
   127  					}
   128  					// The image registry is not stable, which sometimes causes the test to fail. Add retry mechanism to make this
   129  					// less flaky.
   130  					const flakeRetry = 3
   131  					for i := 1; i <= flakeRetry; i++ {
   132  						var err error
   133  						ginkgo.By("create the container")
   134  						container.Create(ctx)
   135  						ginkgo.By("check the container status")
   136  						for start := time.Now(); time.Since(start) < node.ContainerStatusRetryTimeout; time.Sleep(node.ContainerStatusPollInterval) {
   137  							if err = checkContainerStatus(ctx); err == nil {
   138  								break
   139  							}
   140  						}
   141  						ginkgo.By("delete the container")
   142  						_ = container.Delete(ctx)
   143  						if err == nil {
   144  							break
   145  						}
   146  						if i < flakeRetry {
   147  							framework.Logf("No.%d attempt failed: %v, retrying...", i, err)
   148  						} else {
   149  							framework.Failf("All %d attempts failed: %v", flakeRetry, err)
   150  						}
   151  					}
   152  				})
   153  			}
   154  		})
   155  	})
   156  })
   157
View as plain text