Source file src/k8s.io/kubernetes/test/e2e/apimachinery/aggregator.go

Documentation: k8s.io/kubernetes/test/e2e/apimachinery

     1  /*
     2  Copyright 2017 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package apimachinery
    18  
    19  import (
    20  	"context"
    21  	"crypto/rand"
    22  	"encoding/json"
    23  	"fmt"
    24  	"math/big"
    25  	"net"
    26  	"strings"
    27  	"time"
    28  
    29  	appsv1 "k8s.io/api/apps/v1"
    30  	v1 "k8s.io/api/core/v1"
    31  	rbacv1 "k8s.io/api/rbac/v1"
    32  	apierrors "k8s.io/apimachinery/pkg/api/errors"
    33  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    34  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    35  	"k8s.io/apimachinery/pkg/labels"
    36  	"k8s.io/apimachinery/pkg/runtime/schema"
    37  	"k8s.io/apimachinery/pkg/types"
    38  	"k8s.io/apimachinery/pkg/util/intstr"
    39  	"k8s.io/apimachinery/pkg/util/wait"
    40  	"k8s.io/client-go/discovery"
    41  	clientset "k8s.io/client-go/kubernetes"
    42  	"k8s.io/client-go/util/retry"
    43  	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
    44  	aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
    45  	rbacv1helpers "k8s.io/kubernetes/pkg/apis/rbac/v1"
    46  	"k8s.io/kubernetes/test/e2e/framework"
    47  	e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
    48  	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
    49  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
    50  	"k8s.io/kubernetes/test/utils/format"
    51  	imageutils "k8s.io/kubernetes/test/utils/image"
    52  	admissionapi "k8s.io/pod-security-admission/api"
    53  	samplev1alpha1 "k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
    54  	"k8s.io/utils/pointer"
    55  
    56  	"github.com/onsi/ginkgo/v2"
    57  	"github.com/onsi/gomega"
    58  )
    59  
    60  const (
    61  	aggregatorServicePort = 7443
    62  
    63  	apiServiceRetryPeriod  = 1 * time.Second
    64  	apiServiceRetryTimeout = 2 * time.Minute
    65  
    66  	defaultApiServiceGroupName = samplev1alpha1.GroupName
    67  	defaultApiServiceVersion   = "v1alpha1"
    68  )
    69  
    70  var _ = SIGDescribe("Aggregator", func() {
    71  	var aggrclient *aggregatorclient.Clientset
    72  
    73  	f := framework.NewDefaultFramework("aggregator")
    74  	f.NamespacePodSecurityLevel = admissionapi.LevelBaseline
    75  
    76  	// We want namespace initialization BeforeEach inserted by
    77  	// NewDefaultFramework to happen before this, so we put this BeforeEach
    78  	// after NewDefaultFramework.
    79  	ginkgo.BeforeEach(func() {
    80  		config, err := framework.LoadConfig()
    81  		if err != nil {
    82  			framework.Failf("could not load config: %v", err)
    83  		}
    84  		aggrclient, err = aggregatorclient.NewForConfig(config)
    85  		if err != nil {
    86  			framework.Failf("could not create aggregator client: %v", err)
    87  		}
    88  		apiServiceName := defaultApiServiceVersion + "." + defaultApiServiceGroupName
    89  		ginkgo.DeferCleanup(cleanupSampleAPIServer, f.ClientSet, aggrclient, generateSampleAPIServerObjectNames(f.Namespace.Name), apiServiceName)
    90  	})
    91  
    92  	/*
    93  		Release: v1.17, v1.21, v1.27
    94  		Testname: aggregator-supports-the-sample-apiserver
    95  		Description: Ensure that the sample-apiserver code from 1.17 and compiled against 1.17
    96  		will work on the current Aggregator/API-Server.
    97  	*/
    98  	framework.ConformanceIt("Should be able to support the 1.17 Sample API Server using the current Aggregator", func(ctx context.Context) {
    99  		// Testing a 1.17 version of the sample-apiserver
   100  		TestSampleAPIServer(ctx, f, aggrclient, imageutils.GetE2EImage(imageutils.APIServer), defaultApiServiceGroupName, defaultApiServiceVersion)
   101  	})
   102  })
   103  
   104  func cleanupSampleAPIServer(ctx context.Context, client clientset.Interface, aggrclient *aggregatorclient.Clientset, n sampleAPIServerObjectNames, apiServiceName string) {
   105  	// delete the APIService first to avoid causing discovery errors
   106  	_ = aggrclient.ApiregistrationV1().APIServices().Delete(ctx, apiServiceName, metav1.DeleteOptions{})
   107  
   108  	_ = client.AppsV1().Deployments(n.namespace).Delete(ctx, "sample-apiserver-deployment", metav1.DeleteOptions{})
   109  	_ = client.CoreV1().Secrets(n.namespace).Delete(ctx, "sample-apiserver-secret", metav1.DeleteOptions{})
   110  	_ = client.CoreV1().Services(n.namespace).Delete(ctx, "sample-api", metav1.DeleteOptions{})
   111  	_ = client.CoreV1().ServiceAccounts(n.namespace).Delete(ctx, "sample-apiserver", metav1.DeleteOptions{})
   112  	_ = client.RbacV1().RoleBindings("kube-system").Delete(ctx, n.roleBinding, metav1.DeleteOptions{})
   113  	_ = client.RbacV1().ClusterRoleBindings().Delete(ctx, "wardler:"+n.namespace+":auth-delegator", metav1.DeleteOptions{})
   114  	_ = client.RbacV1().ClusterRoles().Delete(ctx, n.clusterRole, metav1.DeleteOptions{})
   115  	_ = client.RbacV1().ClusterRoleBindings().Delete(ctx, n.clusterRoleBinding, metav1.DeleteOptions{})
   116  }
   117  
   118  type sampleAPIServerObjectNames struct {
   119  	namespace          string
   120  	roleBinding        string
   121  	clusterRole        string
   122  	clusterRoleBinding string
   123  }
   124  
   125  func generateSampleAPIServerObjectNames(namespace string) sampleAPIServerObjectNames {
   126  	return sampleAPIServerObjectNames{
   127  		namespace:          namespace,
   128  		roleBinding:        "wardler-auth-reader-" + namespace,
   129  		clusterRole:        "sample-apiserver-reader-" + namespace,
   130  		clusterRoleBinding: "wardler:" + namespace + "sample-apiserver-reader-" + namespace,
   131  	}
   132  }
   133  
   134  func SetUpSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image string, n sampleAPIServerObjectNames, apiServiceGroupName, apiServiceVersion string) {
   135  	ginkgo.By("Registering the sample API server.")
   136  	client := f.ClientSet
   137  	restClient := client.Discovery().RESTClient()
   138  	certCtx := setupServerCert(n.namespace, "sample-api")
   139  	apiServiceName := apiServiceVersion + "." + apiServiceGroupName
   140  
   141  	// kubectl create -f namespace.yaml
   142  	// NOTE: aggregated apis should generally be set up in their own namespace. As the test framework is setting up a new namespace, we are just using that.
   143  
   144  	// kubectl create -f secret.yaml
   145  	secretName := "sample-apiserver-secret"
   146  	secret := &v1.Secret{
   147  		ObjectMeta: metav1.ObjectMeta{
   148  			Name: secretName,
   149  		},
   150  		Type: v1.SecretTypeOpaque,
   151  		Data: map[string][]byte{
   152  			"tls.crt": certCtx.cert,
   153  			"tls.key": certCtx.key,
   154  		},
   155  	}
   156  	_, err := client.CoreV1().Secrets(n.namespace).Create(ctx, secret, metav1.CreateOptions{})
   157  	framework.ExpectNoError(err, "creating secret %s in.namespace %s", secretName, n.namespace)
   158  
   159  	if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) {
   160  		// kubectl create -f clusterrole.yaml
   161  		_, err = client.RbacV1().ClusterRoles().Create(ctx, &rbacv1.ClusterRole{
   162  
   163  			ObjectMeta: metav1.ObjectMeta{Name: n.clusterRole},
   164  			Rules: []rbacv1.PolicyRule{
   165  				rbacv1helpers.NewRule("get", "list", "watch").Groups("").Resources("namespaces").RuleOrDie(),
   166  				rbacv1helpers.NewRule("get", "list", "watch").Groups("admissionregistration.k8s.io").Resources("*").RuleOrDie(),
   167  				rbacv1helpers.NewRule("get", "list", "watch").Groups("flowcontrol.apiserver.k8s.io").Resources("prioritylevelconfigurations", "flowschemas").RuleOrDie(),
   168  			},
   169  		}, metav1.CreateOptions{})
   170  		framework.ExpectNoError(err, "creating cluster role %s", n.clusterRole)
   171  
   172  		_, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{
   173  			ObjectMeta: metav1.ObjectMeta{
   174  				Name: n.clusterRoleBinding,
   175  			},
   176  			RoleRef: rbacv1.RoleRef{
   177  				APIGroup: "rbac.authorization.k8s.io",
   178  				Kind:     "ClusterRole",
   179  				Name:     n.clusterRole,
   180  			},
   181  			Subjects: []rbacv1.Subject{
   182  				{
   183  					APIGroup:  "",
   184  					Kind:      "ServiceAccount",
   185  					Name:      "default",
   186  					Namespace: n.namespace,
   187  				},
   188  			},
   189  		}, metav1.CreateOptions{})
   190  		framework.ExpectNoError(err, "creating cluster role binding %s", n.clusterRoleBinding)
   191  
   192  		// kubectl create -f authDelegator.yaml
   193  		_, err = client.RbacV1().ClusterRoleBindings().Create(ctx, &rbacv1.ClusterRoleBinding{
   194  			ObjectMeta: metav1.ObjectMeta{
   195  				Name: "wardler:" + n.namespace + ":auth-delegator",
   196  			},
   197  			RoleRef: rbacv1.RoleRef{
   198  				APIGroup: "rbac.authorization.k8s.io",
   199  				Kind:     "ClusterRole",
   200  				Name:     "system:auth-delegator",
   201  			},
   202  			Subjects: []rbacv1.Subject{
   203  				{
   204  					APIGroup:  "",
   205  					Kind:      "ServiceAccount",
   206  					Name:      "default",
   207  					Namespace: n.namespace,
   208  				},
   209  			},
   210  		}, metav1.CreateOptions{})
   211  		framework.ExpectNoError(err, "creating cluster role binding %s", "wardler:"+n.namespace+":auth-delegator")
   212  	}
   213  
   214  	// kubectl create -f deploy.yaml
   215  	deploymentName := "sample-apiserver-deployment"
   216  	etcdImage := imageutils.GetE2EImage(imageutils.Etcd)
   217  	podLabels := map[string]string{"app": "sample-apiserver", "apiserver": "true"}
   218  	replicas := int32(1)
   219  	etcdLocalhostAddress := "127.0.0.1"
   220  	if framework.TestContext.ClusterIsIPv6() {
   221  		etcdLocalhostAddress = "::1"
   222  	}
   223  	etcdURL := fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, "2379"))
   224  
   225  	mounts := []v1.VolumeMount{
   226  		{
   227  			Name:      "apiserver-certs",
   228  			ReadOnly:  true,
   229  			MountPath: "/apiserver.local.config/certificates",
   230  		},
   231  	}
   232  	volumes := []v1.Volume{
   233  		{
   234  			Name: "apiserver-certs",
   235  			VolumeSource: v1.VolumeSource{
   236  				Secret: &v1.SecretVolumeSource{SecretName: secretName},
   237  			},
   238  		},
   239  	}
   240  	containers := []v1.Container{
   241  		{
   242  			Name:         "sample-apiserver",
   243  			VolumeMounts: mounts,
   244  			Args: []string{
   245  				fmt.Sprintf("--etcd-servers=%s", etcdURL),
   246  				"--tls-cert-file=/apiserver.local.config/certificates/tls.crt",
   247  				"--tls-private-key-file=/apiserver.local.config/certificates/tls.key",
   248  				"--audit-log-path=-",
   249  				"--audit-log-maxage=0",
   250  				"--audit-log-maxbackup=0",
   251  			},
   252  			Image: image,
   253  			ReadinessProbe: &v1.Probe{
   254  				ProbeHandler: v1.ProbeHandler{
   255  					HTTPGet: &v1.HTTPGetAction{
   256  						Scheme: v1.URISchemeHTTPS,
   257  						Port:   intstr.FromInt32(443),
   258  						Path:   "/readyz",
   259  					},
   260  				},
   261  				InitialDelaySeconds: 20,
   262  				PeriodSeconds:       1,
   263  				SuccessThreshold:    1,
   264  				FailureThreshold:    3,
   265  			},
   266  		},
   267  		{
   268  			Name:  "etcd",
   269  			Image: etcdImage,
   270  			Command: []string{
   271  				"/usr/local/bin/etcd",
   272  				"--listen-client-urls",
   273  				etcdURL,
   274  				"--advertise-client-urls",
   275  				etcdURL,
   276  			},
   277  		},
   278  	}
   279  	d := e2edeployment.NewDeployment(deploymentName, replicas, podLabels, "", "", appsv1.RollingUpdateDeploymentStrategyType)
   280  	d.Spec.Template.Spec.Containers = containers
   281  	d.Spec.Template.Spec.Volumes = volumes
   282  
   283  	deployment, err := client.AppsV1().Deployments(n.namespace).Create(ctx, d, metav1.CreateOptions{})
   284  	framework.ExpectNoError(err, "creating deployment %s in namespace %s", deploymentName, n.namespace)
   285  
   286  	err = e2edeployment.WaitForDeploymentRevisionAndImage(client, n.namespace, deploymentName, "1", image)
   287  	framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", image, deploymentName, n.namespace)
   288  
   289  	err = e2edeployment.WaitForDeploymentRevisionAndImage(client, n.namespace, deploymentName, "1", etcdImage)
   290  	framework.ExpectNoError(err, "waiting for the deployment of image %s in %s in %s to complete", etcdImage, deploymentName, n.namespace)
   291  
   292  	// kubectl create -f service.yaml
   293  	serviceLabels := map[string]string{"apiserver": "true"}
   294  	service := &v1.Service{
   295  		ObjectMeta: metav1.ObjectMeta{
   296  			Namespace: n.namespace,
   297  			Name:      "sample-api",
   298  			Labels:    map[string]string{"test": "aggregator"},
   299  		},
   300  		Spec: v1.ServiceSpec{
   301  			Selector: serviceLabels,
   302  			Ports: []v1.ServicePort{
   303  				{
   304  					Protocol:   v1.ProtocolTCP,
   305  					Port:       aggregatorServicePort,
   306  					TargetPort: intstr.FromInt32(443),
   307  				},
   308  			},
   309  		},
   310  	}
   311  	_, err = client.CoreV1().Services(n.namespace).Create(ctx, service, metav1.CreateOptions{})
   312  	framework.ExpectNoError(err, "creating service %s in namespace %s", "sample-api", n.namespace)
   313  
   314  	// kubectl create -f serviceAccount.yaml
   315  	sa := &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: "sample-apiserver"}}
   316  	_, err = client.CoreV1().ServiceAccounts(n.namespace).Create(ctx, sa, metav1.CreateOptions{})
   317  	framework.ExpectNoError(err, "creating service account %s in namespace %s", "sample-apiserver", n.namespace)
   318  
   319  	if e2eauth.IsRBACEnabled(ctx, client.RbacV1()) {
   320  		// kubectl create -f auth-reader.yaml
   321  		_, err = client.RbacV1().RoleBindings("kube-system").Create(ctx, &rbacv1.RoleBinding{
   322  			ObjectMeta: metav1.ObjectMeta{
   323  				Name: n.roleBinding,
   324  				Annotations: map[string]string{
   325  					rbacv1.AutoUpdateAnnotationKey: "true",
   326  				},
   327  			},
   328  			RoleRef: rbacv1.RoleRef{
   329  				APIGroup: "",
   330  				Kind:     "Role",
   331  				Name:     "extension-apiserver-authentication-reader",
   332  			},
   333  			Subjects: []rbacv1.Subject{
   334  				{
   335  					Kind:      "ServiceAccount",
   336  					Name:      "default",
   337  					Namespace: n.namespace,
   338  				},
   339  			},
   340  		}, metav1.CreateOptions{})
   341  		framework.ExpectNoError(err, "creating role binding %s in namespace %s", n.roleBinding, "kube-system")
   342  	}
   343  
   344  	// Wait for the extension apiserver to be up and healthy
   345  	// kubectl get deployments -n <aggregated-api-namespace> && status == Running
   346  	// NOTE: aggregated apis should generally be set up in their own namespace (<aggregated-api-namespace>). As the test framework
   347  	// is setting up a new namespace, we are just using that.
   348  	err = e2edeployment.WaitForDeploymentComplete(client, deployment)
   349  	framework.ExpectNoError(err, "deploying extension apiserver in namespace %s", n.namespace)
   350  
   351  	// kubectl create -f apiservice.yaml
   352  	_, err = aggrclient.ApiregistrationV1().APIServices().Create(ctx, &apiregistrationv1.APIService{
   353  		ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
   354  		Spec: apiregistrationv1.APIServiceSpec{
   355  			Service: &apiregistrationv1.ServiceReference{
   356  				Namespace: n.namespace,
   357  				Name:      "sample-api",
   358  				Port:      pointer.Int32(aggregatorServicePort),
   359  			},
   360  			Group:                apiServiceGroupName,
   361  			Version:              apiServiceVersion,
   362  			CABundle:             certCtx.signingCert,
   363  			GroupPriorityMinimum: 2000,
   364  			VersionPriority:      200,
   365  		},
   366  	}, metav1.CreateOptions{})
   367  	framework.ExpectNoError(err, "creating apiservice %s", apiServiceName)
   368  
   369  	var (
   370  		currentAPIService *apiregistrationv1.APIService
   371  		currentPods       *v1.PodList
   372  	)
   373  
   374  	err = pollTimed(ctx, 100*time.Millisecond, 60*time.Second, func(ctx context.Context) (bool, error) {
   375  
   376  		currentAPIService, _ = aggrclient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{})
   377  		currentPods, _ = client.CoreV1().Pods(n.namespace).List(ctx, metav1.ListOptions{})
   378  
   379  		request := restClient.Get().AbsPath("/apis/" + apiServiceGroupName + "/" + apiServiceVersion + "/namespaces/default/flunders")
   380  		request.SetHeader("Accept", "application/json")
   381  		_, err := request.DoRaw(ctx)
   382  		if err != nil {
   383  			status, ok := err.(*apierrors.StatusError)
   384  			if !ok {
   385  				return false, err
   386  			}
   387  			if status.Status().Code == 403 || status.Status().Code == 503 {
   388  				return false, nil
   389  			}
   390  			if status.Status().Code == 404 && strings.HasPrefix(err.Error(), "the server could not find the requested resource") {
   391  				return false, nil
   392  			}
   393  			return false, err
   394  		}
   395  		return true, nil
   396  	}, "Waited %s for the sample-apiserver to be ready to handle requests.")
   397  	if err != nil {
   398  		currentAPIServiceJSON, _ := json.Marshal(currentAPIService)
   399  		framework.Logf("current APIService: %s", string(currentAPIServiceJSON))
   400  
   401  		currentPodsJSON, _ := json.Marshal(currentPods)
   402  		framework.Logf("current pods: %s", string(currentPodsJSON))
   403  
   404  		if currentPods != nil {
   405  			for _, pod := range currentPods.Items {
   406  				for _, container := range pod.Spec.Containers {
   407  					logs, err := e2epod.GetPodLogs(ctx, client, n.namespace, pod.Name, container.Name)
   408  					framework.Logf("logs of %s/%s (error: %v): %s", pod.Name, container.Name, err, logs)
   409  				}
   410  			}
   411  		}
   412  	}
   413  	framework.ExpectNoError(err, "gave up waiting for apiservice wardle to come up successfully")
   414  }
   415  
   416  // TestSampleAPIServer is a basic test if the sample-apiserver code from 1.29 and compiled against 1.29
   417  // will work on the current Aggregator/API-Server.
   418  func TestSampleAPIServer(ctx context.Context, f *framework.Framework, aggrclient *aggregatorclient.Clientset, image, apiServiceGroupName, apiServiceVersion string) {
   419  	n := generateSampleAPIServerObjectNames(f.Namespace.Name)
   420  	SetUpSampleAPIServer(ctx, f, aggrclient, image, n, apiServiceGroupName, apiServiceVersion)
   421  	client := f.ClientSet
   422  	restClient := client.Discovery().RESTClient()
   423  
   424  	flunderName := generateFlunderName("rest-flunder")
   425  	apiServiceName := apiServiceVersion + "." + apiServiceGroupName
   426  
   427  	// kubectl create -f flunders-1.yaml -v 9
   428  	// curl -k -v -XPOST https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders
   429  	// Request Body: {"apiVersion":"wardle.example.com/v1alpha1","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"test-flunder","namespace":"default"}}
   430  	flunder := `{"apiVersion":"` + apiServiceGroupName + `/` + apiServiceVersion + `","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"` + flunderName + `","namespace":"default"}}`
   431  	result := restClient.Post().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").Body([]byte(flunder)).SetHeader("Accept", "application/json").Do(ctx)
   432  	framework.ExpectNoError(result.Error(), "creating a new flunders resource")
   433  	var statusCode int
   434  	result.StatusCode(&statusCode)
   435  	if statusCode != 201 {
   436  		framework.Failf("Flunders client creation response was status %d, not 201", statusCode)
   437  	}
   438  	u := &unstructured.Unstructured{}
   439  	if err := result.Into(u); err != nil {
   440  		framework.ExpectNoError(err, "reading created response")
   441  	}
   442  
   443  	gomega.Expect(u.GetAPIVersion()).To(gomega.Equal(apiServiceGroupName + "/" + apiServiceVersion))
   444  	gomega.Expect(u.GetKind()).To(gomega.Equal("Flunder"))
   445  	gomega.Expect(u.GetName()).To(gomega.Equal(flunderName))
   446  
   447  	pods, err := client.CoreV1().Pods(n.namespace).List(ctx, metav1.ListOptions{})
   448  	framework.ExpectNoError(err, "getting pods for flunders service")
   449  
   450  	// kubectl get flunders -v 9
   451  	// curl -k -v -XGET https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders
   452  	contents, err := restClient.Get().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx)
   453  	framework.ExpectNoError(err, "attempting to get a newly created flunders resource")
   454  	var flundersList samplev1alpha1.FlunderList
   455  	err = json.Unmarshal(contents, &flundersList)
   456  	validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/"+apiServiceGroupName+"/"+apiServiceVersion)
   457  	if len(flundersList.Items) != 1 {
   458  		framework.Failf("failed to get back the correct flunders list %v", flundersList)
   459  	}
   460  
   461  	// kubectl delete flunder test-flunder -v 9
   462  	// curl -k -v -XDELETE  https://35.193.112.40/apis/wardle.example.com/v1alpha1/namespaces/default/flunders/test-flunder
   463  	_, err = restClient.Delete().AbsPath("/apis/" + apiServiceGroupName + "/" + apiServiceVersion + "/namespaces/default/flunders/" + flunderName).DoRaw(ctx)
   464  	validateErrorWithDebugInfo(ctx, f, err, pods, "attempting to delete a newly created flunders(%v) resource", flundersList.Items)
   465  
   466  	// kubectl get flunders -v 9
   467  	// curl -k -v -XGET https://localhost/apis/wardle.example.com/v1alpha1/namespaces/default/flunders
   468  	contents, err = restClient.Get().AbsPath("/apis/"+apiServiceGroupName+"/"+apiServiceVersion+"/namespaces/default/flunders").SetHeader("Accept", "application/json").DoRaw(ctx)
   469  	framework.ExpectNoError(err, "confirming delete of a newly created flunders resource")
   470  	err = json.Unmarshal(contents, &flundersList)
   471  	validateErrorWithDebugInfo(ctx, f, err, pods, "Error in unmarshalling %T response from server %s", contents, "/apis/"+apiServiceGroupName+"/"+apiServiceVersion)
   472  	if len(flundersList.Items) != 0 {
   473  		framework.Failf("failed to get back the correct deleted flunders list %v", flundersList)
   474  	}
   475  
   476  	flunderName = generateFlunderName("dynamic-flunder")
   477  
   478  	// Rerun the Create/List/Delete tests using the Dynamic client.
   479  	resources, discoveryErr := client.Discovery().ServerPreferredNamespacedResources()
   480  	groupVersionResources, err := discovery.GroupVersionResources(resources)
   481  	framework.ExpectNoError(err, "getting group version resources for dynamic client")
   482  	gvr := schema.GroupVersionResource{Group: apiServiceGroupName, Version: apiServiceVersion, Resource: "flunders"}
   483  	_, ok := groupVersionResources[gvr]
   484  	if !ok {
   485  		framework.Failf("could not find group version resource for dynamic client and wardle/flunders (discovery error: %v, discovery results: %#v)", discoveryErr, groupVersionResources)
   486  	}
   487  	dynamicClient := f.DynamicClient.Resource(gvr).Namespace(n.namespace)
   488  
   489  	// kubectl create -f flunders-1.yaml
   490  	// Request Body: {"apiVersion":"wardle.example.com/v1alpha1","kind":"Flunder","metadata":{"labels":{"sample-label":"true"},"name":"test-flunder","namespace":"default"}}
   491  	testFlunder := samplev1alpha1.Flunder{
   492  		TypeMeta: metav1.TypeMeta{
   493  			Kind:       "Flunder",
   494  			APIVersion: apiServiceGroupName + "/" + apiServiceVersion,
   495  		},
   496  		ObjectMeta: metav1.ObjectMeta{Name: flunderName},
   497  		Spec:       samplev1alpha1.FlunderSpec{},
   498  	}
   499  	jsonFlunder, err := json.Marshal(testFlunder)
   500  	framework.ExpectNoError(err, "marshalling test-flunder for create using dynamic client")
   501  	unstruct := &unstructured.Unstructured{}
   502  	err = unstruct.UnmarshalJSON(jsonFlunder)
   503  	framework.ExpectNoError(err, "unmarshalling test-flunder as unstructured for create using dynamic client")
   504  	_, err = dynamicClient.Create(ctx, unstruct, metav1.CreateOptions{})
   505  	framework.ExpectNoError(err, "listing flunders using dynamic client")
   506  
   507  	// kubectl get flunders
   508  	unstructuredList, err := dynamicClient.List(ctx, metav1.ListOptions{})
   509  	framework.ExpectNoError(err, "listing flunders using dynamic client")
   510  	if len(unstructuredList.Items) != 1 {
   511  		framework.Failf("failed to get back the correct flunders list %v from the dynamic client", unstructuredList)
   512  	}
   513  
   514  	ginkgo.By("Read Status for " + apiServiceName)
   515  	statusContent, err := restClient.Get().
   516  		AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status").
   517  		SetHeader("Accept", "application/json").DoRaw(ctx)
   518  	framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err)
   519  
   520  	var jr *apiregistrationv1.APIService
   521  	err = json.Unmarshal([]byte(statusContent), &jr)
   522  	framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err)
   523  	gomega.Expect(jr.Status.Conditions[0].Message).To(gomega.Equal("all checks passed"), "The Message returned was %v", jr.Status.Conditions[0].Message)
   524  
   525  	ginkgo.By("kubectl patch apiservice " + apiServiceName + " -p '{\"spec\":{\"versionPriority\": 400}}'")
   526  	patchContent, err := restClient.Patch(types.MergePatchType).
   527  		AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName).
   528  		SetHeader("Accept", "application/json").
   529  		Body([]byte(`{"spec":{"versionPriority": 400}}`)).DoRaw(ctx)
   530  
   531  	framework.ExpectNoError(err, "Patch failed for .../apiservices/"+apiServiceName+". Error: %v", err)
   532  	err = json.Unmarshal([]byte(patchContent), &jr)
   533  	framework.ExpectNoError(err, "Failed to process patchContent: %v | err: %v ", string(patchContent), err)
   534  	gomega.Expect(jr.Spec.VersionPriority).To(gomega.Equal(int32(400)), "The VersionPriority returned was %d", jr.Spec.VersionPriority)
   535  
   536  	ginkgo.By("List APIServices")
   537  	listApiservices, err := restClient.Get().
   538  		AbsPath("/apis/apiregistration.k8s.io/v1/apiservices").
   539  		SetHeader("Accept", "application/json").DoRaw(ctx)
   540  
   541  	framework.ExpectNoError(err, "No response for /apis/apiregistration.k8s.io/v1/apiservices Error: %v", err)
   542  
   543  	var list *apiregistrationv1.APIServiceList
   544  	err = json.Unmarshal([]byte(listApiservices), &list)
   545  	framework.ExpectNoError(err, "Failed to process APIServiceList: %v | err: %v ", list, err)
   546  
   547  	locatedWardle := false
   548  	for _, item := range list.Items {
   549  		if item.Name == apiServiceName {
   550  			framework.Logf("Found " + apiServiceName + " in APIServiceList")
   551  			locatedWardle = true
   552  			break
   553  		}
   554  	}
   555  	if !locatedWardle {
   556  		framework.Failf("Unable to find " + apiServiceName + " in APIServiceList")
   557  	}
   558  
   559  	// As the APIService doesn't have any labels currently set we need to
   560  	// set one so that we can select it later when we call deleteCollection
   561  	ginkgo.By("Adding a label to the APIService")
   562  	apiServiceClient := aggrclient.ApiregistrationV1().APIServices()
   563  	apiServiceLabel := map[string]string{"e2e-apiservice": "patched"}
   564  	apiServicePatch, err := json.Marshal(map[string]interface{}{
   565  		"metadata": map[string]interface{}{
   566  			"labels": apiServiceLabel,
   567  		},
   568  	})
   569  	framework.ExpectNoError(err, "failed to Marshal APIService JSON patch")
   570  	_, err = apiServiceClient.Patch(ctx, apiServiceName, types.StrategicMergePatchType, []byte(apiServicePatch), metav1.PatchOptions{})
   571  	framework.ExpectNoError(err, "failed to patch APIService")
   572  
   573  	patchedApiService, err := apiServiceClient.Get(ctx, apiServiceName, metav1.GetOptions{})
   574  	framework.ExpectNoError(err, "Unable to retrieve api service %s", apiServiceName)
   575  	framework.Logf("APIService labels: %v", patchedApiService.Labels)
   576  
   577  	ginkgo.By("Updating APIService Status")
   578  	var updatedStatus, wardle *apiregistrationv1.APIService
   579  
   580  	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
   581  		var statusToUpdate *apiregistrationv1.APIService
   582  		statusContent, err = restClient.Get().
   583  			AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status").
   584  			SetHeader("Accept", "application/json").DoRaw(ctx)
   585  		framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err)
   586  
   587  		err = json.Unmarshal([]byte(statusContent), &statusToUpdate)
   588  		framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err)
   589  
   590  		statusToUpdate.Status.Conditions = append(statusToUpdate.Status.Conditions, apiregistrationv1.APIServiceCondition{
   591  			Type:    "StatusUpdated",
   592  			Status:  "True",
   593  			Reason:  "E2E",
   594  			Message: "Set from e2e test",
   595  		})
   596  
   597  		updatedStatus, err = apiServiceClient.UpdateStatus(ctx, statusToUpdate, metav1.UpdateOptions{})
   598  		return err
   599  	})
   600  	framework.ExpectNoError(err, "Failed to update status. %v", err)
   601  	framework.Logf("updatedStatus.Conditions: %#v", updatedStatus.Status.Conditions)
   602  
   603  	ginkgo.By("Confirm that " + apiServiceName + " /status was updated")
   604  	statusContent, err = restClient.Get().
   605  		AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status").
   606  		SetHeader("Accept", "application/json").DoRaw(ctx)
   607  	framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err)
   608  
   609  	err = json.Unmarshal([]byte(statusContent), &wardle)
   610  	framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err)
   611  
   612  	foundUpdatedStatusCondition := false
   613  	for _, cond := range wardle.Status.Conditions {
   614  		if cond.Type == "StatusUpdated" && cond.Reason == "E2E" && cond.Message == "Set from e2e test" {
   615  			framework.Logf("Found APIService %v with Labels: %v & Condition: %v", wardle.ObjectMeta.Name, wardle.Labels, cond)
   616  			foundUpdatedStatusCondition = true
   617  			break
   618  		} else {
   619  			framework.Logf("Observed APIService %v with Labels: %v & Condition: %v", wardle.ObjectMeta.Name, wardle.Labels, cond)
   620  		}
   621  	}
   622  	if !foundUpdatedStatusCondition {
   623  		framework.Failf("The updated status condition was not found in:\n%s", format.Object(wardle.Status.Conditions, 1))
   624  	}
   625  	framework.Logf("Found updated status condition for %s", wardle.ObjectMeta.Name)
   626  
   627  	ginkgo.By(fmt.Sprintf("Replace APIService %s", apiServiceName))
   628  	var updatedApiService *apiregistrationv1.APIService
   629  
   630  	err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
   631  		currentApiService, err := apiServiceClient.Get(ctx, apiServiceName, metav1.GetOptions{})
   632  		framework.ExpectNoError(err, "Unable to get APIService %s", apiServiceName)
   633  		currentApiService.Labels = map[string]string{
   634  			apiServiceName: "updated",
   635  		}
   636  		updatedApiService, err = apiServiceClient.Update(ctx, currentApiService, metav1.UpdateOptions{})
   637  		return err
   638  	})
   639  	framework.ExpectNoError(err)
   640  	gomega.Expect(updatedApiService.Labels).To(gomega.HaveKeyWithValue(apiServiceName, "updated"), "should have the updated label but have %q", updatedApiService.Labels[apiServiceName])
   641  	framework.Logf("Found updated apiService label for %q", apiServiceName)
   642  
   643  	// kubectl delete flunder test-flunder
   644  	ginkgo.By(fmt.Sprintf("Delete flunders resource %q", flunderName))
   645  	err = dynamicClient.Delete(ctx, flunderName, metav1.DeleteOptions{})
   646  	validateErrorWithDebugInfo(ctx, f, err, pods, "deleting flunders(%v) using dynamic client", unstructuredList.Items)
   647  
   648  	// kubectl get flunders
   649  	unstructuredList, err = dynamicClient.List(ctx, metav1.ListOptions{})
   650  	framework.ExpectNoError(err, "listing flunders using dynamic client")
   651  	if len(unstructuredList.Items) != 0 {
   652  		framework.Failf("failed to get back the correct deleted flunders list %v from the dynamic client", unstructuredList)
   653  	}
   654  
   655  	ginkgo.By("Recreating test-flunder before removing endpoint via deleteCollection")
   656  	jsonFlunder, err = json.Marshal(testFlunder)
   657  	framework.ExpectNoError(err, "marshalling test-flunder for create using dynamic client")
   658  	unstruct = &unstructured.Unstructured{}
   659  	err = unstruct.UnmarshalJSON(jsonFlunder)
   660  	framework.ExpectNoError(err, "unmarshalling test-flunder as unstructured for create using dynamic client")
   661  	_, err = dynamicClient.Create(ctx, unstruct, metav1.CreateOptions{})
   662  	framework.ExpectNoError(err, "listing flunders using dynamic client")
   663  
   664  	// kubectl get flunders
   665  	unstructuredList, err = dynamicClient.List(ctx, metav1.ListOptions{})
   666  	framework.ExpectNoError(err, "listing flunders using dynamic client")
   667  	if len(unstructuredList.Items) != 1 {
   668  		framework.Failf("failed to get back the correct flunders list %v from the dynamic client", unstructuredList)
   669  	}
   670  
   671  	ginkgo.By("Read " + apiServiceName + " /status before patching it")
   672  	statusContent, err = restClient.Get().
   673  		AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status").
   674  		SetHeader("Accept", "application/json").DoRaw(ctx)
   675  	framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err)
   676  
   677  	wardle.Reset()
   678  	err = json.Unmarshal([]byte(statusContent), &wardle)
   679  	framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err)
   680  
   681  	ginkgo.By("Patch APIService Status")
   682  	patch := map[string]interface{}{
   683  		"status": map[string]interface{}{
   684  			"conditions": append(wardle.Status.Conditions, apiregistrationv1.APIServiceCondition{
   685  				Type:    "StatusPatched",
   686  				Status:  "True",
   687  				Reason:  "E2E",
   688  				Message: "Set by e2e test",
   689  			}),
   690  		},
   691  	}
   692  	payload, err := json.Marshal(patch)
   693  	framework.ExpectNoError(err, "Failed to marshal JSON. %v", err)
   694  
   695  	_, err = restClient.Patch(types.MergePatchType).
   696  		AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status").
   697  		SetHeader("Accept", "application/json").
   698  		Body([]byte(payload)).
   699  		DoRaw(ctx)
   700  	framework.ExpectNoError(err, "Patch failed for .../apiservices/"+apiServiceName+"/status. Error: %v", err)
   701  
   702  	ginkgo.By("Confirm that " + apiServiceName + " /status was patched")
   703  	statusContent, err = restClient.Get().
   704  		AbsPath("/apis/apiregistration.k8s.io/v1/apiservices/"+apiServiceName+"/status").
   705  		SetHeader("Accept", "application/json").DoRaw(ctx)
   706  	framework.ExpectNoError(err, "No response for .../apiservices/"+apiServiceName+"/status. Error: %v", err)
   707  
   708  	wardle.Reset()
   709  	err = json.Unmarshal([]byte(statusContent), &wardle)
   710  	framework.ExpectNoError(err, "Failed to process statusContent: %v | err: %v ", string(statusContent), err)
   711  
   712  	foundPatchedStatusCondition := false
   713  	for _, cond := range wardle.Status.Conditions {
   714  		if cond.Type == "StatusPatched" && cond.Reason == "E2E" && cond.Message == "Set by e2e test" {
   715  			framework.Logf("Found APIService %v with Labels: %v & Conditions: %v", wardle.ObjectMeta.Name, wardle.Labels, cond)
   716  			foundPatchedStatusCondition = true
   717  			break
   718  		} else {
   719  			framework.Logf("Observed APIService %v with Labels: %v & Conditions: %v", wardle.ObjectMeta.Name, wardle.Labels, cond)
   720  		}
   721  	}
   722  	if !foundPatchedStatusCondition {
   723  		framework.Failf("The patched status condition was not found in:\n%s", format.Object(wardle.Status.Conditions, 1))
   724  	}
   725  	framework.Logf("Found patched status condition for %s", wardle.ObjectMeta.Name)
   726  
   727  	apiServiceLabelSelector := labels.SelectorFromSet(updatedApiService.Labels).String()
   728  	ginkgo.By(fmt.Sprintf("APIService deleteCollection with labelSelector: %q", apiServiceLabelSelector))
   729  
   730  	err = aggrclient.ApiregistrationV1().APIServices().DeleteCollection(ctx,
   731  		metav1.DeleteOptions{},
   732  		metav1.ListOptions{LabelSelector: apiServiceLabelSelector})
   733  	framework.ExpectNoError(err, "Unable to delete apiservice %s", apiServiceName)
   734  
   735  	ginkgo.By("Confirm that the generated APIService has been deleted")
   736  	err = wait.PollImmediate(apiServiceRetryPeriod, apiServiceRetryTimeout, checkApiServiceListQuantity(ctx, aggrclient, apiServiceLabelSelector, 0))
   737  	framework.ExpectNoError(err, "failed to count the required APIServices")
   738  	framework.Logf("APIService %s has been deleted.", apiServiceName)
   739  
   740  	cleanupSampleAPIServer(ctx, client, aggrclient, n, apiServiceName)
   741  }
   742  
   743  // pollTimed will call Poll but time how long Poll actually took.
   744  // It will then framework.Logf the msg with the duration of the Poll.
   745  // It is assumed that msg will contain one %s for the elapsed time.
   746  func pollTimed(ctx context.Context, interval, timeout time.Duration, condition wait.ConditionWithContextFunc, msg string) error {
   747  	defer func(start time.Time, msg string) {
   748  		elapsed := time.Since(start)
   749  		framework.Logf(msg, elapsed)
   750  	}(time.Now(), msg)
   751  	return wait.PollWithContext(ctx, interval, timeout, condition)
   752  }
   753  
   754  func validateErrorWithDebugInfo(ctx context.Context, f *framework.Framework, err error, pods *v1.PodList, msg string, fields ...interface{}) {
   755  	if err != nil {
   756  		namespace := f.Namespace.Name
   757  		msg := fmt.Sprintf(msg, fields...)
   758  		msg += fmt.Sprintf(" but received unexpected error:\n%v", err)
   759  		client := f.ClientSet
   760  		ep, err := client.CoreV1().Endpoints(namespace).Get(ctx, "sample-api", metav1.GetOptions{})
   761  		if err == nil {
   762  			msg += fmt.Sprintf("\nFound endpoints for sample-api:\n%v", ep)
   763  		}
   764  		pds, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{})
   765  		if err == nil {
   766  			msg += fmt.Sprintf("\nFound pods in %s:\n%v", namespace, pds)
   767  			msg += fmt.Sprintf("\nOriginal pods in %s:\n%v", namespace, pods)
   768  		}
   769  
   770  		framework.Failf(msg)
   771  	}
   772  }
   773  
   774  func generateFlunderName(base string) string {
   775  	id, err := rand.Int(rand.Reader, big.NewInt(2147483647))
   776  	if err != nil {
   777  		return base
   778  	}
   779  	return fmt.Sprintf("%s-%d", base, id)
   780  }
   781  
   782  func checkApiServiceListQuantity(ctx context.Context, aggrclient *aggregatorclient.Clientset, label string, quantity int) func() (bool, error) {
   783  	return func() (bool, error) {
   784  		var err error
   785  
   786  		framework.Logf("Requesting list of APIServices to confirm quantity")
   787  
   788  		list, err := aggrclient.ApiregistrationV1().APIServices().List(ctx, metav1.ListOptions{LabelSelector: label})
   789  		if err != nil {
   790  			return false, err
   791  		}
   792  
   793  		if len(list.Items) != quantity {
   794  			return false, err
   795  		}
   796  		framework.Logf("Found %d APIService with label %q", quantity, label)
   797  		return true, nil
   798  	}
   799  }
   800  

View as plain text