exec_test.go

Documentation: k8s.io/kubernetes/test/integration/client

     1  /*
     2  Copyright 2021 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package client
    18  
    19  import (
    20  	"context"
    21  	"crypto/tls"
    22  	"crypto/x509"
    23  	"encoding/base64"
    24  	"errors"
    25  	"fmt"
    26  	"net"
    27  	"net/http"
    28  	"os"
    29  	"reflect"
    30  	"strconv"
    31  	"strings"
    32  	"sync"
    33  	"testing"
    34  	"time"
    35  
    36  	"github.com/google/go-cmp/cmp"
    37  	corev1 "k8s.io/api/core/v1"
    38  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    39  	"k8s.io/apimachinery/pkg/util/dump"
    40  	"k8s.io/apimachinery/pkg/util/rand"
    41  	"k8s.io/apimachinery/pkg/util/sets"
    42  	"k8s.io/apimachinery/pkg/util/wait"
    43  	"k8s.io/client-go/informers"
    44  	clientset "k8s.io/client-go/kubernetes"
    45  	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
    46  	"k8s.io/client-go/plugin/pkg/client/auth/exec"
    47  	"k8s.io/client-go/rest"
    48  	"k8s.io/client-go/tools/cache"
    49  	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
    50  	"k8s.io/client-go/tools/metrics"
    51  	"k8s.io/client-go/transport"
    52  	"k8s.io/client-go/util/cert"
    53  	"k8s.io/client-go/util/connrotation"
    54  	kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
    55  	"k8s.io/kubernetes/test/integration/framework"
    56  )
    57  
    58  // This file tests the client-go credential plugin feature.
    59  
    60  // These constants are used to communicate behavior to the testdata/exec-plugin.sh test fixture.
    61  const (
    62  	exitCodeEnvVar   = "EXEC_PLUGIN_EXEC_CODE"
    63  	outputEnvVar     = "EXEC_PLUGIN_OUTPUT"
    64  	outputFileEnvVar = "EXEC_PLUGIN_OUTPUT_FILE"
    65  )
    66  
    67  type roundTripperFunc func(*http.Request) (*http.Response, error)
    68  
    69  func (f roundTripperFunc) RoundTrip(req *http.Request) (*http.Response, error) {
    70  	return f(req)
    71  }
    72  
    73  type syncedHeaderValues struct {
    74  	mu   sync.Mutex
    75  	data [][]string
    76  }
    77  
    78  func (s *syncedHeaderValues) append(values []string) {
    79  	s.mu.Lock()
    80  	defer s.mu.Unlock()
    81  	s.data = append(s.data, values)
    82  }
    83  
    84  func (s *syncedHeaderValues) get() [][]string {
    85  	s.mu.Lock()
    86  	defer s.mu.Unlock()
    87  	return s.data
    88  }
    89  
    90  type execPluginCall struct {
    91  	exitCode   int
    92  	callStatus string
    93  }
    94  
    95  type execPluginMetrics struct {
    96  	calls []execPluginCall
    97  }
    98  
    99  func (m *execPluginMetrics) Increment(exitCode int, callStatus string) {
   100  	m.calls = append(m.calls, execPluginCall{exitCode: exitCode, callStatus: callStatus})
   101  }
   102  
   103  var execPluginMetricsComparer = cmp.Comparer(func(a, b *execPluginMetrics) bool {
   104  	return reflect.DeepEqual(a, b)
   105  })
   106  
   107  type execPluginClientTestData struct {
   108  	name                          string
   109  	clientConfigFunc              func(*rest.Config)
   110  	wantAuthorizationHeaderValues [][]string
   111  	wantCertificate               *tls.Certificate
   112  	wantGetCertificateErrorPrefix string
   113  	wantClientErrorPrefix         string
   114  	wantMetrics                   *execPluginMetrics
   115  }
   116  
   117  func execPluginClientTests(t *testing.T, unauthorizedCert, unauthorizedKey []byte, clientAuthorizedToken, clientCertFileName, clientKeyFileName string) []execPluginClientTestData {
   118  	v1Tests := []execPluginClientTestData{
   119  		{
   120  			name: "unauthorized token",
   121  			clientConfigFunc: func(c *rest.Config) {
   122  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   123  					{
   124  						Name: outputEnvVar,
   125  						Value: `{
   126  										"kind": "ExecCredential",
   127  										"apiVersion": "client.authentication.k8s.io/v1",
   128  										"status": {
   129  											"token": "unauthorized"
   130  										}
   131  									}`,
   132  					},
   133  				}
   134  			},
   135  			wantAuthorizationHeaderValues: [][]string{{"Bearer unauthorized"}},
   136  			wantCertificate:               &tls.Certificate{},
   137  			wantClientErrorPrefix:         "Unauthorized",
   138  			wantMetrics: &execPluginMetrics{
   139  				calls: []execPluginCall{
   140  					// 2 calls since we preemptively refresh the creds upon a 401 HTTP response.
   141  					{exitCode: 0, callStatus: "no_error"},
   142  					{exitCode: 0, callStatus: "no_error"},
   143  				},
   144  			},
   145  		},
   146  		{
   147  			name: "unauthorized certificate",
   148  			clientConfigFunc: func(c *rest.Config) {
   149  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   150  					{
   151  						Name: outputEnvVar,
   152  						Value: fmt.Sprintf(`{
   153  							"kind": "ExecCredential",
   154  							"apiVersion": "client.authentication.k8s.io/v1",
   155  							"status": {
   156  								"clientCertificateData": %q,
   157  								"clientKeyData": %q
   158  							}
   159  						}`, unauthorizedCert, unauthorizedKey),
   160  					},
   161  				}
   162  			},
   163  			wantAuthorizationHeaderValues: [][]string{nil},
   164  			wantCertificate:               x509KeyPair(unauthorizedCert, unauthorizedKey, true),
   165  			wantClientErrorPrefix:         "Unauthorized",
   166  			wantMetrics: &execPluginMetrics{
   167  				calls: []execPluginCall{
   168  					// 2 calls since we preemptively refresh the creds upon a 401 HTTP response.
   169  					{exitCode: 0, callStatus: "no_error"},
   170  					{exitCode: 0, callStatus: "no_error"},
   171  				},
   172  			},
   173  		},
   174  		{
   175  			name: "authorized token",
   176  			clientConfigFunc: func(c *rest.Config) {
   177  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   178  					{
   179  						Name: outputEnvVar,
   180  						Value: fmt.Sprintf(`{
   181  						"kind": "ExecCredential",
   182  						"apiVersion": "client.authentication.k8s.io/v1",
   183  						"status": {
   184  							"token": "%s"
   185  						}
   186  					}`, clientAuthorizedToken),
   187  					},
   188  				}
   189  			},
   190  			wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientAuthorizedToken}},
   191  			wantCertificate:               &tls.Certificate{},
   192  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 0, callStatus: "no_error"}}},
   193  		},
   194  		{
   195  			name: "authorized certificate",
   196  			clientConfigFunc: func(c *rest.Config) {
   197  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   198  					{
   199  						Name: outputEnvVar,
   200  						Value: fmt.Sprintf(`{
   201  							"kind": "ExecCredential",
   202  							"apiVersion": "client.authentication.k8s.io/v1",
   203  							"status": {
   204  								"clientCertificateData": %s,
   205  								"clientKeyData": %s
   206  							}
   207  						}`, read(t, clientCertFileName), read(t, clientKeyFileName)),
   208  					},
   209  				}
   210  			},
   211  			wantAuthorizationHeaderValues: [][]string{nil},
   212  			wantCertificate:               loadX509KeyPair(clientCertFileName, clientKeyFileName),
   213  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 0, callStatus: "no_error"}}},
   214  		},
   215  		{
   216  			name: "authorized token and certificate",
   217  			clientConfigFunc: func(c *rest.Config) {
   218  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   219  					{
   220  						Name: outputEnvVar,
   221  						Value: fmt.Sprintf(`{
   222  							"kind": "ExecCredential",
   223  							"apiVersion": "client.authentication.k8s.io/v1",
   224  							"status": {
   225  								"token": "%s",
   226  								"clientCertificateData": %s,
   227  								"clientKeyData": %s
   228  							}
   229  						}`, clientAuthorizedToken, read(t, clientCertFileName), read(t, clientKeyFileName)),
   230  					},
   231  				}
   232  			},
   233  			wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientAuthorizedToken}},
   234  			wantCertificate:               loadX509KeyPair(clientCertFileName, clientKeyFileName),
   235  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 0, callStatus: "no_error"}}},
   236  		},
   237  		{
   238  			name: "unauthorized token and authorized certificate favors authorized certificate",
   239  			clientConfigFunc: func(c *rest.Config) {
   240  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   241  					{
   242  						Name: outputEnvVar,
   243  						Value: fmt.Sprintf(`{
   244  							"kind": "ExecCredential",
   245  							"apiVersion": "client.authentication.k8s.io/v1",
   246  							"status": {
   247  								"token": "%s",
   248  								"clientCertificateData": %s,
   249  								"clientKeyData": %s
   250  							}
   251  						}`, "client-unauthorized-token", read(t, clientCertFileName), read(t, clientKeyFileName)),
   252  					},
   253  				}
   254  			},
   255  			wantAuthorizationHeaderValues: [][]string{{"Bearer client-unauthorized-token"}},
   256  			wantCertificate:               loadX509KeyPair(clientCertFileName, clientKeyFileName),
   257  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 0, callStatus: "no_error"}}},
   258  		},
   259  		{
   260  			name: "authorized token and unauthorized certificate favors authorized token",
   261  			clientConfigFunc: func(c *rest.Config) {
   262  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   263  					{
   264  						Name: outputEnvVar,
   265  						Value: fmt.Sprintf(`{
   266  							"kind": "ExecCredential",
   267  							"apiVersion": "client.authentication.k8s.io/v1",
   268  							"status": {
   269  								"token": "%s",
   270  								"clientCertificateData": %q,
   271  								"clientKeyData": %q
   272  							}
   273  						}`, clientAuthorizedToken, string(unauthorizedCert), string(unauthorizedKey)),
   274  					},
   275  				}
   276  			},
   277  			wantAuthorizationHeaderValues: [][]string{{"Bearer " + clientAuthorizedToken}},
   278  			wantCertificate:               x509KeyPair([]byte(unauthorizedCert), []byte(unauthorizedKey), true),
   279  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 0, callStatus: "no_error"}}},
   280  		},
   281  		{
   282  			name: "unauthorized token and unauthorized certificate",
   283  			clientConfigFunc: func(c *rest.Config) {
   284  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   285  					{
   286  						Name: outputEnvVar,
   287  						Value: fmt.Sprintf(`{
   288  							"kind": "ExecCredential",
   289  							"apiVersion": "client.authentication.k8s.io/v1",
   290  							"status": {
   291  								"token": "%s",
   292  								"clientCertificateData": %q,
   293  								"clientKeyData": %q
   294  							}
   295  						}`, "client-unauthorized-token", string(unauthorizedCert), string(unauthorizedKey)),
   296  					},
   297  				}
   298  			},
   299  			wantAuthorizationHeaderValues: [][]string{{"Bearer client-unauthorized-token"}},
   300  			wantCertificate:               x509KeyPair(unauthorizedCert, unauthorizedKey, true),
   301  			wantClientErrorPrefix:         "Unauthorized",
   302  			wantMetrics: &execPluginMetrics{
   303  				calls: []execPluginCall{
   304  					// 2 calls since we preemptively refresh the creds upon a 401 HTTP response.
   305  					{exitCode: 0, callStatus: "no_error"},
   306  					{exitCode: 0, callStatus: "no_error"},
   307  				},
   308  			},
   309  		},
   310  		{
   311  			name: "good token with static auth basic creds favors static auth basic creds",
   312  			clientConfigFunc: func(c *rest.Config) {
   313  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   314  					{
   315  						Name: outputEnvVar,
   316  						Value: fmt.Sprintf(`{
   317  							"kind": "ExecCredential",
   318  							"apiVersion": "client.authentication.k8s.io/v1",
   319  							"status": {
   320  								"token": "%s"
   321  							}
   322  						}`, clientAuthorizedToken),
   323  					},
   324  				}
   325  				c.Username = "unauthorized"
   326  				c.Password = "unauthorized"
   327  			},
   328  			wantAuthorizationHeaderValues: [][]string{{"Basic " + basicAuthHeaderValue("unauthorized", "unauthorized")}},
   329  			wantClientErrorPrefix:         "Unauthorized",
   330  			wantMetrics:                   &execPluginMetrics{},
   331  		},
   332  		{
   333  			name: "good token with static auth bearer token favors static auth bearer token",
   334  			clientConfigFunc: func(c *rest.Config) {
   335  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   336  					{
   337  						Name: outputEnvVar,
   338  						Value: fmt.Sprintf(`{
   339  							"kind": "ExecCredential",
   340  							"apiVersion": "client.authentication.k8s.io/v1",
   341  							"status": {
   342  								"token": "%s"
   343  							}
   344  						}`, clientAuthorizedToken),
   345  					},
   346  				}
   347  				c.BearerToken = "some-unauthorized-token"
   348  			},
   349  			wantAuthorizationHeaderValues: [][]string{{"Bearer some-unauthorized-token"}},
   350  			wantClientErrorPrefix:         "Unauthorized",
   351  			wantMetrics:                   &execPluginMetrics{},
   352  		},
   353  		{
   354  			name: "good token with static auth cert and key favors static cert",
   355  			clientConfigFunc: func(c *rest.Config) {
   356  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   357  					{
   358  						Name: outputEnvVar,
   359  						Value: fmt.Sprintf(`{
   360  							"kind": "ExecCredential",
   361  							"apiVersion": "client.authentication.k8s.io/v1",
   362  							"status": {
   363  								"token": "%s"
   364  							}
   365  						}`, clientAuthorizedToken),
   366  					},
   367  				}
   368  				c.CertData = unauthorizedCert
   369  				c.KeyData = unauthorizedKey
   370  			},
   371  			wantAuthorizationHeaderValues: [][]string{nil},
   372  			wantClientErrorPrefix:         "Unauthorized",
   373  			wantCertificate:               x509KeyPair(unauthorizedCert, unauthorizedKey, false),
   374  			wantMetrics:                   &execPluginMetrics{},
   375  		},
   376  		{
   377  			name: "unknown binary",
   378  			clientConfigFunc: func(c *rest.Config) {
   379  				c.ExecProvider.Command = "does not exist"
   380  			},
   381  			wantGetCertificateErrorPrefix: "exec: executable does not exist not found",
   382  			wantClientErrorPrefix:         `Get "https`,
   383  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 1, callStatus: "plugin_not_found_error"}}},
   384  		},
   385  		{
   386  			name: "binary not executable",
   387  			clientConfigFunc: func(c *rest.Config) {
   388  				c.ExecProvider.Command = "./testdata/exec-plugin-not-executable.sh"
   389  			},
   390  			wantGetCertificateErrorPrefix: "exec: fork/exec ./testdata/exec-plugin-not-executable.sh: permission denied",
   391  			wantClientErrorPrefix:         `Get "https`,
   392  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 1, callStatus: "plugin_not_found_error"}}},
   393  		},
   394  		{
   395  			name: "binary fails",
   396  			clientConfigFunc: func(c *rest.Config) {
   397  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   398  					{
   399  						Name:  exitCodeEnvVar,
   400  						Value: "10",
   401  					},
   402  				}
   403  			},
   404  			wantGetCertificateErrorPrefix: "exec: executable testdata/exec-plugin.sh failed with exit code 10",
   405  			wantClientErrorPrefix:         `Get "https`,
   406  			wantMetrics:                   &execPluginMetrics{calls: []execPluginCall{{exitCode: 10, callStatus: "plugin_execution_error"}}},
   407  		},
   408  	}
   409  	return append(v1Tests, v1beta1TestsFromV1Tests(v1Tests)...)
   410  }
   411  
   412  func v1beta1TestsFromV1Tests(v1Tests []execPluginClientTestData) []execPluginClientTestData {
   413  	v1beta1Tests := make([]execPluginClientTestData, 0, len(v1Tests))
   414  	for _, v1Test := range v1Tests {
   415  		v1Test := v1Test
   416  
   417  		v1beta1Test := v1Test
   418  		v1beta1Test.name = fmt.Sprintf("%s v1beta1", v1Test.name)
   419  		v1beta1Test.clientConfigFunc = func(c *rest.Config) {
   420  			v1Test.clientConfigFunc(c)
   421  			c.ExecProvider.APIVersion = "client.authentication.k8s.io/v1beta1"
   422  			for j, oldOutputEnvVar := range c.ExecProvider.Env {
   423  				if oldOutputEnvVar.Name == outputEnvVar {
   424  					c.ExecProvider.Env[j].Value = strings.Replace(oldOutputEnvVar.Value, "client.authentication.k8s.io/v1", "client.authentication.k8s.io/v1beta1", 1)
   425  					break
   426  				}
   427  			}
   428  		}
   429  
   430  		v1beta1Tests = append(v1beta1Tests, v1beta1Test)
   431  	}
   432  	return v1beta1Tests
   433  }
   434  
   435  func TestExecPluginViaClient(t *testing.T) {
   436  	result, clientAuthorizedToken, clientCertFileName, clientKeyFileName := startTestServer(t)
   437  
   438  	unauthorizedCert, unauthorizedKey, err := cert.GenerateSelfSignedCertKey("some-host", nil, nil)
   439  	if err != nil {
   440  		t.Fatal(err)
   441  	}
   442  
   443  	tests := execPluginClientTests(t, unauthorizedCert, unauthorizedKey, clientAuthorizedToken, clientCertFileName, clientKeyFileName)
   444  
   445  	for _, test := range tests {
   446  		test := test
   447  		t.Run(test.name, func(t *testing.T) {
   448  			actualMetrics := captureMetrics(t)
   449  
   450  			var authorizationHeaderValues syncedHeaderValues
   451  			clientConfig := rest.AnonymousClientConfig(result.ClientConfig)
   452  			clientConfig.ExecProvider = &clientcmdapi.ExecConfig{
   453  				Command:    "testdata/exec-plugin.sh",
   454  				APIVersion: "client.authentication.k8s.io/v1",
   455  				Args: []string{
   456  					// If we didn't have this arg, then some metrics assertions might fail because
   457  					// the authenticator may be pulled from a globalCache and therefore it may have
   458  					// already fetched a valid credential.
   459  					"--random-arg-to-avoid-authenticator-cache-hits",
   460  					rand.String(10),
   461  				},
   462  				InteractiveMode: clientcmdapi.IfAvailableExecInteractiveMode,
   463  			}
   464  			clientConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
   465  				return roundTripperFunc(func(req *http.Request) (*http.Response, error) {
   466  					authorizationHeaderValues.append(req.Header.Values("Authorization"))
   467  					return rt.RoundTrip(req)
   468  				})
   469  			})
   470  
   471  			if test.clientConfigFunc != nil {
   472  				test.clientConfigFunc(clientConfig)
   473  			}
   474  			client := clientset.NewForConfigOrDie(clientConfig)
   475  
   476  			ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
   477  			defer cancel()
   478  
   479  			// Validate that the client works as expected on its own.
   480  			_, err = client.CoreV1().ConfigMaps("default").List(ctx, metav1.ListOptions{})
   481  			if test.wantClientErrorPrefix != "" {
   482  				if err == nil || !strings.HasPrefix(err.Error(), test.wantClientErrorPrefix) {
   483  					t.Fatalf(`got %v, wanted "%s..."`, err, test.wantClientErrorPrefix)
   484  				}
   485  			} else if err != nil {
   486  				t.Fatal(err)
   487  			}
   488  
   489  			// Validate that the proper metrics were set.
   490  			if diff := cmp.Diff(test.wantMetrics, actualMetrics, execPluginMetricsComparer); diff != "" {
   491  				t.Error("unexpected metrics; -want, +got:\n" + diff)
   492  			}
   493  
   494  			// Validate that the right token is used.
   495  			if diff := cmp.Diff(test.wantAuthorizationHeaderValues, authorizationHeaderValues.get()); diff != "" {
   496  				t.Error("unexpected authorization header values; -want, +got:\n" + diff)
   497  			}
   498  
   499  			// Validate that the right certs are used.
   500  			tlsConfig, err := rest.TLSConfigFor(clientConfig)
   501  			if err != nil {
   502  				t.Fatal(err)
   503  			}
   504  			if tlsConfig.GetClientCertificate == nil {
   505  				if test.wantCertificate != nil {
   506  					t.Error("GetClientCertificate is nil, but we expected a certificate")
   507  				}
   508  			} else {
   509  				cert, err := tlsConfig.GetClientCertificate(&tls.CertificateRequestInfo{})
   510  				if len(test.wantGetCertificateErrorPrefix) != 0 {
   511  					if err == nil || !strings.HasPrefix(err.Error(), test.wantGetCertificateErrorPrefix) {
   512  						t.Fatalf(`got %q, wanted "%s..."`, err, test.wantGetCertificateErrorPrefix)
   513  					}
   514  				} else if err != nil {
   515  					t.Fatal(err)
   516  				}
   517  				if diff := cmp.Diff(test.wantCertificate, cert); diff != "" {
   518  					t.Error("unexpected certificate; -want, +got:\n" + diff)
   519  				}
   520  			}
   521  		})
   522  	}
   523  }
   524  
   525  func captureMetrics(t *testing.T) *execPluginMetrics {
   526  	previousCallsMetric := metrics.ExecPluginCalls
   527  	t.Cleanup(func() {
   528  		metrics.ExecPluginCalls = previousCallsMetric
   529  	})
   530  
   531  	actualMetrics := &execPluginMetrics{}
   532  	metrics.ExecPluginCalls = actualMetrics
   533  	return actualMetrics
   534  }
   535  
   536  // objectMetaSansResourceVersionComparer compares two metav1.ObjectMeta's except for their resource
   537  // versions. Since the underlying integration test etcd is shared, these resource versions may jump
   538  // past the next sequential number for sequential API calls in the test.
   539  var objectMetaSansResourceVersionComparer = cmp.Comparer(func(a, b metav1.ObjectMeta) bool {
   540  	aa := a.DeepCopy()
   541  	bb := b.DeepCopy()
   542  
   543  	aa.ResourceVersion = ""
   544  	bb.ResourceVersion = ""
   545  
   546  	return cmp.Equal(aa, bb)
   547  })
   548  
   549  type oldNew struct {
   550  	old, new interface{}
   551  }
   552  
   553  var oldNewComparer = cmp.Comparer(func(a, b oldNew) bool {
   554  	return cmp.Equal(a.old, b.old, objectMetaSansResourceVersionComparer) &&
   555  		cmp.Equal(a.new, a.new, objectMetaSansResourceVersionComparer)
   556  })
   557  
   558  type informerSpy struct {
   559  	mu      sync.Mutex
   560  	adds    []interface{}
   561  	updates []oldNew
   562  	deletes []interface{}
   563  }
   564  
   565  func (is *informerSpy) OnAdd(obj interface{}, isInInitialList bool) {
   566  	is.mu.Lock()
   567  	defer is.mu.Unlock()
   568  	is.adds = append(is.adds, obj)
   569  }
   570  
   571  func (is *informerSpy) OnUpdate(old, new interface{}) {
   572  	is.mu.Lock()
   573  	defer is.mu.Unlock()
   574  	is.updates = append(is.updates, oldNew{old: old, new: new})
   575  }
   576  
   577  func (is *informerSpy) OnDelete(obj interface{}) {
   578  	is.mu.Lock()
   579  	defer is.mu.Unlock()
   580  	is.deletes = append(is.deletes, obj)
   581  }
   582  
   583  func (is *informerSpy) clear() {
   584  	is.mu.Lock()
   585  	defer is.mu.Unlock()
   586  	is.adds = []interface{}{}
   587  	is.updates = []oldNew{}
   588  	is.deletes = []interface{}{}
   589  }
   590  
   591  // waitForEvents waits for adds, updates, and deletes to be populated with at least one event.
   592  func (is *informerSpy) waitForEvents(t *testing.T, wantEvents bool) {
   593  	t.Helper()
   594  	// wait for create/update/delete 3 events for 30 seconds
   595  	waitTimeout := time.Second * 30
   596  	if !wantEvents {
   597  		// wait just 15 seconds for no events
   598  		waitTimeout = time.Second * 15
   599  	}
   600  
   601  	err := wait.PollImmediate(time.Second, waitTimeout, func() (bool, error) {
   602  		is.mu.Lock()
   603  		defer is.mu.Unlock()
   604  		return len(is.adds) > 0 && len(is.updates) > 0 && len(is.deletes) > 0, nil
   605  	})
   606  	if wantEvents {
   607  		if err != nil {
   608  			t.Fatalf("wanted events, but got error: %v", err)
   609  		}
   610  	} else {
   611  		if !errors.Is(err, wait.ErrWaitTimeout) {
   612  			if err != nil {
   613  				t.Fatalf("wanted no events, but got error: %v", err)
   614  			} else {
   615  				t.Fatalf("wanted no events, but got some: %s", dump.Pretty(is))
   616  			}
   617  		}
   618  	}
   619  }
   620  
   621  func TestExecPluginViaInformer(t *testing.T) {
   622  	result, clientAuthorizedToken, clientCertFileName, clientKeyFileName := startTestServer(t)
   623  
   624  	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
   625  	defer cancel()
   626  
   627  	adminClient := clientset.NewForConfigOrDie(result.ClientConfig)
   628  	ns := createNamespace(ctx, t, adminClient)
   629  
   630  	tests := []struct {
   631  		name                          string
   632  		clientConfigFunc              func(*rest.Config)
   633  		wantAuthorizationHeaderValues [][]string
   634  		wantCertificate               *tls.Certificate
   635  	}{
   636  		{
   637  			name: "authorized token",
   638  			clientConfigFunc: func(c *rest.Config) {
   639  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   640  					{
   641  						Name: outputEnvVar,
   642  						Value: fmt.Sprintf(`{
   643  							"kind": "ExecCredential",
   644  							"apiVersion": "client.authentication.k8s.io/v1",
   645  							"status": {
   646  								"token": %q
   647  							}
   648  						}`, clientAuthorizedToken),
   649  					},
   650  				}
   651  			},
   652  		},
   653  		{
   654  			name: "authorized certificate",
   655  			clientConfigFunc: func(c *rest.Config) {
   656  				c.ExecProvider.Env = []clientcmdapi.ExecEnvVar{
   657  					{
   658  						Name: outputEnvVar,
   659  						Value: fmt.Sprintf(`{
   660  							"kind": "ExecCredential",
   661  							"apiVersion": "client.authentication.k8s.io/v1",
   662  							"status": {
   663  								"clientCertificateData": %s,
   664  								"clientKeyData": %s
   665  							}
   666  						}`, read(t, clientCertFileName), read(t, clientKeyFileName)),
   667  					},
   668  				}
   669  			},
   670  		},
   671  	}
   672  	for _, test := range tests {
   673  		t.Run(test.name, func(t *testing.T) {
   674  			clientConfig := rest.AnonymousClientConfig(result.ClientConfig)
   675  			clientConfig.ExecProvider = &clientcmdapi.ExecConfig{
   676  				Command:         "testdata/exec-plugin.sh",
   677  				APIVersion:      "client.authentication.k8s.io/v1",
   678  				InteractiveMode: clientcmdapi.IfAvailableExecInteractiveMode,
   679  			}
   680  
   681  			if test.clientConfigFunc != nil {
   682  				test.clientConfigFunc(clientConfig)
   683  			}
   684  
   685  			informer, informerSpy := startConfigMapInformer(ctx, t, clientset.NewForConfigOrDie(clientConfig), ns.Name)
   686  			waitForInformerSync(ctx, t, informer, true, "")
   687  			createdCM, updatedCM, deletedCM := createUpdateDeleteConfigMap(ctx, t, adminClient.CoreV1().ConfigMaps(ns.Name))
   688  			informerSpy.waitForEvents(t, true)
   689  			assertInformerEvents(t, informerSpy, createdCM, updatedCM, deletedCM)
   690  		})
   691  	}
   692  }
   693  
   694  type execPlugin struct {
   695  	t          *testing.T
   696  	outputFile *os.File
   697  }
   698  
   699  func newExecPlugin(t *testing.T) *execPlugin {
   700  	t.Helper()
   701  	outputFile, err := os.CreateTemp("", "kubernetes-client-exec-test-plugin-output-file-*")
   702  	if err != nil {
   703  		t.Fatal(err)
   704  	}
   705  	return &execPlugin{t: t, outputFile: outputFile}
   706  }
   707  
   708  func (e *execPlugin) config() *clientcmdapi.ExecConfig {
   709  	return &clientcmdapi.ExecConfig{
   710  		Command:         "testdata/exec-plugin.sh",
   711  		APIVersion:      "client.authentication.k8s.io/v1",
   712  		InteractiveMode: clientcmdapi.IfAvailableExecInteractiveMode,
   713  		Env: []clientcmdapi.ExecEnvVar{
   714  			{
   715  				Name:  outputFileEnvVar,
   716  				Value: e.outputFile.Name(),
   717  			},
   718  		},
   719  	}
   720  }
   721  
   722  func (e *execPlugin) rotateToken(newToken string, lifetime time.Duration) {
   723  	e.t.Helper()
   724  
   725  	expirationTimestamp := metav1.NewTime(time.Now().Add(lifetime)).Format(time.RFC3339Nano)
   726  	newOutput := fmt.Sprintf(`{
   727  		"kind": "ExecCredential",
   728  		"apiVersion": "client.authentication.k8s.io/v1",
   729  		"status": {
   730  			"expirationTimestamp": %q,
   731  			"token": %q
   732  		}
   733  	}`, expirationTimestamp, newToken)
   734  	if err := os.WriteFile(e.outputFile.Name(), []byte(newOutput), 0644); err != nil {
   735  		e.t.Fatal(err)
   736  	}
   737  }
   738  
   739  func TestExecPluginRotationViaInformer(t *testing.T) {
   740  	t.Parallel()
   741  
   742  	result, clientAuthorizedToken, _, _ := startTestServer(t)
   743  	const clientUnauthorizedToken = "invalid-token"
   744  	const tokenLifetime = time.Second * 5
   745  
   746  	ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5)
   747  	defer cancel()
   748  
   749  	adminClient := clientset.NewForConfigOrDie(result.ClientConfig)
   750  	ns := createNamespace(ctx, t, adminClient)
   751  
   752  	clientDialer := connrotation.NewDialer((&net.Dialer{
   753  		Timeout:   30 * time.Second,
   754  		KeepAlive: 30 * time.Second,
   755  	}).DialContext)
   756  
   757  	execPlugin := newExecPlugin(t)
   758  
   759  	clientConfig := rest.AnonymousClientConfig(result.ClientConfig)
   760  	clientConfig.ExecProvider = execPlugin.config()
   761  	clientConfig.Dial = clientDialer.DialContext
   762  	clientConfig.Wrap(func(rt http.RoundTripper) http.RoundTripper {
   763  		// This makes it helpful to see what is happening with the informer's client.
   764  		return transport.NewDebuggingRoundTripper(rt, transport.DebugCurlCommand, transport.DebugURLTiming)
   765  	})
   766  
   767  	// Initialize informer spy wth invalid token.
   768  	// Make sure informer never syncs because it can't authenticate.
   769  	execPlugin.rotateToken(clientUnauthorizedToken, tokenLifetime)
   770  	informer, informerSpy := startConfigMapInformer(ctx, t, clientset.NewForConfigOrDie(clientConfig), ns.Name)
   771  	waitForInformerSync(ctx, t, informer, false, "")
   772  	createUpdateDeleteConfigMap(ctx, t, adminClient.CoreV1().ConfigMaps(ns.Name))
   773  	informerSpy.waitForEvents(t, false)
   774  
   775  	// Rotate token to valid token.
   776  	// Make sure informer sees events because it now has a valid token with which it can authenticate.
   777  	execPlugin.rotateToken(clientAuthorizedToken, tokenLifetime)
   778  	waitForInformerSync(ctx, t, informer, true, "")
   779  	informerSpy.clear()
   780  	createdCM, updatedCM, deletedCM := createUpdateDeleteConfigMap(ctx, t, adminClient.CoreV1().ConfigMaps(ns.Name))
   781  	informerSpy.waitForEvents(t, true)
   782  	assertInformerEvents(t, informerSpy, createdCM, updatedCM, deletedCM)
   783  
   784  	// Rotate token to something invalid and clip watch connection.
   785  	// Informer should recreate connection with invalid token.
   786  	// Make sure informer does not see events since it is using the invalid token.
   787  	execPlugin.rotateToken(clientUnauthorizedToken, tokenLifetime)
   788  	time.Sleep(tokenLifetime) // wait for old token to expire to make sure the watch is restarted with clientUnauthorizedToken
   789  	clientDialer.CloseAll()
   790  	waitForInformerSync(ctx, t, informer, true, "")
   791  	informerSpy.clear()
   792  	createUpdateDeleteConfigMap(ctx, t, adminClient.CoreV1().ConfigMaps(ns.Name))
   793  	informerSpy.waitForEvents(t, false)
   794  
   795  	// Rotate token to valid token.
   796  	// Make sure informer sees events because it now has a valid token with which it can authenticate.
   797  	lastSyncResourceVersion := informer.LastSyncResourceVersion()
   798  	execPlugin.rotateToken(clientAuthorizedToken, tokenLifetime)
   799  	waitForInformerSync(ctx, t, informer, true, lastSyncResourceVersion)
   800  	informerSpy.clear()
   801  	createdCM, updatedCM, deletedCM = createUpdateDeleteConfigMap(ctx, t, adminClient.CoreV1().ConfigMaps(ns.Name))
   802  	informerSpy.waitForEvents(t, true)
   803  	assertInformerEvents(t, informerSpy, createdCM, updatedCM, deletedCM)
   804  }
   805  
   806  func startTestServer(t *testing.T) (result *kubeapiservertesting.TestServer, clientAuthorizedToken string, clientCertFileName string, clientKeyFileName string) {
   807  	certDir, err := os.MkdirTemp("", "kubernetes-client-exec-test-cert-dir-*")
   808  	if err != nil {
   809  		t.Fatal(err)
   810  	}
   811  	t.Cleanup(func() {
   812  		if err := os.RemoveAll(certDir); err != nil {
   813  			t.Error(err)
   814  		}
   815  	})
   816  
   817  	clientAuthorizedToken = "client-authorized-token"
   818  	tokenFileName := writeTokenFile(t, clientAuthorizedToken)
   819  	clientCAFileName, clientSigningCert, clientSigningKey := writeCACertFiles(t, certDir)
   820  	clientCertFileName, clientKeyFileName = writeCerts(t, clientSigningCert, clientSigningKey, certDir, time.Hour)
   821  	result = kubeapiservertesting.StartTestServerOrDie(
   822  		t,
   823  		nil,
   824  		[]string{
   825  			"--token-auth-file", tokenFileName,
   826  			"--client-ca-file=" + clientCAFileName,
   827  		},
   828  		framework.SharedEtcd(),
   829  	)
   830  	t.Cleanup(result.TearDownFn)
   831  
   832  	return
   833  }
   834  
   835  func writeTokenFile(t *testing.T, goodToken string) string {
   836  	t.Helper()
   837  
   838  	tokenFile, err := os.CreateTemp("", "kubernetes-client-exec-test-token-file-*")
   839  	if err != nil {
   840  		t.Fatal(err)
   841  	}
   842  
   843  	if _, err := tokenFile.WriteString(fmt.Sprintf(`%s,admin,uid1,"system:masters"`, goodToken)); err != nil {
   844  		t.Fatal(err)
   845  	}
   846  
   847  	if err := tokenFile.Close(); err != nil {
   848  		t.Fatal(err)
   849  	}
   850  
   851  	return tokenFile.Name()
   852  }
   853  
   854  func read(t *testing.T, fileName string) string {
   855  	t.Helper()
   856  	data, err := os.ReadFile(fileName)
   857  	if err != nil {
   858  		t.Fatal(err)
   859  	}
   860  	return fmt.Sprintf("%q", string(data))
   861  }
   862  
   863  func basicAuthHeaderValue(username, password string) string {
   864  	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", username, password)))
   865  }
   866  
   867  func x509KeyPair(certPEMBlock, keyPEMBlock []byte, leaf bool) *tls.Certificate {
   868  	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
   869  	if err != nil {
   870  		panic(err)
   871  	}
   872  	if leaf {
   873  		cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
   874  		if err != nil {
   875  			panic(err)
   876  		}
   877  	}
   878  	return &cert
   879  }
   880  
   881  func loadX509KeyPair(certFile, keyFile string) *tls.Certificate {
   882  	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
   883  	if err != nil {
   884  		panic(err)
   885  	}
   886  	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
   887  	if err != nil {
   888  		panic(err)
   889  	}
   890  	return &cert
   891  }
   892  
   893  func createNamespace(ctx context.Context, t *testing.T, client clientset.Interface) *corev1.Namespace {
   894  	t.Helper()
   895  
   896  	ns, err := client.CoreV1().Namespaces().Create(
   897  		ctx,
   898  		&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test-exec-plugin-with-informer-ns"}},
   899  		metav1.CreateOptions{},
   900  	)
   901  	if err != nil {
   902  		t.Fatal(err)
   903  	}
   904  	t.Cleanup(func() {
   905  		// Use a new context since the one passed to this function would have timed out.
   906  		ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
   907  		defer cancel()
   908  		if err := client.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}); err != nil {
   909  			t.Error(err)
   910  		}
   911  	})
   912  
   913  	return ns
   914  }
   915  
   916  func startConfigMapInformer(ctx context.Context, t *testing.T, client clientset.Interface, namespace string) (cache.SharedIndexInformer, *informerSpy) {
   917  	t.Helper()
   918  
   919  	var informerSpy informerSpy
   920  	informerFactory := informers.NewSharedInformerFactoryWithOptions(client, 0, informers.WithNamespace(namespace))
   921  	cmInformer := informerFactory.Core().V1().ConfigMaps().Informer()
   922  	cmInformer.AddEventHandler(&informerSpy)
   923  	if err := cmInformer.SetWatchErrorHandler(func(r *cache.Reflector, err error) {
   924  		// t.Logf("watch error handler: failure in reflector %#v: %v", r, err) // Uncomment for more verbose logging
   925  	}); err != nil {
   926  		t.Fatalf("could not set watch error handler: %v", err)
   927  	}
   928  	informerFactory.Start(ctx.Done())
   929  
   930  	return cmInformer, &informerSpy
   931  }
   932  
   933  func waitForInformerSync(ctx context.Context, t *testing.T, informer cache.SharedIndexInformer, wantSynced bool, lastSyncResourceVersion string) {
   934  	t.Helper()
   935  
   936  	syncCtx, cancel := context.WithTimeout(ctx, time.Second*60)
   937  	defer cancel()
   938  	if gotSynced := cache.WaitForCacheSync(syncCtx.Done(), informer.HasSynced); wantSynced != gotSynced {
   939  		t.Fatalf("wanted sync %t, got sync %t", wantSynced, gotSynced)
   940  	}
   941  
   942  	if len(lastSyncResourceVersion) != 0 {
   943  		if err := wait.PollImmediate(time.Second, time.Second*60, func() (bool, error) {
   944  			return informer.LastSyncResourceVersion() != lastSyncResourceVersion, nil
   945  		}); err != nil {
   946  			t.Fatalf("informer never changed resource versions from %q: %v", lastSyncResourceVersion, err)
   947  		}
   948  	}
   949  }
   950  
   951  func createUpdateDeleteConfigMap(ctx context.Context, t *testing.T, cms v1.ConfigMapInterface) (created, updated, deleted *corev1.ConfigMap) {
   952  	t.Helper()
   953  
   954  	var err error
   955  	created, err = cms.Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "cm"}}, metav1.CreateOptions{})
   956  	if err != nil {
   957  		t.Fatal("could not create ConfigMap:", err)
   958  	}
   959  
   960  	updated = created.DeepCopy()
   961  	updated.Annotations = map[string]string{"tuna": "fish"}
   962  	updated, err = cms.Update(ctx, updated, metav1.UpdateOptions{})
   963  	if err != nil {
   964  		t.Fatal("could not update ConfigMap:", err)
   965  	}
   966  
   967  	if err := cms.Delete(ctx, updated.Name, metav1.DeleteOptions{}); err != nil {
   968  		t.Fatal("could not delete ConfigMap:", err)
   969  	}
   970  
   971  	deleted = updated.DeepCopy()
   972  
   973  	return created, updated, deleted
   974  }
   975  
   976  func assertInformerEvents(t *testing.T, informerSpy *informerSpy, created, updated, deleted interface{}) {
   977  	t.Helper()
   978  
   979  	// Validate that the informer was called correctly.
   980  	if diff := cmp.Diff([]interface{}{created}, informerSpy.adds, objectMetaSansResourceVersionComparer); diff != "" {
   981  		t.Errorf("unexpected add event(s), -want, +got:\n%s", diff)
   982  	}
   983  	if diff := cmp.Diff([]oldNew{{created, updated}}, informerSpy.updates, oldNewComparer); diff != "" {
   984  		t.Errorf("unexpected update event(s), -want, +got:\n%s", diff)
   985  	}
   986  	if diff := cmp.Diff([]interface{}{deleted}, informerSpy.deletes, objectMetaSansResourceVersionComparer); diff != "" {
   987  		t.Errorf("unexpected deleted event(s), -want, +got:\n%s", diff)
   988  	}
   989  
   990  }
   991  
   992  func TestExecPluginGlobalCache(t *testing.T) {
   993  	// we do not really need the server for this test but this allows us to easily share the test data
   994  	result, clientAuthorizedToken, clientCertFileName, clientKeyFileName := startTestServer(t)
   995  
   996  	unauthorizedCert, unauthorizedKey, err := cert.GenerateSelfSignedCertKey("some-host", nil, nil)
   997  	if err != nil {
   998  		t.Fatal(err)
   999  	}
  1000  
  1001  	testsFirstRun := execPluginClientTests(t, unauthorizedCert, unauthorizedKey, clientAuthorizedToken, clientCertFileName, clientKeyFileName)
  1002  	testsSecondRun := execPluginClientTests(t, unauthorizedCert, unauthorizedKey, clientAuthorizedToken, clientCertFileName, clientKeyFileName)
  1003  
  1004  	randStrings := make([]string, 0, len(testsFirstRun))
  1005  	for range testsFirstRun {
  1006  		randStrings = append(randStrings, rand.String(10))
  1007  	}
  1008  
  1009  	getTestExecClientAddresses := func(t *testing.T, tests []execPluginClientTestData, suffix string) []string {
  1010  		var addresses []string
  1011  		for i, test := range tests {
  1012  			test := test
  1013  			t.Run(test.name+" "+suffix, func(t *testing.T) {
  1014  				clientConfig := rest.AnonymousClientConfig(result.ClientConfig)
  1015  				clientConfig.ExecProvider = &clientcmdapi.ExecConfig{
  1016  					Command:    "testdata/exec-plugin.sh",
  1017  					APIVersion: "client.authentication.k8s.io/v1",
  1018  					Args: []string{
  1019  						// carefully control what the global cache sees as the same exec plugin
  1020  						"--random-arg-to-avoid-authenticator-cache-hits",
  1021  						randStrings[i],
  1022  					},
  1023  				}
  1024  
  1025  				if test.clientConfigFunc != nil {
  1026  					test.clientConfigFunc(clientConfig)
  1027  				}
  1028  
  1029  				addresses = append(addresses, execPluginMemoryAddress(t, clientConfig, i))
  1030  			})
  1031  		}
  1032  		return addresses
  1033  	}
  1034  
  1035  	addressesFirstRun := getTestExecClientAddresses(t, testsFirstRun, "first")
  1036  	addressesSecondRun := getTestExecClientAddresses(t, testsSecondRun, "second")
  1037  
  1038  	if diff := cmp.Diff(addressesFirstRun, addressesSecondRun); diff != "" {
  1039  		t.Error("unexpected addresses; -want, +got:\n" + diff)
  1040  	}
  1041  
  1042  	if want, got := len(testsFirstRun), len(addressesFirstRun); want != got {
  1043  		t.Errorf("expected %d addresses but got %d", want, got)
  1044  	}
  1045  
  1046  	if want, got := len(addressesFirstRun), sets.NewString(addressesFirstRun...).Len(); want != got {
  1047  		t.Errorf("expected %d distinct authenticators but got %d", want, got)
  1048  	}
  1049  }
  1050  
  1051  func execPluginMemoryAddress(t *testing.T, config *rest.Config, i int) string {
  1052  	t.Helper()
  1053  
  1054  	wantType := reflect.TypeOf(&exec.Authenticator{})
  1055  
  1056  	tc, err := config.TransportConfig()
  1057  	if err != nil {
  1058  		t.Fatal(err)
  1059  	}
  1060  
  1061  	if tc.WrapTransport == nil {
  1062  		return "<nil> " + strconv.Itoa(i)
  1063  	}
  1064  
  1065  	rt := tc.WrapTransport(nil)
  1066  
  1067  	val := reflect.Indirect(reflect.ValueOf(rt))
  1068  	for i := 0; i < val.NumField(); i++ {
  1069  		field := val.Field(i)
  1070  		if field.Type() == wantType {
  1071  			return strconv.FormatUint(uint64(field.Pointer()), 10)
  1072  		}
  1073  	}
  1074  
  1075  	t.Fatal("unable to find authenticator in rest config")
  1076  	return ""
  1077  }
  1078
View as plain text