secrets.go

Documentation: github.com/emissary-ingress/emissary/v3/cmd/entrypoint

     1  package entrypoint
     2  
     3  import (
     4  	"context"
     5  	"crypto/x509"
     6  	"encoding/json"
     7  	"encoding/pem"
     8  	"fmt"
     9  	"os"
    10  	"strconv"
    11  	"strings"
    12  
    13  	v1 "k8s.io/api/core/v1"
    14  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    15  
    16  	"github.com/datawire/dlib/derror"
    17  	"github.com/datawire/dlib/dlog"
    18  	amb "github.com/emissary-ingress/emissary/v3/pkg/api/getambassador.io/v3alpha1"
    19  	"github.com/emissary-ingress/emissary/v3/pkg/kates"
    20  	"github.com/emissary-ingress/emissary/v3/pkg/snapshot/v1"
    21  	snapshotTypes "github.com/emissary-ingress/emissary/v3/pkg/snapshot/v1"
    22  )
    23  
    24  // checkSecret checks whether a secret is valid, and adds it to the list of secrets
    25  // in this snapshot if so.
    26  func checkSecret(
    27  	ctx context.Context,
    28  	sh *SnapshotHolder,
    29  	what string,
    30  	ref snapshotTypes.SecretRef,
    31  	secret *v1.Secret,
    32  ) {
    33  	forceSecretValidation, _ := strconv.ParseBool(os.Getenv("AMBASSADOR_FORCE_SECRET_VALIDATION"))
    34  	// Make it more convenient to consistently refer to this secret.
    35  	secretName := fmt.Sprintf("%s secret %s.%s", what, ref.Name, ref.Namespace)
    36  
    37  	if secret == nil {
    38  		// This is "impossible". Arguably it should be a panic...
    39  		dlog.Debugf(ctx, "%s not found", secretName)
    40  		return
    41  	}
    42  
    43  	// Assume that the secret is valid...
    44  	isValid := true
    45  
    46  	// ...and that we have no errors.
    47  	var errs derror.MultiError
    48  
    49  	// OK, do we have a TLS private key?
    50  	privKeyPEMBytes, ok := secret.Data[v1.TLSPrivateKeyKey]
    51  
    52  	if ok && len(privKeyPEMBytes) > 0 {
    53  		// Yes. We need to be able to decode it.
    54  		caKeyBlock, _ := pem.Decode(privKeyPEMBytes)
    55  
    56  		if caKeyBlock != nil {
    57  			dlog.Debugf(ctx, "%s has private key, block type %s", secretName, caKeyBlock.Type)
    58  
    59  			// First try PKCS1.
    60  			_, err := x509.ParsePKCS1PrivateKey(caKeyBlock.Bytes)
    61  
    62  			if err != nil {
    63  				// Try PKCS8? (No, = instead of := is not a typo here: we're overwriting the
    64  				// earlier error.)
    65  				_, err = x509.ParsePKCS8PrivateKey(caKeyBlock.Bytes)
    66  			}
    67  
    68  			if err != nil {
    69  				// Try EC? (No, = instead of := is not a typo here: we're overwriting the
    70  				// earlier error.)
    71  				_, err = x509.ParseECPrivateKey(caKeyBlock.Bytes)
    72  			}
    73  
    74  			// Any issues here?
    75  			if err != nil {
    76  				errs = append(errs,
    77  					fmt.Errorf("%s %s cannot be parsed as PKCS1, PKCS8, or EC: %s", secretName, v1.TLSPrivateKeyKey, err.Error()))
    78  				isValid = false
    79  			}
    80  		} else {
    81  			errs = append(errs,
    82  				fmt.Errorf("%s %s is not a PEM-encoded key", secretName, v1.TLSPrivateKeyKey))
    83  			isValid = false
    84  		}
    85  	}
    86  
    87  	// How about a TLS cert bundle?
    88  	caCertPEMBytes, ok := secret.Data[v1.TLSCertKey]
    89  
    90  	if ok && len(caCertPEMBytes) > 0 {
    91  		caCertBlock, _ := pem.Decode(caCertPEMBytes)
    92  
    93  		if caCertBlock != nil {
    94  			dlog.Debugf(ctx, "%s has public key, block type %s", secretName, caCertBlock.Type)
    95  
    96  			_, err := x509.ParseCertificate(caCertBlock.Bytes)
    97  
    98  			if err != nil {
    99  				errs = append(errs,
   100  					fmt.Errorf("%s %s cannot be parsed as x.509: %s", secretName, v1.TLSCertKey, err.Error()))
   101  				isValid = false
   102  			}
   103  		} else {
   104  			errs = append(errs,
   105  				fmt.Errorf("%s %s is not a PEM-encoded certificate", secretName, v1.TLSCertKey))
   106  			isValid = false
   107  		}
   108  	}
   109  
   110  	if isValid || !forceSecretValidation {
   111  		dlog.Debugf(ctx, "taking %s", secretName)
   112  		sh.k8sSnapshot.Secrets = append(sh.k8sSnapshot.Secrets, secret)
   113  	}
   114  	if !isValid {
   115  		// This secret is invalid, but we're not going to log about it -- instead, it'll go into the
   116  		// list of Invalid resources.
   117  		dlog.Debugf(ctx, "%s is not valid, skipping: %s", secretName, errs.Error())
   118  
   119  		// We need to add this to our set of invalid resources. Sadly, this means we need to convert it
   120  		// to an Unstructured and redact various bits.
   121  		secretBytes, err := json.Marshal(secret)
   122  
   123  		if err != nil {
   124  			// This we'll log about, since it's impossible.
   125  			dlog.Errorf(ctx, "unable to marshal invalid %s: %s", secretName, err)
   126  			return
   127  		}
   128  
   129  		var unstructuredSecret kates.Unstructured
   130  		err = json.Unmarshal(secretBytes, &unstructuredSecret)
   131  
   132  		if err != nil {
   133  			// This we'll log about, since it's impossible.
   134  			dlog.Errorf(ctx, "unable to unmarshal invalid %s: %s", secretName, err)
   135  			return
   136  		}
   137  
   138  		// Construct a redacted version of things in the original data map.
   139  		redactedData := map[string]interface{}{}
   140  
   141  		for key := range secret.Data {
   142  			redactedData[key] = "-redacted-"
   143  		}
   144  
   145  		unstructuredSecret.Object["data"] = redactedData
   146  
   147  		// We have to toss the last-applied-configuration as well... and we may as well toss the
   148  		// managedFields.
   149  
   150  		metadata, ok := unstructuredSecret.Object["metadata"].(map[string]interface{})
   151  
   152  		if ok {
   153  			delete(metadata, "managedFields")
   154  
   155  			annotations, ok := metadata["annotations"].(map[string]interface{})
   156  
   157  			if ok {
   158  				delete(annotations, "kubectl.kubernetes.io/last-applied-configuration")
   159  
   160  				if len(annotations) == 0 {
   161  					delete(metadata, "annotations")
   162  				}
   163  			}
   164  
   165  			if len(metadata) == 0 {
   166  				delete(unstructuredSecret.Object, "metadata")
   167  			}
   168  		}
   169  
   170  		// Finally, mark it invalid.
   171  		sh.validator.addInvalid(ctx, &unstructuredSecret, errs.Error())
   172  	}
   173  }
   174  
   175  // ReconcileSecrets figures out which secrets we're actually using,
   176  // since we don't want to send secrets to Ambassador unless we're
   177  // using them, since any secret we send will be saved to disk.
   178  func ReconcileSecrets(ctx context.Context, sh *SnapshotHolder) error {
   179  	envAmbID := GetAmbassadorID()
   180  
   181  	// Start by building up a list of all the K8s objects that are
   182  	// allowed to mention secrets. Note that we vet the ambassador_id
   183  	// for all of these before putting them on the list.
   184  	var resources []kates.Object
   185  
   186  	// Annotations are straightforward, although honestly we should
   187  	// be filtering annotations by type here (or, even better, unfold
   188  	// them earlier so that we can treat them like any other resource
   189  	// here).
   190  
   191  	for _, list := range sh.k8sSnapshot.Annotations {
   192  		for _, a := range list {
   193  			if _, isInvalid := a.(*kates.Unstructured); isInvalid {
   194  				continue
   195  			}
   196  			if GetAmbID(ctx, a).Matches(envAmbID) {
   197  				resources = append(resources, a)
   198  			}
   199  		}
   200  	}
   201  
   202  	// Hosts are a little weird, because we have two ways to find the
   203  	// ambassador_id. Sorry about that.
   204  	for _, h := range sh.k8sSnapshot.Hosts {
   205  		var id amb.AmbassadorID
   206  		if len(h.Spec.AmbassadorID) > 0 {
   207  			id = h.Spec.AmbassadorID
   208  		}
   209  		if id.Matches(envAmbID) {
   210  			resources = append(resources, h)
   211  		}
   212  	}
   213  
   214  	// TLSContexts, Modules, and Ingresses are all straightforward.
   215  	for _, t := range sh.k8sSnapshot.TLSContexts {
   216  		if t.Spec.AmbassadorID.Matches(envAmbID) {
   217  			resources = append(resources, t)
   218  		}
   219  	}
   220  	for _, m := range sh.k8sSnapshot.Modules {
   221  		if m.Spec.AmbassadorID.Matches(envAmbID) {
   222  			resources = append(resources, m)
   223  		}
   224  	}
   225  	for _, i := range sh.k8sSnapshot.Ingresses {
   226  		resources = append(resources, i)
   227  	}
   228  
   229  	// OK. Once that's done, we can check to see if we should be
   230  	// doing secret namespacing or not -- this requires a look into
   231  	// the Ambassador Module, if it's present.
   232  	//
   233  	// XXX Linear searches suck, but whatever, it's just not gonna
   234  	// be all that many things. We won't bother optimizing this unless
   235  	// a profiler shows that it's a problem.
   236  
   237  	secretNamespacing := true
   238  	for _, resource := range resources {
   239  		mod, ok := resource.(*amb.Module)
   240  		// We don't need to recheck ambassador_id on this Module because
   241  		// the Module can't have made it into the resources list without
   242  		// its ambassador_id being checked.
   243  
   244  		if ok && mod.GetName() == "ambassador" {
   245  			// XXX ModuleSecrets is a _godawful_ hack. See the comment on
   246  			// ModuleSecrets itself for more.
   247  			secs := ModuleSecrets{}
   248  			err := convert(mod.Spec.Config, &secs)
   249  			if err != nil {
   250  				dlog.Errorf(ctx, "error parsing module: %v", err)
   251  				continue
   252  			}
   253  			secretNamespacing = secs.Defaults.TLSSecretNamespacing
   254  			break
   255  		}
   256  	}
   257  
   258  	// Once we have our list of secrets, go figure out the names of all
   259  	// the secrets we need. We'll use this "refs" map to hold all the names...
   260  	refs := map[snapshotTypes.SecretRef]bool{}
   261  
   262  	// ...and, uh, this "action" function is really just a closure to avoid
   263  	// needing to pass "refs" to find SecretRefs. Shrug. Arguably more
   264  	// complex than needed, but meh.
   265  	action := func(ref snapshotTypes.SecretRef) {
   266  		refs[ref] = true
   267  	}
   268  
   269  	// So. Walk the list of resources...
   270  	for _, resource := range resources {
   271  		// ...and for each resource, dig out any secrets being referenced.
   272  		findSecretRefs(ctx, resource, secretNamespacing, action)
   273  	}
   274  
   275  	// We _always_ have an implicit references to the cloud-connec-token secret...
   276  	secretRef(GetCloudConnectTokenResourceNamespace(), GetCloudConnectTokenResourceName(), false, action)
   277  
   278  	// We _always_ have an implicit references to the fallback cert secret...
   279  	secretRef(GetAmbassadorNamespace(), "fallback-self-signed-cert", false, action)
   280  
   281  	isEdgeStack, err := IsEdgeStack()
   282  	if err != nil {
   283  		return err
   284  	}
   285  	if isEdgeStack {
   286  		// ...and for Edge Stack, we _always_ have an implicit reference to the
   287  		// license secret.
   288  		secretRef(GetLicenseSecretNamespace(), GetLicenseSecretName(), false, action)
   289  		// We also want to grab any secrets referenced by Edge-Stack filters for use in Edge-Stack
   290  		// the Filters are unstructured because Emissary does not have their type definition
   291  		for _, f := range sh.k8sSnapshot.Filters {
   292  			err := findFilterSecret(f, action)
   293  			if err != nil {
   294  				dlog.Errorf(ctx, "Error gathering secret reference from Filter: %v", err)
   295  			}
   296  		}
   297  	}
   298  
   299  	// OK! After all that, go copy all the matching secrets from FSSecrets and
   300  	// K8sSecrets to Secrets.
   301  	//
   302  	// The way this works is kind of simple: first we check everything in
   303  	// FSSecrets. Then, when we check K8sSecrets, we skip any secrets that are
   304  	// also in FSSecrets. End result: FSSecrets wins if there are any conflicts.
   305  	sh.k8sSnapshot.Secrets = make([]*kates.Secret, 0, len(refs))
   306  
   307  	for ref, secret := range sh.k8sSnapshot.FSSecrets {
   308  		if refs[ref] {
   309  			checkSecret(ctx, sh, "FSSecret", ref, secret)
   310  		}
   311  	}
   312  
   313  	for _, secret := range sh.k8sSnapshot.K8sSecrets {
   314  		ref := snapshotTypes.SecretRef{Namespace: secret.GetNamespace(), Name: secret.GetName()}
   315  
   316  		_, found := sh.k8sSnapshot.FSSecrets[ref]
   317  		if found {
   318  			dlog.Debugf(ctx, "Conflict! skipping K8sSecret %#v", ref)
   319  			continue
   320  		}
   321  
   322  		if refs[ref] {
   323  			checkSecret(ctx, sh, "K8sSecret", ref, secret)
   324  		}
   325  	}
   326  	return nil
   327  }
   328  
   329  // Returns secretName, secretNamespace from a provided (unstructured) filter if it contains a secret
   330  // Returns empty strings when the secret name and/or namespace could not be found
   331  func findFilterSecret(filter *unstructured.Unstructured, action func(snapshotTypes.SecretRef)) error {
   332  	// Just making extra sure this is actually a Filter
   333  	if filter.GetKind() != "Filter" {
   334  		return fmt.Errorf("non-Filter object in Snapshot.Filters: %s", filter.GetKind())
   335  	}
   336  	// Only OAuth2 Filters have secrets, although they don't need to have them.
   337  	// This is overly contrived because Filters are unstructured to Emissary since we don't have the type definitions
   338  	// Yes this is disgusting. It is what it is...
   339  	filterContents := filter.UnstructuredContent()
   340  	filterSpec := filterContents["spec"]
   341  	if filterSpec != nil {
   342  		mapFilters, ok := filterSpec.(map[string]interface{})
   343  		// We need to check if all these type assertions fail since we shouldnt rely on CRD validation to protect us from a panic state
   344  		// I cant imagine a scenario where this would realisticly happen, but we generate a unique log message for tracability and skip processing it
   345  		if !ok {
   346  			// We bail early any time we detect bogus contents for any of these fields
   347  			// and let the APIServer, apiext, and amb-sidecar handle the error reporting
   348  			return nil
   349  		}
   350  
   351  		findOAuthFilterSecret(mapFilters, filter.GetNamespace(), action)
   352  		findAPIKeyFilterSecret(mapFilters, filter.GetNamespace(), action)
   353  	}
   354  	return nil
   355  }
   356  
   357  func findOAuthFilterSecret(
   358  	mapFilters map[string]interface{},
   359  	filterNamespace string,
   360  	action func(snapshotTypes.SecretRef),
   361  ) {
   362  	oAuthFilter := mapFilters["OAuth2"]
   363  	if oAuthFilter == nil {
   364  		return
   365  	}
   366  
   367  	secretName, secretNamespace := "", ""
   368  	// Check if we have a secretName
   369  	mapOAuth, ok := oAuthFilter.(map[string]interface{})
   370  	if !ok {
   371  		return
   372  	}
   373  	sName := mapOAuth["secretName"]
   374  	if sName == nil {
   375  		return
   376  	}
   377  	secretName, ok = sName.(string)
   378  	// This is a weird check, but we have to handle the case where secretName is not provided, and when its explicitly set to ""
   379  	if !ok || secretName == "" {
   380  		// Bail out early since there is no secret
   381  		return
   382  	}
   383  	sNamespace := mapOAuth["secretNamespace"]
   384  	if sNamespace == nil {
   385  		secretNamespace = filterNamespace
   386  	} else {
   387  		secretNamespace, ok = sNamespace.(string)
   388  		if !ok {
   389  			return
   390  		} else if secretNamespace == "" {
   391  			secretNamespace = filterNamespace
   392  		}
   393  	}
   394  	secretRef(secretNamespace, secretName, false, action)
   395  
   396  }
   397  
   398  func findAPIKeyFilterSecret(
   399  	mapFilters map[string]interface{},
   400  	filterNamespace string,
   401  	action func(snapshotTypes.SecretRef),
   402  ) {
   403  	apiKeyFilter := mapFilters["APIKey"]
   404  	if apiKeyFilter != nil {
   405  		mapKeyFilter, ok := apiKeyFilter.(map[string]interface{})
   406  
   407  		if !ok {
   408  			return
   409  		}
   410  
   411  		apiKeys := mapKeyFilter["keys"].([]interface{})
   412  
   413  		for i := range apiKeys {
   414  			secretName := ""
   415  			mapKey, ok := apiKeys[i].(map[string]interface{})
   416  
   417  			if !ok {
   418  				continue
   419  			}
   420  
   421  			sName := mapKey["secretName"]
   422  			if sName == nil {
   423  				continue
   424  			}
   425  
   426  			secretName, ok = sName.(string)
   427  			// This is a weird check, but we have to handle the case where secretName is not provided, and when its explicitly set to ""
   428  			if !ok || secretName == "" {
   429  				// Continue with the next key since there is no secret
   430  				continue
   431  			}
   432  
   433  			secretRef(filterNamespace, secretName, false, action)
   434  		}
   435  	}
   436  }
   437  
   438  // Find all the secrets a given Ambassador resource references.
   439  func findSecretRefs(ctx context.Context, resource kates.Object, secretNamespacing bool, action func(snapshotTypes.SecretRef)) {
   440  	switch r := resource.(type) {
   441  	case *amb.Host:
   442  		// The Host resource is a little odd. Host.spec.tls, Host.spec.tlsSecret, and
   443  		// host.spec.acmeProvider.privateKeySecret can all refer to secrets.
   444  		if r.Spec == nil {
   445  			return
   446  		}
   447  
   448  		if r.Spec.TLS != nil {
   449  			// Host.spec.tls.caSecret is the thing to worry about here.
   450  			secretRef(r.GetNamespace(), r.Spec.TLS.CASecret, secretNamespacing, action)
   451  
   452  			if r.Spec.TLS.CRLSecret != "" {
   453  				secretRef(r.GetNamespace(), r.Spec.TLS.CRLSecret, secretNamespacing, action)
   454  			}
   455  		}
   456  
   457  		// Host.spec.tlsSecret and Host.spec.acmeProvider.privateKeySecret are native-Kubernetes-style
   458  		// `core.v1.LocalObjectReference`s, not Ambassador-style `{name}.{namespace}` strings.  If we
   459  		// ever decide that they should support cross-namespace references, we would do it by adding a
   460  		// `namespace:` field (i.e. changing them to `core.v1.SecretReference`s) rather than by
   461  		// adopting the `{name}.{namespace}` notation.
   462  		if r.Spec.TLSSecret != nil && r.Spec.TLSSecret.Name != "" {
   463  			if r.Spec.TLSSecret.Namespace != "" {
   464  				secretRef(r.Spec.TLSSecret.Namespace, r.Spec.TLSSecret.Name, false, action)
   465  			} else {
   466  				secretRef(r.GetNamespace(), r.Spec.TLSSecret.Name, false, action)
   467  			}
   468  		}
   469  
   470  		if r.Spec.AcmeProvider != nil && r.Spec.AcmeProvider.PrivateKeySecret != nil &&
   471  			r.Spec.AcmeProvider.PrivateKeySecret.Name != "" {
   472  			secretRef(r.GetNamespace(), r.Spec.AcmeProvider.PrivateKeySecret.Name, false, action)
   473  		}
   474  
   475  	case *amb.TLSContext:
   476  		// TLSContext.spec.secret and TLSContext.spec.ca_secret are the things to worry about --
   477  		// but note well that TLSContexts can override the global secretNamespacing setting.
   478  		if r.Spec.Secret != "" {
   479  			if r.Spec.SecretNamespacing != nil {
   480  				secretNamespacing = *r.Spec.SecretNamespacing
   481  			}
   482  			secretRef(r.GetNamespace(), r.Spec.Secret, secretNamespacing, action)
   483  		}
   484  
   485  		if r.Spec.CASecret != "" {
   486  			if r.Spec.SecretNamespacing != nil {
   487  				secretNamespacing = *r.Spec.SecretNamespacing
   488  			}
   489  			secretRef(r.GetNamespace(), r.Spec.CASecret, secretNamespacing, action)
   490  		}
   491  
   492  		if r.Spec.CRLSecret != "" {
   493  			if r.Spec.SecretNamespacing != nil {
   494  				secretNamespacing = *r.Spec.SecretNamespacing
   495  			}
   496  			secretRef(r.GetNamespace(), r.Spec.CRLSecret, secretNamespacing, action)
   497  		}
   498  
   499  	case *amb.Module:
   500  		// This whole thing is a hack. We probably _should_ check to make sure that
   501  		// this is an Ambassador Module or a TLS Module, but, well, those're the only
   502  		// supported kinds now, anyway...
   503  		//
   504  		// XXX ModuleSecrets is a godawful hack. See its comment for more.
   505  		secs := ModuleSecrets{}
   506  		err := convert(r.Spec.Config, &secs)
   507  		if err != nil {
   508  			// XXX
   509  			dlog.Errorf(ctx, "error extracting secrets from module: %v", err)
   510  			return
   511  		}
   512  
   513  		// XXX Technically, this is wrong -- _any_ element named in the module can
   514  		// refer to a secret. Hmmm.
   515  		if secs.Upstream.Secret != "" {
   516  			secretRef(r.GetNamespace(), secs.Upstream.Secret, secretNamespacing, action)
   517  		}
   518  		if secs.Server.Secret != "" {
   519  			secretRef(r.GetNamespace(), secs.Server.Secret, secretNamespacing, action)
   520  		}
   521  		if secs.Client.Secret != "" {
   522  			secretRef(r.GetNamespace(), secs.Client.Secret, secretNamespacing, action)
   523  		}
   524  
   525  	case *snapshot.Ingress:
   526  		// Ingress is pretty straightforward, too, just look in spec.tls.
   527  		for _, itls := range r.Spec.TLS {
   528  			if itls.SecretName != "" {
   529  				secretRef(r.GetNamespace(), itls.SecretName, secretNamespacing, action)
   530  			}
   531  		}
   532  	}
   533  }
   534  
   535  // Mark a secret as one we reference, handling secretNamespacing correctly.
   536  func secretRef(namespace, name string, secretNamespacing bool, action func(snapshotTypes.SecretRef)) {
   537  	if secretNamespacing {
   538  		parts := strings.Split(name, ".")
   539  		if len(parts) > 1 {
   540  			namespace = parts[len(parts)-1]
   541  			name = strings.Join(parts[:len(parts)-1], ".")
   542  		}
   543  	}
   544  
   545  	action(snapshotTypes.SecretRef{Namespace: namespace, Name: name})
   546  }
   547  
   548  // ModuleSecrets is... a hack. It's sort of a mashup of the chunk of the Ambassador
   549  // Module and the chunk of the TLS Module that are common, because they're able to
   550  // specify secrets. However... first, I don't think the TLS Module actually supported
   551  // tls_secret_namespacing. Second, the Ambassador Module at least supports arbitrary
   552  // origination context names -- _any_ key in the TLS dictionary will get turned into
   553  // an origination context.
   554  //
   555  // I seriously doubt that either of these will actually affect anyone at this remove,
   556  // but... yeah.
   557  type ModuleSecrets struct {
   558  	Defaults struct {
   559  		TLSSecretNamespacing bool `json:"tls_secret_namespacing"`
   560  	} `json:"defaults"`
   561  	Upstream struct {
   562  		Secret string `json:"secret"`
   563  	} `json:"upstream"`
   564  	Server struct {
   565  		Secret string `json:"secret"`
   566  	} `json:"server"`
   567  	Client struct {
   568  		Secret string `json:"secret"`
   569  	} `json:"client"`
   570  }
   571
View as plain text