secrets.go

Documentation: github.com/datawire/ambassador/v2/cmd/entrypoint

     1  package entrypoint
     2  
     3  import (
     4  	"context"
     5  	"crypto/x509"
     6  	"encoding/json"
     7  	"encoding/pem"
     8  	"fmt"
     9  	"os"
    10  	"strconv"
    11  	"strings"
    12  
    13  	v1 "k8s.io/api/core/v1"
    14  	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
    15  
    16  	amb "github.com/datawire/ambassador/v2/pkg/api/getambassador.io/v3alpha1"
    17  	"github.com/datawire/ambassador/v2/pkg/kates"
    18  	"github.com/datawire/ambassador/v2/pkg/kates/k8s_resource_types"
    19  	snapshotTypes "github.com/datawire/ambassador/v2/pkg/snapshot/v1"
    20  	"github.com/datawire/dlib/derror"
    21  	"github.com/datawire/dlib/dlog"
    22  )
    23  
    24  // checkSecret checks whether a secret is valid, and adds it to the list of secrets
    25  // in this snapshot if so.
    26  func checkSecret(
    27  	ctx context.Context,
    28  	sh *SnapshotHolder,
    29  	what string,
    30  	ref snapshotTypes.SecretRef,
    31  	secret *v1.Secret) {
    32  	forceSecretValidation, _ := strconv.ParseBool(os.Getenv("AMBASSADOR_FORCE_SECRET_VALIDATION"))
    33  	// Make it more convenient to consistently refer to this secret.
    34  	secretName := fmt.Sprintf("%s secret %s.%s", what, ref.Name, ref.Namespace)
    35  
    36  	if secret == nil {
    37  		// This is "impossible". Arguably it should be a panic...
    38  		dlog.Debugf(ctx, "%s not found", secretName)
    39  		return
    40  	}
    41  
    42  	// Assume that the secret is valid...
    43  	isValid := true
    44  
    45  	// ...and that we have no errors.
    46  	var errs derror.MultiError
    47  
    48  	// OK, do we have a TLS private key?
    49  	privKeyPEMBytes, ok := secret.Data[v1.TLSPrivateKeyKey]
    50  
    51  	if ok && len(privKeyPEMBytes) > 0 {
    52  		// Yes. We need to be able to decode it.
    53  		caKeyBlock, _ := pem.Decode(privKeyPEMBytes)
    54  
    55  		if caKeyBlock != nil {
    56  			dlog.Debugf(ctx, "%s has private key, block type %s", secretName, caKeyBlock.Type)
    57  
    58  			// First try PKCS1.
    59  			_, err := x509.ParsePKCS1PrivateKey(caKeyBlock.Bytes)
    60  
    61  			if err != nil {
    62  				// Try PKCS8? (No, = instead of := is not a typo here: we're overwriting the
    63  				// earlier error.)
    64  				_, err = x509.ParsePKCS8PrivateKey(caKeyBlock.Bytes)
    65  			}
    66  
    67  			if err != nil {
    68  				// Try EC? (No, = instead of := is not a typo here: we're overwriting the
    69  				// earlier error.)
    70  				_, err = x509.ParseECPrivateKey(caKeyBlock.Bytes)
    71  			}
    72  
    73  			// Any issues here?
    74  			if err != nil {
    75  				errs = append(errs,
    76  					fmt.Errorf("%s %s cannot be parsed as PKCS1, PKCS8, or EC: %s", secretName, v1.TLSPrivateKeyKey, err.Error()))
    77  				isValid = false
    78  			}
    79  		} else {
    80  			errs = append(errs,
    81  				fmt.Errorf("%s %s is not a PEM-encoded key", secretName, v1.TLSPrivateKeyKey))
    82  			isValid = false
    83  		}
    84  	}
    85  
    86  	// How about a TLS cert bundle?
    87  	caCertPEMBytes, ok := secret.Data[v1.TLSCertKey]
    88  
    89  	if ok && len(caCertPEMBytes) > 0 {
    90  		caCertBlock, _ := pem.Decode(caCertPEMBytes)
    91  
    92  		if caCertBlock != nil {
    93  			dlog.Debugf(ctx, "%s has public key, block type %s", secretName, caCertBlock.Type)
    94  
    95  			_, err := x509.ParseCertificate(caCertBlock.Bytes)
    96  
    97  			if err != nil {
    98  				errs = append(errs,
    99  					fmt.Errorf("%s %s cannot be parsed as x.509: %s", secretName, v1.TLSCertKey, err.Error()))
   100  				isValid = false
   101  			}
   102  		} else {
   103  			errs = append(errs,
   104  				fmt.Errorf("%s %s is not a PEM-encoded certificate", secretName, v1.TLSCertKey))
   105  			isValid = false
   106  		}
   107  	}
   108  
   109  	if isValid || !forceSecretValidation {
   110  		dlog.Debugf(ctx, "taking %s", secretName)
   111  		sh.k8sSnapshot.Secrets = append(sh.k8sSnapshot.Secrets, secret)
   112  	}
   113  	if !isValid {
   114  		// This secret is invalid, but we're not going to log about it -- instead, it'll go into the
   115  		// list of Invalid resources.
   116  		dlog.Debugf(ctx, "%s is not valid, skipping: %s", secretName, errs.Error())
   117  
   118  		// We need to add this to our set of invalid resources. Sadly, this means we need to convert it
   119  		// to an Unstructured and redact various bits.
   120  		secretBytes, err := json.Marshal(secret)
   121  
   122  		if err != nil {
   123  			// This we'll log about, since it's impossible.
   124  			dlog.Errorf(ctx, "unable to marshal invalid %s: %s", secretName, err)
   125  			return
   126  		}
   127  
   128  		var unstructuredSecret kates.Unstructured
   129  		err = json.Unmarshal(secretBytes, &unstructuredSecret)
   130  
   131  		if err != nil {
   132  			// This we'll log about, since it's impossible.
   133  			dlog.Errorf(ctx, "unable to unmarshal invalid %s: %s", secretName, err)
   134  			return
   135  		}
   136  
   137  		// Construct a redacted version of things in the original data map.
   138  		redactedData := map[string]interface{}{}
   139  
   140  		for key := range secret.Data {
   141  			redactedData[key] = "-redacted-"
   142  		}
   143  
   144  		unstructuredSecret.Object["data"] = redactedData
   145  
   146  		// We have to toss the last-applied-configuration as well... and we may as well toss the
   147  		// managedFields.
   148  
   149  		metadata, ok := unstructuredSecret.Object["metadata"].(map[string]interface{})
   150  
   151  		if ok {
   152  			delete(metadata, "managedFields")
   153  
   154  			annotations, ok := metadata["annotations"].(map[string]interface{})
   155  
   156  			if ok {
   157  				delete(annotations, "kubectl.kubernetes.io/last-applied-configuration")
   158  
   159  				if len(annotations) == 0 {
   160  					delete(metadata, "annotations")
   161  				}
   162  			}
   163  
   164  			if len(metadata) == 0 {
   165  				delete(unstructuredSecret.Object, "metadata")
   166  			}
   167  		}
   168  
   169  		// Finally, mark it invalid.
   170  		sh.validator.addInvalid(ctx, &unstructuredSecret, errs.Error())
   171  	}
   172  }
   173  
   174  // ReconcileSecrets figures out which secrets we're actually using,
   175  // since we don't want to send secrets to Ambassador unless we're
   176  // using them, since any secret we send will be saved to disk.
   177  func ReconcileSecrets(ctx context.Context, sh *SnapshotHolder) error {
   178  	// Start by building up a list of all the K8s objects that are
   179  	// allowed to mention secrets. Note that we vet the ambassador_id
   180  	// for all of these before putting them on the list.
   181  	var resources []kates.Object
   182  
   183  	// Annotations are straightforward, although honestly we should
   184  	// be filtering annotations by type here (or, even better, unfold
   185  	// them earlier so that we can treat them like any other resource
   186  	// here).
   187  
   188  	for _, list := range sh.k8sSnapshot.Annotations {
   189  		for _, a := range list {
   190  			if _, isInvalid := a.(*kates.Unstructured); isInvalid {
   191  				continue
   192  			}
   193  			if include(GetAmbID(ctx, a)) {
   194  				resources = append(resources, a)
   195  			}
   196  		}
   197  	}
   198  
   199  	// Hosts are a little weird, because we have two ways to find the
   200  	// ambassador_id. Sorry about that.
   201  	for _, h := range sh.k8sSnapshot.Hosts {
   202  		var id amb.AmbassadorID
   203  		if len(h.Spec.AmbassadorID) > 0 {
   204  			id = h.Spec.AmbassadorID
   205  		}
   206  		if include(id) {
   207  			resources = append(resources, h)
   208  		}
   209  	}
   210  
   211  	// TLSContexts, Modules, and Ingresses are all straightforward.
   212  	for _, t := range sh.k8sSnapshot.TLSContexts {
   213  		if include(t.Spec.AmbassadorID) {
   214  			resources = append(resources, t)
   215  		}
   216  	}
   217  	for _, m := range sh.k8sSnapshot.Modules {
   218  		if include(m.Spec.AmbassadorID) {
   219  			resources = append(resources, m)
   220  		}
   221  	}
   222  	for _, i := range sh.k8sSnapshot.Ingresses {
   223  		resources = append(resources, i)
   224  	}
   225  
   226  	// OK. Once that's done, we can check to see if we should be
   227  	// doing secret namespacing or not -- this requires a look into
   228  	// the Ambassador Module, if it's present.
   229  	//
   230  	// XXX Linear searches suck, but whatever, it's just not gonna
   231  	// be all that many things. We won't bother optimizing this unless
   232  	// a profiler shows that it's a problem.
   233  
   234  	secretNamespacing := true
   235  	for _, resource := range resources {
   236  		mod, ok := resource.(*amb.Module)
   237  		// We don't need to recheck ambassador_id on this Module because
   238  		// the Module can't have made it into the resources list without
   239  		// its ambassador_id being checked.
   240  
   241  		if ok && mod.GetName() == "ambassador" {
   242  			// XXX ModuleSecrets is a _godawful_ hack. See the comment on
   243  			// ModuleSecrets itself for more.
   244  			secs := ModuleSecrets{}
   245  			err := convert(mod.Spec.Config, &secs)
   246  			if err != nil {
   247  				dlog.Errorf(ctx, "error parsing module: %v", err)
   248  				continue
   249  			}
   250  			secretNamespacing = secs.Defaults.TLSSecretNamespacing
   251  			break
   252  		}
   253  	}
   254  
   255  	// Once we have our list of secrets, go figure out the names of all
   256  	// the secrets we need. We'll use this "refs" map to hold all the names...
   257  	refs := map[snapshotTypes.SecretRef]bool{}
   258  
   259  	// ...and, uh, this "action" function is really just a closure to avoid
   260  	// needing to pass "refs" to find SecretRefs. Shrug. Arguably more
   261  	// complex than needed, but meh.
   262  	action := func(ref snapshotTypes.SecretRef) {
   263  		refs[ref] = true
   264  	}
   265  
   266  	// So. Walk the list of resources...
   267  	for _, resource := range resources {
   268  		// ...and for each resource, dig out any secrets being referenced.
   269  		findSecretRefs(ctx, resource, secretNamespacing, action)
   270  	}
   271  
   272  	// We _always_ have an implicit references to the cloud-connec-token secret...
   273  	secretRef(GetCloudConnectTokenResourceNamespace(), GetCloudConnectTokenResourceName(), false, action)
   274  
   275  	// We _always_ have an implicit references to the fallback cert secret...
   276  	secretRef(GetAmbassadorNamespace(), "fallback-self-signed-cert", false, action)
   277  
   278  	isEdgeStack, err := IsEdgeStack()
   279  	if err != nil {
   280  		return err
   281  	}
   282  	if isEdgeStack {
   283  		// ...and for Edge Stack, we _always_ have an implicit reference to the
   284  		// license secret.
   285  		secretRef(GetLicenseSecretNamespace(), GetLicenseSecretName(), false, action)
   286  		// We also want to grab any secrets referenced by Edge-Stack filters for use in Edge-Stack
   287  		// the Filters are unstructured because Emissary does not have their type definition
   288  		for _, f := range sh.k8sSnapshot.Filters {
   289  			err := findFilterSecret(f, action)
   290  			if err != nil {
   291  				dlog.Errorf(ctx, "Error gathering secret reference from Filter: %v", err)
   292  			}
   293  		}
   294  	}
   295  
   296  	// OK! After all that, go copy all the matching secrets from FSSecrets and
   297  	// K8sSecrets to Secrets.
   298  	//
   299  	// The way this works is kind of simple: first we check everything in
   300  	// FSSecrets. Then, when we check K8sSecrets, we skip any secrets that are
   301  	// also in FSSecrets. End result: FSSecrets wins if there are any conflicts.
   302  	sh.k8sSnapshot.Secrets = make([]*kates.Secret, 0, len(refs))
   303  
   304  	for ref, secret := range sh.k8sSnapshot.FSSecrets {
   305  		if refs[ref] {
   306  			checkSecret(ctx, sh, "FSSecret", ref, secret)
   307  		}
   308  	}
   309  
   310  	for _, secret := range sh.k8sSnapshot.K8sSecrets {
   311  		ref := snapshotTypes.SecretRef{Namespace: secret.GetNamespace(), Name: secret.GetName()}
   312  
   313  		_, found := sh.k8sSnapshot.FSSecrets[ref]
   314  		if found {
   315  			dlog.Debugf(ctx, "Conflict! skipping K8sSecret %#v", ref)
   316  			continue
   317  		}
   318  
   319  		if refs[ref] {
   320  			checkSecret(ctx, sh, "K8sSecret", ref, secret)
   321  		}
   322  	}
   323  	return nil
   324  }
   325  
   326  // Returns secretName, secretNamespace from a provided (unstructured) filter if it contains a secret
   327  // Returns empty strings when the secret name and/or namespace could not be found
   328  func findFilterSecret(filter *unstructured.Unstructured, action func(snapshotTypes.SecretRef)) error {
   329  	// Just making extra sure this is actually a Filter
   330  	if filter.GetKind() != "Filter" {
   331  		return fmt.Errorf("non-Filter object in Snapshot.Filters: %s", filter.GetKind())
   332  	}
   333  	// Only OAuth2 Filters have secrets, although they don't need to have them.
   334  	// This is overly contrived because Filters are unstructured to Emissary since we don't have the type definitions
   335  	// Yes this is disgusting. It is what it is...
   336  	filterContents := filter.UnstructuredContent()
   337  	filterSpec := filterContents["spec"]
   338  	if filterSpec != nil {
   339  		mapOAuth, ok := filterSpec.(map[string]interface{})
   340  		// We need to check if all these type assertions fail since we shouldnt rely on CRD validation to protect us from a panic state
   341  		// I cant imagine a scenario where this would realisticly happen, but we generate a unique log message for tracability and skip processing it
   342  		if !ok {
   343  			// We bail early any time we detect bogus contents for any of these fields
   344  			// and let the APIServer, apiext, and amb-sidecar handle the error reporting
   345  			return nil
   346  		}
   347  		oAuthFilter := mapOAuth["OAuth2"]
   348  		if oAuthFilter != nil {
   349  			secretName, secretNamespace := "", ""
   350  			// Check if we have a secretName
   351  			mapOAuth, ok := oAuthFilter.(map[string]interface{})
   352  			if !ok {
   353  				return nil
   354  			}
   355  			sName := mapOAuth["secretName"]
   356  			if sName == nil {
   357  				return nil
   358  			}
   359  			secretName, ok = sName.(string)
   360  			// This is a weird check, but we have to handle the case where secretName is not provided, and when its explicitly set to ""
   361  			if !ok || secretName == "" {
   362  				// Bail out early since there is no secret
   363  				return nil
   364  			}
   365  			sNamespace := mapOAuth["secretNamespace"]
   366  			if sNamespace == nil {
   367  				secretNamespace = filter.GetNamespace()
   368  			} else {
   369  				secretNamespace, ok = sNamespace.(string)
   370  				if !ok {
   371  					return nil
   372  				} else if secretNamespace == "" {
   373  					secretNamespace = filter.GetNamespace()
   374  				}
   375  			}
   376  			secretRef(secretNamespace, secretName, false, action)
   377  		}
   378  	}
   379  	return nil
   380  }
   381  
   382  // Find all the secrets a given Ambassador resource references.
   383  func findSecretRefs(ctx context.Context, resource kates.Object, secretNamespacing bool, action func(snapshotTypes.SecretRef)) {
   384  	switch r := resource.(type) {
   385  	case *amb.Host:
   386  		// The Host resource is a little odd. Host.spec.tls, Host.spec.tlsSecret, and
   387  		// host.spec.acmeProvider.privateKeySecret can all refer to secrets.
   388  		if r.Spec == nil {
   389  			return
   390  		}
   391  
   392  		if r.Spec.TLS != nil {
   393  			// Host.spec.tls.caSecret is the thing to worry about here.
   394  			secretRef(r.GetNamespace(), r.Spec.TLS.CASecret, secretNamespacing, action)
   395  
   396  			if r.Spec.TLS.CRLSecret != "" {
   397  				secretRef(r.GetNamespace(), r.Spec.TLS.CRLSecret, secretNamespacing, action)
   398  			}
   399  		}
   400  
   401  		// Host.spec.tlsSecret and Host.spec.acmeProvider.privateKeySecret are native-Kubernetes-style
   402  		// `core.v1.LocalObjectReference`s, not Ambassador-style `{name}.{namespace}` strings.  If we
   403  		// ever decide that they should support cross-namespace references, we would do it by adding a
   404  		// `namespace:` field (i.e. changing them to `core.v1.SecretReference`s) rather than by
   405  		// adopting the `{name}.{namespace}` notation.
   406  		if r.Spec.TLSSecret != nil && r.Spec.TLSSecret.Name != "" {
   407  			if r.Spec.TLSSecret.Namespace != "" {
   408  				secretRef(r.Spec.TLSSecret.Namespace, r.Spec.TLSSecret.Name, false, action)
   409  			} else {
   410  				secretRef(r.GetNamespace(), r.Spec.TLSSecret.Name, false, action)
   411  			}
   412  		}
   413  
   414  		if r.Spec.AcmeProvider != nil && r.Spec.AcmeProvider.PrivateKeySecret != nil &&
   415  			r.Spec.AcmeProvider.PrivateKeySecret.Name != "" {
   416  			secretRef(r.GetNamespace(), r.Spec.AcmeProvider.PrivateKeySecret.Name, false, action)
   417  		}
   418  
   419  	case *amb.TLSContext:
   420  		// TLSContext.spec.secret and TLSContext.spec.ca_secret are the things to worry about --
   421  		// but note well that TLSContexts can override the global secretNamespacing setting.
   422  		if r.Spec.Secret != "" {
   423  			if r.Spec.SecretNamespacing != nil {
   424  				secretNamespacing = *r.Spec.SecretNamespacing
   425  			}
   426  			secretRef(r.GetNamespace(), r.Spec.Secret, secretNamespacing, action)
   427  		}
   428  
   429  		if r.Spec.CASecret != "" {
   430  			if r.Spec.SecretNamespacing != nil {
   431  				secretNamespacing = *r.Spec.SecretNamespacing
   432  			}
   433  			secretRef(r.GetNamespace(), r.Spec.CASecret, secretNamespacing, action)
   434  		}
   435  
   436  		if r.Spec.CRLSecret != "" {
   437  			if r.Spec.SecretNamespacing != nil {
   438  				secretNamespacing = *r.Spec.SecretNamespacing
   439  			}
   440  			secretRef(r.GetNamespace(), r.Spec.CRLSecret, secretNamespacing, action)
   441  		}
   442  
   443  	case *amb.Module:
   444  		// This whole thing is a hack. We probably _should_ check to make sure that
   445  		// this is an Ambassador Module or a TLS Module, but, well, those're the only
   446  		// supported kinds now, anyway...
   447  		//
   448  		// XXX ModuleSecrets is a godawful hack. See its comment for more.
   449  		secs := ModuleSecrets{}
   450  		err := convert(r.Spec.Config, &secs)
   451  		if err != nil {
   452  			// XXX
   453  			dlog.Errorf(ctx, "error extracting secrets from module: %v", err)
   454  			return
   455  		}
   456  
   457  		// XXX Technically, this is wrong -- _any_ element named in the module can
   458  		// refer to a secret. Hmmm.
   459  		if secs.Upstream.Secret != "" {
   460  			secretRef(r.GetNamespace(), secs.Upstream.Secret, secretNamespacing, action)
   461  		}
   462  		if secs.Server.Secret != "" {
   463  			secretRef(r.GetNamespace(), secs.Server.Secret, secretNamespacing, action)
   464  		}
   465  		if secs.Client.Secret != "" {
   466  			secretRef(r.GetNamespace(), secs.Client.Secret, secretNamespacing, action)
   467  		}
   468  
   469  	case *k8s_resource_types.Ingress:
   470  		// Ingress is pretty straightforward, too, just look in spec.tls.
   471  		for _, itls := range r.Spec.TLS {
   472  			if itls.SecretName != "" {
   473  				secretRef(r.GetNamespace(), itls.SecretName, secretNamespacing, action)
   474  			}
   475  		}
   476  	}
   477  }
   478  
   479  // Mark a secret as one we reference, handling secretNamespacing correctly.
   480  func secretRef(namespace, name string, secretNamespacing bool, action func(snapshotTypes.SecretRef)) {
   481  	if secretNamespacing {
   482  		parts := strings.Split(name, ".")
   483  		if len(parts) > 1 {
   484  			namespace = parts[len(parts)-1]
   485  			name = strings.Join(parts[:len(parts)-1], ".")
   486  		}
   487  	}
   488  
   489  	action(snapshotTypes.SecretRef{Namespace: namespace, Name: name})
   490  }
   491  
   492  // ModuleSecrets is... a hack. It's sort of a mashup of the chunk of the Ambassador
   493  // Module and the chunk of the TLS Module that are common, because they're able to
   494  // specify secrets. However... first, I don't think the TLS Module actually supported
   495  // tls_secret_namespacing. Second, the Ambassador Module at least supports arbitrary
   496  // origination context names -- _any_ key in the TLS dictionary will get turned into
   497  // an origination context.
   498  //
   499  // I seriously doubt that either of these will actually affect anyone at this remove,
   500  // but... yeah.
   501  type ModuleSecrets struct {
   502  	Defaults struct {
   503  		TLSSecretNamespacing bool `json:"tls_secret_namespacing"`
   504  	} `json:"defaults"`
   505  	Upstream struct {
   506  		Secret string `json:"secret"`
   507  	} `json:"upstream"`
   508  	Server struct {
   509  		Secret string `json:"secret"`
   510  	} `json:"server"`
   511  	Client struct {
   512  		Secret string `json:"secret"`
   513  	} `json:"client"`
   514  }
   515
View as plain text