package trustanchor import ( "context" "fmt" "os" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" "edge-infra.dev/pkg/edge/linkerd" l5dv1alpha1 "edge-infra.dev/pkg/edge/linkerd/k8s/apis/linkerd/v1alpha1" "edge-infra.dev/test/f2" "edge-infra.dev/test/f2/x/ktest" ) var f f2.Framework var linkerdNamespace = &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: linkerd.Namespace}} func TestMain(m *testing.M) { f = f2.New(context.Background(), f2.WithExtensions( ktest.New(), )). Setup(func(ctx f2.Context) (f2.Context, error) { k, err := ktest.FromContext(ctx) if err != nil { return ctx, err } // Override timeouts if we aren't using a live cluster if !*k.Env.UseExistingCluster { k.Timeout = 5 * time.Second k.Tick = 10 * time.Millisecond } return ctx, nil }).Teardown() os.Exit(f.Run(m)) } func TestCreateIfNotExists(t *testing.T) { var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd emptyTrustAnchorSecret *corev1.Secret trustAnchorSecret *corev1.Secret ) feature := f2.NewFeature("CreateIfNotExists"). Setup("create linkerd resources", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) l5d = createL5d() emptyTrustAnchorSecret = buildSecret(l5d, []byte{}, []byte{}) trustAnchorSecret = buildSecret(l5d, []byte("fake-cert"), []byte{}) return ctx }). Test("unable to create trust anchor secret", func(ctx f2.Context, t *testing.T) f2.Context { cert, err := CreateIfNotExists(ctx, k.Client, l5d) assert.EqualError(t, err, "failed to create secret") assert.Equal(t, "", cert) assert.EqualValues(t, l5d.Status.Conditions[0].Status, corev1.ConditionFalse) assert.Equal(t, l5d.Status.Conditions[0].Reason, l5dv1alpha1.TrustAnchorSecretSetupFailedReason) return ctx }). Test("unable to determine if trust anchor exists", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Create(ctx, linkerdNamespace)) require.NoError(t, k.Client.Create(ctx, emptyTrustAnchorSecret)) cert, err := CreateIfNotExists(ctx, k.Client, l5d) assert.Error(t, err) assert.Equal(t, "", cert) assert.EqualValues(t, l5d.Status.Conditions[0].Status, corev1.ConditionFalse) assert.Equal(t, l5d.Status.Conditions[0].Reason, l5dv1alpha1.TrustAnchorSecretSetupFailedReason) return ctx }). Test("successfully create new certificate", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, emptyTrustAnchorSecret)) cert, err := CreateIfNotExists(ctx, k.Client, l5d) assert.NoError(t, err) assert.Contains(t, cert, "-----BEGIN CERTIFICATE-----") assert.Contains(t, cert, "-----END CERTIFICATE-----") assert.EqualValues(t, l5d.Status.Conditions[0].Status, corev1.ConditionTrue) assert.Equal(t, l5d.Status.Conditions[0].Reason, l5dv1alpha1.SucceededReason) return ctx }). Test("succesfullly return existing certificate", func(ctx f2.Context, t *testing.T) f2.Context { cert, err := CreateIfNotExists(ctx, k.Client, l5d) assert.NoError(t, err) assert.Contains(t, cert, "-----BEGIN CERTIFICATE-----") assert.Contains(t, cert, "-----END CERTIFICATE-----") assert.EqualValues(t, l5d.Status.Conditions[0].Status, corev1.ConditionTrue) assert.Equal(t, l5d.Status.Conditions[0].Reason, l5dv1alpha1.SucceededReason) return ctx }). Teardown("delete trust anchor secret", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, trustAnchorSecret)) return ctx }).Feature() f.Test(t, feature) } func TestSecretExists(t *testing.T) { var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd emptyTrustAnchorSecret *corev1.Secret trustAnchorSecret *corev1.Secret ) feature := f2.NewFeature("secretExists"). Setup("create linkerd resources", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) l5d = createL5d() emptyTrustAnchorSecret = buildSecret(l5d, []byte{}, []byte{}) trustAnchorSecret = buildSecret(l5d, []byte("fake-cert"), []byte{}) return ctx }). Test("return empty certificate if secret doesnt exist", func(ctx f2.Context, t *testing.T) f2.Context { cert, err := SecretExists(ctx, k.Client) assert.NoError(t, err) assert.Equal(t, "", cert) return ctx }). Test("error if certifcate key is empty for existing secret", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Create(ctx, emptyTrustAnchorSecret)) cert, err := SecretExists(ctx, k.Client) assert.Equal(t, "", cert) assert.EqualError(t, err, fmt.Sprintf("secret was present, but %s was not present or empty", corev1.TLSCertKey)) return ctx }). Test("return certificate string if secret exists", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, emptyTrustAnchorSecret)) require.NoError(t, k.Client.Create(ctx, trustAnchorSecret)) cert, err := SecretExists(ctx, k.Client) assert.Equal(t, "fake-cert", cert) assert.NoError(t, err) return ctx }). Teardown("delete trust anchor secret", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, trustAnchorSecret)) return ctx }).Feature() f.Test(t, feature) } func TestCreateSecret(t *testing.T) { var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd ) feature := f2.NewFeature("createSecret"). Setup("create linkerd CR", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) l5d = createL5d() return ctx }). Test("create trust anchor seret if secret does not exist", func(ctx f2.Context, t *testing.T) f2.Context { cert, err := createSecret(ctx, k.Client, l5d) assert.NoError(t, err) assert.NotEqual(t, "", cert) assert.Contains(t, cert, "-----BEGIN CERTIFICATE-----") assert.Contains(t, cert, "-----END CERTIFICATE-----") return ctx }). Test("return certificate if secret already exists", func(ctx f2.Context, t *testing.T) f2.Context { cert, err := createSecret(ctx, k.Client, l5d) assert.NoError(t, err) assert.NotEqual(t, "", cert) assert.Contains(t, cert, "-----BEGIN CERTIFICATE-----") assert.Contains(t, cert, "-----END CERTIFICATE-----") return ctx }). Teardown("delete trust anchor secret", func(ctx f2.Context, t *testing.T) f2.Context { trustAnchorSecret := buildSecret(l5d, []byte{}, []byte{}) require.NoError(t, k.Client.Delete(ctx, trustAnchorSecret)) return ctx }).Feature() f.Test(t, feature) } func TestRotate(t *testing.T) { var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd trustAnchorSecret *corev1.Secret trustAnchorCert string trustRootsCM *corev1.ConfigMap ) feature := f2.NewFeature("rotate"). Setup("setup", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) l5d = createL5d() trustAnchorCert = "fake-cert" trustAnchorSecret = buildSecret(l5d, []byte(trustAnchorCert), []byte{}) // create identity trust roots configmap trustRootsCM = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: linkerd.LinkerdIdentityConfigMap, Namespace: linkerd.Namespace, }, } require.NoError(t, k.Client.Create(ctx, trustRootsCM)) return ctx }). Test("secret doesn't exist", func(ctx f2.Context, t *testing.T) f2.Context { require.Error(t, k.Client.Get(ctx, linkerd.TrustAnchorKey(), trustAnchorSecret)) assert.NoError(t, Rotate(ctx, k.Client, l5d)) assert.EqualValues(t, l5d.Status.Conditions[0].Status, corev1.ConditionTrue) assert.Equal(t, l5d.Status.Conditions[0].Reason, l5dv1alpha1.SucceededReason) assert.NoError(t, k.Client.Get(ctx, linkerd.TrustAnchorKey(), trustAnchorSecret)) // assert trust roots configmap is correct require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.LinkerdIdentityConfigMap, Namespace: linkerd.Namespace}, trustRootsCM)) caBundle, exists := trustRootsCM.Data[CaBundleKey] assert.True(t, exists) assert.NotEqual(t, "", caBundle) assert.Contains(t, caBundle, "-----BEGIN CERTIFICATE-----") assert.Contains(t, caBundle, "-----END CERTIFICATE-----") return ctx }). Test("secret already exists", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Get(ctx, linkerd.TrustAnchorKey(), trustAnchorSecret)) assert.NoError(t, Rotate(ctx, k.Client, l5d)) assert.EqualValues(t, l5d.Status.Conditions[0].Status, corev1.ConditionTrue) assert.Equal(t, l5d.Status.Conditions[0].Reason, l5dv1alpha1.DualAnchor) // assert trust roots configmap require.NoError(t, k.Client.Get(ctx, types.NamespacedName{Name: linkerd.LinkerdIdentityConfigMap, Namespace: linkerd.Namespace}, trustRootsCM)) caBundle, exists := trustRootsCM.Data[CaBundleKey] assert.True(t, exists) assert.Contains(t, caBundle, "-----BEGIN CERTIFICATE-----") assert.Contains(t, caBundle, "-----END CERTIFICATE-----") return ctx }). Teardown("delete trust anchor secret", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, trustAnchorSecret)) require.NoError(t, k.Client.Delete(ctx, trustRootsCM)) return ctx }). Feature() f.Test(t, feature) } func TestIsRotated(t *testing.T) { var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd trustAnchorSecret *corev1.Secret ) feature := f2.NewFeature("IsRotated"). Setup("setup", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) l5d = createL5d() trustAnchorSecret = buildSecret(l5d, []byte("fake-cert"), []byte{}) return ctx }). Test("nil secret", func(ctx f2.Context, t *testing.T) f2.Context { // assert we get a nil secret trustAnchorSecret, err := getSecret(ctx, k.Client) assert.NoError(t, err) assert.Nil(t, trustAnchorSecret) // assert IsRotated returns false assert.False(t, IsRotated(ctx, k.Client)) return ctx }). Test("annotation doesn't exist", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Create(ctx, trustAnchorSecret)) assert.Empty(t, trustAnchorSecret.Annotations[trustAnchorRotated]) assert.False(t, IsRotated(ctx, k.Client)) return ctx }). Test("annotation set", func(ctx f2.Context, t *testing.T) f2.Context { secret := trustAnchorSecret.DeepCopy() secret.Annotations = map[string]string{trustAnchorRotated: "true"} require.NoError(t, k.Client.Patch(ctx, secret, client.StrategicMergeFrom(trustAnchorSecret.DeepCopy()))) assert.True(t, IsRotated(ctx, k.Client)) return ctx }). Teardown("delete trust anchor secret", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, trustAnchorSecret)) return ctx }). Feature() f.Test(t, feature) } func TestCaBundle(t *testing.T) { var ( k *ktest.K8s l5d *l5dv1alpha1.Linkerd cmObj = types.NamespacedName{Namespace: linkerd.Namespace, Name: linkerd.LinkerdIdentityConfigMap} identityCM = &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: linkerd.LinkerdIdentityConfigMap, Namespace: linkerd.Namespace, }, } caSecret = "secret" caExtraSecret = "extra-secret" trustCert = "test-cert" trustAnchorSecret = &corev1.Secret{} ) featureUpdateCaBundle := f2.NewFeature("updateCaBundle"). Setup("create identity configmap", func(ctx f2.Context, t *testing.T) f2.Context { k = ktest.FromContextT(ctx, t) require.NoError(t, k.Client.Create(ctx, identityCM.DeepCopy())) return ctx }). Test("no data", func(ctx f2.Context, t *testing.T) f2.Context { newCaBundle, err := updateCaBundle(ctx, k.Client, "", "fake-secret") require.NoError(t, err) cm := &corev1.ConfigMap{} require.NoError(t, k.Client.Get(ctx, cmObj, cm)) assert.NotEmpty(t, cm.Data[CaBundleKey]) assert.Equal(t, newCaBundle, cm.Data[CaBundleKey]) return ctx }). Test("ca bundle updated", func(ctx f2.Context, t *testing.T) f2.Context { newCaBundle, err := updateCaBundle(ctx, k.Client, caSecret, caExtraSecret) require.NoError(t, err) cm := &corev1.ConfigMap{} require.NoError(t, k.Client.Get(ctx, cmObj, cm)) assert.NotEmpty(t, cm.Data[CaBundleKey]) assert.Equal(t, newCaBundle, cm.Data[CaBundleKey]) return ctx }).Feature() featureGetCaBundle := f2.NewFeature("getCaBundle"). Setup("setup trust anchor secret", func(ctx f2.Context, t *testing.T) f2.Context { l5d = createL5d() trustAnchorSecret := buildSecret(l5d, []byte(trustCert), []byte{}) require.NoError(t, k.Client.Create(ctx, trustAnchorSecret)) return ctx }). Test("ca bundle exists", func(ctx f2.Context, t *testing.T) f2.Context { caBundle, err := GetCaBundle(ctx, k.Client) assert.NoError(t, err) assert.Equal(t, fmt.Sprintf("%s%s", caSecret, caExtraSecret), caBundle) return ctx }). Test("configmap doesn't exist", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, identityCM)) caBundle, err := GetCaBundle(ctx, k.Client) assert.NoError(t, err) assert.True(t, errors.IsNotFound(k.Client.Get(ctx, cmObj, &corev1.ConfigMap{}))) assert.Equal(t, trustCert, caBundle) return ctx }). Test("ca bundle doesn't exist", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Create(ctx, identityCM)) _, err := GetCaBundle(ctx, k.Client) assert.ErrorContains(t, err, "ca bundle not found in configmap") return ctx }). Feature() featureCheckCaBundle := f2.NewFeature("checkCaBundle"). Setup("setup check", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Get(ctx, linkerd.TrustAnchorKey(), trustAnchorSecret)) return ctx }). Test("secret does not exist", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, trustAnchorSecret)) caBundleOK, err := CheckCaBundle(ctx, k.Client) assert.ErrorContains(t, err, "trust anchor secret does not exist") assert.False(t, caBundleOK) return ctx }). Test("root ca not in bundle", func(ctx f2.Context, t *testing.T) f2.Context { trustAnchorSecret = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: linkerd.TrustAnchorName, Namespace: linkerd.Namespace}} require.NoError(t, k.Client.Create(ctx, trustAnchorSecret)) caBundleOK, err := CheckCaBundle(ctx, k.Client) assert.ErrorContains(t, err, "secret was present, but tls.crt was not present or empty") assert.False(t, caBundleOK) return ctx }). Test("root ca in bundle", func(ctx f2.Context, t *testing.T) f2.Context { require.NoError(t, k.Client.Delete(ctx, trustAnchorSecret)) cert, err := Create(ctx, k.Client, l5d) require.NoError(t, err) _, err = updateCaBundle(ctx, k.Client, "", cert) require.NoError(t, err) caBundleOK, err := CheckCaBundle(ctx, k.Client) assert.NoError(t, err) assert.True(t, caBundleOK) return ctx }). Feature() featureGetLastValidCa := f2.NewFeature("getLastValidCa"). Test("empty ca bundle", func(ctx f2.Context, t *testing.T) f2.Context { lastCa, err := getLastValidCa("") assert.NoError(t, err) assert.Empty(t, lastCa) return ctx }). Test("single cert in bundle", func(ctx f2.Context, t *testing.T) f2.Context { cert, _, err := generateSecret(ctx, l5d) require.NoError(t, err) lastCa, err := getLastValidCa(cert) assert.NoError(t, err) assert.Equal(t, cert, lastCa) return ctx }). Test("multiple certs in bundle", func(ctx f2.Context, t *testing.T) f2.Context { certFirst, _, err := generateSecret(ctx, l5d) require.NoError(t, err) // ensure that certFirst and certLast are created with different times time.Sleep(time.Second) certLast, _, err := generateSecret(ctx, l5d) require.NoError(t, err) lastCa, err := getLastValidCa(fmt.Sprintf("%s%s", certLast, certFirst)) assert.NoError(t, err) assert.Equal(t, certLast, lastCa) return ctx }). Feature() f.Test(t, featureUpdateCaBundle, featureGetCaBundle, featureCheckCaBundle, featureGetLastValidCa, ) } func createL5d() *l5dv1alpha1.Linkerd { return &l5dv1alpha1.Linkerd{ ObjectMeta: metav1.ObjectMeta{ Name: "linkerd", Namespace: linkerd.Namespace, UID: "12345", }, } }