1 package opaqueports
2
3 import (
4 "bytes"
5 "context"
6 "fmt"
7 "html/template"
8 "os"
9 "testing"
10 "time"
11
12 "github.com/linkerd/linkerd2/testutil"
13 v1 "k8s.io/api/core/v1"
14 )
15
16 var TestHelper *testutil.TestHelper
17
18 var opaquePortsClientTemplate = template.Must(template.New("opaque_ports_client.yaml").ParseFiles("testdata/opaque_ports_client.yaml"))
19
20 var (
21 opaquePodApp = "opaque-pod"
22 opaquePodSC = "slow-cooker-opaque-pod"
23 opaqueSvcApp = "opaque-service"
24 opaqueSvcSC = "slow-cooker-opaque-service"
25 opaqueUnmeshedSvcApp = "opaque-unmeshed"
26 opaqueUnmeshedSvcPod = "opaque-unmeshed-svc"
27 opaqueUnmeshedSvcSC = "slow-cooker-opaque-unmeshed-svc"
28 )
29
30 type testCase struct {
31 name string
32 appName string
33 appChecks []check
34 scName string
35 scChecks []check
36 }
37
38 type check func(metrics, ns string) error
39
40 func checks(c ...check) []check { return c }
41
42 func TestMain(m *testing.M) {
43 TestHelper = testutil.NewTestHelper()
44
45 TestHelper.WaitUntilDeployReady(testutil.LinkerdDeployReplicasEdge)
46 os.Exit(m.Run())
47 }
48
49
50
51 type clientTemplateArgs struct {
52 ServiceCookerOpaqueServiceTargetHost string
53 ServiceCookerOpaquePodTargetHost string
54 ServiceCookerOpaqueUnmeshedSVCTargetHost string
55 }
56
57 func serviceName(n string) string {
58 return fmt.Sprintf("svc-%s", n)
59 }
60
61
62
63
64
65 func TestOpaquePortsCalledByServiceTarget(t *testing.T) {
66 ctx := context.Background()
67 TestHelper.WithDataPlaneNamespace(ctx, "opaque-ports-called-by-service-name-test", map[string]string{}, t, func(t *testing.T, opaquePortsNs string) {
68 checks := func(c ...check) []check { return c }
69
70 if err := deployApplications(opaquePortsNs); err != nil {
71 testutil.AnnotatedFatal(t, "failed to deploy applications", err)
72 }
73 waitForAppDeploymentReady(t, opaquePortsNs)
74
75 tmplArgs := clientTemplateArgs{
76 ServiceCookerOpaqueServiceTargetHost: serviceName(opaqueSvcApp),
77 ServiceCookerOpaquePodTargetHost: serviceName(opaquePodApp),
78 ServiceCookerOpaqueUnmeshedSVCTargetHost: serviceName(opaqueUnmeshedSvcApp),
79 }
80 if err := deployTemplate(opaquePortsNs, opaquePortsClientTemplate, tmplArgs); err != nil {
81 testutil.AnnotatedFatal(t, "failed to deploy client pods", err)
82 }
83 waitForClientDeploymentReady(t, opaquePortsNs)
84
85 runTests(ctx, t, opaquePortsNs, []testCase{
86 {
87 name: "calling a meshed service when opaque annotation is on receiving pod",
88 scName: opaquePodSC,
89 scChecks: checks(
90 hasNoOutboundHTTPRequest,
91 hasOutboundTCPWithTLSAndNoAuthority,
92 ),
93 appName: opaquePodApp,
94 appChecks: checks(hasInboundTCPTrafficWithTLS),
95 },
96 {
97 name: "calling a meshed service when opaque annotation is on receiving service",
98 scName: opaqueSvcSC,
99 scChecks: checks(
100 hasNoOutboundHTTPRequest,
101 hasOutboundTCPWithTLSAndAuthority,
102 ),
103 appName: opaqueSvcApp,
104 appChecks: checks(hasInboundTCPTrafficWithTLS),
105 },
106 {
107 name: "calling an unmeshed service when opaque annotation is on service",
108 scName: opaqueUnmeshedSvcSC,
109 scChecks: checks(
110 hasNoOutboundHTTPRequest,
111 hasOutboundTCPWithAuthorityAndNoTLS,
112 ),
113 },
114 })
115 })
116 }
117
118 func TestOpaquePortsCalledByPodTarget(t *testing.T) {
119 ctx := context.Background()
120 TestHelper.WithDataPlaneNamespace(ctx, "opaque-ports-called-by-pod-ip-test", map[string]string{}, t, func(t *testing.T, opaquePortsNs string) {
121
122 if err := deployApplications(opaquePortsNs); err != nil {
123 testutil.AnnotatedFatal(t, "failed to deploy applications", err)
124 }
125 waitForAppDeploymentReady(t, opaquePortsNs)
126
127 tmplArgs, err := templateArgsPodIP(ctx, opaquePortsNs)
128 if err != nil {
129 testutil.AnnotatedFatal(t, "failed to fetch pod IPs", err)
130 }
131
132 if err := deployTemplate(opaquePortsNs, opaquePortsClientTemplate, tmplArgs); err != nil {
133 testutil.AnnotatedFatal(t, "failed to deploy client pods", err)
134 }
135 waitForClientDeploymentReady(t, opaquePortsNs)
136
137 runTests(ctx, t, opaquePortsNs, []testCase{
138 {
139 name: "calling a meshed service when opaque annotation is on receiving pod",
140 scName: opaquePodSC,
141 scChecks: checks(
142 hasNoOutboundHTTPRequest,
143 hasOutboundTCPWithTLSAndNoAuthority,
144 ),
145 appName: opaquePodApp,
146 appChecks: checks(hasInboundTCPTrafficWithTLS),
147 },
148 {
149 name: "calling a meshed service when opaque annotation is on receiving service",
150 scName: opaqueSvcSC,
151 scChecks: checks(
152
153 hasOutboundHTTPRequestWithTLS,
154
155 hasOutboundTCPWithTLSAndNoAuthority,
156 ),
157 appName: opaqueSvcApp,
158 appChecks: checks(hasInboundTCPTrafficWithTLS),
159 },
160 {
161 name: "calling an unmeshed service",
162 scName: opaqueUnmeshedSvcSC,
163 scChecks: checks(
164
165 hasOutboundHTTPRequestNoTLS,
166
167 hasOutboundTCPWithNoTLSAndNoAuthority,
168 ),
169 },
170 })
171 })
172 }
173
174 func waitForAppDeploymentReady(t *testing.T, opaquePortsNs string) {
175 TestHelper.WaitRollout(t, map[string]testutil.DeploySpec{
176 opaquePodApp: {
177 Namespace: opaquePortsNs,
178 Replicas: 1,
179 },
180 opaqueSvcApp: {
181 Namespace: opaquePortsNs,
182 Replicas: 1,
183 },
184 opaqueUnmeshedSvcPod: {
185 Namespace: opaquePortsNs,
186 Replicas: 1,
187 },
188 })
189 }
190
191 func waitForClientDeploymentReady(t *testing.T, opaquePortsNs string) {
192 TestHelper.WaitRollout(t, map[string]testutil.DeploySpec{
193 opaquePodSC: {
194 Namespace: opaquePortsNs,
195 Replicas: 1,
196 },
197 opaqueSvcSC: {
198 Namespace: opaquePortsNs,
199 Replicas: 1,
200 },
201 opaqueUnmeshedSvcSC: {
202 Namespace: opaquePortsNs,
203 Replicas: 1,
204 },
205 })
206 }
207
208 func templateArgsPodIP(ctx context.Context, ns string) (clientTemplateArgs, error) {
209 opaquePodSCPodIP, err := getPodIPByAppLabel(ctx, ns, opaquePodApp)
210 if err != nil {
211 return clientTemplateArgs{}, fmt.Errorf("failed to fetch pod IP for %q: %w", opaquePodApp, err)
212 }
213 opaqueSvcSCPodIP, err := getPodIPByAppLabel(ctx, ns, opaqueSvcApp)
214 if err != nil {
215 return clientTemplateArgs{}, fmt.Errorf("failed to fetch pod IP for %q: %w", opaqueSvcApp, err)
216 }
217 opaqueUnmeshedSvcPodIP, err := getPodIPByAppLabel(ctx, ns, opaqueUnmeshedSvcPod)
218 if err != nil {
219 return clientTemplateArgs{}, fmt.Errorf("failed to fetch pod IP for %q: %w", opaqueUnmeshedSvcPod, err)
220 }
221 return clientTemplateArgs{
222 ServiceCookerOpaquePodTargetHost: opaquePodSCPodIP,
223 ServiceCookerOpaqueServiceTargetHost: opaqueSvcSCPodIP,
224 ServiceCookerOpaqueUnmeshedSVCTargetHost: opaqueUnmeshedSvcPodIP,
225 }, nil
226 }
227
228 func runTests(ctx context.Context, t *testing.T, ns string, tcs []testCase) {
229 t.Helper()
230 for _, tc := range tcs {
231 t.Run(tc.name, func(t *testing.T) {
232 err := testutil.RetryFor(30*time.Second, func() error {
233 if err := checkPodMetrics(ctx, ns, tc.scName, tc.scChecks); err != nil {
234 return fmt.Errorf("failed to check metrics for client pod: %w", err)
235 }
236 if tc.appName == "" {
237 return nil
238 }
239 if err := checkPodMetrics(ctx, ns, tc.appName, tc.appChecks); err != nil {
240 return fmt.Errorf("failed to check metrics for app pod: %w", err)
241 }
242 return nil
243 })
244 if err != nil {
245 testutil.AnnotatedFatalf(t, "unexpected metric for pod", "unexpected metric for pod: %s", err)
246 }
247 })
248 }
249 }
250
251 func checkPodMetrics(ctx context.Context, opaquePortsNs string, podAppLabel string, checks []check) error {
252 pods, err := TestHelper.GetPods(ctx, opaquePortsNs, map[string]string{"app": podAppLabel})
253 if err != nil {
254 return fmt.Errorf("error getting pods for label 'app: %q': %w", podAppLabel, err)
255 }
256 if len(pods) == 0 {
257 return fmt.Errorf("no pods found for label 'app: %q'", podAppLabel)
258 }
259 metrics, err := getPodMetrics(pods[0], opaquePortsNs)
260 if err != nil {
261 return fmt.Errorf("error getting metrics for pod %q: %w", pods[0].Name, err)
262 }
263 for _, check := range checks {
264 if err := check(metrics, opaquePortsNs); err != nil {
265 return fmt.Errorf("validation of pod metrics failed: %w", err)
266 }
267 }
268 return nil
269 }
270
271 func deployApplications(ns string) error {
272 out, err := TestHelper.Kubectl("", "apply", "-f", "testdata/opaque_ports_application.yaml", "-n", ns)
273 if err != nil {
274 return fmt.Errorf("failed apply deployment file %q: %w", out, err)
275 }
276 return nil
277 }
278
279 func deployTemplate(ns string, tmpl *template.Template, templateArgs interface{}) error {
280 bb := &bytes.Buffer{}
281 if err := tmpl.Execute(bb, templateArgs); err != nil {
282 return fmt.Errorf("failed to write deployment template: %w", err)
283 }
284 out, err := TestHelper.KubectlApply(bb.String(), ns)
285 if err != nil {
286 return fmt.Errorf("failed apply deployment file %q: %w", out, err)
287 }
288 return nil
289 }
290
291 func getPodMetrics(pod v1.Pod, ns string) (string, error) {
292 podName := fmt.Sprintf("pod/%s", pod.Name)
293 cmd := []string{"diagnostics", "proxy-metrics", "--namespace", ns, podName}
294 metrics, err := TestHelper.LinkerdRun(cmd...)
295 if err != nil {
296 return "", err
297 }
298 return metrics, nil
299 }
300
301 func getPodIPByAppLabel(ctx context.Context, ns string, app string) (string, error) {
302 labels := map[string]string{"app": app}
303 pods, err := TestHelper.GetPods(ctx, ns, labels)
304 if err != nil {
305 return "", fmt.Errorf("failed to get pod by labels %v: %w", labels, err)
306 }
307 if len(pods) == 0 {
308 return "", fmt.Errorf("no pods found for labels %v", labels)
309 }
310 return pods[0].Status.PodIP, nil
311 }
312
View as plain text