...

Source file src/github.com/linkerd/linkerd2/test/integration/external/stat/stat_test.go

Documentation: github.com/linkerd/linkerd2/test/integration/external/stat

     1  package get
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"os"
     7  	"strconv"
     8  	"strings"
     9  	"testing"
    10  	"time"
    11  
    12  	"github.com/linkerd/linkerd2/testutil"
    13  )
    14  
    15  //////////////////////
    16  ///   TEST SETUP   ///
    17  //////////////////////
    18  
    19  var TestHelper *testutil.TestHelper
    20  
    21  func TestMain(m *testing.M) {
    22  	TestHelper = testutil.NewTestHelper()
    23  	// Block test execution until viz extension pods are running
    24  	TestHelper.WaitUntilDeployReady(testutil.ExternalVizDeployReplicas)
    25  	os.Exit(m.Run())
    26  }
    27  
    28  //////////////////////
    29  /// TEST EXECUTION ///
    30  //////////////////////
    31  
    32  // These tests retry for up to 20 seconds, since each call to "linkerd stat"
    33  // generates traffic to the components in the linkerd namespace, and we're
    34  // testing that those components are properly reporting stats. It's ok if the
    35  // first few attempts fail due to missing stats, since the requests from those
    36  // failed attempts will eventually be recorded in the stats that we're
    37  // requesting, and the test will pass.
    38  func TestCliStatForLinkerdNamespace(t *testing.T) {
    39  	ctx := context.Background()
    40  	var prometheusPod, prometheusAuthority, prometheusNamespace, prometheusDeployment, metricsPod string
    41  	// Get Metrics Pod
    42  	pods, err := TestHelper.GetPodNamesForDeployment(ctx, TestHelper.GetVizNamespace(), "metrics-api")
    43  	if err != nil {
    44  		testutil.AnnotatedFatalf(t, "failed to get pods for metrics-api",
    45  			"failed to get pods for metrics-api: %s", err)
    46  	}
    47  	if len(pods) != 1 {
    48  		testutil.Fatalf(t, "expected 1 pod for metrics-api, got %d", len(pods))
    49  	}
    50  	metricsPod = pods[0]
    51  
    52  	// Retrieve Prometheus pod details
    53  	prometheusNamespace = "external-prometheus"
    54  	prometheusDeployment = "prometheus"
    55  
    56  	pods, err = TestHelper.GetPodNamesForDeployment(ctx, prometheusNamespace, prometheusDeployment)
    57  	if err != nil {
    58  		testutil.AnnotatedFatalf(t, "failed to get pods for prometheus",
    59  			"failed to get pods for prometheus: %s", err)
    60  	}
    61  	if len(pods) != 1 {
    62  		testutil.Fatalf(t, "expected 1 pod for prometheus, got %d", len(pods))
    63  	}
    64  	prometheusPod = pods[0]
    65  	prometheusAuthority = prometheusDeployment + "." + prometheusNamespace + ".svc.cluster.local:9090"
    66  
    67  	testCases := []struct {
    68  		args         []string
    69  		expectedRows map[string]string
    70  		status       string
    71  		isAuthority  bool
    72  	}{
    73  		{
    74  			args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetLinkerdNamespace()},
    75  			expectedRows: map[string]string{
    76  				"linkerd-destination":    "1/1",
    77  				"linkerd-identity":       "1/1",
    78  				"linkerd-proxy-injector": "1/1",
    79  			},
    80  		},
    81  		{
    82  			args: []string{"viz", "stat", "ns", TestHelper.GetLinkerdNamespace()},
    83  			expectedRows: map[string]string{
    84  				TestHelper.GetLinkerdNamespace(): "3/3",
    85  			},
    86  		},
    87  		{
    88  			args: []string{"viz", "stat", fmt.Sprintf("po/%s", prometheusPod), "-n", prometheusNamespace, "--from", fmt.Sprintf("po/%s", metricsPod), "--from-namespace", TestHelper.GetVizNamespace()},
    89  			expectedRows: map[string]string{
    90  				prometheusPod: "1/1",
    91  			},
    92  			status: "Running",
    93  		},
    94  		{
    95  			args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("po/%s", prometheusPod), "--to-namespace", prometheusNamespace},
    96  			expectedRows: map[string]string{
    97  				"metrics-api": "1/1",
    98  			},
    99  		},
   100  		{
   101  			args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("svc/%s", prometheusDeployment), "--to-namespace", prometheusNamespace},
   102  			expectedRows: map[string]string{
   103  				"metrics-api": "1/1",
   104  			},
   105  		},
   106  		{
   107  			args: []string{"viz", "stat", "po", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("au/%s", prometheusAuthority), "--to-namespace", prometheusNamespace},
   108  			expectedRows: map[string]string{
   109  				metricsPod: "1/1",
   110  			},
   111  			status: "Running",
   112  		},
   113  		{
   114  			args: []string{"viz", "stat", "au", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("po/%s", prometheusPod), "--to-namespace", prometheusNamespace},
   115  			expectedRows: map[string]string{
   116  				prometheusAuthority: "-",
   117  			},
   118  			isAuthority: true,
   119  		},
   120  		{
   121  			args: []string{"viz", "stat", "au", "-n", TestHelper.GetVizNamespace(), "--to", fmt.Sprintf("po/%s", prometheusPod), "--to-namespace", prometheusNamespace},
   122  			expectedRows: map[string]string{
   123  				prometheusAuthority: "-",
   124  			},
   125  			isAuthority: true,
   126  		},
   127  	}
   128  
   129  	if !TestHelper.ExternalPrometheus() {
   130  		testCases = append(testCases, []struct {
   131  			args         []string
   132  			expectedRows map[string]string
   133  			status       string
   134  			isAuthority  bool
   135  		}{
   136  			{
   137  				args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace()},
   138  				expectedRows: map[string]string{
   139  					"metrics-api":  "1/1",
   140  					"tap":          "1/1",
   141  					"web":          "1/1",
   142  					"tap-injector": "1/1",
   143  				},
   144  			},
   145  			{
   146  				args: []string{"viz", "stat", "ns", TestHelper.GetVizNamespace()},
   147  				expectedRows: map[string]string{
   148  					TestHelper.GetVizNamespace(): "4/4",
   149  				},
   150  			},
   151  			{
   152  				args: []string{"viz", "stat", "svc", "prometheus", "-n", prometheusNamespace, "--from", "deploy/metrics-api", "--from-namespace", TestHelper.GetVizNamespace()},
   153  
   154  				expectedRows: map[string]string{
   155  					"prometheus": "-",
   156  				},
   157  			},
   158  		}...,
   159  		)
   160  	} else {
   161  		testCases = append(testCases, []struct {
   162  			args         []string
   163  			expectedRows map[string]string
   164  			status       string
   165  			isAuthority  bool
   166  		}{
   167  			{
   168  				args: []string{"viz", "stat", "deploy", "-n", TestHelper.GetVizNamespace()},
   169  				expectedRows: map[string]string{
   170  					"metrics-api":  "1/1",
   171  					"tap":          "1/1",
   172  					"web":          "1/1",
   173  					"tap-injector": "1/1",
   174  				},
   175  			},
   176  			{
   177  				args: []string{"viz", "stat", "ns", TestHelper.GetVizNamespace()},
   178  				expectedRows: map[string]string{
   179  					TestHelper.GetVizNamespace(): "4/4",
   180  				},
   181  			},
   182  		}...,
   183  		)
   184  	}
   185  
   186  	// Apply a sample application
   187  	TestHelper.WithDataPlaneNamespace(ctx, "stat-test", map[string]string{}, t, func(t *testing.T, prefixedNs string) {
   188  		out, err := TestHelper.LinkerdRun("inject", "--manual", "./testdata/application.yaml")
   189  		if err != nil {
   190  			testutil.AnnotatedFatal(t, "'linkerd inject' command failed", err)
   191  		}
   192  
   193  		out, err = TestHelper.KubectlApply(out, prefixedNs)
   194  		if err != nil {
   195  			testutil.AnnotatedFatalf(t, "'kubectl apply' command failed",
   196  				"'kubectl apply' command failed\n%s", out)
   197  		}
   198  
   199  		// wait for deployments to start
   200  		for _, deploy := range []string{"backend", "failing", "slow-cooker"} {
   201  			if err := TestHelper.CheckPods(ctx, prefixedNs, deploy, 1); err != nil {
   202  				//nolint:errorlint
   203  				if rce, ok := err.(*testutil.RestartCountError); ok {
   204  					testutil.AnnotatedWarn(t, "CheckPods timed-out", rce)
   205  				} else {
   206  					testutil.AnnotatedError(t, "CheckPods timed-out", err)
   207  				}
   208  			}
   209  		}
   210  
   211  		testCases = append(testCases, []struct {
   212  			args         []string
   213  			expectedRows map[string]string
   214  			status       string
   215  			isAuthority  bool
   216  		}{
   217  			{
   218  				args: []string{"viz", "stat", "svc", "-n", prefixedNs},
   219  				expectedRows: map[string]string{
   220  					"backend-svc": "-",
   221  				},
   222  			},
   223  			{
   224  				args: []string{"viz", "stat", "svc/backend-svc", "-n", prefixedNs, "--from", "deploy/slow-cooker"},
   225  				expectedRows: map[string]string{
   226  					"backend-svc": "-",
   227  				},
   228  			},
   229  		}...,
   230  		)
   231  
   232  		for _, tt := range testCases {
   233  			tt := tt // pin
   234  			timeout := 20 * time.Second
   235  			t.Run("linkerd "+strings.Join(tt.args, " "), func(t *testing.T) {
   236  				err := testutil.RetryFor(timeout, func() error {
   237  					// Use a short time window so that transient errors at startup
   238  					// fall out of the window.
   239  					tt.args = append(tt.args, "-t", "30s")
   240  					out, err := TestHelper.LinkerdRun(tt.args...)
   241  					if err != nil {
   242  						testutil.AnnotatedFatalf(t, "unexpected stat error",
   243  							"unexpected stat error: %s\n%s", err, out)
   244  					}
   245  
   246  					expectedColumnCount := 8
   247  					if tt.isAuthority {
   248  						expectedColumnCount = 7
   249  					}
   250  					if tt.status != "" {
   251  						expectedColumnCount++
   252  					}
   253  					rowStats, err := testutil.ParseRows(out, len(tt.expectedRows), expectedColumnCount)
   254  					if err != nil {
   255  						return err
   256  					}
   257  
   258  					for name, meshed := range tt.expectedRows {
   259  						if err := validateRowStats(name, meshed, tt.status, rowStats, tt.isAuthority); err != nil {
   260  							return err
   261  						}
   262  					}
   263  
   264  					return nil
   265  				})
   266  				if err != nil {
   267  					testutil.AnnotatedFatal(t, fmt.Sprintf("timed-out checking stats (%s)", timeout), err)
   268  				}
   269  			})
   270  		}
   271  	})
   272  }
   273  
   274  func validateRowStats(name, expectedMeshCount, expectedStatus string, rowStats map[string]*testutil.RowStat, isAuthority bool) error {
   275  	stat, ok := rowStats[name]
   276  	if !ok {
   277  		return fmt.Errorf("no stats found for [%s]", name)
   278  	}
   279  
   280  	if stat.Status != expectedStatus {
   281  		return fmt.Errorf("expected status '%s' for '%s', got '%s'",
   282  			expectedStatus, name, stat.Status)
   283  	}
   284  
   285  	if stat.Meshed != expectedMeshCount {
   286  		return fmt.Errorf("expected mesh count [%s] for [%s], got [%s]",
   287  			expectedMeshCount, name, stat.Meshed)
   288  	}
   289  
   290  	expectedSuccessRate := "100.00%"
   291  	if stat.Success != expectedSuccessRate {
   292  		return fmt.Errorf("expected success rate [%s] for [%s], got [%s]",
   293  			expectedSuccessRate, name, stat.Success)
   294  	}
   295  
   296  	if !strings.HasSuffix(stat.Rps, "rps") {
   297  		return fmt.Errorf("unexpected rps for [%s], got [%s]",
   298  			name, stat.Rps)
   299  	}
   300  
   301  	if !strings.HasSuffix(stat.P50Latency, "ms") {
   302  		return fmt.Errorf("unexpected p50 latency for [%s], got [%s]",
   303  			name, stat.P50Latency)
   304  	}
   305  
   306  	if !strings.HasSuffix(stat.P95Latency, "ms") {
   307  		return fmt.Errorf("unexpected p95 latency for [%s], got [%s]",
   308  			name, stat.P95Latency)
   309  	}
   310  
   311  	if !strings.HasSuffix(stat.P99Latency, "ms") {
   312  		return fmt.Errorf("unexpected p99 latency for [%s], got [%s]",
   313  			name, stat.P99Latency)
   314  	}
   315  
   316  	if stat.TCPOpenConnections != "-" && !isAuthority {
   317  		_, err := strconv.Atoi(stat.TCPOpenConnections)
   318  		if err != nil {
   319  			return fmt.Errorf("error parsing number of TCP connections [%s]: %w", stat.TCPOpenConnections, err)
   320  		}
   321  	}
   322  
   323  	return nil
   324  }
   325  

View as plain text