1use linkerd_policy_test::{
2 annotate_service, bb, create, create_ready_pod, curl, with_temp_ns, LinkerdInject,
3};
4
5#[tokio::test(flavor = "current_thread")]
6async fn consecutive_failures() {
7 const MAX_FAILS: usize = 5;
8 with_temp_ns(|client, ns| async move {
9 // Create a bb service with one pod that always returns 500s.
10 let bad_pod = bb::Terminus::new(&ns)
11 .named("bb-bad")
12 .percent_failure(100)
13 .to_pod();
14 let svc = {
15 let svc = bb::Terminus::service(&ns);
16 annotate_service(svc, maplit::btreemap!{
17 "balancer.linkerd.io/failure-accrual" => "consecutive".to_string(),
18 "balancer.linkerd.io/failure-accrual-consecutive-max-failures" => MAX_FAILS.to_string(),
19 // don't allow the failing pod to enter probation during the test.
20 "balancer.linkerd.io/failure-accrual-consecutive-min-penalty" => "5m".to_string(),
21 "balancer.linkerd.io/failure-accrual-consecutive-max-penalty" => "10m".to_string(),
22 })
23 };
24 tokio::join!(
25 create(&client, svc),
26 create_ready_pod(&client, bad_pod),
27 );
28
29 let curl = curl::Runner::init(&client, &ns)
30 .await
31 .run_execable("curl", LinkerdInject::Enabled)
32 .await;
33
34 let url = format!("http://{}", bb::Terminus::SERVICE_NAME);
35 for request in 0..MAX_FAILS * 2 {
36 tracing::info!("Sending request {request}...");
37 let status = curl
38 .get(&url)
39 .await
40 .expect("curl command should succeed");
41 tracing::info!(request, ?status);
42 if request < MAX_FAILS {
43 assert_eq!(status, hyper::StatusCode::INTERNAL_SERVER_ERROR);
44 } else {
45 // Once the circuit breaker has tripped, any in flight request
46 // will fail with a 504 due to failfast, and subsequent requests
47 // will fail with a 503 because the balancer has no available
48 // endpoints.
49 assert!(
50 matches!(status, hyper::StatusCode::GATEWAY_TIMEOUT | hyper::StatusCode::SERVICE_UNAVAILABLE),
51 "expected 503 or 504, got {status:?}"
52 );
53 }
54 }
55 })
56 .await;
57}
View as plain text