1 package ambex 2 3 import ( 4 // standard library 5 "context" 6 "fmt" 7 8 // third-party libraries 9 "google.golang.org/protobuf/proto" 10 "google.golang.org/protobuf/types/known/anypb" 11 12 // envoy api v2 13 apiv2 "github.com/datawire/ambassador/v2/pkg/api/envoy/api/v2" 14 apiv2_core "github.com/datawire/ambassador/v2/pkg/api/envoy/api/v2/core" 15 apiv2_endpoint "github.com/datawire/ambassador/v2/pkg/api/envoy/api/v2/endpoint" 16 apiv2_listener "github.com/datawire/ambassador/v2/pkg/api/envoy/api/v2/listener" 17 apiv2_httpman "github.com/datawire/ambassador/v2/pkg/api/envoy/config/filter/network/http_connection_manager/v2" 18 19 // envoy api v3 20 apiv3_cluster "github.com/datawire/ambassador/v2/pkg/api/envoy/config/cluster/v3" 21 apiv3_core "github.com/datawire/ambassador/v2/pkg/api/envoy/config/core/v3" 22 apiv3_endpoint "github.com/datawire/ambassador/v2/pkg/api/envoy/config/endpoint/v3" 23 apiv3_listener "github.com/datawire/ambassador/v2/pkg/api/envoy/config/listener/v3" 24 apiv3_route "github.com/datawire/ambassador/v2/pkg/api/envoy/config/route/v3" 25 apiv3_httpman "github.com/datawire/ambassador/v2/pkg/api/envoy/extensions/filters/network/http_connection_manager/v3" 26 27 // envoy control plane 28 ecp_cache_types "github.com/datawire/ambassador/v2/pkg/envoy-control-plane/cache/types" 29 ecp_v2_resource "github.com/datawire/ambassador/v2/pkg/envoy-control-plane/resource/v2" 30 ecp_v3_resource "github.com/datawire/ambassador/v2/pkg/envoy-control-plane/resource/v3" 31 ecp_wellknown "github.com/datawire/ambassador/v2/pkg/envoy-control-plane/wellknown" 32 33 // first-party libraries 34 "github.com/datawire/dlib/dlog" 35 ) 36 37 // ListenerToRdsListener will take a listener definition and extract any inline RouteConfigurations 38 // replacing them with a reference to an RDS supplied route configuration. It does not modify the 39 // supplied listener, any configuration included in the result is copied from the input. 40 // 41 // If the input listener does not match the expected form it is simply copied, i.e. it is the 42 // identity transform for any inputs not matching the expected form. 43 // 44 // Example Input (that will get transformed in a non-identity way): 45 // 46 // - a listener configured with an http connection manager 47 // 48 // - that specifies an http router 49 // 50 // - that supplies its RouteConfiguration inline via the route_config field 51 // 52 // { 53 // "name": "...", 54 // ..., 55 // "filter_chains": [ 56 // { 57 // "filter_chain_match": {...}, 58 // "filters": [ 59 // { 60 // "name": "envoy.filters.network.http_connection_manager", 61 // "typed_config": { 62 // "@type": "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager", 63 // "http_filters": [...], 64 // "route_config": { 65 // "virtual_hosts": [ 66 // { 67 // "name": "ambassador-listener-8443-*", 68 // "domains": ["*"], 69 // "routes": [...], 70 // } 71 // ] 72 // } 73 // } 74 // } 75 // ] 76 // } 77 // ] 78 // } 79 // 80 // Example Output: 81 // 82 // - a duplicate listener that defines the "rds" field instead of the "route_config" field 83 // 84 // - and a list of route configurations 85 // 86 // - with route_config_name supplied in such a way as to correlate the two together 87 // 88 // lnr, routes, err := ListenerToRdsListener(...) 89 // 90 // lnr = { 91 // "name": "...", 92 // ..., 93 // "filter_chains": [ 94 // { 95 // "filter_chain_match": {...}, 96 // "filters": [ 97 // { 98 // "name": "envoy.filters.network.http_connection_manager", 99 // "typed_config": { 100 // "@type": "type.googleapis.com/envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager", 101 // "http_filters": [...], 102 // "rds": { 103 // "config_source": { 104 // "ads": {} 105 // }, 106 // "route_config_name": "ambassador-listener-8443-routeconfig-0" 107 // } 108 // } 109 // } 110 // ] 111 // } 112 // ] 113 // } 114 // 115 // routes = [ 116 // { 117 // "name": "ambassador-listener-8443-routeconfig-0", 118 // "virtual_hosts": [ 119 // { 120 // "name": "ambassador-listener-8443-*", 121 // "domains": ["*"], 122 // "routes": [...], 123 // } 124 // ] 125 // } 126 // ] 127 func ListenerToRdsListener(lnr *apiv2.Listener) (*apiv2.Listener, []*apiv2.RouteConfiguration, error) { 128 l := proto.Clone(lnr).(*apiv2.Listener) 129 var routes []*apiv2.RouteConfiguration 130 for _, fc := range l.FilterChains { 131 for _, f := range fc.Filters { 132 if f.Name != ecp_wellknown.HTTPConnectionManager { 133 // We only know how to create an rds listener for HttpConnectionManager 134 // listeners. We must ignore all other listeners. 135 continue 136 } 137 138 // Note that the hcm configuration is stored in a protobuf any, so √the 139 // GetHTTPConnectionManager is actually returning an unmarshalled copy. 140 hcm := ecp_v2_resource.GetHTTPConnectionManager(f) 141 if hcm != nil { 142 // RouteSpecifier is a protobuf oneof that corresponds to the rds, route_config, and 143 // scoped_routes fields. Only one of those may be set at a time. 144 rs, ok := hcm.RouteSpecifier.(*apiv2_httpman.HttpConnectionManager_RouteConfig) 145 if ok { 146 rc := rs.RouteConfig 147 if rc.Name == "" { 148 // Generate a unique name for the RouteConfiguration that we can use to 149 // correlate the listener to the RDS record. We use the listener name plus 150 // an index because there can be more than one route configuration 151 // associated with a given listener. 152 rc.Name = fmt.Sprintf("%s-routeconfig-%d", l.Name, len(routes)) 153 } 154 routes = append(routes, rc) 155 // Now that we have extracted and named the RouteConfiguration, we change the 156 // RouteSpecifier from the inline RouteConfig variation to RDS via ADS. This 157 // will cause it to use whatever ADS source is defined in the bootstrap 158 // configuration. 159 hcm.RouteSpecifier = &apiv2_httpman.HttpConnectionManager_Rds{ 160 Rds: &apiv2_httpman.Rds{ 161 ConfigSource: &apiv2_core.ConfigSource{ 162 ConfigSourceSpecifier: &apiv2_core.ConfigSource_Ads{ 163 Ads: &apiv2_core.AggregatedConfigSource{}, 164 }, 165 }, 166 RouteConfigName: rc.Name, 167 }, 168 } 169 } 170 171 // Because the hcm is a protobuf any, we need to remarshal it, we can't simply 172 // expect the above modifications to take effect on our clone of the input. There is 173 // also a protobuf oneof that includes the deprecated config and typed_config 174 // fields. 175 any, err := anypb.New(hcm) 176 if err != nil { 177 return nil, nil, err 178 } 179 f.ConfigType = &apiv2_listener.Filter_TypedConfig{TypedConfig: any} 180 } 181 } 182 } 183 184 return l, routes, nil 185 } 186 187 // V3ListenerToRdsListener is the v3 variety of ListnerToRdsListener 188 func V3ListenerToRdsListener(lnr *apiv3_listener.Listener) (*apiv3_listener.Listener, []*apiv3_route.RouteConfiguration, error) { 189 l := proto.Clone(lnr).(*apiv3_listener.Listener) 190 var routes []*apiv3_route.RouteConfiguration 191 for _, fc := range l.FilterChains { 192 for _, f := range fc.Filters { 193 if f.Name != ecp_wellknown.HTTPConnectionManager { 194 // We only know how to create an rds listener for HttpConnectionManager 195 // listeners. We must ignore all other listeners. 196 continue 197 } 198 199 // Note that the hcm configuration is stored in a protobuf any, so √the 200 // GetHTTPConnectionManager is actually returning an unmarshalled copy. 201 hcm := ecp_v3_resource.GetHTTPConnectionManager(f) 202 if hcm != nil { 203 // RouteSpecifier is a protobuf oneof that corresponds to the rds, route_config, and 204 // scoped_routes fields. Only one of those may be set at a time. 205 rs, ok := hcm.RouteSpecifier.(*apiv3_httpman.HttpConnectionManager_RouteConfig) 206 if ok { 207 rc := rs.RouteConfig 208 if rc.Name == "" { 209 // Generate a unique name for the RouteConfiguration that we can use to 210 // correlate the listener to the RDS record. We use the listener name plus 211 // an index because there can be more than one route configuration 212 // associated with a given listener. 213 rc.Name = fmt.Sprintf("%s-routeconfig-%d", l.Name, len(routes)) 214 } 215 routes = append(routes, rc) 216 // Now that we have extracted and named the RouteConfiguration, we change the 217 // RouteSpecifier from the inline RouteConfig variation to RDS via ADS. This 218 // will cause it to use whatever ADS source is defined in the bootstrap 219 // configuration. 220 hcm.RouteSpecifier = &apiv3_httpman.HttpConnectionManager_Rds{ 221 Rds: &apiv3_httpman.Rds{ 222 ConfigSource: &apiv3_core.ConfigSource{ 223 ConfigSourceSpecifier: &apiv3_core.ConfigSource_Ads{ 224 Ads: &apiv3_core.AggregatedConfigSource{}, 225 }, 226 ResourceApiVersion: apiv3_core.ApiVersion_V3, 227 }, 228 RouteConfigName: rc.Name, 229 }, 230 } 231 } 232 233 // Because the hcm is a protobuf any, we need to remarshal it, we can't simply 234 // expect the above modifications to take effect on our clone of the input. There is 235 // also a protobuf oneof that includes the deprecated config and typed_config 236 // fields. 237 any, err := anypb.New(hcm) 238 if err != nil { 239 return nil, nil, err 240 } 241 f.ConfigType = &apiv3_listener.Filter_TypedConfig{TypedConfig: any} 242 } 243 } 244 } 245 246 return l, routes, nil 247 } 248 249 // JoinEdsClusters will perform an outer join operation between the eds clusters in the supplied 250 // clusterlist and the eds endpoint data in the supplied map. It will return a slice of 251 // ClusterLoadAssignments (cast to []ecp_cache_types.Resource) with endpoint data for all the eds clusters in 252 // the supplied list. If there is no map entry for a given cluster, an empty ClusterLoadAssignment 253 // will be synthesized. The result is a set of endpoints that are consistent (by the 254 // go-control-plane's definition of consistent) with the input clusters. 255 func JoinEdsClusters(ctx context.Context, clusters []ecp_cache_types.Resource, edsEndpoints map[string]*apiv2.ClusterLoadAssignment, edsBypass bool) (endpoints []ecp_cache_types.Resource) { 256 for _, clu := range clusters { 257 c := clu.(*apiv2.Cluster) 258 // Don't mess with non EDS clusters. 259 if c.EdsClusterConfig == nil { 260 continue 261 } 262 263 // By default, envoy will use the cluster name to lookup ClusterLoadAssignments unless the 264 // ServiceName is supplied in the EdsClusterConfig. 265 ref := c.EdsClusterConfig.ServiceName 266 if ref == "" { 267 ref = c.Name 268 } 269 270 // This change was introduced as a stop gap solution to mitigate the 503 issues when certificates are rotated. 271 // The issue is CDS gets updated and waits for EDS to send ClusterLoadAssignment. 272 // During this wait period calls that are coming through get hit with a 503 since the cluster is in a warming state. 273 // The solution is to "hijack" the cluster and insert all the endpoints instead of relying on EDS. 274 // Now there will be a discrepancy between envoy/envoy.json and the config envoy. 275 if edsBypass { 276 c.EdsClusterConfig = nil 277 // Type 0 is STATIC 278 c.ClusterDiscoveryType = &apiv2.Cluster_Type{Type: 0} 279 280 if ep, ok := edsEndpoints[ref]; ok { 281 c.LoadAssignment = ep 282 } else { 283 c.LoadAssignment = &apiv2.ClusterLoadAssignment{ 284 ClusterName: ref, 285 Endpoints: []*apiv2_endpoint.LocalityLbEndpoints{}, 286 } 287 } 288 } else { 289 var source string 290 ep, ok := edsEndpoints[ref] 291 if ok { 292 source = "found" 293 } else { 294 ep = &apiv2.ClusterLoadAssignment{ 295 ClusterName: ref, 296 Endpoints: []*apiv2_endpoint.LocalityLbEndpoints{}, 297 } 298 source = "synthesized" 299 } 300 301 dlog.Debugf(ctx, "%s envoy v2 ClusterLoadAssignment for cluster %s: %v", source, c.Name, ep) 302 endpoints = append(endpoints, ep) 303 } 304 } 305 306 return 307 } 308 309 // JoinEdsClustersV3 will perform an outer join operation between the eds clusters in the supplied 310 // clusterlist and the eds endpoint data in the supplied map. It will return a slice of 311 // ClusterLoadAssignments (cast to []ecp_cache_types.Resource) with endpoint data for all the eds clusters in 312 // the supplied list. If there is no map entry for a given cluster, an empty ClusterLoadAssignment 313 // will be synthesized. The result is a set of endpoints that are consistent (by the 314 // go-control-plane's definition of consistent) with the input clusters. 315 func JoinEdsClustersV3(ctx context.Context, clusters []ecp_cache_types.Resource, edsEndpoints map[string]*apiv3_endpoint.ClusterLoadAssignment, edsBypass bool) (endpoints []ecp_cache_types.Resource) { 316 for _, clu := range clusters { 317 c := clu.(*apiv3_cluster.Cluster) 318 // Don't mess with non EDS clusters. 319 if c.EdsClusterConfig == nil { 320 continue 321 } 322 323 // By default, envoy will use the cluster name to lookup ClusterLoadAssignments unless the 324 // ServiceName is supplied in the EdsClusterConfig. 325 ref := c.EdsClusterConfig.ServiceName 326 if ref == "" { 327 ref = c.Name 328 } 329 330 // This change was introduced as a stop gap solution to mitigate the 503 issues when certificates are rotated. 331 // The issue is CDS gets updated and waits for EDS to send ClusterLoadAssignment. 332 // During this wait period calls that are coming through get hit with a 503 since the cluster is in a warming state. 333 // The solution is to "hijack" the cluster and insert all the endpoints instead of relying on EDS. 334 // Now there will be a discrepancy between envoy/envoy.json and the config envoy. 335 if edsBypass { 336 c.EdsClusterConfig = nil 337 // Type 0 is STATIC 338 c.ClusterDiscoveryType = &apiv3_cluster.Cluster_Type{Type: 0} 339 340 if ep, ok := edsEndpoints[ref]; ok { 341 c.LoadAssignment = ep 342 } else { 343 c.LoadAssignment = &apiv3_endpoint.ClusterLoadAssignment{ 344 ClusterName: ref, 345 Endpoints: []*apiv3_endpoint.LocalityLbEndpoints{}, 346 } 347 } 348 } else { 349 var source string 350 ep, ok := edsEndpoints[ref] 351 if ok { 352 source = "found" 353 } else { 354 ep = &apiv3_endpoint.ClusterLoadAssignment{ 355 ClusterName: ref, 356 Endpoints: []*apiv3_endpoint.LocalityLbEndpoints{}, 357 } 358 source = "synthesized" 359 } 360 361 dlog.Debugf(ctx, "%s envoy v3 ClusterLoadAssignment for cluster %s: %v", source, c.Name, ep) 362 endpoints = append(endpoints, ep) 363 } 364 365 } 366 367 return 368 } 369