descriptors.go

Documentation: edge-infra.dev/pkg/lib/gcp/monitoring/metrics

     1  package metrics
     2  
     3  import (
     4  	"fmt"
     5  	"regexp"
     6  
     7  	monitoringpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
     8  	"google.golang.org/api/iterator"
     9  	labelpb "google.golang.org/genproto/googleapis/api/label"
    10  
    11  	"edge-infra.dev/pkg/lib/gcp/monitoring/monutil"
    12  )
    13  
    14  const (
    15  	mdRegex  = `projects/(?P<project_id>[\S-]+)/metricDescriptors/(?P<prefix>(custom|external|prometheus|workload)\.googleapis\.com)/(?P<metric_name>[\w_]+/[\w:]+)`
    16  	mdPrefix = `(custom|external|prometheus|workload)\.googleapis\.com/([\w_]+/[\w:]+)`
    17  )
    18  
    19  type Descriptor struct {
    20  	Name         string
    21  	DisplayName  string
    22  	Labels       []string `json:",omitempty"`
    23  	MetricKind   string
    24  	ValueType    string
    25  	Type         string
    26  	Units        string   `json:",omitempty"`
    27  	CommonLabels []string `json:",omitempty"`
    28  	DiffLabels   []string `json:",omitempty"`
    29  	AllLabels    []string `json:",omitempty"`
    30  }
    31  
    32  type Descriptors struct {
    33  	Descriptor []Descriptor
    34  }
    35  
    36  type Metrics []Metric
    37  type Projects []Project
    38  
    39  type Metric struct {
    40  	Type         string
    41  	Labels       []string `json:",omitempty"`
    42  	*Projects    `json:",omitempty"`
    43  	DiffLabels   []string `json:",omitempty"`
    44  	CommonLabels []string `json:",omitempty"`
    45  }
    46  type Project struct {
    47  	ID         string
    48  	Labels     []string `json:",omitempty"`
    49  	DiffLabels []string `json:",omitempty"`
    50  	*Metrics   `json:",omitempty"`
    51  }
    52  
    53  type descriptorLabelMap map[string]labelMap
    54  type labelMap map[string][]string
    55  
    56  type AlignReport struct {
    57  	Misaligned int
    58  	Attempted  int
    59  	Completed  int
    60  	Aligned    []string `json:",omitempty"`
    61  	Skipped    []string `json:",omitempty"`
    62  	Failed     []string `json:",omitempty"`
    63  }
    64  
    65  type DeleteReport struct {
    66  	Attempted int
    67  	Completed int
    68  	Deleted   []string `json:",omitempty"`
    69  	Skipped   []string `json:",omitempty"`
    70  	Failed    []string `json:",omitempty"`
    71  }
    72  
    73  type AlignDescriptor struct {
    74  	Name         string
    75  	CommonLabels []string `json:",omitempty"`
    76  	DiffLabels   []string `json:",omitempty"`
    77  }
    78  
    79  // delete metric descriptor for project
    80  func (c *Client) DeleteDescriptor(prefix string, metricName string) error {
    81  	req := &monitoringpb.DeleteMetricDescriptorRequest{
    82  		Name: fmt.Sprintf("projects/%s/metricDescriptors/%s/%s", c.ProjectID, prefix, metricName),
    83  	}
    84  
    85  	return c.client.DeleteMetricDescriptor(c.ctx, req)
    86  }
    87  
    88  // get metric descriptor for project
    89  func (c *Client) GetDescriptor(prefix string, metricName string) (*Descriptor, error) {
    90  	req := &monitoringpb.GetMetricDescriptorRequest{
    91  		Name: fmt.Sprintf("projects/%s/metricDescriptors/%s/%s", c.ProjectID, prefix, metricName),
    92  	}
    93  	resp, err := c.client.GetMetricDescriptor(c.ctx, req)
    94  	if err != nil {
    95  		return &Descriptor{}, err
    96  	}
    97  
    98  	return &Descriptor{
    99  		Name:        resp.Name,
   100  		DisplayName: resp.DisplayName,
   101  		Labels:      getLabelNames(resp.Labels),
   102  		MetricKind:  resp.MetricKind.String(),
   103  		ValueType:   resp.ValueType.String(),
   104  		Type:        resp.Type,
   105  	}, nil
   106  }
   107  
   108  // queries the monitoring api to retrieve a list of matching metric descriptors
   109  func (c *Client) ListDescriptors(url string, metricName string) (*Descriptors, error) {
   110  	req := &monitoringpb.ListMetricDescriptorsRequest{
   111  		Name:   fmt.Sprintf("projects/%s", c.ProjectID),
   112  		Filter: fmt.Sprintf("metric.type = starts_with(\"%s/%s\")", url, metricName),
   113  	}
   114  	var metricDescriptors Descriptors
   115  
   116  	iter := c.client.ListMetricDescriptors(c.ctx, req)
   117  	for {
   118  		resp, err := iter.Next()
   119  		if err == iterator.Done {
   120  			break
   121  		}
   122  		if err != nil {
   123  			return nil, err
   124  		}
   125  
   126  		metricDescriptors.Descriptor = append(metricDescriptors.Descriptor,
   127  			Descriptor{
   128  				Name:        resp.Name,
   129  				DisplayName: resp.DisplayName,
   130  				Labels:      getLabelNames(resp.Labels),
   131  				MetricKind:  resp.MetricKind.String(),
   132  				ValueType:   resp.ValueType.String(),
   133  				Type:        resp.Type,
   134  			})
   135  	}
   136  	return &metricDescriptors, nil
   137  }
   138  
   139  // returns a list of all labels found for each unique metric descriptor
   140  func (ds *Descriptors) allLabels() labelMap {
   141  	descLabels := *monutil.MapSlice(ds.metricNames())
   142  
   143  	for i := 0; i < len(ds.Descriptor); i++ {
   144  		mName := ParseMetricName(ds.Descriptor[i].Name)
   145  		if diff := monutil.DiffStrSlice(descLabels[mName], ds.Descriptor[i].Labels); len(diff) > 0 {
   146  			descLabels[mName] = append(descLabels[mName], diff...)
   147  		}
   148  	}
   149  	return descLabels
   150  }
   151  
   152  // returns a list of all labels found for each unique metric descriptor
   153  func (ds *Descriptors) commonLabels() labelMap {
   154  	dLabels := ds.allLabels()
   155  
   156  	for i := 0; i < len(ds.Descriptor); i++ {
   157  		mName := ParseMetricName(ds.Descriptor[i].Name)
   158  		if diff := monutil.DiffStrSlice(ds.Descriptor[i].Labels, dLabels[mName]); len(diff) > 0 {
   159  			dLabels[mName] = monutil.RemoveStrSlice(dLabels[mName], diff)
   160  		}
   161  	}
   162  	return dLabels
   163  }
   164  
   165  // sorts metricDescriptors by (project|metric|label)
   166  func sorter(ds *Descriptors, method string) descriptorLabelMap {
   167  	sDesc := make(descriptorLabelMap)
   168  
   169  	for i := 0; i < len(ds.Descriptor); i++ {
   170  		var val, subVal string
   171  		p := ParseProjectID(ds.Descriptor[i].Name)
   172  		m := ParseMetricName(ds.Descriptor[i].Name)
   173  		switch method {
   174  		case "project":
   175  			val = p
   176  			subVal = m
   177  		case "metric":
   178  			val = m
   179  			subVal = p
   180  		case "label":
   181  			// TBD - not supported yet - coming soon
   182  			return sDesc
   183  		}
   184  
   185  		if sDesc[val] == nil {
   186  			sDesc[val] = make(labelMap)
   187  		}
   188  		sDesc[val][subVal] = ds.Descriptor[i].Labels
   189  	}
   190  	return sDesc
   191  }
   192  
   193  // returns a sorted by project descriptorLabelMap
   194  func (ds *Descriptors) byProject() descriptorLabelMap {
   195  	pList := sorter(ds, "project")
   196  	return pList
   197  }
   198  
   199  // returns a sorted by metric type descriptorLabelMap
   200  func (ds *Descriptors) byMetric() descriptorLabelMap {
   201  	mList := sorter(ds, "metric")
   202  	return mList
   203  }
   204  
   205  // returns a list of unique Types from the list of metricDescriptor objects
   206  func (ds *Descriptors) metricNames() []string {
   207  	var mNames []string
   208  
   209  	for i := 0; i < len(ds.Descriptor); i++ {
   210  		mName := ParseMetricName(ds.Descriptor[i].Name)
   211  		if mName != "" && !monutil.InStrList(mName, mNames) {
   212  			mNames = append(mNames, mName)
   213  		}
   214  	}
   215  	return mNames
   216  }
   217  
   218  // returns a list of unique ProjectIDs from the list of metricDescriptor objects
   219  func (ds *Descriptors) getProjectIDs() []string {
   220  	var projectIDs []string
   221  
   222  	for i := 0; i < len(ds.Descriptor); i++ {
   223  		pID := ParseProjectID(ds.Descriptor[i].Name)
   224  		if pID != "" && !monutil.InStrList(pID, projectIDs) {
   225  			projectIDs = append(projectIDs, pID)
   226  		}
   227  	}
   228  	return projectIDs
   229  }
   230  
   231  // merges descriptor lists into a single object
   232  func AppendDescriptors(d1 []Descriptor, d2 []Descriptor) (Descriptors, error) {
   233  	desc := append(d1, d2...)
   234  	if len(desc) == len(d1)+len(d2) {
   235  		return Descriptors{desc}, nil
   236  	}
   237  	return Descriptors{}, fmt.Errorf("AppendDescriptors: unable to merge descriptor lists")
   238  }
   239  
   240  // returns a slice of metrics sorted descriptors
   241  func (ds *Descriptors) ListMetrics() Metrics {
   242  	dlm := ds.byMetric()
   243  	var mList Metrics
   244  
   245  	for k, v := range dlm {
   246  		mList = append(mList, Metric{
   247  			Type:     k,
   248  			Projects: v.Projects(),
   249  		})
   250  	}
   251  	return mList
   252  }
   253  
   254  // returns a slice of projects sorted descriptors
   255  func (ds *Descriptors) ListProjects() Projects {
   256  	dlm := ds.byProject()
   257  	var pList Projects
   258  
   259  	for k, v := range dlm {
   260  		pList = append(pList, Project{
   261  			ID:      k,
   262  			Metrics: v.Metrics(),
   263  		})
   264  	}
   265  	return pList
   266  }
   267  
   268  // returns a slice of compared descriptors
   269  func (ds *Descriptors) Compare() Descriptors {
   270  	var dList []Descriptor
   271  	cLabels := ds.commonLabels()
   272  	aLabels := ds.allLabels()
   273  
   274  	for i := 0; i < len(ds.Descriptor); i++ {
   275  		d := ds.Descriptor[i]
   276  		mName := ParseMetricName(d.Name)
   277  		dLabels := monutil.RemoveStrSlice(aLabels[mName], d.Labels)
   278  
   279  		if len(dLabels) == 0 {
   280  			continue
   281  		}
   282  
   283  		dList = append(dList, Descriptor{
   284  			Name:         d.Name,
   285  			DisplayName:  d.DisplayName,
   286  			Labels:       d.Labels,
   287  			MetricKind:   d.MetricKind,
   288  			Type:         d.Type,
   289  			Units:        d.Units,
   290  			DiffLabels:   dLabels,
   291  			CommonLabels: cLabels[mName],
   292  			AllLabels:    aLabels[mName],
   293  		})
   294  	}
   295  	return Descriptors{dList}
   296  }
   297  
   298  // returns a slice of compared descriptors sorted by metrics
   299  func (ds *Descriptors) CompareMetrics() Metrics {
   300  	dlm := ds.byMetric()
   301  	var mList Metrics
   302  	cLabels := ds.commonLabels()
   303  
   304  	for k, v := range dlm {
   305  		dLabels := monutil.RemoveStrSlice(ds.allLabels()[k], cLabels[k])
   306  		if len(dLabels) == 0 {
   307  			continue
   308  		}
   309  
   310  		mList = append(mList, Metric{
   311  			Type:         k,
   312  			DiffLabels:   dLabels,
   313  			CommonLabels: cLabels[k],
   314  			Projects:     v.Projects().compare(cLabels[k]),
   315  		})
   316  	}
   317  	return mList
   318  }
   319  
   320  // returns a slice of compared descriptors sorted by project
   321  func (ds *Descriptors) CompareProjects() Projects {
   322  	dlm := ds.byProject()
   323  	var pList Projects
   324  
   325  	for k, v := range dlm {
   326  		pList = append(pList, Project{
   327  			ID:      k,
   328  			Metrics: v.Metrics().compare(ds.commonLabels()),
   329  		})
   330  	}
   331  	return pList
   332  }
   333  
   334  // returns metrics slice with compared descriptor label values
   335  func (ms Metrics) compare(cLabels labelMap) *Metrics {
   336  	for i := 0; i < len(ms); i++ {
   337  		diff := monutil.DiffStrSlice(cLabels[ms[i].Type], ms[i].Labels)
   338  		if len(diff) == 0 {
   339  			continue
   340  		}
   341  		ms[i].DiffLabels = diff
   342  		ms[i].CommonLabels = cLabels[ms[i].Type]
   343  		ms[i].Labels = nil
   344  	}
   345  	return &ms
   346  }
   347  
   348  // returns projects slice with compared descriptor label values
   349  func (ps Projects) compare(cLabels []string) *Projects {
   350  	for i := 0; i < len(ps); i++ {
   351  		diff := monutil.DiffStrSlice(cLabels, ps[i].Labels)
   352  		ps[i].DiffLabels = diff
   353  	}
   354  	return &ps
   355  }
   356  
   357  // returns a slice of Project from a labelMap
   358  func (lm labelMap) Projects() *Projects {
   359  	var projects Projects
   360  	for k, v := range lm {
   361  		projects = append(projects, Project{
   362  			ID:     k,
   363  			Labels: v,
   364  		})
   365  	}
   366  	return &projects
   367  }
   368  
   369  // returns a slice of Metric from a labelMap
   370  func (lm labelMap) Metrics() *Metrics {
   371  	var metrics Metrics
   372  	for k, v := range lm {
   373  		metrics = append(metrics, Metric{
   374  			Type:   k,
   375  			Labels: v,
   376  		})
   377  	}
   378  	return &metrics
   379  }
   380  
   381  // removes any descriptors from the list that do not match one of the provided project ID
   382  func (ds Descriptors) FilterProjects(project []string) (*Descriptors, error) {
   383  	var filtered Descriptors
   384  
   385  	if !monutil.InSlice(ds.getProjectIDs(), project) {
   386  		return nil, fmt.Errorf("unable to locate any of the provided projects in descriptors list")
   387  	}
   388  
   389  	for i := 0; i < len(ds.Descriptor); i++ {
   390  		if monutil.InStrList(ParseProjectID(ds.Descriptor[i].Name), project) {
   391  			filtered.Descriptor = append(filtered.Descriptor, ds.Descriptor[i])
   392  		}
   393  	}
   394  	return &filtered, nil
   395  }
   396  
   397  // removes any descriptors from the list that do not match one of the provided metric type
   398  func (ds Descriptors) FilterMetrics(metric []string) (*Descriptors, error) {
   399  	var filtered Descriptors
   400  
   401  	if !monutil.InSlice(ds.metricNames(), metric) {
   402  		return nil, fmt.Errorf("unable to locate any of the provided metric types in descriptors list")
   403  	}
   404  
   405  	for i := 0; i < len(ds.Descriptor); i++ {
   406  		if monutil.InStrList(ParseMetricName(ds.Descriptor[i].Name), metric) {
   407  			filtered.Descriptor = append(filtered.Descriptor, ds.Descriptor[i])
   408  		}
   409  	}
   410  	return &filtered, nil
   411  }
   412  
   413  // removes any descriptors from the list that do not have one of the specified labels
   414  func (ds Descriptors) FilterLabels(labels []string) (*Descriptors, error) {
   415  	var filtered Descriptors
   416  
   417  	for i := 0; i < len(ds.Descriptor); i++ {
   418  		if monutil.InSlice(ds.Descriptor[i].Labels, labels) {
   419  			filtered.Descriptor = append(filtered.Descriptor, ds.Descriptor[i])
   420  		}
   421  	}
   422  	if len(filtered.Descriptor) == 0 {
   423  		return nil, fmt.Errorf("unable to locate any of the provided labels names in descriptors list")
   424  	}
   425  
   426  	return &filtered, nil
   427  }
   428  
   429  // returns the list of label key names from a label object
   430  func getLabelNames(ld []*labelpb.LabelDescriptor) []string {
   431  	var labels []string
   432  	for i := 0; i < len(ld); i++ {
   433  		labels = append(labels, ld[i].Key)
   434  	}
   435  	return labels
   436  }
   437  
   438  // returns true if the provided string contains a prefix
   439  func HasMetricPrefix(m string) bool {
   440  	r := regexp.MustCompile(mdPrefix)
   441  	return r.MatchString(m)
   442  }
   443  
   444  // returns the project ID from a descriptor
   445  func ParseProjectID(n string) string {
   446  	r := regexp.MustCompile(mdRegex)
   447  	match := monutil.RegexNamedMatches(r, n)
   448  	return match["project_id"]
   449  }
   450  
   451  func ParseMetricName(n string) string {
   452  	r := regexp.MustCompile(mdRegex)
   453  	match := monutil.RegexNamedMatches(r, n)
   454  	return match["metric_name"]
   455  }
   456  
   457  func ParsePrefix(n string) string {
   458  	r := regexp.MustCompile(mdRegex)
   459  	match := monutil.RegexNamedMatches(r, n)
   460  	return match["prefix"]
   461  }
   462
View as plain text