package metrics import ( "fmt" "regexp" monitoringpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb" "google.golang.org/api/iterator" labelpb "google.golang.org/genproto/googleapis/api/label" "edge-infra.dev/pkg/lib/gcp/monitoring/monutil" ) const ( mdRegex = `projects/(?P[\S-]+)/metricDescriptors/(?P(custom|external|prometheus|workload)\.googleapis\.com)/(?P[\w_]+/[\w:]+)` mdPrefix = `(custom|external|prometheus|workload)\.googleapis\.com/([\w_]+/[\w:]+)` ) type Descriptor struct { Name string DisplayName string Labels []string `json:",omitempty"` MetricKind string ValueType string Type string Units string `json:",omitempty"` CommonLabels []string `json:",omitempty"` DiffLabels []string `json:",omitempty"` AllLabels []string `json:",omitempty"` } type Descriptors struct { Descriptor []Descriptor } type Metrics []Metric type Projects []Project type Metric struct { Type string Labels []string `json:",omitempty"` *Projects `json:",omitempty"` DiffLabels []string `json:",omitempty"` CommonLabels []string `json:",omitempty"` } type Project struct { ID string Labels []string `json:",omitempty"` DiffLabels []string `json:",omitempty"` *Metrics `json:",omitempty"` } type descriptorLabelMap map[string]labelMap type labelMap map[string][]string type AlignReport struct { Misaligned int Attempted int Completed int Aligned []string `json:",omitempty"` Skipped []string `json:",omitempty"` Failed []string `json:",omitempty"` } type DeleteReport struct { Attempted int Completed int Deleted []string `json:",omitempty"` Skipped []string `json:",omitempty"` Failed []string `json:",omitempty"` } type AlignDescriptor struct { Name string CommonLabels []string `json:",omitempty"` DiffLabels []string `json:",omitempty"` } // delete metric descriptor for project func (c *Client) DeleteDescriptor(prefix string, metricName string) error { req := &monitoringpb.DeleteMetricDescriptorRequest{ Name: fmt.Sprintf("projects/%s/metricDescriptors/%s/%s", c.ProjectID, prefix, metricName), } return c.client.DeleteMetricDescriptor(c.ctx, req) } // get metric descriptor for project func (c *Client) GetDescriptor(prefix string, metricName string) (*Descriptor, error) { req := &monitoringpb.GetMetricDescriptorRequest{ Name: fmt.Sprintf("projects/%s/metricDescriptors/%s/%s", c.ProjectID, prefix, metricName), } resp, err := c.client.GetMetricDescriptor(c.ctx, req) if err != nil { return &Descriptor{}, err } return &Descriptor{ Name: resp.Name, DisplayName: resp.DisplayName, Labels: getLabelNames(resp.Labels), MetricKind: resp.MetricKind.String(), ValueType: resp.ValueType.String(), Type: resp.Type, }, nil } // queries the monitoring api to retrieve a list of matching metric descriptors func (c *Client) ListDescriptors(url string, metricName string) (*Descriptors, error) { req := &monitoringpb.ListMetricDescriptorsRequest{ Name: fmt.Sprintf("projects/%s", c.ProjectID), Filter: fmt.Sprintf("metric.type = starts_with(\"%s/%s\")", url, metricName), } var metricDescriptors Descriptors iter := c.client.ListMetricDescriptors(c.ctx, req) for { resp, err := iter.Next() if err == iterator.Done { break } if err != nil { return nil, err } metricDescriptors.Descriptor = append(metricDescriptors.Descriptor, Descriptor{ Name: resp.Name, DisplayName: resp.DisplayName, Labels: getLabelNames(resp.Labels), MetricKind: resp.MetricKind.String(), ValueType: resp.ValueType.String(), Type: resp.Type, }) } return &metricDescriptors, nil } // returns a list of all labels found for each unique metric descriptor func (ds *Descriptors) allLabels() labelMap { descLabels := *monutil.MapSlice(ds.metricNames()) for i := 0; i < len(ds.Descriptor); i++ { mName := ParseMetricName(ds.Descriptor[i].Name) if diff := monutil.DiffStrSlice(descLabels[mName], ds.Descriptor[i].Labels); len(diff) > 0 { descLabels[mName] = append(descLabels[mName], diff...) } } return descLabels } // returns a list of all labels found for each unique metric descriptor func (ds *Descriptors) commonLabels() labelMap { dLabels := ds.allLabels() for i := 0; i < len(ds.Descriptor); i++ { mName := ParseMetricName(ds.Descriptor[i].Name) if diff := monutil.DiffStrSlice(ds.Descriptor[i].Labels, dLabels[mName]); len(diff) > 0 { dLabels[mName] = monutil.RemoveStrSlice(dLabels[mName], diff) } } return dLabels } // sorts metricDescriptors by (project|metric|label) func sorter(ds *Descriptors, method string) descriptorLabelMap { sDesc := make(descriptorLabelMap) for i := 0; i < len(ds.Descriptor); i++ { var val, subVal string p := ParseProjectID(ds.Descriptor[i].Name) m := ParseMetricName(ds.Descriptor[i].Name) switch method { case "project": val = p subVal = m case "metric": val = m subVal = p case "label": // TBD - not supported yet - coming soon return sDesc } if sDesc[val] == nil { sDesc[val] = make(labelMap) } sDesc[val][subVal] = ds.Descriptor[i].Labels } return sDesc } // returns a sorted by project descriptorLabelMap func (ds *Descriptors) byProject() descriptorLabelMap { pList := sorter(ds, "project") return pList } // returns a sorted by metric type descriptorLabelMap func (ds *Descriptors) byMetric() descriptorLabelMap { mList := sorter(ds, "metric") return mList } // returns a list of unique Types from the list of metricDescriptor objects func (ds *Descriptors) metricNames() []string { var mNames []string for i := 0; i < len(ds.Descriptor); i++ { mName := ParseMetricName(ds.Descriptor[i].Name) if mName != "" && !monutil.InStrList(mName, mNames) { mNames = append(mNames, mName) } } return mNames } // returns a list of unique ProjectIDs from the list of metricDescriptor objects func (ds *Descriptors) getProjectIDs() []string { var projectIDs []string for i := 0; i < len(ds.Descriptor); i++ { pID := ParseProjectID(ds.Descriptor[i].Name) if pID != "" && !monutil.InStrList(pID, projectIDs) { projectIDs = append(projectIDs, pID) } } return projectIDs } // merges descriptor lists into a single object func AppendDescriptors(d1 []Descriptor, d2 []Descriptor) (Descriptors, error) { desc := append(d1, d2...) if len(desc) == len(d1)+len(d2) { return Descriptors{desc}, nil } return Descriptors{}, fmt.Errorf("AppendDescriptors: unable to merge descriptor lists") } // returns a slice of metrics sorted descriptors func (ds *Descriptors) ListMetrics() Metrics { dlm := ds.byMetric() var mList Metrics for k, v := range dlm { mList = append(mList, Metric{ Type: k, Projects: v.Projects(), }) } return mList } // returns a slice of projects sorted descriptors func (ds *Descriptors) ListProjects() Projects { dlm := ds.byProject() var pList Projects for k, v := range dlm { pList = append(pList, Project{ ID: k, Metrics: v.Metrics(), }) } return pList } // returns a slice of compared descriptors func (ds *Descriptors) Compare() Descriptors { var dList []Descriptor cLabels := ds.commonLabels() aLabels := ds.allLabels() for i := 0; i < len(ds.Descriptor); i++ { d := ds.Descriptor[i] mName := ParseMetricName(d.Name) dLabels := monutil.RemoveStrSlice(aLabels[mName], d.Labels) if len(dLabels) == 0 { continue } dList = append(dList, Descriptor{ Name: d.Name, DisplayName: d.DisplayName, Labels: d.Labels, MetricKind: d.MetricKind, Type: d.Type, Units: d.Units, DiffLabels: dLabels, CommonLabels: cLabels[mName], AllLabels: aLabels[mName], }) } return Descriptors{dList} } // returns a slice of compared descriptors sorted by metrics func (ds *Descriptors) CompareMetrics() Metrics { dlm := ds.byMetric() var mList Metrics cLabels := ds.commonLabels() for k, v := range dlm { dLabels := monutil.RemoveStrSlice(ds.allLabels()[k], cLabels[k]) if len(dLabels) == 0 { continue } mList = append(mList, Metric{ Type: k, DiffLabels: dLabels, CommonLabels: cLabels[k], Projects: v.Projects().compare(cLabels[k]), }) } return mList } // returns a slice of compared descriptors sorted by project func (ds *Descriptors) CompareProjects() Projects { dlm := ds.byProject() var pList Projects for k, v := range dlm { pList = append(pList, Project{ ID: k, Metrics: v.Metrics().compare(ds.commonLabels()), }) } return pList } // returns metrics slice with compared descriptor label values func (ms Metrics) compare(cLabels labelMap) *Metrics { for i := 0; i < len(ms); i++ { diff := monutil.DiffStrSlice(cLabels[ms[i].Type], ms[i].Labels) if len(diff) == 0 { continue } ms[i].DiffLabels = diff ms[i].CommonLabels = cLabels[ms[i].Type] ms[i].Labels = nil } return &ms } // returns projects slice with compared descriptor label values func (ps Projects) compare(cLabels []string) *Projects { for i := 0; i < len(ps); i++ { diff := monutil.DiffStrSlice(cLabels, ps[i].Labels) ps[i].DiffLabels = diff } return &ps } // returns a slice of Project from a labelMap func (lm labelMap) Projects() *Projects { var projects Projects for k, v := range lm { projects = append(projects, Project{ ID: k, Labels: v, }) } return &projects } // returns a slice of Metric from a labelMap func (lm labelMap) Metrics() *Metrics { var metrics Metrics for k, v := range lm { metrics = append(metrics, Metric{ Type: k, Labels: v, }) } return &metrics } // removes any descriptors from the list that do not match one of the provided project ID func (ds Descriptors) FilterProjects(project []string) (*Descriptors, error) { var filtered Descriptors if !monutil.InSlice(ds.getProjectIDs(), project) { return nil, fmt.Errorf("unable to locate any of the provided projects in descriptors list") } for i := 0; i < len(ds.Descriptor); i++ { if monutil.InStrList(ParseProjectID(ds.Descriptor[i].Name), project) { filtered.Descriptor = append(filtered.Descriptor, ds.Descriptor[i]) } } return &filtered, nil } // removes any descriptors from the list that do not match one of the provided metric type func (ds Descriptors) FilterMetrics(metric []string) (*Descriptors, error) { var filtered Descriptors if !monutil.InSlice(ds.metricNames(), metric) { return nil, fmt.Errorf("unable to locate any of the provided metric types in descriptors list") } for i := 0; i < len(ds.Descriptor); i++ { if monutil.InStrList(ParseMetricName(ds.Descriptor[i].Name), metric) { filtered.Descriptor = append(filtered.Descriptor, ds.Descriptor[i]) } } return &filtered, nil } // removes any descriptors from the list that do not have one of the specified labels func (ds Descriptors) FilterLabels(labels []string) (*Descriptors, error) { var filtered Descriptors for i := 0; i < len(ds.Descriptor); i++ { if monutil.InSlice(ds.Descriptor[i].Labels, labels) { filtered.Descriptor = append(filtered.Descriptor, ds.Descriptor[i]) } } if len(filtered.Descriptor) == 0 { return nil, fmt.Errorf("unable to locate any of the provided labels names in descriptors list") } return &filtered, nil } // returns the list of label key names from a label object func getLabelNames(ld []*labelpb.LabelDescriptor) []string { var labels []string for i := 0; i < len(ld); i++ { labels = append(labels, ld[i].Key) } return labels } // returns true if the provided string contains a prefix func HasMetricPrefix(m string) bool { r := regexp.MustCompile(mdPrefix) return r.MatchString(m) } // returns the project ID from a descriptor func ParseProjectID(n string) string { r := regexp.MustCompile(mdRegex) match := monutil.RegexNamedMatches(r, n) return match["project_id"] } func ParseMetricName(n string) string { r := regexp.MustCompile(mdRegex) match := monutil.RegexNamedMatches(r, n) return match["metric_name"] } func ParsePrefix(n string) string { r := regexp.MustCompile(mdRegex) match := monutil.RegexNamedMatches(r, n) return match["prefix"] }