text_parse_test.go

Documentation: github.com/prometheus/common/expfmt

     1  // Copyright 2014 The Prometheus Authors
     2  // Licensed under the Apache License, Version 2.0 (the "License");
     3  // you may not use this file except in compliance with the License.
     4  // You may obtain a copy of the License at
     5  //
     6  // http://www.apache.org/licenses/LICENSE-2.0
     7  //
     8  // Unless required by applicable law or agreed to in writing, software
     9  // distributed under the License is distributed on an "AS IS" BASIS,
    10  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package expfmt
    15  
    16  import (
    17  	"errors"
    18  	"math"
    19  	"strings"
    20  	"testing"
    21  
    22  	dto "github.com/prometheus/client_model/go"
    23  	"google.golang.org/protobuf/proto"
    24  )
    25  
    26  func testTextParse(t testing.TB) {
    27  	scenarios := []struct {
    28  		in  string
    29  		out []*dto.MetricFamily
    30  	}{
    31  		// 0: Empty lines as input.
    32  		{
    33  			in: `
    34  
    35  `,
    36  			out: []*dto.MetricFamily{},
    37  		},
    38  		// 1: Minimal case.
    39  		{
    40  			in: `
    41  minimal_metric 1.234
    42  another_metric -3e3 103948
    43  # Even that:
    44  no_labels{} 3
    45  # HELP line for non-existing metric will be ignored.
    46  `,
    47  			out: []*dto.MetricFamily{
    48  				{
    49  					Name: proto.String("minimal_metric"),
    50  					Type: dto.MetricType_UNTYPED.Enum(),
    51  					Metric: []*dto.Metric{
    52  						{
    53  							Untyped: &dto.Untyped{
    54  								Value: proto.Float64(1.234),
    55  							},
    56  						},
    57  					},
    58  				},
    59  				{
    60  					Name: proto.String("another_metric"),
    61  					Type: dto.MetricType_UNTYPED.Enum(),
    62  					Metric: []*dto.Metric{
    63  						{
    64  							Untyped: &dto.Untyped{
    65  								Value: proto.Float64(-3e3),
    66  							},
    67  							TimestampMs: proto.Int64(103948),
    68  						},
    69  					},
    70  				},
    71  				{
    72  					Name: proto.String("no_labels"),
    73  					Type: dto.MetricType_UNTYPED.Enum(),
    74  					Metric: []*dto.Metric{
    75  						{
    76  							Untyped: &dto.Untyped{
    77  								Value: proto.Float64(3),
    78  							},
    79  						},
    80  					},
    81  				},
    82  			},
    83  		},
    84  		// 2: Counters & gauges, docstrings, various whitespace, escape sequences.
    85  		{
    86  			in: `
    87  # A normal comment.
    88  #
    89  # TYPE name counter
    90  name{labelname="val1",basename="basevalue"} NaN
    91  name {labelname="val2",basename="base\"v\\al\nue"} 0.23 1234567890
    92  # HELP name two-line\n doc  str\\ing
    93  
    94   # HELP  name2  	doc str"ing 2
    95    #    TYPE    name2 gauge
    96  name2{labelname="val2"	,basename   =   "basevalue2"		} +Inf 54321
    97  name2{ labelname = "val1" , }-Inf
    98  `,
    99  			out: []*dto.MetricFamily{
   100  				{
   101  					Name: proto.String("name"),
   102  					Help: proto.String("two-line\n doc  str\\ing"),
   103  					Type: dto.MetricType_COUNTER.Enum(),
   104  					Metric: []*dto.Metric{
   105  						{
   106  							Label: []*dto.LabelPair{
   107  								{
   108  									Name:  proto.String("labelname"),
   109  									Value: proto.String("val1"),
   110  								},
   111  								{
   112  									Name:  proto.String("basename"),
   113  									Value: proto.String("basevalue"),
   114  								},
   115  							},
   116  							Counter: &dto.Counter{
   117  								Value: proto.Float64(math.NaN()),
   118  							},
   119  						},
   120  						{
   121  							Label: []*dto.LabelPair{
   122  								{
   123  									Name:  proto.String("labelname"),
   124  									Value: proto.String("val2"),
   125  								},
   126  								{
   127  									Name:  proto.String("basename"),
   128  									Value: proto.String("base\"v\\al\nue"),
   129  								},
   130  							},
   131  							Counter: &dto.Counter{
   132  								Value: proto.Float64(.23),
   133  							},
   134  							TimestampMs: proto.Int64(1234567890),
   135  						},
   136  					},
   137  				},
   138  				{
   139  					Name: proto.String("name2"),
   140  					Help: proto.String("doc str\"ing 2"),
   141  					Type: dto.MetricType_GAUGE.Enum(),
   142  					Metric: []*dto.Metric{
   143  						{
   144  							Label: []*dto.LabelPair{
   145  								{
   146  									Name:  proto.String("labelname"),
   147  									Value: proto.String("val2"),
   148  								},
   149  								{
   150  									Name:  proto.String("basename"),
   151  									Value: proto.String("basevalue2"),
   152  								},
   153  							},
   154  							Gauge: &dto.Gauge{
   155  								Value: proto.Float64(math.Inf(+1)),
   156  							},
   157  							TimestampMs: proto.Int64(54321),
   158  						},
   159  						{
   160  							Label: []*dto.LabelPair{
   161  								{
   162  									Name:  proto.String("labelname"),
   163  									Value: proto.String("val1"),
   164  								},
   165  							},
   166  							Gauge: &dto.Gauge{
   167  								Value: proto.Float64(math.Inf(-1)),
   168  							},
   169  						},
   170  					},
   171  				},
   172  			},
   173  		},
   174  		// 3: The evil summary, mixed with other types and funny comments.
   175  		{
   176  			in: `
   177  # TYPE my_summary summary
   178  my_summary{n1="val1",quantile="0.5"} 110
   179  decoy -1 -2
   180  my_summary{n1="val1",quantile="0.9"} 140 1
   181  my_summary_count{n1="val1"} 42
   182  # Latest timestamp wins in case of a summary.
   183  my_summary_sum{n1="val1"} 4711 2
   184  fake_sum{n1="val1"} 2001
   185  # TYPE another_summary summary
   186  another_summary_count{n2="val2",n1="val1"} 20
   187  my_summary_count{n2="val2",n1="val1"} 5 5
   188  another_summary{n1="val1",n2="val2",quantile=".3"} -1.2
   189  my_summary_sum{n1="val2"} 08 15
   190  my_summary{n1="val3", quantile="0.2"} 4711
   191    my_summary{n1="val1",n2="val2",quantile="-12.34",} NaN
   192  # some
   193  # funny comments
   194  # HELP 
   195  # HELP
   196  # HELP my_summary
   197  # HELP my_summary 
   198  `,
   199  			out: []*dto.MetricFamily{
   200  				{
   201  					Name: proto.String("fake_sum"),
   202  					Type: dto.MetricType_UNTYPED.Enum(),
   203  					Metric: []*dto.Metric{
   204  						{
   205  							Label: []*dto.LabelPair{
   206  								{
   207  									Name:  proto.String("n1"),
   208  									Value: proto.String("val1"),
   209  								},
   210  							},
   211  							Untyped: &dto.Untyped{
   212  								Value: proto.Float64(2001),
   213  							},
   214  						},
   215  					},
   216  				},
   217  				{
   218  					Name: proto.String("decoy"),
   219  					Type: dto.MetricType_UNTYPED.Enum(),
   220  					Metric: []*dto.Metric{
   221  						{
   222  							Untyped: &dto.Untyped{
   223  								Value: proto.Float64(-1),
   224  							},
   225  							TimestampMs: proto.Int64(-2),
   226  						},
   227  					},
   228  				},
   229  				{
   230  					Name: proto.String("my_summary"),
   231  					Type: dto.MetricType_SUMMARY.Enum(),
   232  					Metric: []*dto.Metric{
   233  						{
   234  							Label: []*dto.LabelPair{
   235  								{
   236  									Name:  proto.String("n1"),
   237  									Value: proto.String("val1"),
   238  								},
   239  							},
   240  							Summary: &dto.Summary{
   241  								SampleCount: proto.Uint64(42),
   242  								SampleSum:   proto.Float64(4711),
   243  								Quantile: []*dto.Quantile{
   244  									{
   245  										Quantile: proto.Float64(0.5),
   246  										Value:    proto.Float64(110),
   247  									},
   248  									{
   249  										Quantile: proto.Float64(0.9),
   250  										Value:    proto.Float64(140),
   251  									},
   252  								},
   253  							},
   254  							TimestampMs: proto.Int64(2),
   255  						},
   256  						{
   257  							Label: []*dto.LabelPair{
   258  								{
   259  									Name:  proto.String("n2"),
   260  									Value: proto.String("val2"),
   261  								},
   262  								{
   263  									Name:  proto.String("n1"),
   264  									Value: proto.String("val1"),
   265  								},
   266  							},
   267  							Summary: &dto.Summary{
   268  								SampleCount: proto.Uint64(5),
   269  								Quantile: []*dto.Quantile{
   270  									{
   271  										Quantile: proto.Float64(-12.34),
   272  										Value:    proto.Float64(math.NaN()),
   273  									},
   274  								},
   275  							},
   276  							TimestampMs: proto.Int64(5),
   277  						},
   278  						{
   279  							Label: []*dto.LabelPair{
   280  								{
   281  									Name:  proto.String("n1"),
   282  									Value: proto.String("val2"),
   283  								},
   284  							},
   285  							Summary: &dto.Summary{
   286  								SampleSum: proto.Float64(8),
   287  							},
   288  							TimestampMs: proto.Int64(15),
   289  						},
   290  						{
   291  							Label: []*dto.LabelPair{
   292  								{
   293  									Name:  proto.String("n1"),
   294  									Value: proto.String("val3"),
   295  								},
   296  							},
   297  							Summary: &dto.Summary{
   298  								Quantile: []*dto.Quantile{
   299  									{
   300  										Quantile: proto.Float64(0.2),
   301  										Value:    proto.Float64(4711),
   302  									},
   303  								},
   304  							},
   305  						},
   306  					},
   307  				},
   308  				{
   309  					Name: proto.String("another_summary"),
   310  					Type: dto.MetricType_SUMMARY.Enum(),
   311  					Metric: []*dto.Metric{
   312  						{
   313  							Label: []*dto.LabelPair{
   314  								{
   315  									Name:  proto.String("n2"),
   316  									Value: proto.String("val2"),
   317  								},
   318  								{
   319  									Name:  proto.String("n1"),
   320  									Value: proto.String("val1"),
   321  								},
   322  							},
   323  							Summary: &dto.Summary{
   324  								SampleCount: proto.Uint64(20),
   325  								Quantile: []*dto.Quantile{
   326  									{
   327  										Quantile: proto.Float64(0.3),
   328  										Value:    proto.Float64(-1.2),
   329  									},
   330  								},
   331  							},
   332  						},
   333  					},
   334  				},
   335  			},
   336  		},
   337  		// 4: The histogram.
   338  		{
   339  			in: `
   340  # HELP request_duration_microseconds The response latency.
   341  # TYPE request_duration_microseconds histogram
   342  request_duration_microseconds_bucket{le="100"} 123
   343  request_duration_microseconds_bucket{le="120"} 412
   344  request_duration_microseconds_bucket{le="144"} 592
   345  request_duration_microseconds_bucket{le="172.8"} 1524
   346  request_duration_microseconds_bucket{le="+Inf"} 2693
   347  request_duration_microseconds_sum 1.7560473e+06
   348  request_duration_microseconds_count 2693
   349  `,
   350  			out: []*dto.MetricFamily{
   351  				{
   352  					Name: proto.String("request_duration_microseconds"),
   353  					Help: proto.String("The response latency."),
   354  					Type: dto.MetricType_HISTOGRAM.Enum(),
   355  					Metric: []*dto.Metric{
   356  						{
   357  							Histogram: &dto.Histogram{
   358  								SampleCount: proto.Uint64(2693),
   359  								SampleSum:   proto.Float64(1756047.3),
   360  								Bucket: []*dto.Bucket{
   361  									{
   362  										UpperBound:      proto.Float64(100),
   363  										CumulativeCount: proto.Uint64(123),
   364  									},
   365  									{
   366  										UpperBound:      proto.Float64(120),
   367  										CumulativeCount: proto.Uint64(412),
   368  									},
   369  									{
   370  										UpperBound:      proto.Float64(144),
   371  										CumulativeCount: proto.Uint64(592),
   372  									},
   373  									{
   374  										UpperBound:      proto.Float64(172.8),
   375  										CumulativeCount: proto.Uint64(1524),
   376  									},
   377  									{
   378  										UpperBound:      proto.Float64(math.Inf(+1)),
   379  										CumulativeCount: proto.Uint64(2693),
   380  									},
   381  								},
   382  							},
   383  						},
   384  					},
   385  				},
   386  			},
   387  		},
   388  	}
   389  
   390  	for i, scenario := range scenarios {
   391  		out, err := parser.TextToMetricFamilies(strings.NewReader(scenario.in))
   392  		if err != nil {
   393  			t.Errorf("%d. error: %s", i, err)
   394  			continue
   395  		}
   396  		if expected, got := len(scenario.out), len(out); expected != got {
   397  			t.Errorf(
   398  				"%d. expected %d MetricFamilies, got %d",
   399  				i, expected, got,
   400  			)
   401  		}
   402  		for _, expected := range scenario.out {
   403  			got, ok := out[expected.GetName()]
   404  			if !ok {
   405  				t.Errorf(
   406  					"%d. expected MetricFamily %q, found none",
   407  					i, expected.GetName(),
   408  				)
   409  				continue
   410  			}
   411  			if expected.String() != got.String() {
   412  				t.Errorf(
   413  					"%d. expected MetricFamily %s, got %s",
   414  					i, expected, got,
   415  				)
   416  			}
   417  		}
   418  	}
   419  }
   420  
   421  func TestTextParse(t *testing.T) {
   422  	testTextParse(t)
   423  }
   424  
   425  func BenchmarkTextParse(b *testing.B) {
   426  	for i := 0; i < b.N; i++ {
   427  		testTextParse(b)
   428  	}
   429  }
   430  
   431  func testTextParseError(t testing.TB) {
   432  	scenarios := []struct {
   433  		in  string
   434  		err string
   435  	}{
   436  		// 0: No new-line at end of input.
   437  		{
   438  			in: `
   439  bla 3.14
   440  blubber 42`,
   441  			err: "text format parsing error in line 3: unexpected end of input stream",
   442  		},
   443  		// 1: Invalid escape sequence in label value.
   444  		{
   445  			in:  `metric{label="\t"} 3.14`,
   446  			err: "text format parsing error in line 1: invalid escape sequence",
   447  		},
   448  		// 2: Newline in label value.
   449  		{
   450  			in: `
   451  metric{label="new
   452  line"} 3.14
   453  `,
   454  			err: `text format parsing error in line 2: label value "new" contains unescaped new-line`,
   455  		},
   456  		// 3:
   457  		{
   458  			in:  `metric{@="bla"} 3.14`,
   459  			err: "text format parsing error in line 1: invalid label name for metric",
   460  		},
   461  		// 4:
   462  		{
   463  			in:  `metric{__name__="bla"} 3.14`,
   464  			err: `text format parsing error in line 1: label name "__name__" is reserved`,
   465  		},
   466  		// 5:
   467  		{
   468  			in:  `metric{label+="bla"} 3.14`,
   469  			err: "text format parsing error in line 1: expected '=' after label name",
   470  		},
   471  		// 6:
   472  		{
   473  			in:  `metric{label=bla} 3.14`,
   474  			err: "text format parsing error in line 1: expected '\"' at start of label value",
   475  		},
   476  		// 7:
   477  		{
   478  			in: `
   479  # TYPE metric summary
   480  metric{quantile="bla"} 3.14
   481  `,
   482  			err: "text format parsing error in line 3: expected float as value for 'quantile' label",
   483  		},
   484  		// 8:
   485  		{
   486  			in:  `metric{label="bla"+} 3.14`,
   487  			err: "text format parsing error in line 1: unexpected end of label value",
   488  		},
   489  		// 9:
   490  		{
   491  			in: `metric{label="bla"} 3.14 2.72
   492  `,
   493  			err: "text format parsing error in line 1: expected integer as timestamp",
   494  		},
   495  		// 10:
   496  		{
   497  			in: `metric{label="bla"} 3.14 2 3
   498  `,
   499  			err: "text format parsing error in line 1: spurious string after timestamp",
   500  		},
   501  		// 11:
   502  		{
   503  			in: `metric{label="bla"} blubb
   504  `,
   505  			err: "text format parsing error in line 1: expected float as value",
   506  		},
   507  		// 12:
   508  		{
   509  			in: `
   510  # HELP metric one
   511  # HELP metric two
   512  `,
   513  			err: "text format parsing error in line 3: second HELP line for metric name",
   514  		},
   515  		// 13:
   516  		{
   517  			in: `
   518  # TYPE metric counter
   519  # TYPE metric untyped
   520  `,
   521  			err: `text format parsing error in line 3: second TYPE line for metric name "metric", or TYPE reported after samples`,
   522  		},
   523  		// 14:
   524  		{
   525  			in: `
   526  metric 4.12
   527  # TYPE metric counter
   528  `,
   529  			err: `text format parsing error in line 3: second TYPE line for metric name "metric", or TYPE reported after samples`,
   530  		},
   531  		// 14:
   532  		{
   533  			in: `
   534  # TYPE metric bla
   535  `,
   536  			err: "text format parsing error in line 2: unknown metric type",
   537  		},
   538  		// 15:
   539  		{
   540  			in: `
   541  # TYPE met-ric
   542  `,
   543  			err: "text format parsing error in line 2: invalid metric name in comment",
   544  		},
   545  		// 16:
   546  		{
   547  			in:  `@invalidmetric{label="bla"} 3.14 2`,
   548  			err: "text format parsing error in line 1: invalid metric name",
   549  		},
   550  		// 17:
   551  		{
   552  			in:  `{label="bla"} 3.14 2`,
   553  			err: "text format parsing error in line 1: invalid metric name",
   554  		},
   555  		// 18:
   556  		{
   557  			in: `
   558  # TYPE metric histogram
   559  metric_bucket{le="bla"} 3.14
   560  `,
   561  			err: "text format parsing error in line 3: expected float as value for 'le' label",
   562  		},
   563  		// 19: Invalid UTF-8 in label value.
   564  		{
   565  			in:  "metric{l=\"\xbd\"} 3.14\n",
   566  			err: "text format parsing error in line 1: invalid label value \"\\xbd\"",
   567  		},
   568  		// 20: Go 1.13 sometimes allows underscores in numbers.
   569  		{
   570  			in:  "foo 1_2\n",
   571  			err: "text format parsing error in line 1: expected float as value",
   572  		},
   573  		// 21: Go 1.13 supports hex floating point.
   574  		{
   575  			in:  "foo 0x1p-3\n",
   576  			err: "text format parsing error in line 1: expected float as value",
   577  		},
   578  		// 22: Check for various other literals variants, just in case.
   579  		{
   580  			in:  "foo 0x1P-3\n",
   581  			err: "text format parsing error in line 1: expected float as value",
   582  		},
   583  		// 23:
   584  		{
   585  			in:  "foo 0B1\n",
   586  			err: "text format parsing error in line 1: expected float as value",
   587  		},
   588  		// 24:
   589  		{
   590  			in:  "foo 0O1\n",
   591  			err: "text format parsing error in line 1: expected float as value",
   592  		},
   593  		// 25:
   594  		{
   595  			in:  "foo 0X1\n",
   596  			err: "text format parsing error in line 1: expected float as value",
   597  		},
   598  		// 26:
   599  		{
   600  			in:  "foo 0x1\n",
   601  			err: "text format parsing error in line 1: expected float as value",
   602  		},
   603  		// 27:
   604  		{
   605  			in:  "foo 0b1\n",
   606  			err: "text format parsing error in line 1: expected float as value",
   607  		},
   608  		// 28:
   609  		{
   610  			in:  "foo 0o1\n",
   611  			err: "text format parsing error in line 1: expected float as value",
   612  		},
   613  		// 29:
   614  		{
   615  			in:  "foo 0x1\n",
   616  			err: "text format parsing error in line 1: expected float as value",
   617  		},
   618  		// 30:
   619  		{
   620  			in:  "foo 0x1\n",
   621  			err: "text format parsing error in line 1: expected float as value",
   622  		},
   623  		// 31: Check histogram label.
   624  		{
   625  			in: `
   626  # TYPE metric histogram
   627  metric_bucket{le="0x1p-3"} 3.14
   628  `,
   629  			err: "text format parsing error in line 3: expected float as value for 'le' label",
   630  		},
   631  		// 32: Check quantile label.
   632  		{
   633  			in: `
   634  # TYPE metric summary
   635  metric{quantile="0x1p-3"} 3.14
   636  `,
   637  			err: "text format parsing error in line 3: expected float as value for 'quantile' label",
   638  		},
   639  		// 33: Check duplicate label.
   640  		{
   641  			in:  `metric{label="bla",label="bla"} 3.14`,
   642  			err: "text format parsing error in line 1: duplicate label names for metric",
   643  		},
   644  	}
   645  
   646  	for i, scenario := range scenarios {
   647  		_, err := parser.TextToMetricFamilies(strings.NewReader(scenario.in))
   648  		if err == nil {
   649  			t.Errorf("%d. expected error, got nil", i)
   650  			continue
   651  		}
   652  		if expected, got := scenario.err, err.Error(); strings.Index(got, expected) != 0 {
   653  			t.Errorf(
   654  				"%d. expected error starting with %q, got %q",
   655  				i, expected, got,
   656  			)
   657  		}
   658  	}
   659  }
   660  
   661  func TestTextParseError(t *testing.T) {
   662  	testTextParseError(t)
   663  }
   664  
   665  func BenchmarkParseError(b *testing.B) {
   666  	for i := 0; i < b.N; i++ {
   667  		testTextParseError(b)
   668  	}
   669  }
   670  
   671  func TestTextParserStartOfLine(t *testing.T) {
   672  	t.Run("EOF", func(t *testing.T) {
   673  		p := TextParser{}
   674  		in := strings.NewReader("")
   675  		p.reset(in)
   676  		fn := p.startOfLine()
   677  		if fn != nil {
   678  			t.Errorf("Unexpected non-nil function: %v", fn)
   679  		}
   680  		if p.err != nil {
   681  			t.Errorf("Unexpected error: %v", p.err)
   682  		}
   683  	})
   684  
   685  	t.Run("OtherError", func(t *testing.T) {
   686  		p := TextParser{}
   687  		in := &errReader{err: errors.New("unexpected error")}
   688  		p.reset(in)
   689  		fn := p.startOfLine()
   690  		if fn != nil {
   691  			t.Errorf("Unexpected non-nil function: %v", fn)
   692  		}
   693  		if p.err != nil && !errors.Is(p.err, in.err) {
   694  			t.Errorf("Unexpected error: %v, expected %v", p.err, in.err)
   695  		}
   696  	})
   697  }
   698  
   699  type errReader struct {
   700  	err error
   701  }
   702  
   703  func (r *errReader) Read(p []byte) (int, error) {
   704  	return 0, r.err
   705  }
   706
View as plain text