protobenchmarks_test.go

Documentation: cloud.google.com/go/bigquery/storage/managedwriter/adapt

     1  // Copyright 2021 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     https://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package adapt_test
    16  
    17  import (
    18  	"fmt"
    19  	"testing"
    20  	"time"
    21  
    22  	"cloud.google.com/go/bigquery"
    23  	"cloud.google.com/go/bigquery/storage/managedwriter/adapt"
    24  	"cloud.google.com/go/bigquery/storage/managedwriter/testdata"
    25  
    26  	"google.golang.org/protobuf/proto"
    27  	"google.golang.org/protobuf/reflect/protoreflect"
    28  	"google.golang.org/protobuf/types/known/wrapperspb"
    29  )
    30  
    31  var benchDescriptor protoreflect.Descriptor
    32  
    33  func BenchmarkStorageSchemaToDescriptor(b *testing.B) {
    34  	syntaxLabels := []string{"proto2", "proto3"}
    35  	for _, bm := range []struct {
    36  		name string
    37  		in   bigquery.Schema
    38  	}{
    39  		{
    40  			name: "SingleField",
    41  			in: bigquery.Schema{
    42  				{Name: "field", Type: bigquery.StringFieldType},
    43  			},
    44  		},
    45  		{
    46  			name: "NestedRecord",
    47  			in: bigquery.Schema{
    48  				{Name: "field1", Type: bigquery.StringFieldType},
    49  				{Name: "field2", Type: bigquery.IntegerFieldType},
    50  				{Name: "field3", Type: bigquery.BooleanFieldType},
    51  				{
    52  					Name: "field4",
    53  					Type: bigquery.RecordFieldType,
    54  					Schema: bigquery.Schema{
    55  						{Name: "recordfield1", Type: bigquery.GeographyFieldType},
    56  						{Name: "recordfield2", Type: bigquery.TimestampFieldType},
    57  					},
    58  				},
    59  			},
    60  		},
    61  		{
    62  			name: "SimpleMessage",
    63  			in:   testdata.SimpleMessageSchema,
    64  		},
    65  		{
    66  			name: "GithubArchiveSchema",
    67  			in:   testdata.GithubArchiveSchema,
    68  		},
    69  	} {
    70  		for _, s := range syntaxLabels {
    71  			b.Run(fmt.Sprintf("%s-%s", bm.name, s), func(b *testing.B) {
    72  				convSchema, err := adapt.BQSchemaToStorageTableSchema(bm.in)
    73  				if err != nil {
    74  					b.Errorf("%q: schema conversion fail: %v", bm.name, err)
    75  				}
    76  				for n := 0; n < b.N; n++ {
    77  					if s == "proto3" {
    78  						benchDescriptor, err = adapt.StorageSchemaToProto3Descriptor(convSchema, "root")
    79  					} else {
    80  						benchDescriptor, err = adapt.StorageSchemaToProto2Descriptor(convSchema, "root")
    81  					}
    82  					if err != nil {
    83  						b.Errorf("failed to convert %q: %v", bm.name, err)
    84  					}
    85  				}
    86  			})
    87  		}
    88  	}
    89  }
    90  
    91  var staticBytes []byte
    92  
    93  func BenchmarkStaticProtoSerialization(b *testing.B) {
    94  	for _, bm := range []struct {
    95  		name    string
    96  		in      bigquery.Schema
    97  		syntax  string
    98  		setterF func() protoreflect.ProtoMessage
    99  	}{
   100  		{
   101  			name: "SimpleMessageProto2",
   102  			setterF: func() protoreflect.ProtoMessage {
   103  				return &testdata.SimpleMessageProto2{
   104  					Name:  proto.String(fmt.Sprintf("test-%d", time.Now().UnixNano())),
   105  					Value: proto.Int64(time.Now().UnixNano()),
   106  				}
   107  			},
   108  		},
   109  		{
   110  			name: "SimpleMessageProto3",
   111  			setterF: func() protoreflect.ProtoMessage {
   112  				return &testdata.SimpleMessageProto3{
   113  					Name:  fmt.Sprintf("test-%d", time.Now().UnixNano()),
   114  					Value: &wrapperspb.Int64Value{Value: time.Now().UnixNano()},
   115  				}
   116  			},
   117  		},
   118  		{
   119  			name: "GithubArchiveProto2",
   120  			setterF: func() protoreflect.ProtoMessage {
   121  				nowNano := time.Now().UnixNano()
   122  				return &testdata.GithubArchiveMessageProto2{
   123  					Type:    proto.String("SomeEvent"),
   124  					Public:  proto.Bool(nowNano%2 == 0),
   125  					Payload: proto.String(fmt.Sprintf("stuff %d", nowNano)),
   126  					Repo: &testdata.GithubArchiveRepoProto2{
   127  						Id:   proto.Int64(nowNano),
   128  						Name: proto.String("staticname"),
   129  						Url:  proto.String(fmt.Sprintf("foo.com/%d", nowNano)),
   130  					},
   131  					Actor: &testdata.GithubArchiveEntityProto2{
   132  						Id:         proto.Int64(nowNano % 1000),
   133  						Login:      proto.String(fmt.Sprintf("login-%d", nowNano%1000)),
   134  						GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)),
   135  						AvatarUrl:  proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)),
   136  						Url:        proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)),
   137  					},
   138  					Org: &testdata.GithubArchiveEntityProto2{
   139  						Id:         proto.Int64(nowNano % 1000),
   140  						Login:      proto.String(fmt.Sprintf("login-%d", nowNano%1000)),
   141  						GravatarId: proto.String(fmt.Sprintf("grav-%d", nowNano%1000000)),
   142  						AvatarUrl:  proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)),
   143  						Url:        proto.String(fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)),
   144  					},
   145  					CreatedAt: proto.Int64(nowNano),
   146  					Id:        proto.String(fmt.Sprintf("id%d", nowNano)),
   147  					Other:     proto.String("other"),
   148  				}
   149  			},
   150  		},
   151  		{
   152  			// Only set a single top-level field in a larger message.
   153  			name: "GithubArchiveProto2_Sparse",
   154  			setterF: func() protoreflect.ProtoMessage {
   155  				nowNano := time.Now().UnixNano()
   156  				return &testdata.GithubArchiveMessageProto2{
   157  					Id: proto.String(fmt.Sprintf("id%d", nowNano)),
   158  				}
   159  			},
   160  		},
   161  		{
   162  			name: "GithubArchiveProto3",
   163  			setterF: func() protoreflect.ProtoMessage {
   164  				nowNano := time.Now().UnixNano()
   165  				return &testdata.GithubArchiveMessageProto3{
   166  					Type:    &wrapperspb.StringValue{Value: "SomeEvent"},
   167  					Public:  &wrapperspb.BoolValue{Value: nowNano%2 == 0},
   168  					Payload: &wrapperspb.StringValue{Value: fmt.Sprintf("stuff %d", nowNano)},
   169  					Repo: &testdata.GithubArchiveRepoProto3{
   170  						Id:   &wrapperspb.Int64Value{Value: nowNano},
   171  						Name: &wrapperspb.StringValue{Value: "staticname"},
   172  						Url:  &wrapperspb.StringValue{Value: fmt.Sprintf("foo.com/%d", nowNano)},
   173  					},
   174  					Actor: &testdata.GithubArchiveEntityProto3{
   175  						Id:         &wrapperspb.Int64Value{Value: nowNano % 1000},
   176  						Login:      &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)},
   177  						GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)},
   178  						AvatarUrl:  &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)},
   179  						Url:        &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)},
   180  					},
   181  					Org: &testdata.GithubArchiveEntityProto3{
   182  						Id:         &wrapperspb.Int64Value{Value: nowNano % 1000},
   183  						Login:      &wrapperspb.StringValue{Value: fmt.Sprintf("login-%d", nowNano%1000)},
   184  						GravatarId: &wrapperspb.StringValue{Value: fmt.Sprintf("grav-%d", nowNano%1000000)},
   185  						AvatarUrl:  &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)},
   186  						Url:        &wrapperspb.StringValue{Value: fmt.Sprintf("https://something.com/img/%d", nowNano%10000000)},
   187  					},
   188  					CreatedAt: &wrapperspb.Int64Value{Value: nowNano},
   189  					Id:        &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)},
   190  					Other:     &wrapperspb.StringValue{Value: "other"},
   191  				}
   192  			},
   193  		},
   194  		{
   195  			// Only set a single field in a larger message.
   196  			name: "GithubArchiveProto3_Sparse",
   197  			setterF: func() protoreflect.ProtoMessage {
   198  				nowNano := time.Now().UnixNano()
   199  				return &testdata.GithubArchiveMessageProto3{
   200  					Id: &wrapperspb.StringValue{Value: fmt.Sprintf("id%d", nowNano)},
   201  				}
   202  			},
   203  		},
   204  	} {
   205  		b.Run(bm.name, func(b *testing.B) {
   206  			var totalBytes int64
   207  			for n := 0; n < b.N; n++ {
   208  				m := bm.setterF()
   209  				out, err := proto.Marshal(m)
   210  				if err != nil {
   211  					b.Errorf("%q %q: Marshal: %v", bm.name, bm.syntax, err)
   212  				}
   213  				totalBytes = totalBytes + int64(len(out))
   214  				staticBytes = out
   215  			}
   216  			b.Logf("N=%d, avg bytes/message: %d", b.N, totalBytes/int64(b.N))
   217  		})
   218  	}
   219  }
   220
View as plain text