schemaconversion_test.go

Documentation: cloud.google.com/go/bigquery/storage/managedwriter/adapt

     1  // Copyright 2021 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     https://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package adapt
    16  
    17  import (
    18  	"testing"
    19  
    20  	"cloud.google.com/go/bigquery"
    21  	"cloud.google.com/go/bigquery/storage/apiv1/storagepb"
    22  	"cloud.google.com/go/internal/testutil"
    23  	"github.com/google/go-cmp/cmp"
    24  	"google.golang.org/protobuf/testing/protocmp"
    25  )
    26  
    27  func TestFieldConversions(t *testing.T) {
    28  	testCases := []struct {
    29  		desc  string
    30  		bq    *bigquery.FieldSchema
    31  		proto *storagepb.TableFieldSchema
    32  	}{
    33  		{
    34  			desc:  "nil",
    35  			bq:    nil,
    36  			proto: nil,
    37  		},
    38  		{
    39  			desc: "string field",
    40  			bq: &bigquery.FieldSchema{
    41  				Name:        "name",
    42  				Type:        bigquery.StringFieldType,
    43  				Description: "description",
    44  			},
    45  			proto: &storagepb.TableFieldSchema{
    46  				Name:        "name",
    47  				Type:        storagepb.TableFieldSchema_STRING,
    48  				Description: "description",
    49  				Mode:        storagepb.TableFieldSchema_NULLABLE,
    50  			},
    51  		},
    52  		{
    53  			desc: "required integer field",
    54  			bq: &bigquery.FieldSchema{
    55  				Name:        "name",
    56  				Type:        bigquery.IntegerFieldType,
    57  				Description: "description",
    58  				Required:    true,
    59  			},
    60  			proto: &storagepb.TableFieldSchema{
    61  				Name:        "name",
    62  				Type:        storagepb.TableFieldSchema_INT64,
    63  				Description: "description",
    64  				Mode:        storagepb.TableFieldSchema_REQUIRED,
    65  			},
    66  		},
    67  		{
    68  			desc: "struct with repeated bytes subfield",
    69  			bq: &bigquery.FieldSchema{
    70  				Name:        "name",
    71  				Type:        bigquery.RecordFieldType,
    72  				Description: "description",
    73  				Required:    true,
    74  				Schema: bigquery.Schema{
    75  					&bigquery.FieldSchema{
    76  						Name:        "inner1",
    77  						Repeated:    true,
    78  						Description: "repeat",
    79  						Type:        bigquery.BytesFieldType,
    80  					},
    81  				},
    82  			},
    83  			proto: &storagepb.TableFieldSchema{
    84  				Name:        "name",
    85  				Type:        storagepb.TableFieldSchema_STRUCT,
    86  				Description: "description",
    87  				Mode:        storagepb.TableFieldSchema_REQUIRED,
    88  				Fields: []*storagepb.TableFieldSchema{
    89  					{
    90  						Name:        "inner1",
    91  						Mode:        storagepb.TableFieldSchema_REPEATED,
    92  						Description: "repeat",
    93  						Type:        storagepb.TableFieldSchema_BYTES,
    94  					},
    95  				},
    96  			},
    97  		},
    98  		{
    99  			desc: "range type",
   100  			bq: &bigquery.FieldSchema{
   101  				Name:        "name",
   102  				Type:        bigquery.RangeFieldType,
   103  				Description: "description",
   104  				Required:    true,
   105  				RangeElementType: &bigquery.RangeElementType{
   106  					Type: bigquery.TimestampFieldType,
   107  				},
   108  			},
   109  			proto: &storagepb.TableFieldSchema{
   110  				Name:        "name",
   111  				Type:        storagepb.TableFieldSchema_RANGE,
   112  				Description: "description",
   113  				Mode:        storagepb.TableFieldSchema_REQUIRED,
   114  				RangeElementType: &storagepb.TableFieldSchema_FieldElementType{
   115  					Type: storagepb.TableFieldSchema_TIMESTAMP,
   116  				},
   117  			},
   118  		},
   119  	}
   120  
   121  	for _, tc := range testCases {
   122  		// first, bq to proto
   123  		converted, err := bqFieldToProto(tc.bq)
   124  		if err != nil {
   125  			t.Errorf("case (%s) failed conversion from bq: %v", tc.desc, err)
   126  		}
   127  		if diff := cmp.Diff(converted, tc.proto, protocmp.Transform()); diff != "" {
   128  			t.Errorf("conversion to proto diff (%s):\n%v", tc.desc, diff)
   129  		}
   130  		// reverse conversion, proto to bq
   131  		reverse, err := protoToBQField(tc.proto)
   132  		if err != nil {
   133  			t.Errorf("case (%s) failed conversion from proto: %v", tc.desc, err)
   134  		}
   135  		if diff := cmp.Diff(reverse, tc.bq); diff != "" {
   136  			t.Errorf("conversion to BQ diff (%s):\n%v", tc.desc, diff)
   137  		}
   138  	}
   139  }
   140  
   141  func TestSchemaConversion(t *testing.T) {
   142  
   143  	testCases := []struct {
   144  		description   string
   145  		bqSchema      bigquery.Schema
   146  		storageSchema *storagepb.TableSchema
   147  	}{
   148  		{
   149  			description:   "nil",
   150  			bqSchema:      nil,
   151  			storageSchema: nil,
   152  		},
   153  		{
   154  			description: "scalars",
   155  			bqSchema: bigquery.Schema{
   156  				{Name: "f1", Type: bigquery.StringFieldType},
   157  				{Name: "f2", Type: bigquery.IntegerFieldType},
   158  				{Name: "f3", Type: bigquery.BooleanFieldType},
   159  			},
   160  			storageSchema: &storagepb.TableSchema{
   161  				Fields: []*storagepb.TableFieldSchema{
   162  					{Name: "f1", Type: storagepb.TableFieldSchema_STRING, Mode: storagepb.TableFieldSchema_NULLABLE},
   163  					{Name: "f2", Type: storagepb.TableFieldSchema_INT64, Mode: storagepb.TableFieldSchema_NULLABLE},
   164  					{Name: "f3", Type: storagepb.TableFieldSchema_BOOL, Mode: storagepb.TableFieldSchema_NULLABLE},
   165  				},
   166  			},
   167  		},
   168  		{
   169  			description: "array",
   170  			bqSchema: bigquery.Schema{
   171  				{Name: "arr", Type: bigquery.NumericFieldType, Repeated: true},
   172  				{Name: "big", Type: bigquery.BigNumericFieldType, Required: true},
   173  			},
   174  			storageSchema: &storagepb.TableSchema{
   175  				Fields: []*storagepb.TableFieldSchema{
   176  					{Name: "arr", Type: storagepb.TableFieldSchema_NUMERIC, Mode: storagepb.TableFieldSchema_REPEATED},
   177  					{Name: "big", Type: storagepb.TableFieldSchema_BIGNUMERIC, Mode: storagepb.TableFieldSchema_REQUIRED},
   178  				},
   179  			},
   180  		},
   181  		{
   182  			description: "nested",
   183  			bqSchema: bigquery.Schema{
   184  				{Name: "struct1", Type: bigquery.RecordFieldType, Schema: []*bigquery.FieldSchema{
   185  					{Name: "leaf1", Type: bigquery.DateFieldType},
   186  					{Name: "leaf2", Type: bigquery.DateTimeFieldType},
   187  				}},
   188  				{Name: "field2", Type: bigquery.StringFieldType},
   189  			},
   190  			storageSchema: &storagepb.TableSchema{
   191  				Fields: []*storagepb.TableFieldSchema{
   192  					{Name: "struct1",
   193  						Type: storagepb.TableFieldSchema_STRUCT,
   194  						Mode: storagepb.TableFieldSchema_NULLABLE,
   195  						Fields: []*storagepb.TableFieldSchema{
   196  							{Name: "leaf1", Type: storagepb.TableFieldSchema_DATE, Mode: storagepb.TableFieldSchema_NULLABLE},
   197  							{Name: "leaf2", Type: storagepb.TableFieldSchema_DATETIME, Mode: storagepb.TableFieldSchema_NULLABLE},
   198  						}},
   199  					{Name: "field2", Type: storagepb.TableFieldSchema_STRING, Mode: storagepb.TableFieldSchema_NULLABLE},
   200  				},
   201  			},
   202  		},
   203  		{
   204  			description: "range types",
   205  			bqSchema: bigquery.Schema{
   206  				{Name: "rangedate", Type: bigquery.RangeFieldType, RangeElementType: &bigquery.RangeElementType{Type: bigquery.DateFieldType}},
   207  				{Name: "rangedatetime", Type: bigquery.RangeFieldType, RangeElementType: &bigquery.RangeElementType{Type: bigquery.DateTimeFieldType}},
   208  				{Name: "rangetimestamp", Type: bigquery.RangeFieldType, RangeElementType: &bigquery.RangeElementType{Type: bigquery.TimestampFieldType}},
   209  			},
   210  			storageSchema: &storagepb.TableSchema{
   211  				Fields: []*storagepb.TableFieldSchema{
   212  					{Name: "rangedate",
   213  						Type:             storagepb.TableFieldSchema_RANGE,
   214  						Mode:             storagepb.TableFieldSchema_NULLABLE,
   215  						RangeElementType: &storagepb.TableFieldSchema_FieldElementType{Type: storagepb.TableFieldSchema_DATE}},
   216  					{Name: "rangedatetime",
   217  						Type:             storagepb.TableFieldSchema_RANGE,
   218  						Mode:             storagepb.TableFieldSchema_NULLABLE,
   219  						RangeElementType: &storagepb.TableFieldSchema_FieldElementType{Type: storagepb.TableFieldSchema_DATETIME}},
   220  					{Name: "rangetimestamp",
   221  						Type:             storagepb.TableFieldSchema_RANGE,
   222  						Mode:             storagepb.TableFieldSchema_NULLABLE,
   223  						RangeElementType: &storagepb.TableFieldSchema_FieldElementType{Type: storagepb.TableFieldSchema_TIMESTAMP}},
   224  				},
   225  			},
   226  		},
   227  	}
   228  	for _, tc := range testCases {
   229  		t.Run(tc.description, func(t *testing.T) {
   230  			// BQ -> Storage
   231  			storageS, err := BQSchemaToStorageTableSchema(tc.bqSchema)
   232  			if err != nil {
   233  				t.Errorf("BQSchemaToStorageTableSchema(%s): %v", tc.description, err)
   234  			}
   235  			if diff := testutil.Diff(storageS, tc.storageSchema); diff != "" {
   236  				t.Fatalf("BQSchemaToStorageTableSchema(%s): -got, +want:\n%s", tc.description, diff)
   237  			}
   238  
   239  			// Storage -> BQ
   240  			bqS, err := StorageTableSchemaToBQSchema(tc.storageSchema)
   241  			if err != nil {
   242  				t.Errorf("StorageTableSchemaToBQSchema(%s): %v", tc.description, err)
   243  			}
   244  			if diff := testutil.Diff(bqS, tc.bqSchema); diff != "" {
   245  				t.Fatalf("StorageTableSchemaToBQSchema(%s): -got, +want:\n%s", tc.description, diff)
   246  			}
   247  		})
   248  	}
   249  }
   250
View as plain text