external_test.go

Documentation: cloud.google.com/go/bigquery

     1  // Copyright 2017 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bigquery
    16  
    17  import (
    18  	"testing"
    19  
    20  	"cloud.google.com/go/internal/pretty"
    21  	"cloud.google.com/go/internal/testutil"
    22  )
    23  
    24  func TestExternalDataConfig(t *testing.T) {
    25  	// Round-trip of ExternalDataConfig to underlying representation.
    26  	for i, want := range []*ExternalDataConfig{
    27  		{
    28  			SourceFormat:        CSV,
    29  			SourceURIs:          []string{"uri"},
    30  			Schema:              Schema{{Name: "n", Type: IntegerFieldType}},
    31  			AutoDetect:          true,
    32  			Compression:         Gzip,
    33  			IgnoreUnknownValues: true,
    34  			MaxBadRecords:       17,
    35  			Options: &CSVOptions{
    36  				AllowJaggedRows:     true,
    37  				AllowQuotedNewlines: true,
    38  				Encoding:            UTF_8,
    39  				FieldDelimiter:      "f",
    40  				Quote:               "q",
    41  				SkipLeadingRows:     3,
    42  				NullMarker:          "marker",
    43  			},
    44  			ConnectionID: "connection",
    45  		},
    46  		{
    47  			SourceFormat: GoogleSheets,
    48  			Options: &GoogleSheetsOptions{
    49  				SkipLeadingRows: 4,
    50  				Range:           "sheet1!A1:Z10",
    51  			},
    52  		},
    53  		{
    54  			SourceFormat: Avro,
    55  			HivePartitioningOptions: &HivePartitioningOptions{
    56  				Mode:                   AutoHivePartitioningMode,
    57  				SourceURIPrefix:        "gs://somebucket/a/b/c",
    58  				RequirePartitionFilter: true,
    59  			},
    60  		},
    61  		{
    62  			SourceFormat: Bigtable,
    63  			Options: &BigtableOptions{
    64  				IgnoreUnspecifiedColumnFamilies: true,
    65  				ReadRowkeyAsString:              true,
    66  				ColumnFamilies: []*BigtableColumnFamily{
    67  					{
    68  						FamilyID:       "f1",
    69  						Encoding:       "TEXT",
    70  						OnlyReadLatest: true,
    71  						Type:           "FLOAT",
    72  						Columns: []*BigtableColumn{
    73  							{
    74  								Qualifier:      "valid-utf-8",
    75  								FieldName:      "fn",
    76  								OnlyReadLatest: true,
    77  								Encoding:       "BINARY",
    78  								Type:           "STRING",
    79  							},
    80  						},
    81  					},
    82  				},
    83  			},
    84  		},
    85  		{
    86  			SourceFormat: Parquet,
    87  			Options: &ParquetOptions{
    88  				EnumAsString:        true,
    89  				EnableListInference: true,
    90  			},
    91  		},
    92  		{
    93  			SourceFormat:       Parquet,
    94  			DecimalTargetTypes: []DecimalTargetType{BigNumericTargetType, NumericTargetType, StringTargetType},
    95  		},
    96  		{
    97  			SourceFormat: Avro,
    98  			Options: &AvroOptions{
    99  				UseAvroLogicalTypes: true,
   100  			},
   101  		},
   102  	} {
   103  		q := want.toBQ()
   104  		got, err := bqToExternalDataConfig(&q)
   105  		if err != nil {
   106  			t.Fatal(err)
   107  		}
   108  		if diff := testutil.Diff(got, want); diff != "" {
   109  			t.Errorf("#%d: got=-, want=+:\n%s", i, diff)
   110  		}
   111  	}
   112  }
   113  
   114  func TestQuote(t *testing.T) {
   115  	ptr := func(s string) *string { return &s }
   116  
   117  	for _, test := range []struct {
   118  		quote string
   119  		force bool
   120  		want  *string
   121  	}{
   122  		{"", false, nil},
   123  		{"", true, ptr("")},
   124  		{"-", false, ptr("-")},
   125  		{"-", true, ptr("")},
   126  	} {
   127  		o := CSVOptions{
   128  			Quote:          test.quote,
   129  			ForceZeroQuote: test.force,
   130  		}
   131  		got := o.quote()
   132  		if (got == nil) != (test.want == nil) {
   133  			t.Errorf("%+v\ngot %v\nwant %v", test, pretty.Value(got), pretty.Value(test.want))
   134  		}
   135  		if got != nil && test.want != nil && *got != *test.want {
   136  			t.Errorf("%+v: got %q, want %q", test, *got, *test.want)
   137  		}
   138  	}
   139  }
   140  
   141  func TestQualifier(t *testing.T) {
   142  	b := BigtableColumn{Qualifier: "a"}
   143  	q := b.toBQ()
   144  	if q.QualifierString != b.Qualifier || q.QualifierEncoded != "" {
   145  		t.Errorf("got (%q, %q), want (%q, %q)",
   146  			q.QualifierString, q.QualifierEncoded, b.Qualifier, "")
   147  	}
   148  	b2, err := bqToBigtableColumn(q)
   149  	if err != nil {
   150  		t.Fatal(err)
   151  	}
   152  	if got, want := b2.Qualifier, b.Qualifier; got != want {
   153  		t.Errorf("got %q, want %q", got, want)
   154  	}
   155  
   156  	const (
   157  		invalidUTF8    = "\xDF\xFF"
   158  		invalidEncoded = "3/8"
   159  	)
   160  	b = BigtableColumn{Qualifier: invalidUTF8}
   161  	q = b.toBQ()
   162  	if q.QualifierString != "" || q.QualifierEncoded != invalidEncoded {
   163  		t.Errorf("got (%q, %q), want (%q, %q)",
   164  			q.QualifierString, "", b.Qualifier, invalidEncoded)
   165  	}
   166  	b2, err = bqToBigtableColumn(q)
   167  	if err != nil {
   168  		t.Fatal(err)
   169  	}
   170  	if got, want := b2.Qualifier, b.Qualifier; got != want {
   171  		t.Errorf("got %q, want %q", got, want)
   172  	}
   173  }
   174
View as plain text