file_test.go

Documentation: cloud.google.com/go/bigquery

     1  // Copyright 2016 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bigquery
    16  
    17  import (
    18  	"testing"
    19  
    20  	"cloud.google.com/go/internal/testutil"
    21  	bq "google.golang.org/api/bigquery/v2"
    22  )
    23  
    24  var (
    25  	hyphen = "-"
    26  	fc     = FileConfig{
    27  		SourceFormat:        CSV,
    28  		AutoDetect:          true,
    29  		MaxBadRecords:       7,
    30  		IgnoreUnknownValues: true,
    31  		Schema: Schema{
    32  			stringFieldSchema(),
    33  			nestedFieldSchema(),
    34  		},
    35  		CSVOptions: CSVOptions{
    36  			Quote:                          hyphen,
    37  			FieldDelimiter:                 "\t",
    38  			SkipLeadingRows:                8,
    39  			AllowJaggedRows:                true,
    40  			AllowQuotedNewlines:            true,
    41  			Encoding:                       UTF_8,
    42  			NullMarker:                     "marker",
    43  			PreserveASCIIControlCharacters: true,
    44  		},
    45  	}
    46  )
    47  
    48  func TestFileConfigPopulateLoadConfig(t *testing.T) {
    49  	testcases := []struct {
    50  		description string
    51  		fileConfig  *FileConfig
    52  		want        *bq.JobConfigurationLoad
    53  	}{
    54  		{
    55  			description: "default json",
    56  			fileConfig: &FileConfig{
    57  				SourceFormat: JSON,
    58  			},
    59  			want: &bq.JobConfigurationLoad{
    60  				SourceFormat: "NEWLINE_DELIMITED_JSON",
    61  			},
    62  		},
    63  		{
    64  			description: "csv",
    65  			fileConfig:  &fc,
    66  			want: &bq.JobConfigurationLoad{
    67  				SourceFormat:                   "CSV",
    68  				FieldDelimiter:                 "\t",
    69  				SkipLeadingRows:                8,
    70  				AllowJaggedRows:                true,
    71  				AllowQuotedNewlines:            true,
    72  				Autodetect:                     true,
    73  				Encoding:                       "UTF-8",
    74  				MaxBadRecords:                  7,
    75  				IgnoreUnknownValues:            true,
    76  				NullMarker:                     "marker",
    77  				PreserveAsciiControlCharacters: true,
    78  				Schema: &bq.TableSchema{
    79  					Fields: []*bq.TableFieldSchema{
    80  						bqStringFieldSchema(),
    81  						bqNestedFieldSchema(),
    82  					}},
    83  				Quote: &hyphen,
    84  			},
    85  		},
    86  		{
    87  			description: "parquet",
    88  			fileConfig: &FileConfig{
    89  				SourceFormat: Parquet,
    90  				ParquetOptions: &ParquetOptions{
    91  					EnumAsString:        true,
    92  					EnableListInference: true,
    93  				},
    94  			},
    95  			want: &bq.JobConfigurationLoad{
    96  				SourceFormat: "PARQUET",
    97  				ParquetOptions: &bq.ParquetOptions{
    98  					EnumAsString:        true,
    99  					EnableListInference: true,
   100  				},
   101  			},
   102  		},
   103  		{
   104  			description: "avro",
   105  			fileConfig: &FileConfig{
   106  				SourceFormat: Avro,
   107  				AvroOptions: &AvroOptions{
   108  					UseAvroLogicalTypes: true,
   109  				},
   110  			},
   111  			want: &bq.JobConfigurationLoad{
   112  				SourceFormat:        "AVRO",
   113  				UseAvroLogicalTypes: true,
   114  			},
   115  		},
   116  	}
   117  	for _, tc := range testcases {
   118  		got := &bq.JobConfigurationLoad{}
   119  		tc.fileConfig.populateLoadConfig(got)
   120  		if diff := testutil.Diff(got, tc.want); diff != "" {
   121  			t.Errorf("case %s, got=-, want=+:\n%s", tc.description, diff)
   122  		}
   123  	}
   124  }
   125  
   126  func TestFileConfigPopulateExternalDataConfig(t *testing.T) {
   127  	testcases := []struct {
   128  		description string
   129  		fileConfig  *FileConfig
   130  		want        *bq.ExternalDataConfiguration
   131  	}{
   132  		{
   133  			description: "json defaults",
   134  			fileConfig: &FileConfig{
   135  				SourceFormat: JSON,
   136  			},
   137  			want: &bq.ExternalDataConfiguration{
   138  				SourceFormat: "NEWLINE_DELIMITED_JSON",
   139  			},
   140  		},
   141  		{
   142  			description: "csv fileconfig",
   143  			fileConfig:  &fc,
   144  			want: &bq.ExternalDataConfiguration{
   145  				SourceFormat:        "CSV",
   146  				Autodetect:          true,
   147  				MaxBadRecords:       7,
   148  				IgnoreUnknownValues: true,
   149  				Schema: &bq.TableSchema{
   150  					Fields: []*bq.TableFieldSchema{
   151  						bqStringFieldSchema(),
   152  						bqNestedFieldSchema(),
   153  					}},
   154  				CsvOptions: &bq.CsvOptions{
   155  					AllowJaggedRows:                true,
   156  					AllowQuotedNewlines:            true,
   157  					Encoding:                       "UTF-8",
   158  					FieldDelimiter:                 "\t",
   159  					Quote:                          &hyphen,
   160  					SkipLeadingRows:                8,
   161  					NullMarker:                     "marker",
   162  					PreserveAsciiControlCharacters: true,
   163  				},
   164  			},
   165  		},
   166  		{
   167  			description: "parquet",
   168  			fileConfig: &FileConfig{
   169  				SourceFormat: Parquet,
   170  				ParquetOptions: &ParquetOptions{
   171  					EnumAsString:        true,
   172  					EnableListInference: true,
   173  				},
   174  			},
   175  			want: &bq.ExternalDataConfiguration{
   176  				SourceFormat: "PARQUET",
   177  				ParquetOptions: &bq.ParquetOptions{
   178  					EnumAsString:        true,
   179  					EnableListInference: true,
   180  				},
   181  			},
   182  		},
   183  	}
   184  	for _, tc := range testcases {
   185  		got := &bq.ExternalDataConfiguration{}
   186  		tc.fileConfig.populateExternalDataConfig(got)
   187  		if diff := testutil.Diff(got, tc.want); diff != "" {
   188  			t.Errorf("case %s, got=-, want=+:\n%s", tc.description, diff)
   189  		}
   190  	}
   191  
   192  }
   193
View as plain text