inserter_test.go

Documentation: cloud.google.com/go/bigquery

     1  // Copyright 2015 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bigquery
    16  
    17  import (
    18  	"errors"
    19  	"fmt"
    20  	"strconv"
    21  	"testing"
    22  
    23  	"cloud.google.com/go/internal/pretty"
    24  	"cloud.google.com/go/internal/testutil"
    25  	"github.com/google/go-cmp/cmp"
    26  	bq "google.golang.org/api/bigquery/v2"
    27  )
    28  
    29  type testSaver struct {
    30  	row      map[string]Value
    31  	insertID string
    32  	err      error
    33  }
    34  
    35  func (ts testSaver) Save() (map[string]Value, string, error) {
    36  	return ts.row, ts.insertID, ts.err
    37  }
    38  
    39  func TestNewInsertRequest(t *testing.T) {
    40  	prev := randomIDFn
    41  	n := 0
    42  	randomIDFn = func() string { n++; return strconv.Itoa(n) }
    43  	defer func() { randomIDFn = prev }()
    44  
    45  	tests := []struct {
    46  		ul     *Uploader
    47  		savers []ValueSaver
    48  		req    *bq.TableDataInsertAllRequest
    49  	}{
    50  		{
    51  			ul:     &Uploader{},
    52  			savers: nil,
    53  			req:    nil,
    54  		},
    55  		{
    56  			ul: &Uploader{},
    57  			savers: []ValueSaver{
    58  				testSaver{row: map[string]Value{"one": 1}},
    59  				testSaver{row: map[string]Value{"two": 2}},
    60  				testSaver{insertID: NoDedupeID, row: map[string]Value{"three": 3}},
    61  			},
    62  			req: &bq.TableDataInsertAllRequest{
    63  				Rows: []*bq.TableDataInsertAllRequestRows{
    64  					{InsertId: "1", Json: map[string]bq.JsonValue{"one": 1}},
    65  					{InsertId: "2", Json: map[string]bq.JsonValue{"two": 2}},
    66  					{InsertId: "", Json: map[string]bq.JsonValue{"three": 3}},
    67  				},
    68  			},
    69  		},
    70  		{
    71  			ul: &Uploader{
    72  				TableTemplateSuffix: "suffix",
    73  				IgnoreUnknownValues: true,
    74  				SkipInvalidRows:     true,
    75  			},
    76  			savers: []ValueSaver{
    77  				testSaver{insertID: "a", row: map[string]Value{"one": 1}},
    78  				testSaver{insertID: "", row: map[string]Value{"two": 2}},
    79  			},
    80  			req: &bq.TableDataInsertAllRequest{
    81  				Rows: []*bq.TableDataInsertAllRequestRows{
    82  					{InsertId: "a", Json: map[string]bq.JsonValue{"one": 1}},
    83  					{InsertId: "3", Json: map[string]bq.JsonValue{"two": 2}},
    84  				},
    85  				TemplateSuffix:      "suffix",
    86  				SkipInvalidRows:     true,
    87  				IgnoreUnknownValues: true,
    88  			},
    89  		},
    90  	}
    91  	for i, tc := range tests {
    92  		got, err := tc.ul.newInsertRequest(tc.savers)
    93  		if err != nil {
    94  			t.Fatal(err)
    95  		}
    96  		want := tc.req
    97  		if !testutil.Equal(got, want) {
    98  			t.Errorf("%d: %#v: got %#v, want %#v", i, tc.ul, got, want)
    99  		}
   100  	}
   101  }
   102  
   103  func TestNewInsertRequestErrors(t *testing.T) {
   104  	var u Uploader
   105  	_, err := u.newInsertRequest([]ValueSaver{testSaver{err: errors.New("bang")}})
   106  	if err == nil {
   107  		t.Error("got nil, want error")
   108  	}
   109  }
   110  
   111  func TestHandleInsertErrors(t *testing.T) {
   112  	rows := []*bq.TableDataInsertAllRequestRows{
   113  		{InsertId: "a"},
   114  		{InsertId: "b"},
   115  	}
   116  	for _, test := range []struct {
   117  		description string
   118  		in          []*bq.TableDataInsertAllResponseInsertErrors
   119  		want        error
   120  	}{
   121  		{
   122  			description: "nil error",
   123  			in:          nil,
   124  			want:        nil,
   125  		},
   126  		{
   127  			description: "single error last row",
   128  			in:          []*bq.TableDataInsertAllResponseInsertErrors{{Index: 1}},
   129  			want:        PutMultiError{RowInsertionError{InsertID: "b", RowIndex: 1}},
   130  		},
   131  		{
   132  			description: "single error first row",
   133  			in:          []*bq.TableDataInsertAllResponseInsertErrors{{Index: 0}},
   134  			want:        PutMultiError{RowInsertionError{InsertID: "a", RowIndex: 0}},
   135  		},
   136  		{
   137  			description: "errors with extended message",
   138  			in: []*bq.TableDataInsertAllResponseInsertErrors{
   139  				{Errors: []*bq.ErrorProto{{Message: "m0"}}, Index: 0},
   140  				{Errors: []*bq.ErrorProto{{Message: "m1"}}, Index: 1},
   141  			},
   142  			want: PutMultiError{
   143  				RowInsertionError{InsertID: "a", RowIndex: 0, Errors: []error{&Error{Message: "m0"}}},
   144  				RowInsertionError{InsertID: "b", RowIndex: 1, Errors: []error{&Error{Message: "m1"}}},
   145  			},
   146  		},
   147  		{
   148  			description: "invalid index",
   149  			in: []*bq.TableDataInsertAllResponseInsertErrors{
   150  				{Errors: []*bq.ErrorProto{{Message: "m0"}}, Index: 2},
   151  			},
   152  			want: fmt.Errorf("internal error: unexpected row index: 2"),
   153  		},
   154  	} {
   155  		got := handleInsertErrors(test.in, rows)
   156  		if _, ok := got.(PutMultiError); ok {
   157  			// compare structure of the PutMultiError
   158  			if !testutil.Equal(got, test.want) {
   159  				t.Errorf("(case: %s)\nin %#v\ngot\n%#v\nwant\n%#v", test.description, test.in, got, test.want)
   160  			}
   161  			continue
   162  		}
   163  
   164  		if got != nil && test.want != nil && got.Error() != test.want.Error() {
   165  			// check matching error messages
   166  			t.Errorf("(case: %s)\nin %#v:\ngot\n%#v\nwant\n%#v", test.description, test.in, got, test.want)
   167  		}
   168  
   169  	}
   170  }
   171  
   172  func TestValueSavers(t *testing.T) {
   173  	ts := &testSaver{}
   174  	type T struct{ I int }
   175  	schema, err := InferSchema(T{})
   176  	if err != nil {
   177  		t.Fatal(err)
   178  	}
   179  	for _, test := range []struct {
   180  		in   interface{}
   181  		want []ValueSaver
   182  	}{
   183  		{[]interface{}(nil), nil},
   184  		{[]interface{}{}, nil},
   185  		{ts, []ValueSaver{ts}},
   186  		{T{I: 1}, []ValueSaver{&StructSaver{Schema: schema, Struct: T{I: 1}}}},
   187  		{[]ValueSaver{ts, ts}, []ValueSaver{ts, ts}},
   188  		{[]interface{}{ts, ts}, []ValueSaver{ts, ts}},
   189  		{[]T{{I: 1}, {I: 2}}, []ValueSaver{
   190  			&StructSaver{Schema: schema, Struct: T{I: 1}},
   191  			&StructSaver{Schema: schema, Struct: T{I: 2}},
   192  		}},
   193  		{[]interface{}{T{I: 1}, &T{I: 2}}, []ValueSaver{
   194  			&StructSaver{Schema: schema, Struct: T{I: 1}},
   195  			&StructSaver{Schema: schema, Struct: &T{I: 2}},
   196  		}},
   197  		{&StructSaver{Struct: T{I: 3}, InsertID: "foo"},
   198  			[]ValueSaver{
   199  				&StructSaver{Schema: schema, Struct: T{I: 3}, InsertID: "foo"},
   200  			}},
   201  	} {
   202  		got, err := valueSavers(test.in)
   203  		if err != nil {
   204  			t.Fatal(err)
   205  		}
   206  		if !testutil.Equal(got, test.want, cmp.AllowUnexported(testSaver{})) {
   207  			t.Errorf("%+v: got %v, want %v", test.in, pretty.Value(got), pretty.Value(test.want))
   208  		}
   209  		// Make sure Save is successful.
   210  		for i, vs := range got {
   211  			_, _, err := vs.Save()
   212  			if err != nil {
   213  				t.Fatalf("%+v, #%d: got error %v, want nil", test.in, i, err)
   214  			}
   215  		}
   216  	}
   217  }
   218  
   219  func TestValueSaversErrors(t *testing.T) {
   220  	inputs := []interface{}{
   221  		nil,
   222  		1,
   223  		[]int{1, 2},
   224  		[]interface{}{
   225  			testSaver{row: map[string]Value{"one": 1}, insertID: "a"},
   226  			1,
   227  		},
   228  		StructSaver{},
   229  	}
   230  	for _, in := range inputs {
   231  		if _, err := valueSavers(in); err == nil {
   232  			t.Errorf("%#v: got nil, want error", in)
   233  		}
   234  	}
   235  }
   236
View as plain text