1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package bigquery
16
17 import (
18 "encoding/base64"
19 "errors"
20 "fmt"
21 "math/big"
22 "reflect"
23 "regexp"
24 "strings"
25 "time"
26
27 "cloud.google.com/go/civil"
28 "cloud.google.com/go/internal/fields"
29 bq "google.golang.org/api/bigquery/v2"
30 )
31
32
33 var (
34 timestampFormat = "2006-01-02 15:04:05.999999-07:00"
35 dateTimeFormat = "2006-01-02 15:04:05"
36 )
37
38 var (
39
40 validFieldName = regexp.MustCompile("^[a-zA-Z_][a-zA-Z0-9_]{0,127}$")
41 )
42
43 const (
44 nullableTagOption = "nullable"
45 jsonTagOption = "json"
46 )
47
48 func bqTagParser(t reflect.StructTag) (name string, keep bool, other interface{}, err error) {
49 name, keep, opts, err := fields.ParseStandardTag("bigquery", t)
50 if err != nil {
51 return "", false, nil, err
52 }
53 if name != "" && !validFieldName.MatchString(name) {
54 return "", false, nil, invalidFieldNameError(name)
55 }
56 for _, opt := range opts {
57 if opt != nullableTagOption && opt != jsonTagOption {
58 return "", false, nil, fmt.Errorf(
59 "bigquery: invalid tag option %q. The only valid options are %q and %q",
60 opt, nullableTagOption, jsonTagOption)
61 }
62 }
63 return name, keep, opts, nil
64 }
65
66 type invalidFieldNameError string
67
68 func (e invalidFieldNameError) Error() string {
69 return fmt.Sprintf("bigquery: invalid name %q of field in struct", string(e))
70 }
71
72 var fieldCache = fields.NewCache(bqTagParser, nil, nil)
73
74 var (
75 int64ParamType = &bq.QueryParameterType{Type: "INT64"}
76 float64ParamType = &bq.QueryParameterType{Type: "FLOAT64"}
77 boolParamType = &bq.QueryParameterType{Type: "BOOL"}
78 stringParamType = &bq.QueryParameterType{Type: "STRING"}
79 bytesParamType = &bq.QueryParameterType{Type: "BYTES"}
80 dateParamType = &bq.QueryParameterType{Type: "DATE"}
81 timeParamType = &bq.QueryParameterType{Type: "TIME"}
82 dateTimeParamType = &bq.QueryParameterType{Type: "DATETIME"}
83 timestampParamType = &bq.QueryParameterType{Type: "TIMESTAMP"}
84 numericParamType = &bq.QueryParameterType{Type: "NUMERIC"}
85 bigNumericParamType = &bq.QueryParameterType{Type: "BIGNUMERIC"}
86 geographyParamType = &bq.QueryParameterType{Type: "GEOGRAPHY"}
87 intervalParamType = &bq.QueryParameterType{Type: "INTERVAL"}
88 jsonParamType = &bq.QueryParameterType{Type: "JSON"}
89 rangeParamType = &bq.QueryParameterType{Type: "RANGE"}
90 )
91
92 var (
93 typeOfDate = reflect.TypeOf(civil.Date{})
94 typeOfTime = reflect.TypeOf(civil.Time{})
95 typeOfDateTime = reflect.TypeOf(civil.DateTime{})
96 typeOfGoTime = reflect.TypeOf(time.Time{})
97 typeOfRat = reflect.TypeOf(&big.Rat{})
98 typeOfIntervalValue = reflect.TypeOf(&IntervalValue{})
99 typeOfRangeValue = reflect.TypeOf(&RangeValue{})
100 typeOfQueryParameterValue = reflect.TypeOf(&QueryParameterValue{})
101 )
102
103
104 type QueryParameter struct {
105
106
107 Name string
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148 Value interface{}
149 }
150
151
152 type QueryParameterValue struct {
153
154
155
156 Type StandardSQLDataType
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178 Value interface{}
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227 ArrayValue []QueryParameterValue
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264 StructValue map[string]QueryParameterValue
265 }
266
267 func (p QueryParameterValue) toBQParamType() *bq.QueryParameterType {
268 return p.Type.toBQParamType()
269 }
270
271 func (p QueryParameterValue) toBQParamValue() (*bq.QueryParameterValue, error) {
272 if len(p.ArrayValue) > 0 {
273 pv := &bq.QueryParameterValue{}
274 pv.ArrayValues = []*bq.QueryParameterValue{}
275 for _, v := range p.ArrayValue {
276 val, err := v.toBQParamValue()
277 if err != nil {
278 return nil, err
279 }
280 pv.ArrayValues = append(pv.ArrayValues, val)
281 }
282 return pv, nil
283 }
284 if len(p.StructValue) > 0 {
285 pv := &bq.QueryParameterValue{}
286 pv.StructValues = map[string]bq.QueryParameterValue{}
287 for name, param := range p.StructValue {
288 v, err := param.toBQParamValue()
289 if err != nil {
290 return nil, err
291 }
292 pv.StructValues[name] = *v
293 }
294 return pv, nil
295 }
296 pv, err := paramValue(reflect.ValueOf(p.Value))
297 if err != nil {
298 return nil, err
299 }
300 return pv, nil
301 }
302
303 func (p QueryParameter) toBQ() (*bq.QueryParameter, error) {
304 v := reflect.ValueOf(p.Value)
305 pv, err := paramValue(v)
306 if err != nil {
307 return nil, err
308 }
309 pt, err := paramType(reflect.TypeOf(p.Value), v)
310 if err != nil {
311 return nil, err
312 }
313 return &bq.QueryParameter{
314 Name: p.Name,
315 ParameterValue: pv,
316 ParameterType: pt,
317 }, nil
318 }
319
320 var errNilParam = fmt.Errorf("bigquery: nil parameter")
321
322 func paramType(t reflect.Type, v reflect.Value) (*bq.QueryParameterType, error) {
323 if t == nil {
324 return nil, errNilParam
325 }
326 switch t {
327 case typeOfDate, typeOfNullDate:
328 return dateParamType, nil
329 case typeOfTime, typeOfNullTime:
330 return timeParamType, nil
331 case typeOfDateTime, typeOfNullDateTime:
332 return dateTimeParamType, nil
333 case typeOfGoTime, typeOfNullTimestamp:
334 return timestampParamType, nil
335 case typeOfRat:
336 return numericParamType, nil
337 case typeOfIntervalValue:
338 return intervalParamType, nil
339 case typeOfNullBool:
340 return boolParamType, nil
341 case typeOfNullFloat64:
342 return float64ParamType, nil
343 case typeOfNullInt64:
344 return int64ParamType, nil
345 case typeOfNullString:
346 return stringParamType, nil
347 case typeOfNullGeography:
348 return geographyParamType, nil
349 case typeOfNullJSON:
350 return jsonParamType, nil
351 case typeOfRangeValue:
352 iv := v.Interface().(*RangeValue)
353
354
355 element := iv.Start
356 if element == nil {
357 element = iv.End
358 }
359 if element == nil {
360 return nil, fmt.Errorf("unable to determine range element type from RangeValue without a non-nil start or end value")
361 }
362 elet, err := paramType(reflect.TypeOf(element), reflect.ValueOf(element))
363 if err != nil {
364 return nil, err
365 }
366 return &bq.QueryParameterType{
367 Type: "RANGE",
368 RangeElementType: elet,
369 }, nil
370 case typeOfQueryParameterValue:
371 return v.Interface().(*QueryParameterValue).toBQParamType(), nil
372 }
373 switch t.Kind() {
374 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32:
375 return int64ParamType, nil
376
377 case reflect.Float32, reflect.Float64:
378 return float64ParamType, nil
379
380 case reflect.Bool:
381 return boolParamType, nil
382
383 case reflect.String:
384 return stringParamType, nil
385
386 case reflect.Slice:
387 if t.Elem().Kind() == reflect.Uint8 {
388 return bytesParamType, nil
389 }
390 fallthrough
391
392 case reflect.Array:
393 et, err := paramType(t.Elem(), v)
394 if err != nil {
395 return nil, err
396 }
397 return &bq.QueryParameterType{Type: "ARRAY", ArrayType: et}, nil
398
399 case reflect.Ptr:
400 if t.Elem().Kind() != reflect.Struct {
401 break
402 }
403 t = t.Elem()
404 fallthrough
405
406 case reflect.Struct:
407 var fts []*bq.QueryParameterTypeStructTypes
408 fields, err := fieldCache.Fields(t)
409 if err != nil {
410 return nil, err
411 }
412 for _, f := range fields {
413 prefixes := []string{"*", "[]"}
414 for _, prefix := range prefixes {
415 if strings.TrimPrefix(t.String(), prefix) == strings.TrimPrefix(f.Type.String(), prefix) {
416 return nil, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter due to an attribute cycle/recursion detected", t)
417 }
418 }
419 pt, err := paramType(f.Type, v)
420 if err != nil {
421 return nil, err
422 }
423 fts = append(fts, &bq.QueryParameterTypeStructTypes{
424 Name: f.Name,
425 Type: pt,
426 })
427 }
428 return &bq.QueryParameterType{Type: "STRUCT", StructTypes: fts}, nil
429 }
430 return nil, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter type", t)
431 }
432
433 func paramValue(v reflect.Value) (*bq.QueryParameterValue, error) {
434 res := &bq.QueryParameterValue{}
435 if !v.IsValid() {
436 return res, errNilParam
437 }
438 t := v.Type()
439 switch t {
440
441
442 case typeOfNullInt64,
443 typeOfNullString,
444 typeOfNullGeography,
445 typeOfNullFloat64,
446 typeOfNullBool,
447 typeOfNullTimestamp,
448 typeOfNullDate,
449 typeOfNullTime,
450 typeOfNullDateTime,
451 typeOfNullJSON:
452
453
454
455 if !v.FieldByName("Valid").Bool() {
456
457
458 res.NullFields = append(res.NullFields, "Value")
459 return res, nil
460 }
461
462 switch t {
463 case typeOfNullInt64:
464 res.Value = fmt.Sprint(v.FieldByName("Int64").Interface())
465 case typeOfNullString:
466 res.Value = fmt.Sprint(v.FieldByName("StringVal").Interface())
467 case typeOfNullGeography:
468 res.Value = fmt.Sprint(v.FieldByName("GeographyVal").Interface())
469 case typeOfNullJSON:
470 res.Value = fmt.Sprint(v.FieldByName("JSONVal").Interface())
471 case typeOfNullFloat64:
472 res.Value = fmt.Sprint(v.FieldByName("Float64").Interface())
473 case typeOfNullBool:
474 res.Value = fmt.Sprint(v.FieldByName("Bool").Interface())
475 case typeOfNullTimestamp:
476 res.Value = v.FieldByName("Timestamp").Interface().(time.Time).Format(timestampFormat)
477 case typeOfNullDate:
478 res.Value = v.FieldByName("Date").Interface().(civil.Date).String()
479 case typeOfNullTime:
480 res.Value = CivilTimeString(v.FieldByName("Time").Interface().(civil.Time))
481 case typeOfNullDateTime:
482 res.Value = CivilDateTimeString(v.FieldByName("DateTime").Interface().(civil.DateTime))
483 }
484
485
486 if res.Value == "" {
487 res.ForceSendFields = append(res.ForceSendFields, "Value")
488 }
489 return res, nil
490
491 case typeOfDate:
492 res.Value = v.Interface().(civil.Date).String()
493 return res, nil
494 case typeOfTime:
495
496
497
498 res.Value = CivilTimeString(v.Interface().(civil.Time))
499 return res, nil
500
501 case typeOfDateTime:
502 res.Value = CivilDateTimeString(v.Interface().(civil.DateTime))
503 return res, nil
504
505 case typeOfGoTime:
506 res.Value = v.Interface().(time.Time).Format(timestampFormat)
507 return res, nil
508
509 case typeOfRat:
510
511
512
513 res.Value = NumericString(v.Interface().(*big.Rat))
514 return res, nil
515 case typeOfIntervalValue:
516 res.Value = IntervalString(v.Interface().(*IntervalValue))
517 return res, nil
518 case typeOfRangeValue:
519
520
521 res.RangeValue = &bq.RangeValue{}
522 iv := v.Interface().(*RangeValue)
523 sVal, err := paramValue(reflect.ValueOf(iv.Start))
524 if err != nil {
525 if !errors.Is(err, errNilParam) {
526 return nil, err
527 }
528 } else {
529 res.RangeValue.Start = sVal
530 }
531 eVal, err := paramValue(reflect.ValueOf(iv.End))
532 if err != nil {
533 if !errors.Is(err, errNilParam) {
534 return nil, err
535 }
536 } else {
537 res.RangeValue.End = eVal
538 }
539 return res, nil
540 case typeOfQueryParameterValue:
541 return v.Interface().(*QueryParameterValue).toBQParamValue()
542 }
543 switch t.Kind() {
544 case reflect.Slice:
545 if t.Elem().Kind() == reflect.Uint8 {
546 res.Value = base64.StdEncoding.EncodeToString(v.Interface().([]byte))
547 return res, nil
548 }
549 fallthrough
550
551 case reflect.Array:
552 var vals []*bq.QueryParameterValue
553 for i := 0; i < v.Len(); i++ {
554 val, err := paramValue(v.Index(i))
555 if err != nil {
556 return nil, err
557 }
558 vals = append(vals, val)
559 }
560 return &bq.QueryParameterValue{ArrayValues: vals}, nil
561
562 case reflect.Ptr:
563 if t.Elem().Kind() != reflect.Struct {
564 return res, fmt.Errorf("bigquery: Go type %s cannot be represented as a parameter value", t)
565 }
566 t = t.Elem()
567 v = v.Elem()
568 if !v.IsValid() {
569
570 return res, nil
571 }
572 fallthrough
573
574 case reflect.Struct:
575 fields, err := fieldCache.Fields(t)
576 if err != nil {
577 return nil, err
578 }
579 res.StructValues = map[string]bq.QueryParameterValue{}
580 for _, f := range fields {
581 fv := v.FieldByIndex(f.Index)
582 fp, err := paramValue(fv)
583 if err != nil {
584 return nil, err
585 }
586 res.StructValues[f.Name] = *fp
587 }
588 return res, nil
589 }
590
591
592 res.Value = fmt.Sprint(v.Interface())
593
594 if res.Value == "" {
595 res.ForceSendFields = append(res.ForceSendFields, "Value")
596 }
597 return res, nil
598 }
599
600 func bqToQueryParameter(q *bq.QueryParameter) (QueryParameter, error) {
601 p := QueryParameter{Name: q.Name}
602 val, err := convertParamValue(q.ParameterValue, q.ParameterType)
603 if err != nil {
604 return QueryParameter{}, err
605 }
606 p.Value = val
607 return p, nil
608 }
609
610 var paramTypeToFieldType = map[string]FieldType{
611 int64ParamType.Type: IntegerFieldType,
612 float64ParamType.Type: FloatFieldType,
613 boolParamType.Type: BooleanFieldType,
614 stringParamType.Type: StringFieldType,
615 bytesParamType.Type: BytesFieldType,
616 dateParamType.Type: DateFieldType,
617 timeParamType.Type: TimeFieldType,
618 numericParamType.Type: NumericFieldType,
619 bigNumericParamType.Type: BigNumericFieldType,
620 geographyParamType.Type: GeographyFieldType,
621 intervalParamType.Type: IntervalFieldType,
622 jsonParamType.Type: JSONFieldType,
623 }
624
625
626
627
628 func convertParamValue(qval *bq.QueryParameterValue, qtype *bq.QueryParameterType) (interface{}, error) {
629 switch qtype.Type {
630 case "ARRAY":
631 if qval == nil {
632 return []interface{}(nil), nil
633 }
634 return convertParamArray(qval.ArrayValues, qtype.ArrayType)
635 case "STRUCT":
636 if qval == nil {
637 return map[string]interface{}(nil), nil
638 }
639 return convertParamStruct(qval.StructValues, qtype.StructTypes)
640 case "RANGE":
641 rv := &RangeValue{}
642 if qval.RangeValue == nil {
643 return rv, nil
644 }
645 if qval.RangeValue.Start != nil {
646 startVal, err := convertParamValue(qval.RangeValue.Start, qtype.RangeElementType)
647 if err != nil {
648 return nil, err
649 }
650 rv.Start = startVal
651 }
652 if qval.RangeValue.End != nil {
653 endVal, err := convertParamValue(qval.RangeValue.End, qtype.RangeElementType)
654 if err != nil {
655 return nil, err
656 }
657 rv.End = endVal
658 }
659 return rv, nil
660 case "TIMESTAMP":
661 if isNullScalar(qval) {
662 return NullTimestamp{Valid: false}, nil
663 }
664 formats := []string{timestampFormat, time.RFC3339Nano, dateTimeFormat}
665 var lastParseErr error
666 for _, format := range formats {
667 t, err := time.Parse(format, qval.Value)
668 if err != nil {
669 lastParseErr = err
670 continue
671 }
672 return t, nil
673 }
674 return nil, lastParseErr
675
676 case "DATETIME":
677 if isNullScalar(qval) {
678 return NullDateTime{Valid: false}, nil
679 }
680 return parseCivilDateTime(qval.Value)
681 default:
682 if isNullScalar(qval) {
683 switch qtype.Type {
684 case "INT64":
685 return NullInt64{Valid: false}, nil
686 case "STRING":
687 return NullString{Valid: false}, nil
688 case "FLOAT64":
689 return NullFloat64{Valid: false}, nil
690 case "BOOL":
691 return NullBool{Valid: false}, nil
692 case "DATE":
693 return NullDate{Valid: false}, nil
694 case "TIME":
695 return NullTime{Valid: false}, nil
696 case "GEOGRAPHY":
697 return NullGeography{Valid: false}, nil
698 case "JSON":
699 return NullJSON{Valid: false}, nil
700 }
701
702 }
703 return convertBasicType(qval.Value, paramTypeToFieldType[qtype.Type])
704 }
705 }
706
707
708
709 func isNullScalar(qval *bq.QueryParameterValue) bool {
710 if qval == nil {
711 return true
712 }
713 for _, v := range qval.NullFields {
714 if v == "Value" {
715 return true
716 }
717 }
718 return false
719 }
720
721
722
723 func convertParamArray(elVals []*bq.QueryParameterValue, elType *bq.QueryParameterType) ([]interface{}, error) {
724 var vals []interface{}
725 for _, el := range elVals {
726 val, err := convertParamValue(el, elType)
727 if err != nil {
728 return nil, err
729 }
730 vals = append(vals, val)
731 }
732 return vals, nil
733 }
734
735
736
737 func convertParamStruct(sVals map[string]bq.QueryParameterValue, sTypes []*bq.QueryParameterTypeStructTypes) (map[string]interface{}, error) {
738 vals := map[string]interface{}{}
739 for _, st := range sTypes {
740 if sv, ok := sVals[st.Name]; ok {
741 val, err := convertParamValue(&sv, st.Type)
742 if err != nil {
743 return nil, err
744 }
745 vals[st.Name] = val
746 } else {
747 vals[st.Name] = nil
748 }
749 }
750 return vals, nil
751 }
752
View as plain text