1 // Copyright 2015 Google LLC 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package bigquery 16 17 import ( 18 "io" 19 20 bq "google.golang.org/api/bigquery/v2" 21 ) 22 23 // GCSReference is a reference to one or more Google Cloud Storage objects, which together constitute 24 // an input or output to a BigQuery operation. 25 type GCSReference struct { 26 // URIs refer to Google Cloud Storage objects. 27 URIs []string 28 29 FileConfig 30 31 // DestinationFormat is the format to use when writing exported files. 32 // Allowed values are: CSV, Avro, JSON. The default is CSV. 33 // CSV is not supported for tables with nested or repeated fields. 34 DestinationFormat DataFormat 35 36 // Compression specifies the type of compression to apply when writing data 37 // to Google Cloud Storage, or using this GCSReference as an ExternalData 38 // source with CSV or JSON SourceFormat. Default is None. 39 // 40 // Avro files allow additional compression types: DEFLATE and SNAPPY. 41 Compression Compression 42 } 43 44 // NewGCSReference constructs a reference to one or more Google Cloud Storage objects, which together constitute a data source or destination. 45 // In the simple case, a single URI in the form gs://bucket/object may refer to a single GCS object. 46 // Data may also be split into mutiple files, if multiple URIs or URIs containing wildcards are provided. 47 // Each URI may contain one '*' wildcard character, which (if present) must come after the bucket name. 48 // For more information about the treatment of wildcards and multiple URIs, 49 // see https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple 50 func NewGCSReference(uri ...string) *GCSReference { 51 return &GCSReference{URIs: uri} 52 } 53 54 // Compression is the type of compression to apply when writing data to Google Cloud Storage. 55 type Compression string 56 57 const ( 58 // None specifies no compression. 59 None Compression = "NONE" 60 // Gzip specifies gzip compression. 61 Gzip Compression = "GZIP" 62 // Deflate specifies DEFLATE compression for Avro files. 63 Deflate Compression = "DEFLATE" 64 // Snappy specifies SNAPPY compression for Avro files. 65 Snappy Compression = "SNAPPY" 66 ) 67 68 func (gcs *GCSReference) populateLoadConfig(lc *bq.JobConfigurationLoad) io.Reader { 69 lc.SourceUris = gcs.URIs 70 gcs.FileConfig.populateLoadConfig(lc) 71 return nil 72 } 73 74 func (gcs *GCSReference) toBQ() bq.ExternalDataConfiguration { 75 conf := bq.ExternalDataConfiguration{ 76 Compression: string(gcs.Compression), 77 SourceUris: append([]string{}, gcs.URIs...), 78 } 79 gcs.FileConfig.populateExternalDataConfig(&conf) 80 return conf 81 } 82