// Copyright 2016 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package bigquery import ( "context" "time" bq "google.golang.org/api/bigquery/v2" ) // TableCopyOperationType is used to indicate the type of operation performed by a BigQuery // copy job. type TableCopyOperationType string var ( // CopyOperation indicates normal table to table copying. CopyOperation TableCopyOperationType = "COPY" // SnapshotOperation indicates creating a snapshot from a regular table, which // operates as an immutable copy. SnapshotOperation TableCopyOperationType = "SNAPSHOT" // RestoreOperation indicates creating/restoring a table from a snapshot. RestoreOperation TableCopyOperationType = "RESTORE" // CloneOperation indicates creating a table clone, which creates a writeable // copy of a base table that is billed based on difference from the base table. CloneOperation TableCopyOperationType = "CLONE" ) // CopyConfig holds the configuration for a copy job. type CopyConfig struct { // Srcs are the tables from which data will be copied. Srcs []*Table // Dst is the table into which the data will be copied. Dst *Table // CreateDisposition specifies the circumstances under which the destination table will be created. // The default is CreateIfNeeded. CreateDisposition TableCreateDisposition // WriteDisposition specifies how existing data in the destination table is treated. // The default is WriteEmpty. WriteDisposition TableWriteDisposition // The labels associated with this job. Labels map[string]string // Custom encryption configuration (e.g., Cloud KMS keys). DestinationEncryptionConfig *EncryptionConfig // One of the supported operation types when executing a Table Copy jobs. By default this // copies tables, but can also be set to perform snapshot or restore operations. OperationType TableCopyOperationType // Sets a best-effort deadline on a specific job. If job execution exceeds this // timeout, BigQuery may attempt to cancel this work automatically. // // This deadline cannot be adjusted or removed once the job is created. Consider // using Job.Cancel in situations where you need more dynamic behavior. // // Experimental: this option is experimental and may be modified or removed in future versions, // regardless of any other documented package stability guarantees. JobTimeout time.Duration } func (c *CopyConfig) toBQ() *bq.JobConfiguration { var ts []*bq.TableReference for _, t := range c.Srcs { ts = append(ts, t.toBQ()) } return &bq.JobConfiguration{ Labels: c.Labels, Copy: &bq.JobConfigurationTableCopy{ CreateDisposition: string(c.CreateDisposition), WriteDisposition: string(c.WriteDisposition), DestinationTable: c.Dst.toBQ(), DestinationEncryptionConfiguration: c.DestinationEncryptionConfig.toBQ(), SourceTables: ts, OperationType: string(c.OperationType), }, JobTimeoutMs: c.JobTimeout.Milliseconds(), } } func bqToCopyConfig(q *bq.JobConfiguration, c *Client) *CopyConfig { cc := &CopyConfig{ Labels: q.Labels, CreateDisposition: TableCreateDisposition(q.Copy.CreateDisposition), WriteDisposition: TableWriteDisposition(q.Copy.WriteDisposition), Dst: bqToTable(q.Copy.DestinationTable, c), DestinationEncryptionConfig: bqToEncryptionConfig(q.Copy.DestinationEncryptionConfiguration), OperationType: TableCopyOperationType(q.Copy.OperationType), JobTimeout: time.Duration(q.JobTimeoutMs) * time.Millisecond, } for _, t := range q.Copy.SourceTables { cc.Srcs = append(cc.Srcs, bqToTable(t, c)) } return cc } // A Copier copies data into a BigQuery table from one or more BigQuery tables. type Copier struct { JobIDConfig CopyConfig c *Client } // CopierFrom returns a Copier which can be used to copy data into a // BigQuery table from one or more BigQuery tables. // The returned Copier may optionally be further configured before its Run method is called. func (t *Table) CopierFrom(srcs ...*Table) *Copier { return &Copier{ c: t.c, CopyConfig: CopyConfig{ Srcs: srcs, Dst: t, }, } } // Run initiates a copy job. func (c *Copier) Run(ctx context.Context) (*Job, error) { return c.c.insertJob(ctx, c.newJob(), nil) } func (c *Copier) newJob() *bq.Job { return &bq.Job{ JobReference: c.JobIDConfig.createJobRef(c.c), Configuration: c.CopyConfig.toBQ(), } }