copy.go

Documentation: cloud.google.com/go/bigquery

     1  // Copyright 2016 Google LLC
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bigquery
    16  
    17  import (
    18  	"context"
    19  	"time"
    20  
    21  	bq "google.golang.org/api/bigquery/v2"
    22  )
    23  
    24  // TableCopyOperationType is used to indicate the type of operation performed by a BigQuery
    25  // copy job.
    26  type TableCopyOperationType string
    27  
    28  var (
    29  	// CopyOperation indicates normal table to table copying.
    30  	CopyOperation TableCopyOperationType = "COPY"
    31  	// SnapshotOperation indicates creating a snapshot from a regular table, which
    32  	// operates as an immutable copy.
    33  	SnapshotOperation TableCopyOperationType = "SNAPSHOT"
    34  	// RestoreOperation indicates creating/restoring a table from a snapshot.
    35  	RestoreOperation TableCopyOperationType = "RESTORE"
    36  	// CloneOperation indicates creating a table clone, which creates a writeable
    37  	// copy of a base table that is billed based on difference from the base table.
    38  	CloneOperation TableCopyOperationType = "CLONE"
    39  )
    40  
    41  // CopyConfig holds the configuration for a copy job.
    42  type CopyConfig struct {
    43  	// Srcs are the tables from which data will be copied.
    44  	Srcs []*Table
    45  
    46  	// Dst is the table into which the data will be copied.
    47  	Dst *Table
    48  
    49  	// CreateDisposition specifies the circumstances under which the destination table will be created.
    50  	// The default is CreateIfNeeded.
    51  	CreateDisposition TableCreateDisposition
    52  
    53  	// WriteDisposition specifies how existing data in the destination table is treated.
    54  	// The default is WriteEmpty.
    55  	WriteDisposition TableWriteDisposition
    56  
    57  	// The labels associated with this job.
    58  	Labels map[string]string
    59  
    60  	// Custom encryption configuration (e.g., Cloud KMS keys).
    61  	DestinationEncryptionConfig *EncryptionConfig
    62  
    63  	// One of the supported operation types when executing a Table Copy jobs.  By default this
    64  	// copies tables, but can also be set to perform snapshot or restore operations.
    65  	OperationType TableCopyOperationType
    66  
    67  	// Sets a best-effort deadline on a specific job.  If job execution exceeds this
    68  	// timeout, BigQuery may attempt to cancel this work automatically.
    69  	//
    70  	// This deadline cannot be adjusted or removed once the job is created.  Consider
    71  	// using Job.Cancel in situations where you need more dynamic behavior.
    72  	//
    73  	// Experimental: this option is experimental and may be modified or removed in future versions,
    74  	// regardless of any other documented package stability guarantees.
    75  	JobTimeout time.Duration
    76  }
    77  
    78  func (c *CopyConfig) toBQ() *bq.JobConfiguration {
    79  	var ts []*bq.TableReference
    80  	for _, t := range c.Srcs {
    81  		ts = append(ts, t.toBQ())
    82  	}
    83  	return &bq.JobConfiguration{
    84  		Labels: c.Labels,
    85  		Copy: &bq.JobConfigurationTableCopy{
    86  			CreateDisposition:                  string(c.CreateDisposition),
    87  			WriteDisposition:                   string(c.WriteDisposition),
    88  			DestinationTable:                   c.Dst.toBQ(),
    89  			DestinationEncryptionConfiguration: c.DestinationEncryptionConfig.toBQ(),
    90  			SourceTables:                       ts,
    91  			OperationType:                      string(c.OperationType),
    92  		},
    93  		JobTimeoutMs: c.JobTimeout.Milliseconds(),
    94  	}
    95  }
    96  
    97  func bqToCopyConfig(q *bq.JobConfiguration, c *Client) *CopyConfig {
    98  	cc := &CopyConfig{
    99  		Labels:                      q.Labels,
   100  		CreateDisposition:           TableCreateDisposition(q.Copy.CreateDisposition),
   101  		WriteDisposition:            TableWriteDisposition(q.Copy.WriteDisposition),
   102  		Dst:                         bqToTable(q.Copy.DestinationTable, c),
   103  		DestinationEncryptionConfig: bqToEncryptionConfig(q.Copy.DestinationEncryptionConfiguration),
   104  		OperationType:               TableCopyOperationType(q.Copy.OperationType),
   105  		JobTimeout:                  time.Duration(q.JobTimeoutMs) * time.Millisecond,
   106  	}
   107  	for _, t := range q.Copy.SourceTables {
   108  		cc.Srcs = append(cc.Srcs, bqToTable(t, c))
   109  	}
   110  	return cc
   111  }
   112  
   113  // A Copier copies data into a BigQuery table from one or more BigQuery tables.
   114  type Copier struct {
   115  	JobIDConfig
   116  	CopyConfig
   117  	c *Client
   118  }
   119  
   120  // CopierFrom returns a Copier which can be used to copy data into a
   121  // BigQuery table from one or more BigQuery tables.
   122  // The returned Copier may optionally be further configured before its Run method is called.
   123  func (t *Table) CopierFrom(srcs ...*Table) *Copier {
   124  	return &Copier{
   125  		c: t.c,
   126  		CopyConfig: CopyConfig{
   127  			Srcs: srcs,
   128  			Dst:  t,
   129  		},
   130  	}
   131  }
   132  
   133  // Run initiates a copy job.
   134  func (c *Copier) Run(ctx context.Context) (*Job, error) {
   135  	return c.c.insertJob(ctx, c.newJob(), nil)
   136  }
   137  
   138  func (c *Copier) newJob() *bq.Job {
   139  	return &bq.Job{
   140  		JobReference:  c.JobIDConfig.createJobRef(c.c),
   141  		Configuration: c.CopyConfig.toBQ(),
   142  	}
   143  }
   144
View as plain text