important constants for handling the aes encryption
const (
GcmTagLength = 16
NonceLength = 12
)
Module constants for constructing the AAD bytes, the order here is important as the constants are set via iota.
const (
int8 = iota
ColumnMetaModule
DataPageModule
DictPageModule
DataPageHeaderModule
DictPageHeaderModule
ColumnIndexModule
OffsetIndexModule
)
func CreateFooterAad(aadPrefix string) string
CreateFooterAad takes an aadPrefix and constructs the security AAD bytes for encrypting and decrypting the parquet footer bytes.
func CreateModuleAad(fileAad string, moduleType int8, rowGroupOrdinal, columnOrdinal, pageOrdinal int16) string
CreateModuleAad creates the section AAD security bytes for the file, module, row group, column and page.
This should be used for being passed to the encryptor and decryptor whenever requesting AAD bytes.
func NewAesEncryptor(alg parquet.Cipher, metadata bool) *aesEncryptor
NewAesEncryptor constructs an encryptor for the passed in cipher and whether or not it's being used to encrypt metadata.
func QuickUpdatePageAad(aad []byte, newPageOrdinal int16)
QuickUpdatePageAad updates aad with the new page ordinal, modifying the last two bytes of aad.
Decryptor is the basic interface for any decryptor generated from a FileDecryptor
type Decryptor interface {
// returns the File Level AAD bytes
FileAad() string
// returns the current allocator that was used for any extra allocations of buffers
Allocator() memory.Allocator
// returns the CiphertextSizeDelta from the decryptor
CiphertextSizeDelta() int
// Decrypt just returns the decrypted plaintext from the src ciphertext
Decrypt(src []byte) []byte
// Decrypt just returns the decrypted plaintext from the src ciphertext
DecryptFrom(r io.Reader) []byte
// set the AAD bytes of the decryptor to the provided string
UpdateAad(string)
}
Encryptor is the basic interface for encryptors, for now there's only the single aes encryptor implementation, but having it as an interface allows easy addition manipulation of encryptor implementations in the future.
type Encryptor interface {
// FileAad returns the file level AAD bytes for this encryptor
FileAad() string
// UpdateAad sets the aad bytes for encryption to the provided string
UpdateAad(string)
// Allocator returns the allocator that was used to construct the encryptor
Allocator() memory.Allocator
// CiphertextSizeDelta returns the extra bytes that will be added to the ciphertext
// for a total size of len(plaintext) + CiphertextSizeDelta bytes
CiphertextSizeDelta() int
// Encrypt writes the encrypted ciphertext for src to w and returns the total
// number of bytes written.
Encrypt(w io.Writer, src []byte) int
// EncryptColumnMetaData returns true if the column metadata should be encrypted based on the
// column encryption settings and footer encryption setting.
EncryptColumnMetaData(encryptFooter bool, properties *parquet.ColumnEncryptionProperties) bool
}
FileDecryptor is an interface used by the filereader for decrypting an entire parquet file as we go, usually constructed from the DecryptionProperties
type FileDecryptor interface {
// Returns the key for decrypting the footer if provided
GetFooterKey() string
// Provides the file level AAD security bytes
FileAad() string
// return which algorithm this decryptor was constructed for
Algorithm() parquet.Cipher
// return the FileDecryptionProperties that were used for this decryptor
Properties() *parquet.FileDecryptionProperties
// Clear out the decryption keys, this is automatically called after every
// successfully decrypted file to ensure that keys aren't kept around.
WipeOutDecryptionKeys()
// GetFooterDecryptor returns a Decryptor interface for use to decrypt the footer
// of a parquet file.
GetFooterDecryptor() Decryptor
// GetFooterDecryptorForColumnMeta returns a Decryptor interface for Column Metadata
// in the file footer using the AAD bytes provided.
GetFooterDecryptorForColumnMeta(aad string) Decryptor
// GetFooterDecryptorForColumnData returns the decryptor that can be used for decrypting
// actual column data footer bytes, not column metadata.
GetFooterDecryptorForColumnData(aad string) Decryptor
// GetColumnMetaDecryptor returns a decryptor for the requested column path, key and AAD bytes
// but only for decrypting the row group level metadata
GetColumnMetaDecryptor(columnPath, columnKeyMetadata, aad string) Decryptor
// GetColumnDataDecryptor returns a decryptor for the requested column path, key, and AAD bytes
// but only for the rowgroup column data.
GetColumnDataDecryptor(columnPath, columnKeyMetadata, aad string) Decryptor
}
func NewFileDecryptor(props *parquet.FileDecryptionProperties, fileAad string, alg parquet.Cipher, keymetadata string, mem memory.Allocator) FileDecryptor
NewFileDecryptor constructs a decryptor from the provided configuration of properties, cipher and key metadata. Using the provided memory allocator or the default allocator if one isn't provided.
FileEncryptor is the interface for constructing encryptors for the different sections of a parquet file.
type FileEncryptor interface {
// GetFooterEncryptor returns an encryptor for the footer metadata
GetFooterEncryptor() Encryptor
// GetFooterSigningEncryptor returns an encryptor for creating the signature
// for the footer as opposed to encrypting the footer bytes directly.
GetFooterSigningEncryptor() Encryptor
// GetColumnMetaEncryptor returns an encryptor for the metadata only of the requested
// column path string.
GetColumnMetaEncryptor(columnPath string) Encryptor
// GetColumnDataEncryptor returns an encryptor for the column data ONLY of
// the requested column path string.
GetColumnDataEncryptor(columnPath string) Encryptor
// WipeOutEncryptionKeys deletes the keys that were used for encryption,
// called after every successfully encrypted file to ensure against accidental
// key re-use.
WipeOutEncryptionKeys()
}
func NewFileEncryptor(props *parquet.FileEncryptionProperties, mem memory.Allocator) FileEncryptor
NewFileEncryptor returns a new encryptor using the given encryption properties.
Panics if the properties passed have already been used to construct an encryptor ie: props.IsUtilized returns true. If mem is nil, will default to memory.DefaultAllocator
IntegerKeyIDRetriever is used for using unsigned 32bit integers as key ids.
type IntegerKeyIDRetriever map[uint32]string
func (i IntegerKeyIDRetriever) GetKey(keyMetadata []byte) string
GetKey expects the key metadata bytes to be a little endian uint32 which is then used to retrieve the key bytes. Panics if the key id cannot be found.
func (i IntegerKeyIDRetriever) PutKey(keyID uint32, key string)
PutKey adds keys with uint32 IDs
StringKeyIDRetriever implements the KeyRetriever interface GetKey to allow setting in keys with a string id.
type StringKeyIDRetriever map[string]string
func (s StringKeyIDRetriever) GetKey(keyMetadata []byte) string
GetKey expects the keymetadata to match one of the keys that were added with PutKey and panics if the key cannot be found.
func (s StringKeyIDRetriever) PutKey(keyID, key string)
PutKey adds a key with the given string ID that can be retrieved