write.go

Documentation: github.com/google/go-containerregistry/pkg/legacy/tarball

     1  // Copyright 2019 Google LLC All Rights Reserved.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //    http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package tarball
    16  
    17  import (
    18  	"archive/tar"
    19  	"bytes"
    20  	"encoding/json"
    21  	"fmt"
    22  	"io"
    23  	"sort"
    24  	"strings"
    25  
    26  	"github.com/google/go-containerregistry/pkg/legacy"
    27  	"github.com/google/go-containerregistry/pkg/name"
    28  	v1 "github.com/google/go-containerregistry/pkg/v1"
    29  	"github.com/google/go-containerregistry/pkg/v1/partial"
    30  	"github.com/google/go-containerregistry/pkg/v1/tarball"
    31  )
    32  
    33  // repositoriesTarDescriptor represents the repositories file inside a `docker save` tarball.
    34  type repositoriesTarDescriptor map[string]map[string]string
    35  
    36  // v1Layer represents a layer with metadata needed by the v1 image spec https://github.com/moby/moby/blob/master/image/spec/v1.md.
    37  type v1Layer struct {
    38  	// config is the layer metadata.
    39  	config *legacy.LayerConfigFile
    40  	// layer is the v1.Layer object this v1Layer represents.
    41  	layer v1.Layer
    42  }
    43  
    44  // json returns the raw bytes of the json metadata of the given v1Layer.
    45  func (l *v1Layer) json() ([]byte, error) {
    46  	return json.Marshal(l.config)
    47  }
    48  
    49  // version returns the raw bytes of the "VERSION" file of the given v1Layer.
    50  func (l *v1Layer) version() []byte {
    51  	return []byte("1.0")
    52  }
    53  
    54  // v1LayerID computes the v1 image format layer id for the given v1.Layer with the given v1 parent ID and raw image config.
    55  func v1LayerID(layer v1.Layer, parentID string, rawConfig []byte) (string, error) {
    56  	d, err := layer.Digest()
    57  	if err != nil {
    58  		return "", fmt.Errorf("unable to get layer digest to generate v1 layer ID: %w", err)
    59  	}
    60  	s := fmt.Sprintf("%s %s", d.Hex, parentID)
    61  	if len(rawConfig) != 0 {
    62  		s = fmt.Sprintf("%s %s", s, string(rawConfig))
    63  	}
    64  
    65  	h, _, _ := v1.SHA256(strings.NewReader(s))
    66  	return h.Hex, nil
    67  }
    68  
    69  // newTopV1Layer creates a new v1Layer for a layer other than the top layer in a v1 image tarball.
    70  func newV1Layer(layer v1.Layer, parent *v1Layer, history v1.History) (*v1Layer, error) {
    71  	parentID := ""
    72  	if parent != nil {
    73  		parentID = parent.config.ID
    74  	}
    75  	id, err := v1LayerID(layer, parentID, nil)
    76  	if err != nil {
    77  		return nil, fmt.Errorf("unable to generate v1 layer ID: %w", err)
    78  	}
    79  	result := &v1Layer{
    80  		layer: layer,
    81  		config: &legacy.LayerConfigFile{
    82  			ConfigFile: v1.ConfigFile{
    83  				Created: history.Created,
    84  				Author:  history.Author,
    85  			},
    86  			ContainerConfig: v1.Config{
    87  				Cmd: []string{history.CreatedBy},
    88  			},
    89  			ID:        id,
    90  			Parent:    parentID,
    91  			Throwaway: history.EmptyLayer,
    92  			Comment:   history.Comment,
    93  		},
    94  	}
    95  	return result, nil
    96  }
    97  
    98  // newTopV1Layer creates a new v1Layer for the top layer in a v1 image tarball.
    99  func newTopV1Layer(layer v1.Layer, parent *v1Layer, history v1.History, imgConfig *v1.ConfigFile, rawConfig []byte) (*v1Layer, error) {
   100  	result, err := newV1Layer(layer, parent, history)
   101  	if err != nil {
   102  		return nil, err
   103  	}
   104  	id, err := v1LayerID(layer, result.config.Parent, rawConfig)
   105  	if err != nil {
   106  		return nil, fmt.Errorf("unable to generate v1 layer ID for top layer: %w", err)
   107  	}
   108  	result.config.ID = id
   109  	result.config.Architecture = imgConfig.Architecture
   110  	result.config.Container = imgConfig.Container
   111  	result.config.DockerVersion = imgConfig.DockerVersion
   112  	result.config.OS = imgConfig.OS
   113  	result.config.Config = imgConfig.Config
   114  	result.config.Created = imgConfig.Created
   115  	return result, nil
   116  }
   117  
   118  // splitTag splits the given tagged image name <registry>/<repository>:<tag>
   119  // into <registry>/<repository> and <tag>.
   120  func splitTag(name string) (string, string) {
   121  	// Split on ":"
   122  	parts := strings.Split(name, ":")
   123  	// Verify that we aren't confusing a tag for a hostname w/ port for the purposes of weak validation.
   124  	if len(parts) > 1 && !strings.Contains(parts[len(parts)-1], "/") {
   125  		base := strings.Join(parts[:len(parts)-1], ":")
   126  		tag := parts[len(parts)-1]
   127  		return base, tag
   128  	}
   129  	return name, ""
   130  }
   131  
   132  // addTags adds the given image tags to the given "repositories" file descriptor in a v1 image tarball.
   133  func addTags(repos repositoriesTarDescriptor, tags []string, topLayerID string) {
   134  	for _, t := range tags {
   135  		base, tag := splitTag(t)
   136  		tagToID, ok := repos[base]
   137  		if !ok {
   138  			tagToID = make(map[string]string)
   139  			repos[base] = tagToID
   140  		}
   141  		tagToID[tag] = topLayerID
   142  	}
   143  }
   144  
   145  // updateLayerSources updates the given layer digest to descriptor map with the descriptor of the given layer in the given image if it's an undistributable layer.
   146  func updateLayerSources(layerSources map[v1.Hash]v1.Descriptor, layer v1.Layer, img v1.Image) error {
   147  	d, err := layer.Digest()
   148  	if err != nil {
   149  		return err
   150  	}
   151  	// Add to LayerSources if it's a foreign layer.
   152  	desc, err := partial.BlobDescriptor(img, d)
   153  	if err != nil {
   154  		return err
   155  	}
   156  	if !desc.MediaType.IsDistributable() {
   157  		diffid, err := partial.BlobToDiffID(img, d)
   158  		if err != nil {
   159  			return err
   160  		}
   161  		layerSources[diffid] = *desc
   162  	}
   163  	return nil
   164  }
   165  
   166  // Write is a wrapper to write a single image in V1 format and tag to a tarball.
   167  func Write(ref name.Reference, img v1.Image, w io.Writer) error {
   168  	return MultiWrite(map[name.Reference]v1.Image{ref: img}, w)
   169  }
   170  
   171  // filterEmpty filters out the history corresponding to empty layers from the
   172  // given history.
   173  func filterEmpty(h []v1.History) []v1.History {
   174  	result := []v1.History{}
   175  	for _, i := range h {
   176  		if i.EmptyLayer {
   177  			continue
   178  		}
   179  		result = append(result, i)
   180  	}
   181  	return result
   182  }
   183  
   184  // MultiWrite writes the contents of each image to the provided reader, in the V1 image tarball format.
   185  // The contents are written in the following format:
   186  // One manifest.json file at the top level containing information about several images.
   187  // One repositories file mapping from the image <registry>/<repo name> to <tag> to the id of the top most layer.
   188  // For every layer, a directory named with the layer ID is created with the following contents:
   189  //
   190  //	layer.tar - The uncompressed layer tarball.
   191  //	<layer id>.json- Layer metadata json.
   192  //	VERSION- Schema version string. Always set to "1.0".
   193  //
   194  // One file for the config blob, named after its SHA.
   195  func MultiWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error {
   196  	tf := tar.NewWriter(w)
   197  	defer tf.Close()
   198  
   199  	sortedImages, imageToTags := dedupRefToImage(refToImage)
   200  	var m tarball.Manifest
   201  	repos := make(repositoriesTarDescriptor)
   202  
   203  	seenLayerIDs := make(map[string]struct{})
   204  	for _, img := range sortedImages {
   205  		tags := imageToTags[img]
   206  
   207  		// Write the config.
   208  		cfgName, err := img.ConfigName()
   209  		if err != nil {
   210  			return err
   211  		}
   212  		cfgFileName := fmt.Sprintf("%s.json", cfgName.Hex)
   213  		cfgBlob, err := img.RawConfigFile()
   214  		if err != nil {
   215  			return err
   216  		}
   217  		if err := writeTarEntry(tf, cfgFileName, bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil {
   218  			return err
   219  		}
   220  		cfg, err := img.ConfigFile()
   221  		if err != nil {
   222  			return err
   223  		}
   224  
   225  		// Store foreign layer info.
   226  		layerSources := make(map[v1.Hash]v1.Descriptor)
   227  
   228  		// Write the layers.
   229  		layers, err := img.Layers()
   230  		if err != nil {
   231  			return err
   232  		}
   233  		history := filterEmpty(cfg.History)
   234  		// Create a blank config history if the config didn't have a history.
   235  		if len(history) == 0 && len(layers) != 0 {
   236  			history = make([]v1.History, len(layers))
   237  		} else if len(layers) != len(history) {
   238  			return fmt.Errorf("image config had layer history which did not match the number of layers, got len(history)=%d, len(layers)=%d, want len(history)=len(layers)", len(history), len(layers))
   239  		}
   240  		layerFiles := make([]string, len(layers))
   241  		var prev *v1Layer
   242  		for i, l := range layers {
   243  			if err := updateLayerSources(layerSources, l, img); err != nil {
   244  				return fmt.Errorf("unable to update image metadata to include undistributable layer source information: %w", err)
   245  			}
   246  			var cur *v1Layer
   247  			if i < (len(layers) - 1) {
   248  				cur, err = newV1Layer(l, prev, history[i])
   249  			} else {
   250  				cur, err = newTopV1Layer(l, prev, history[i], cfg, cfgBlob)
   251  			}
   252  			if err != nil {
   253  				return err
   254  			}
   255  			layerFiles[i] = fmt.Sprintf("%s/layer.tar", cur.config.ID)
   256  			if _, ok := seenLayerIDs[cur.config.ID]; ok {
   257  				prev = cur
   258  				continue
   259  			}
   260  			seenLayerIDs[cur.config.ID] = struct{}{}
   261  
   262  			// If the v1.Layer implements UncompressedSize efficiently, use that
   263  			// for the tar header. Otherwise, this iterates over Uncompressed().
   264  			// NOTE: If using a streaming layer, this may consume the layer.
   265  			size, err := partial.UncompressedSize(l)
   266  			if err != nil {
   267  				return err
   268  			}
   269  			u, err := l.Uncompressed()
   270  			if err != nil {
   271  				return err
   272  			}
   273  			defer u.Close()
   274  			if err := writeTarEntry(tf, layerFiles[i], u, size); err != nil {
   275  				return err
   276  			}
   277  
   278  			j, err := cur.json()
   279  			if err != nil {
   280  				return err
   281  			}
   282  			if err := writeTarEntry(tf, fmt.Sprintf("%s/json", cur.config.ID), bytes.NewReader(j), int64(len(j))); err != nil {
   283  				return err
   284  			}
   285  			v := cur.version()
   286  			if err := writeTarEntry(tf, fmt.Sprintf("%s/VERSION", cur.config.ID), bytes.NewReader(v), int64(len(v))); err != nil {
   287  				return err
   288  			}
   289  			prev = cur
   290  		}
   291  
   292  		// Generate the tar descriptor and write it.
   293  		m = append(m, tarball.Descriptor{
   294  			Config:       cfgFileName,
   295  			RepoTags:     tags,
   296  			Layers:       layerFiles,
   297  			LayerSources: layerSources,
   298  		})
   299  		// prev should be the top layer here. Use it to add the image tags
   300  		// to the tarball repositories file.
   301  		addTags(repos, tags, prev.config.ID)
   302  	}
   303  
   304  	mBytes, err := json.Marshal(m)
   305  	if err != nil {
   306  		return err
   307  	}
   308  
   309  	if err := writeTarEntry(tf, "manifest.json", bytes.NewReader(mBytes), int64(len(mBytes))); err != nil {
   310  		return err
   311  	}
   312  	reposBytes, err := json.Marshal(&repos)
   313  	if err != nil {
   314  		return err
   315  	}
   316  	return writeTarEntry(tf, "repositories", bytes.NewReader(reposBytes), int64(len(reposBytes)))
   317  }
   318  
   319  func dedupRefToImage(refToImage map[name.Reference]v1.Image) ([]v1.Image, map[v1.Image][]string) {
   320  	imageToTags := make(map[v1.Image][]string)
   321  
   322  	for ref, img := range refToImage {
   323  		if tag, ok := ref.(name.Tag); ok {
   324  			if tags, ok := imageToTags[img]; ok && tags != nil {
   325  				imageToTags[img] = append(tags, tag.String())
   326  			} else {
   327  				imageToTags[img] = []string{tag.String()}
   328  			}
   329  		} else {
   330  			if _, ok := imageToTags[img]; !ok {
   331  				imageToTags[img] = nil
   332  			}
   333  		}
   334  	}
   335  
   336  	// Force specific order on tags
   337  	imgs := []v1.Image{}
   338  	for img, tags := range imageToTags {
   339  		sort.Strings(tags)
   340  		imgs = append(imgs, img)
   341  	}
   342  
   343  	sort.Slice(imgs, func(i, j int) bool {
   344  		cfI, err := imgs[i].ConfigName()
   345  		if err != nil {
   346  			return false
   347  		}
   348  		cfJ, err := imgs[j].ConfigName()
   349  		if err != nil {
   350  			return false
   351  		}
   352  		return cfI.Hex < cfJ.Hex
   353  	})
   354  
   355  	return imgs, imageToTags
   356  }
   357  
   358  // Writes a file to the provided writer with a corresponding tar header
   359  func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error {
   360  	hdr := &tar.Header{
   361  		Mode:     0644,
   362  		Typeflag: tar.TypeReg,
   363  		Size:     size,
   364  		Name:     path,
   365  	}
   366  	if err := tf.WriteHeader(hdr); err != nil {
   367  		return err
   368  	}
   369  	_, err := io.Copy(tf, r)
   370  	return err
   371  }
   372
View as plain text