bump go-containerregistry dep

This commit is contained in:
Josh Chorlton
2020-08-04 19:41:26 -07:00
parent c480a06347
commit 57ab7e03f1
918 changed files with 150715 additions and 29142 deletions
+65 -22
View File
@@ -6,6 +6,10 @@
//
// This package is intended to be a more powerful and safer alternative to
// reflect.DeepEqual for comparing whether two values are semantically equal.
// It is intended to only be used in tests, as performance is not a goal and
// it may panic if it cannot compare the values. Its propensity towards
// panicking means that its unsuitable for production environments where a
// spurious panic may be fatal.
//
// The primary features of cmp are:
//
@@ -22,8 +26,8 @@
// equality is determined by recursively comparing the primitive kinds on both
// values, much like reflect.DeepEqual. Unlike reflect.DeepEqual, unexported
// fields are not compared by default; they result in panics unless suppressed
// by using an Ignore option (see cmpopts.IgnoreUnexported) or explicitly compared
// using the AllowUnexported option.
// by using an Ignore option (see cmpopts.IgnoreUnexported) or explicitly
// compared using the Exporter option.
package cmp
import (
@@ -62,8 +66,8 @@ import (
//
// Structs are equal if recursively calling Equal on all fields report equal.
// If a struct contains unexported fields, Equal panics unless an Ignore option
// (e.g., cmpopts.IgnoreUnexported) ignores that field or the AllowUnexported
// option explicitly permits comparing the unexported field.
// (e.g., cmpopts.IgnoreUnexported) ignores that field or the Exporter option
// explicitly permits comparing the unexported field.
//
// Slices are equal if they are both nil or both non-nil, where recursively
// calling Equal on all non-ignored slice or array elements report equal.
@@ -80,6 +84,11 @@ import (
// Pointers and interfaces are equal if they are both nil or both non-nil,
// where they have the same underlying concrete type and recursively
// calling Equal on the underlying values reports equal.
//
// Before recursing into a pointer, slice element, or map, the current path
// is checked to detect whether the address has already been visited.
// If there is a cycle, then the pointed at values are considered equal
// only if both addresses were previously visited in the same path step.
func Equal(x, y interface{}, opts ...Option) bool {
vx := reflect.ValueOf(x)
vy := reflect.ValueOf(y)
@@ -137,6 +146,7 @@ type state struct {
// Calling statelessCompare must not result in observable changes to these.
result diff.Result // The current result of comparison
curPath Path // The current path in the value tree
curPtrs pointerPath // The current set of visited pointers
reporters []reporter // Optional reporters
// recChecker checks for infinite cycles applying the same set of
@@ -148,13 +158,14 @@ type state struct {
dynChecker dynChecker
// These fields, once set by processOption, will not change.
exporters map[reflect.Type]bool // Set of structs with unexported field visibility
opts Options // List of all fundamental and filter options
exporters []exporter // List of exporters for structs with unexported fields
opts Options // List of all fundamental and filter options
}
func newState(opts []Option) *state {
// Always ensure a validator option exists to validate the inputs.
s := &state{opts: Options{validator{}}}
s.curPtrs.Init()
s.processOption(Options(opts))
return s
}
@@ -174,13 +185,8 @@ func (s *state) processOption(opt Option) {
panic(fmt.Sprintf("cannot use an unfiltered option: %v", opt))
}
s.opts = append(s.opts, opt)
case visibleStructs:
if s.exporters == nil {
s.exporters = make(map[reflect.Type]bool)
}
for t := range opt {
s.exporters[t] = true
}
case exporter:
s.exporters = append(s.exporters, opt)
case reporter:
s.reporters = append(s.reporters, opt)
default:
@@ -192,9 +198,9 @@ func (s *state) processOption(opt Option) {
// This function is stateless in that it does not alter the current result,
// or output to any registered reporters.
func (s *state) statelessCompare(step PathStep) diff.Result {
// We do not save and restore the curPath because all of the compareX
// methods should properly push and pop from the path.
// It is an implementation bug if the contents of curPath differs from
// We do not save and restore curPath and curPtrs because all of the
// compareX methods should properly push and pop from them.
// It is an implementation bug if the contents of the paths differ from
// when calling this function to when returning from it.
oldResult, oldReporters := s.result, s.reporters
@@ -216,9 +222,17 @@ func (s *state) compareAny(step PathStep) {
}
s.recChecker.Check(s.curPath)
// Obtain the current type and values.
// Cycle-detection for slice elements (see NOTE in compareSlice).
t := step.Type()
vx, vy := step.Values()
if si, ok := step.(SliceIndex); ok && si.isSlice && vx.IsValid() && vy.IsValid() {
px, py := vx.Addr(), vy.Addr()
if eq, visited := s.curPtrs.Push(px, py); visited {
s.report(eq, reportByCycle)
return
}
defer s.curPtrs.Pop(px, py)
}
// Rule 1: Check whether an option applies on this node in the value tree.
if s.tryOptions(t, vx, vy) {
@@ -354,6 +368,7 @@ func sanitizeValue(v reflect.Value, t reflect.Type) reflect.Value {
func (s *state) compareStruct(t reflect.Type, vx, vy reflect.Value) {
var vax, vay reflect.Value // Addressable versions of vx and vy
var mayForce, mayForceInit bool
step := StructField{&structField{}}
for i := 0; i < t.NumField(); i++ {
step.typ = t.Field(i).Type
@@ -375,7 +390,13 @@ func (s *state) compareStruct(t reflect.Type, vx, vy reflect.Value) {
vax = makeAddressable(vx)
vay = makeAddressable(vy)
}
step.mayForce = s.exporters[t]
if !mayForceInit {
for _, xf := range s.exporters {
mayForce = mayForce || xf(t)
}
mayForceInit = true
}
step.mayForce = mayForce
step.pvx = vax
step.pvy = vay
step.field = t.Field(i)
@@ -391,9 +412,21 @@ func (s *state) compareSlice(t reflect.Type, vx, vy reflect.Value) {
return
}
// TODO: Support cyclic data structures.
// NOTE: It is incorrect to call curPtrs.Push on the slice header pointer
// since slices represents a list of pointers, rather than a single pointer.
// The pointer checking logic must be handled on a per-element basis
// in compareAny.
//
// A slice header (see reflect.SliceHeader) in Go is a tuple of a starting
// pointer P, a length N, and a capacity C. Supposing each slice element has
// a memory size of M, then the slice is equivalent to the list of pointers:
// [P+i*M for i in range(N)]
//
// For example, v[:0] and v[:1] are slices with the same starting pointer,
// but they are clearly different values. Using the slice pointer alone
// violates the assumption that equal pointers implies equal values.
step := SliceIndex{&sliceIndex{pathStep: pathStep{typ: t.Elem()}}}
step := SliceIndex{&sliceIndex{pathStep: pathStep{typ: t.Elem()}, isSlice: isSlice}}
withIndexes := func(ix, iy int) SliceIndex {
if ix >= 0 {
step.vx, step.xkey = vx.Index(ix), ix
@@ -470,7 +503,12 @@ func (s *state) compareMap(t reflect.Type, vx, vy reflect.Value) {
return
}
// TODO: Support cyclic data structures.
// Cycle-detection for maps.
if eq, visited := s.curPtrs.Push(vx, vy); visited {
s.report(eq, reportByCycle)
return
}
defer s.curPtrs.Pop(vx, vy)
// We combine and sort the two map keys so that we can perform the
// comparisons in a deterministic order.
@@ -507,7 +545,12 @@ func (s *state) comparePtr(t reflect.Type, vx, vy reflect.Value) {
return
}
// TODO: Support cyclic data structures.
// Cycle-detection for pointers.
if eq, visited := s.curPtrs.Push(vx, vy); visited {
s.report(eq, reportByCycle)
return
}
defer s.curPtrs.Pop(vx, vy)
vx, vy = vx.Elem(), vy.Elem()
s.compareAny(Indirect{&indirect{pathStep{t.Elem(), vx, vy}}})
+2 -2
View File
@@ -8,8 +8,8 @@ package cmp
import "reflect"
const supportAllowUnexported = false
const supportExporters = false
func retrieveUnexportedField(reflect.Value, reflect.StructField) reflect.Value {
panic("retrieveUnexportedField is not implemented")
panic("no support for forcibly accessing unexported fields")
}
+4 -2
View File
@@ -11,7 +11,7 @@ import (
"unsafe"
)
const supportAllowUnexported = true
const supportExporters = true
// retrieveUnexportedField uses unsafe to forcibly retrieve any field from
// a struct such that the value has read-write permissions.
@@ -19,5 +19,7 @@ const supportAllowUnexported = true
// The parent struct, v, must be addressable, while f must be a StructField
// describing the field to retrieve.
func retrieveUnexportedField(v reflect.Value, f reflect.StructField) reflect.Value {
return reflect.NewAt(f.Type, unsafe.Pointer(v.UnsafeAddr()+f.Offset)).Elem()
// See https://github.com/google/go-cmp/issues/167 for discussion of the
// following expression.
return reflect.NewAt(f.Type, unsafe.Pointer(uintptr(unsafe.Pointer(v.UnsafeAddr()))+f.Offset)).Elem()
}
+3 -1
View File
@@ -19,7 +19,7 @@ func SortKeys(vs []reflect.Value) []reflect.Value {
}
// Sort the map keys.
sort.Slice(vs, func(i, j int) bool { return isLess(vs[i], vs[j]) })
sort.SliceStable(vs, func(i, j int) bool { return isLess(vs[i], vs[j]) })
// Deduplicate keys (fails for NaNs).
vs2 := vs[:1]
@@ -42,6 +42,8 @@ func isLess(x, y reflect.Value) bool {
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return x.Uint() < y.Uint()
case reflect.Float32, reflect.Float64:
// NOTE: This does not sort -0 as less than +0
// since Go maps treat -0 and +0 as equal keys.
fx, fy := x.Float(), y.Float()
return fx < fy || math.IsNaN(fx) && !math.IsNaN(fy)
case reflect.Complex64, reflect.Complex128:
+6 -3
View File
@@ -4,7 +4,10 @@
package value
import "reflect"
import (
"math"
"reflect"
)
// IsZero reports whether v is the zero value.
// This does not rely on Interface and so can be used on unexported fields.
@@ -17,9 +20,9 @@ func IsZero(v reflect.Value) bool {
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return v.Uint() == 0
case reflect.Float32, reflect.Float64:
return v.Float() == 0
return math.Float64bits(v.Float()) == 0
case reflect.Complex64, reflect.Complex128:
return v.Complex() == 0
return math.Float64bits(real(v.Complex())) == 0 && math.Float64bits(imag(v.Complex())) == 0
case reflect.String:
return v.String() == ""
case reflect.UnsafePointer:
+40 -15
View File
@@ -225,8 +225,20 @@ func (validator) apply(s *state, vx, vy reflect.Value) {
// Unable to Interface implies unexported field without visibility access.
if !vx.CanInterface() || !vy.CanInterface() {
const help = "consider using a custom Comparer; if you control the implementation of type, you can also consider AllowUnexported or cmpopts.IgnoreUnexported"
panic(fmt.Sprintf("cannot handle unexported field: %#v\n%s", s.curPath, help))
const help = "consider using a custom Comparer; if you control the implementation of type, you can also consider using an Exporter, AllowUnexported, or cmpopts.IgnoreUnexported"
var name string
if t := s.curPath.Index(-2).Type(); t.Name() != "" {
// Named type with unexported fields.
name = fmt.Sprintf("%q.%v", t.PkgPath(), t.Name()) // e.g., "path/to/package".MyType
} else {
// Unnamed type with unexported fields. Derive PkgPath from field.
var pkgPath string
for i := 0; i < t.NumField() && pkgPath == ""; i++ {
pkgPath = t.Field(i).PkgPath
}
name = fmt.Sprintf("%q.(%v)", pkgPath, t.String()) // e.g., "path/to/package".(struct { a int })
}
panic(fmt.Sprintf("cannot handle unexported field at %#v:\n\t%v\n%s", s.curPath, name, help))
}
panic("not reachable")
@@ -360,9 +372,8 @@ func (cm comparer) String() string {
return fmt.Sprintf("Comparer(%s)", function.NameOf(cm.fnc))
}
// AllowUnexported returns an Option that forcibly allows operations on
// unexported fields in certain structs, which are specified by passing in a
// value of each struct type.
// Exporter returns an Option that specifies whether Equal is allowed to
// introspect into the unexported fields of certain struct types.
//
// Users of this option must understand that comparing on unexported fields
// from external packages is not safe since changes in the internal
@@ -386,10 +397,24 @@ func (cm comparer) String() string {
//
// In other cases, the cmpopts.IgnoreUnexported option can be used to ignore
// all unexported fields on specified struct types.
func AllowUnexported(types ...interface{}) Option {
if !supportAllowUnexported {
panic("AllowUnexported is not supported on purego builds, Google App Engine Standard, or GopherJS")
func Exporter(f func(reflect.Type) bool) Option {
if !supportExporters {
panic("Exporter is not supported on purego builds")
}
return exporter(f)
}
type exporter func(reflect.Type) bool
func (exporter) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
panic("not implemented")
}
// AllowUnexported returns an Options that allows Equal to forcibly introspect
// unexported fields of the specified struct types.
//
// See Exporter for the proper use of this option.
func AllowUnexported(types ...interface{}) Option {
m := make(map[reflect.Type]bool)
for _, typ := range types {
t := reflect.TypeOf(typ)
@@ -398,13 +423,7 @@ func AllowUnexported(types ...interface{}) Option {
}
m[t] = true
}
return visibleStructs(m)
}
type visibleStructs map[reflect.Type]bool
func (visibleStructs) filter(_ *state, _ reflect.Type, _, _ reflect.Value) applicableOption {
panic("not implemented")
return exporter(func(t reflect.Type) bool { return m[t] })
}
// Result represents the comparison result for a single node and
@@ -436,6 +455,11 @@ func (r Result) ByFunc() bool {
return r.flags&reportByFunc != 0
}
// ByCycle reports whether a reference cycle was detected.
func (r Result) ByCycle() bool {
return r.flags&reportByCycle != 0
}
type resultFlags uint
const (
@@ -446,6 +470,7 @@ const (
reportByIgnore
reportByMethod
reportByFunc
reportByCycle
)
// Reporter is an Option that can be passed to Equal. When Equal traverses
+70 -1
View File
@@ -10,6 +10,8 @@ import (
"strings"
"unicode"
"unicode/utf8"
"github.com/google/go-cmp/cmp/internal/value"
)
// Path is a list of PathSteps describing the sequence of operations to get
@@ -41,7 +43,7 @@ type PathStep interface {
// In some cases, one or both may be invalid or have restrictions:
// • For StructField, both are not interface-able if the current field
// is unexported and the struct type is not explicitly permitted by
// AllowUnexported to traverse unexported fields.
// an Exporter to traverse unexported fields.
// • For SliceIndex, one may be invalid if an element is missing from
// either the x or y slice.
// • For MapIndex, one may be invalid if an entry is missing from
@@ -207,6 +209,7 @@ type SliceIndex struct{ *sliceIndex }
type sliceIndex struct {
pathStep
xkey, ykey int
isSlice bool // False for reflect.Array
}
func (si SliceIndex) Type() reflect.Type { return si.typ }
@@ -301,6 +304,72 @@ func (tf Transform) Func() reflect.Value { return tf.trans.fnc }
// The == operator can be used to detect the exact option used.
func (tf Transform) Option() Option { return tf.trans }
// pointerPath represents a dual-stack of pointers encountered when
// recursively traversing the x and y values. This data structure supports
// detection of cycles and determining whether the cycles are equal.
// In Go, cycles can occur via pointers, slices, and maps.
//
// The pointerPath uses a map to represent a stack; where descension into a
// pointer pushes the address onto the stack, and ascension from a pointer
// pops the address from the stack. Thus, when traversing into a pointer from
// reflect.Ptr, reflect.Slice element, or reflect.Map, we can detect cycles
// by checking whether the pointer has already been visited. The cycle detection
// uses a seperate stack for the x and y values.
//
// If a cycle is detected we need to determine whether the two pointers
// should be considered equal. The definition of equality chosen by Equal
// requires two graphs to have the same structure. To determine this, both the
// x and y values must have a cycle where the previous pointers were also
// encountered together as a pair.
//
// Semantically, this is equivalent to augmenting Indirect, SliceIndex, and
// MapIndex with pointer information for the x and y values.
// Suppose px and py are two pointers to compare, we then search the
// Path for whether px was ever encountered in the Path history of x, and
// similarly so with py. If either side has a cycle, the comparison is only
// equal if both px and py have a cycle resulting from the same PathStep.
//
// Using a map as a stack is more performant as we can perform cycle detection
// in O(1) instead of O(N) where N is len(Path).
type pointerPath struct {
// mx is keyed by x pointers, where the value is the associated y pointer.
mx map[value.Pointer]value.Pointer
// my is keyed by y pointers, where the value is the associated x pointer.
my map[value.Pointer]value.Pointer
}
func (p *pointerPath) Init() {
p.mx = make(map[value.Pointer]value.Pointer)
p.my = make(map[value.Pointer]value.Pointer)
}
// Push indicates intent to descend into pointers vx and vy where
// visited reports whether either has been seen before. If visited before,
// equal reports whether both pointers were encountered together.
// Pop must be called if and only if the pointers were never visited.
//
// The pointers vx and vy must be a reflect.Ptr, reflect.Slice, or reflect.Map
// and be non-nil.
func (p pointerPath) Push(vx, vy reflect.Value) (equal, visited bool) {
px := value.PointerOf(vx)
py := value.PointerOf(vy)
_, ok1 := p.mx[px]
_, ok2 := p.my[py]
if ok1 || ok2 {
equal = p.mx[px] == py && p.my[py] == px // Pointers paired together
return equal, true
}
p.mx[px] = py
p.my[py] = px
return false, false
}
// Pop ascends from pointers vx and vy.
func (p pointerPath) Pop(vx, vy reflect.Value) {
delete(p.mx, value.PointerOf(vx))
delete(p.my, value.PointerOf(vy))
}
// isExported reports whether the identifier is exported.
func isExported(id string) bool {
r, _ := utf8.DecodeRuneInString(id)
+12 -7
View File
@@ -81,14 +81,19 @@ func (opts formatOptions) FormatDiff(v *valueNode) textNode {
return opts.FormatDiffSlice(v)
}
var withinSlice bool
if v.parent != nil && (v.parent.Type.Kind() == reflect.Slice || v.parent.Type.Kind() == reflect.Array) {
withinSlice = true
}
// For leaf nodes, format the value based on the reflect.Values alone.
if v.MaxDepth == 0 {
switch opts.DiffMode {
case diffUnknown, diffIdentical:
// Format Equal.
if v.NumDiff == 0 {
outx := opts.FormatValue(v.ValueX, visitedPointers{})
outy := opts.FormatValue(v.ValueY, visitedPointers{})
outx := opts.FormatValue(v.ValueX, withinSlice, visitedPointers{})
outy := opts.FormatValue(v.ValueY, withinSlice, visitedPointers{})
if v.NumIgnored > 0 && v.NumSame == 0 {
return textEllipsis
} else if outx.Len() < outy.Len() {
@@ -101,8 +106,8 @@ func (opts formatOptions) FormatDiff(v *valueNode) textNode {
// Format unequal.
assert(opts.DiffMode == diffUnknown)
var list textList
outx := opts.WithTypeMode(elideType).FormatValue(v.ValueX, visitedPointers{})
outy := opts.WithTypeMode(elideType).FormatValue(v.ValueY, visitedPointers{})
outx := opts.WithTypeMode(elideType).FormatValue(v.ValueX, withinSlice, visitedPointers{})
outy := opts.WithTypeMode(elideType).FormatValue(v.ValueY, withinSlice, visitedPointers{})
if outx != nil {
list = append(list, textRecord{Diff: '-', Value: outx})
}
@@ -111,9 +116,9 @@ func (opts formatOptions) FormatDiff(v *valueNode) textNode {
}
return opts.WithTypeMode(emitType).FormatType(v.Type, list)
case diffRemoved:
return opts.FormatValue(v.ValueX, visitedPointers{})
return opts.FormatValue(v.ValueX, withinSlice, visitedPointers{})
case diffInserted:
return opts.FormatValue(v.ValueY, visitedPointers{})
return opts.FormatValue(v.ValueY, withinSlice, visitedPointers{})
default:
panic("invalid diff mode")
}
@@ -168,7 +173,7 @@ func (opts formatOptions) formatDiffList(recs []reportRecord, k reflect.Kind) te
var isZero bool
switch opts.DiffMode {
case diffIdentical:
isZero = value.IsZero(r.Value.ValueX) || value.IsZero(r.Value.ValueX)
isZero = value.IsZero(r.Value.ValueX) || value.IsZero(r.Value.ValueY)
case diffRemoved:
isZero = value.IsZero(r.Value.ValueX)
case diffInserted:
+13 -11
View File
@@ -74,7 +74,7 @@ func (opts formatOptions) FormatType(t reflect.Type, s textNode) textNode {
// FormatValue prints the reflect.Value, taking extra care to avoid descending
// into pointers already in m. As pointers are visited, m is also updated.
func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out textNode) {
func (opts formatOptions) FormatValue(v reflect.Value, withinSlice bool, m visitedPointers) (out textNode) {
if !v.IsValid() {
return nil
}
@@ -108,12 +108,15 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
return textLine(fmt.Sprint(v.Bool()))
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return textLine(fmt.Sprint(v.Int()))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
// Unnamed uints are usually bytes or words, so use hexadecimal.
if t.PkgPath() == "" || t.Kind() == reflect.Uintptr {
case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return textLine(fmt.Sprint(v.Uint()))
case reflect.Uint8:
if withinSlice {
return textLine(formatHex(v.Uint()))
}
return textLine(fmt.Sprint(v.Uint()))
case reflect.Uintptr:
return textLine(formatHex(v.Uint()))
case reflect.Float32, reflect.Float64:
return textLine(fmt.Sprint(v.Float()))
case reflect.Complex64, reflect.Complex128:
@@ -129,7 +132,7 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
if value.IsZero(vv) {
continue // Elide fields with zero values
}
s := opts.WithTypeMode(autoType).FormatValue(vv, m)
s := opts.WithTypeMode(autoType).FormatValue(vv, false, m)
list = append(list, textRecord{Key: t.Field(i).Name, Value: s})
}
return textWrap{"{", list, "}"}
@@ -156,7 +159,7 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
continue
}
}
s := opts.WithTypeMode(elideType).FormatValue(vi, m)
s := opts.WithTypeMode(elideType).FormatValue(vi, true, m)
list = append(list, textRecord{Value: s})
}
return textWrap{ptr + "{", list, "}"}
@@ -171,7 +174,7 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
var list textList
for _, k := range value.SortKeys(v.MapKeys()) {
sk := formatMapKey(k)
sv := opts.WithTypeMode(elideType).FormatValue(v.MapIndex(k), m)
sv := opts.WithTypeMode(elideType).FormatValue(v.MapIndex(k), false, m)
list = append(list, textRecord{Key: sk, Value: sv})
}
if opts.PrintAddresses {
@@ -189,7 +192,7 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
ptr = formatPointer(v)
}
skipType = true // Let the underlying value print the type instead
return textWrap{"&" + ptr, opts.FormatValue(v.Elem(), m), ""}
return textWrap{"&" + ptr, opts.FormatValue(v.Elem(), false, m), ""}
case reflect.Interface:
if v.IsNil() {
return textNil
@@ -197,7 +200,7 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
// Interfaces accept different concrete types,
// so configure the underlying value to explicitly print the type.
skipType = true // Print the concrete type instead
return opts.WithTypeMode(emitType).FormatValue(v.Elem(), m)
return opts.WithTypeMode(emitType).FormatValue(v.Elem(), false, m)
default:
panic(fmt.Sprintf("%v kind not handled", v.Kind()))
}
@@ -208,9 +211,8 @@ func (opts formatOptions) FormatValue(v reflect.Value, m visitedPointers) (out t
func formatMapKey(v reflect.Value) string {
var opts formatOptions
opts.TypeMode = elideType
opts.AvoidStringer = true
opts.ShallowPointers = true
s := opts.FormatValue(v, visitedPointers{}).String()
s := opts.FormatValue(v, false, visitedPointers{}).String()
return strings.TrimSpace(s)
}
+5 -3
View File
@@ -90,7 +90,7 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
}
if r == '\n' {
if maxLineLen < i-lastLineIdx {
lastLineIdx = i - lastLineIdx
maxLineLen = i - lastLineIdx
}
lastLineIdx = i + 1
numLines++
@@ -172,7 +172,9 @@ func (opts formatOptions) FormatDiffSlice(v *valueNode) textNode {
switch t.Elem().Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
ss = append(ss, fmt.Sprint(v.Index(i).Int()))
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
case reflect.Uint, reflect.Uint16, reflect.Uint32, reflect.Uint64:
ss = append(ss, fmt.Sprint(v.Index(i).Uint()))
case reflect.Uint8, reflect.Uintptr:
ss = append(ss, formatHex(v.Index(i).Uint()))
case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
ss = append(ss, fmt.Sprint(v.Index(i).Interface()))
@@ -322,7 +324,7 @@ func coalesceInterveningIdentical(groups []diffStats, windowSize int) []diffStat
hadX, hadY := prev.NumRemoved > 0, prev.NumInserted > 0
hasX, hasY := next.NumRemoved > 0, next.NumInserted > 0
if ((hadX || hasX) && (hadY || hasY)) && curr.NumIdentical <= windowSize {
*prev = (*prev).Append(*curr).Append(*next)
*prev = prev.Append(*curr).Append(*next)
groups = groups[:len(groups)-1] // Truncate off equal group
continue
}
+6 -1
View File
@@ -19,6 +19,11 @@ var randBool = rand.New(rand.NewSource(time.Now().Unix())).Intn(2) == 0
type indentMode int
func (n indentMode) appendIndent(b []byte, d diffMode) []byte {
// The output of Diff is documented as being unstable to provide future
// flexibility in changing the output for more humanly readable reports.
// This logic intentionally introduces instability to the exact output
// so that users can detect accidental reliance on stability early on,
// rather than much later when an actual change to the format occurs.
if flags.Deterministic || randBool {
// Use regular spaces (U+0020).
switch d {
@@ -360,7 +365,7 @@ func (s diffStats) String() string {
// Pluralize the name (adjusting for some obscure English grammar rules).
name := s.Name
if sum > 1 {
name = name + "s"
name += "s"
if strings.HasSuffix(name, "ys") {
name = name[:len(name)-2] + "ies" // e.g., "entrys" => "entries"
}
@@ -16,6 +16,7 @@ package k8schain
import (
"fmt"
"sync"
"github.com/google/go-containerregistry/pkg/authn"
"github.com/google/go-containerregistry/pkg/name"
@@ -40,6 +41,11 @@ type Options struct {
ImagePullSecrets []string
}
var (
keyring credentialprovider.DockerKeyring
once sync.Once
)
// New returns a new authn.Keychain suitable for resolving image references as
// scoped by the provided Options. It speaks to Kubernetes through the provided
// client interface.
@@ -83,8 +89,12 @@ func New(client kubernetes.Interface, opt Options) (authn.Keychain, error) {
}
}
once.Do(func() {
keyring = credentialprovider.NewDockerKeyring()
})
// Third, extend the default keyring with the pull secrets.
kr, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, credentialprovider.NewDockerKeyring())
kr, err := credentialprovidersecrets.MakeDockerKeyring(pullSecrets, keyring)
if err != nil {
return nil, err
}
+1 -2
View File
@@ -54,8 +54,6 @@ func (i *imageOpener) Open() (v1.Image, error) {
return nil, err
}
i.client.NegotiateAPIVersion(context.Background())
tb, err := tarball.Image(opener, nil)
if err != nil {
return nil, err
@@ -117,6 +115,7 @@ func Image(ref name.Reference, options ...ImageOption) (v1.Image, error) {
return nil, err
}
}
i.client.NegotiateAPIVersion(context.Background())
return i.Open()
}
+8
View File
@@ -16,6 +16,7 @@ package layout
import (
"encoding/json"
"errors"
"fmt"
"io"
"io/ioutil"
@@ -131,6 +132,13 @@ func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) {
return nil, err
}
if h == (v1.Hash{}) {
if len(im.Manifests) != 1 {
return nil, errors.New("oci layout must contain only a single image to be used with layout.Image")
}
return &(im.Manifests)[0], nil
}
for _, desc := range im.Manifests {
if desc.Digest == h {
return &desc, nil
+14 -7
View File
@@ -69,13 +69,15 @@ func (i *image) compute() error {
digestMap := make(map[v1.Hash]v1.Layer)
for _, add := range i.adds {
diffID, err := add.Layer.DiffID()
if err != nil {
return err
}
diffIDs = append(diffIDs, diffID)
history = append(history, add.History)
diffIDMap[diffID] = add.Layer
if add.Layer != nil {
diffID, err := add.Layer.DiffID()
if err != nil {
return err
}
diffIDs = append(diffIDs, diffID)
diffIDMap[diffID] = add.Layer
}
}
m, err := i.base.Manifest()
@@ -85,6 +87,11 @@ func (i *image) compute() error {
manifest := m.DeepCopy()
manifestLayers := manifest.Layers
for _, add := range i.adds {
if add.Layer == nil {
// Empty layers include only history in manifest.
continue
}
desc, err := partial.Descriptor(add.Layer)
if err != nil {
return err
@@ -251,7 +258,7 @@ func (i *image) LayerByDiffID(h v1.Hash) (v1.Layer, error) {
func validate(adds []Addendum) error {
for _, add := range adds {
if add.Layer == nil {
if add.Layer == nil && !add.History.EmptyLayer {
return errors.New("unable to add a nil layer to the image")
}
}
+45 -17
View File
@@ -50,6 +50,11 @@ func Rebase(orig, oldBase, newBase v1.Image) (v1.Image, error) {
}
}
oldConfig, err := oldBase.ConfigFile()
if err != nil {
return nil, fmt.Errorf("failed to get config for old base: %v", err)
}
origConfig, err := orig.ConfigFile()
if err != nil {
return nil, fmt.Errorf("failed to get config for original: %v", err)
@@ -73,25 +78,48 @@ func Rebase(orig, oldBase, newBase v1.Image) (v1.Image, error) {
return nil, fmt.Errorf("could not get config for new base: %v", err)
}
// Add new base layers.
for i := range newBaseLayers {
rebasedImage, err = Append(rebasedImage, Addendum{
Layer: newBaseLayers[i],
History: newConfig.History[i],
})
if err != nil {
return nil, fmt.Errorf("failed to append layer %d of new base layers", i)
}
rebasedImage, err = Append(rebasedImage, createAddendums(0, 0, newConfig.History, newBaseLayers)...)
if err != nil {
return nil, fmt.Errorf("failed to append new base image: %v", err)
}
// Add original layers above the old base.
start := len(oldBaseLayers)
for i := range origLayers[start:] {
rebasedImage, err = Append(rebasedImage, Addendum{
Layer: origLayers[start+i],
History: origConfig.History[start+i],
})
if err != nil {
return nil, fmt.Errorf("failed to append layer %d of original layers", i)
}
rebasedImage, err = Append(rebasedImage, createAddendums(len(oldConfig.History), len(oldBaseLayers)+1, origConfig.History, origLayers)...)
if err != nil {
return nil, fmt.Errorf("failed to append original image: %v", err)
}
return rebasedImage, nil
}
// createAddendums makes a list of addendums from a history and layers starting from a specific history and layer
// indexes.
func createAddendums(startHistory, startLayer int, history []v1.History, layers []v1.Layer) []Addendum {
var adds []Addendum
// History should be a superset of layers; empty layers (e.g. ENV statements) only exist in history.
// They cannot be iterated identically but must be walked independently, only advancing the iterator for layers
// when a history entry for a non-empty layer is seen.
layerIndex := 0
for historyIndex := range history {
var layer v1.Layer
emptyLayer := history[historyIndex].EmptyLayer
if !emptyLayer {
layer = layers[layerIndex]
layerIndex++
}
if historyIndex >= startHistory || layerIndex >= startLayer {
adds = append(adds, Addendum{
Layer: layer,
History: history[historyIndex],
})
}
}
// In the event history was malformed or non-existent, append the remaining layers.
for i := layerIndex; i < len(layers); i++ {
if i >= startLayer {
adds = append(adds, Addendum{Layer: layers[layerIndex]})
}
}
return adds
}
+24
View File
@@ -0,0 +1,24 @@
// Copyright 2020 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package v1
// Update representation of an update of transfer progress. Some functions
// in this module can take a channel to which updates will be sent while a
// transfer is in progress.
type Update struct {
Total int64
Complete int64
Error error
}
+1 -1
View File
@@ -69,7 +69,7 @@ func Image(byteSize, layers int64) (v1.Image, error) {
Layer: layer,
History: v1.History{
Author: "random.Image",
Comment: fmt.Sprintf("this is a random history %d", i),
Comment: fmt.Sprintf("this is a random history %d of %d", i, layers),
CreatedBy: "random",
Created: v1.Time{time.Now()},
},
+10 -1
View File
@@ -52,7 +52,11 @@ func CatalogPage(target name.Registry, last string, n int, options ...Option) ([
}
client := http.Client{Transport: tr}
resp, err := client.Get(uri.String())
req, err := http.NewRequest(http.MethodGet, uri.String(), nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req.WithContext(o.context))
if err != nil {
return nil, err
}
@@ -92,6 +96,11 @@ func Catalog(ctx context.Context, target name.Registry, options ...Option) ([]st
client := http.Client{Transport: tr}
// WithContext overrides the ctx passed directly.
if o.context != context.Background() {
ctx = o.context
}
var (
parsed catalog
repoList []string
+4 -2
View File
@@ -1,6 +1,7 @@
package remote
import (
"context"
"fmt"
"net/http"
@@ -34,8 +35,9 @@ func CheckPushPermission(ref name.Reference, kc authn.Keychain, t http.RoundTrip
// authorize a push. Figure out how to return early here when we can,
// to avoid a roundtrip for spec-compliant registries.
w := writer{
repo: ref.Context(),
client: &http.Client{Transport: tr},
repo: ref.Context(),
client: &http.Client{Transport: tr},
context: context.Background(),
}
loc, _, err := w.initiateUpload("", "")
if loc != "" {
+1 -1
View File
@@ -47,7 +47,7 @@ func Delete(ref name.Reference, options ...Option) error {
return err
}
resp, err := c.Do(req)
resp, err := c.Do(req.WithContext(o.context))
if err != nil {
return err
}
+22 -15
View File
@@ -16,6 +16,7 @@ package remote
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
@@ -168,10 +169,7 @@ func (d *Descriptor) ImageIndex() (v1.ImageIndex, error) {
func (d *Descriptor) remoteImage() *remoteImage {
return &remoteImage{
fetcher: fetcher{
Ref: d.Ref,
Client: d.Client,
},
fetcher: d.fetcher,
manifest: d.Manifest,
mediaType: d.MediaType,
descriptor: &d.Descriptor,
@@ -180,10 +178,7 @@ func (d *Descriptor) remoteImage() *remoteImage {
func (d *Descriptor) remoteIndex() *remoteIndex {
return &remoteIndex{
fetcher: fetcher{
Ref: d.Ref,
Client: d.Client,
},
fetcher: d.fetcher,
manifest: d.Manifest,
mediaType: d.MediaType,
descriptor: &d.Descriptor,
@@ -192,8 +187,9 @@ func (d *Descriptor) remoteIndex() *remoteIndex {
// fetcher implements methods for reading from a registry.
type fetcher struct {
Ref name.Reference
Client *http.Client
Ref name.Reference
Client *http.Client
context context.Context
}
func makeFetcher(ref name.Reference, o *options) (*fetcher, error) {
@@ -202,8 +198,9 @@ func makeFetcher(ref name.Reference, o *options) (*fetcher, error) {
return nil, err
}
return &fetcher{
Ref: ref,
Client: &http.Client{Transport: tr},
Ref: ref,
Client: &http.Client{Transport: tr},
context: o.context,
}, nil
}
@@ -228,7 +225,7 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType
}
req.Header.Set("Accept", strings.Join(accept, ","))
resp, err := f.Client.Do(req)
resp, err := f.Client.Do(req.WithContext(f.context))
if err != nil {
return nil, nil, err
}
@@ -282,7 +279,12 @@ func (f *fetcher) fetchManifest(ref name.Reference, acceptable []types.MediaType
func (f *fetcher) fetchBlob(h v1.Hash) (io.ReadCloser, error) {
u := f.url("blobs", h.String())
resp, err := f.Client.Get(u.String())
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
resp, err := f.Client.Do(req.WithContext(f.context))
if err != nil {
return nil, err
}
@@ -297,7 +299,12 @@ func (f *fetcher) fetchBlob(h v1.Hash) (io.ReadCloser, error) {
func (f *fetcher) headBlob(h v1.Hash) (*http.Response, error) {
u := f.url("blobs", h.String())
resp, err := f.Client.Head(u.String())
req, err := http.NewRequest(http.MethodHead, u.String(), nil)
if err != nil {
return nil, err
}
resp, err := f.Client.Do(req.WithContext(f.context))
if err != nil {
return nil, err
}
+6 -1
View File
@@ -156,7 +156,12 @@ func (rl *remoteImageLayer) Compressed() (io.ReadCloser, error) {
// TODO: Maybe we don't want to try pulling from the registry first?
var lastErr error
for _, u := range urls {
resp, err := rl.ri.Client.Get(u.String())
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
resp, err := rl.ri.Client.Do(req.WithContext(rl.ri.context))
if err != nil {
lastErr = err
continue
+3 -2
View File
@@ -180,8 +180,9 @@ func (r *remoteIndex) childDescriptor(child v1.Descriptor, platform v1.Platform)
}
return &Descriptor{
fetcher: fetcher{
Ref: ref,
Client: r.Client,
Ref: ref,
Client: r.Client,
context: r.context,
},
Manifest: manifest,
Descriptor: child,
+7
View File
@@ -58,6 +58,13 @@ func ListWithContext(ctx context.Context, repo name.Repository, options ...Optio
RawQuery: "n=1000",
}
// This is lazy, but I want to make sure List(..., WithContext(ctx)) works
// without calling makeOptions() twice (which can have side effects).
// This means ListWithContext(ctx, ..., WithContext(ctx2)) prefers ctx2.
if o.context != context.Background() {
ctx = o.context
}
client := http.Client{Transport: tr}
tagList := []string{}
parsed := tags{}
+17
View File
@@ -15,6 +15,7 @@
package remote
import (
"context"
"net/http"
"github.com/google/go-containerregistry/pkg/authn"
@@ -31,6 +32,7 @@ type options struct {
keychain authn.Keychain
transport http.RoundTripper
platform v1.Platform
context context.Context
}
func makeOptions(target authn.Resource, opts ...Option) (*options, error) {
@@ -38,6 +40,7 @@ func makeOptions(target authn.Resource, opts ...Option) (*options, error) {
auth: authn.Anonymous,
transport: http.DefaultTransport,
platform: defaultPlatform,
context: context.Background(),
}
for _, option := range opts {
@@ -114,3 +117,17 @@ func WithPlatform(p v1.Platform) Option {
return nil
}
}
// WithContext is a functional option for setting the context in http requests
// performed by a given function. Note that this context is used for _all_
// http requests, not just the initial volley. E.g., for remote.Image, the
// context will be set on http requests generated by subsequent calls to
// RawConfigFile() and even methods on layers returned by Layers().
//
// The default context is context.Background().
func WithContext(ctx context.Context) Option {
return func(o *options) error {
o.context = ctx
return nil
}
}
@@ -89,8 +89,7 @@ func (e *Error) Temporary() bool {
return false
}
for _, d := range e.Errors {
// TODO: Include other error types.
if d.Code != BlobUploadInvalidErrorCode {
if _, ok := temporaryErrorCodes[d.Code]; !ok {
return false
}
}
@@ -149,8 +148,15 @@ const (
UnauthorizedErrorCode ErrorCode = "UNAUTHORIZED"
DeniedErrorCode ErrorCode = "DENIED"
UnsupportedErrorCode ErrorCode = "UNSUPPORTED"
TooManyRequestsErrorCode ErrorCode = "TOOMANYREQUESTS"
)
// TODO: Include other error types.
var temporaryErrorCodes = map[ErrorCode]struct{}{
BlobUploadInvalidErrorCode: struct{}{},
TooManyRequestsErrorCode: struct{}{},
}
// CheckError returns a structured error if the response status is not in codes.
func CheckError(resp *http.Response, codes ...int) error {
for _, code := range codes {
@@ -166,10 +172,14 @@ func CheckError(resp *http.Response, codes ...int) error {
// https://github.com/docker/distribution/blob/master/docs/spec/api.md#errors
structuredError := &Error{}
if err := json.Unmarshal(b, structuredError); err != nil {
structuredError.rawBody = string(b)
}
// This can fail if e.g. the response body is not valid JSON. That's fine,
// we'll construct an appropriate error string from the body and status code.
_ = json.Unmarshal(b, structuredError)
structuredError.rawBody = string(b)
structuredError.StatusCode = resp.StatusCode
structuredError.request = resp.Request
return structuredError
}
@@ -16,6 +16,8 @@ package transport
import (
"fmt"
"io"
"io/ioutil"
"net/http"
"strings"
@@ -83,7 +85,12 @@ func ping(reg name.Registry, t http.RoundTripper) (*pingResp, error) {
// Potentially retry with http.
continue
}
defer resp.Body.Close()
defer func() {
// By draining the body, make sure to reuse the connection made by
// the ping for the following access to the registry
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
}()
switch resp.StatusCode {
case http.StatusOK:
+32 -16
View File
@@ -16,6 +16,7 @@ package remote
import (
"bytes"
"context"
"errors"
"fmt"
"io"
@@ -56,8 +57,9 @@ func Write(ref name.Reference, img v1.Image, options ...Option) error {
return err
}
w := writer{
repo: ref.Context(),
client: &http.Client{Transport: tr},
repo: ref.Context(),
client: &http.Client{Transport: tr},
context: o.context,
}
// Upload individual layers in goroutines and collect any errors.
@@ -130,8 +132,9 @@ func Write(ref name.Reference, img v1.Image, options ...Option) error {
// writer writes the elements of an image to a remote image reference.
type writer struct {
repo name.Repository
client *http.Client
repo name.Repository
client *http.Client
context context.Context
}
// url returns a url.Url for the specified path in the context of this remote image reference.
@@ -166,7 +169,12 @@ func (w *writer) nextLocation(resp *http.Response) (string, error) {
func (w *writer) checkExistingBlob(h v1.Hash) (bool, error) {
u := w.url(fmt.Sprintf("/v2/%s/blobs/%s", w.repo.RepositoryStr(), h.String()))
resp, err := w.client.Head(u.String())
req, err := http.NewRequest(http.MethodHead, u.String(), nil)
if err != nil {
return false, err
}
resp, err := w.client.Do(req.WithContext(w.context))
if err != nil {
return false, err
}
@@ -190,7 +198,7 @@ func (w *writer) checkExistingManifest(h v1.Hash, mt types.MediaType) (bool, err
}
req.Header.Set("Accept", string(mt))
resp, err := w.client.Do(req)
resp, err := w.client.Do(req.WithContext(w.context))
if err != nil {
return false, err
}
@@ -220,7 +228,12 @@ func (w *writer) initiateUpload(from, mount string) (location string, mounted bo
u.RawQuery = uv.Encode()
// Make the request to initiate the blob upload.
resp, err := w.client.Post(u.String(), "application/json", nil)
req, err := http.NewRequest(http.MethodPost, u.String(), nil)
if err != nil {
return "", false, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := w.client.Do(req.WithContext(w.context))
if err != nil {
return "", false, err
}
@@ -255,7 +268,7 @@ func (w *writer) streamBlob(blob io.ReadCloser, streamLocation string) (commitLo
return "", err
}
resp, err := w.client.Do(req)
resp, err := w.client.Do(req.WithContext(w.context))
if err != nil {
return "", err
}
@@ -286,7 +299,7 @@ func (w *writer) commitBlob(location, digest string) error {
return err
}
resp, err := w.client.Do(req)
resp, err := w.client.Do(req.WithContext(w.context))
if err != nil {
return err
}
@@ -426,7 +439,7 @@ func (w *writer) commitImage(t Taggable, ref name.Reference) error {
}
req.Header.Set("Content-Type", string(desc.MediaType))
resp, err := w.client.Do(req)
resp, err := w.client.Do(req.WithContext(w.context))
if err != nil {
return err
}
@@ -485,8 +498,9 @@ func WriteIndex(ref name.Reference, ii v1.ImageIndex, options ...Option) error {
return err
}
w := writer{
repo: ref.Context(),
client: &http.Client{Transport: tr},
repo: ref.Context(),
client: &http.Client{Transport: tr},
context: o.context,
}
for _, desc := range index.Manifests {
@@ -538,8 +552,9 @@ func WriteLayer(repo name.Repository, layer v1.Layer, options ...Option) error {
return err
}
w := writer{
repo: repo,
client: &http.Client{Transport: tr},
repo: repo,
client: &http.Client{Transport: tr},
context: o.context,
}
return w.uploadOne(layer)
@@ -564,8 +579,9 @@ func Tag(tag name.Tag, t Taggable, options ...Option) error {
return err
}
w := writer{
repo: tag.Context(),
client: &http.Client{Transport: tr},
repo: tag.Context(),
client: &http.Client{Transport: tr},
context: o.context,
}
return w.commitImage(t, tag)
@@ -162,6 +162,10 @@ func (i *image) loadTarDescriptorAndConfig() error {
return err
}
if i.manifest == nil {
return errors.New("no valid manifest.json in tarball")
}
i.imgDescriptor, err = i.manifest.findDescriptor(i.tag)
if err != nil {
return err
+257 -42
View File
@@ -18,9 +18,12 @@ import (
"archive/tar"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sort"
"strings"
"github.com/google/go-containerregistry/pkg/name"
v1 "github.com/google/go-containerregistry/pkg/v1"
@@ -29,41 +32,41 @@ import (
// WriteToFile writes in the compressed format to a tarball, on disk.
// This is just syntactic sugar wrapping tarball.Write with a new file.
func WriteToFile(p string, ref name.Reference, img v1.Image) error {
func WriteToFile(p string, ref name.Reference, img v1.Image, opts ...WriteOption) error {
w, err := os.Create(p)
if err != nil {
return err
}
defer w.Close()
return Write(ref, img, w)
return Write(ref, img, w, opts...)
}
// MultiWriteToFile writes in the compressed format to a tarball, on disk.
// This is just syntactic sugar wrapping tarball.MultiWrite with a new file.
func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image) error {
func MultiWriteToFile(p string, tagToImage map[name.Tag]v1.Image, opts ...WriteOption) error {
refToImage := make(map[name.Reference]v1.Image, len(tagToImage))
for i, d := range tagToImage {
refToImage[i] = d
}
return MultiRefWriteToFile(p, refToImage)
return MultiRefWriteToFile(p, refToImage, opts...)
}
// MultiRefWriteToFile writes in the compressed format to a tarball, on disk.
// This is just syntactic sugar wrapping tarball.MultiRefWrite with a new file.
func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image) error {
func MultiRefWriteToFile(p string, refToImage map[name.Reference]v1.Image, opts ...WriteOption) error {
w, err := os.Create(p)
if err != nil {
return err
}
defer w.Close()
return MultiRefWrite(refToImage, w)
return MultiRefWrite(refToImage, w, opts...)
}
// Write is a wrapper to write a single image and tag to a tarball.
func Write(ref name.Reference, img v1.Image, w io.Writer) error {
return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w)
func Write(ref name.Reference, img v1.Image, w io.Writer, opts ...WriteOption) error {
return MultiRefWrite(map[name.Reference]v1.Image{ref: img}, w, opts...)
}
// MultiWrite writes the contents of each image to the provided reader, in the compressed format.
@@ -71,12 +74,12 @@ func Write(ref name.Reference, img v1.Image, w io.Writer) error {
// One manifest.json file at the top level containing information about several images.
// One file for each layer, named after the layer's SHA.
// One file for the config blob, named after its SHA.
func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error {
func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer, opts ...WriteOption) error {
refToImage := make(map[name.Reference]v1.Image, len(tagToImage))
for i, d := range tagToImage {
refToImage[i] = d
}
return MultiRefWrite(refToImage, w)
return MultiRefWrite(refToImage, w, opts...)
}
// MultiRefWrite writes the contents of each image to the provided reader, in the compressed format.
@@ -84,41 +87,102 @@ func MultiWrite(tagToImage map[name.Tag]v1.Image, w io.Writer) error {
// One manifest.json file at the top level containing information about several images.
// One file for each layer, named after the layer's SHA.
// One file for the config blob, named after its SHA.
func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error {
tf := tar.NewWriter(w)
func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer, opts ...WriteOption) error {
// process options
o := &writeOptions{
updates: nil,
}
for _, option := range opts {
if err := option(o); err != nil {
return err
}
}
m, err := calculateManifest(refToImage)
if err != nil {
return sendUpdateReturn(o, fmt.Errorf("error calculating manifest: %v", err))
}
mBytes, err := json.Marshal(m)
if err != nil {
return sendUpdateReturn(o, fmt.Errorf("could not marshall manifest to bytes: %v", err))
}
size, err := calculateTarballSize(refToImage, mBytes)
if err != nil {
return sendUpdateReturn(o, fmt.Errorf("error calculating tarball size: %v", err))
}
return writeImagesToTar(refToImage, mBytes, size, w, o)
}
// sendUpdateReturn return the passed in error message, also sending on update channel, if it exists
func sendUpdateReturn(o *writeOptions, err error) error {
if o != nil && o.updates != nil {
o.updates <- v1.Update{
Error: err,
}
}
return err
}
// sendProgressWriterReturn return the passed in error message, also sending on update channel, if it exists, along with downloaded information
func sendProgressWriterReturn(pw *progressWriter, err error) error {
if pw != nil {
return pw.Error(err)
}
return err
}
// writeImagesToTar writes the images to the tarball
func writeImagesToTar(refToImage map[name.Reference]v1.Image, m []byte, size int64, w io.Writer, o *writeOptions) (err error) {
if w == nil {
return sendUpdateReturn(o, errors.New("must pass valid writer"))
}
imageToTags := dedupRefToImage(refToImage)
tw := w
var pw *progressWriter
// we only calculate the sizes and use a progressWriter if we were provided
// an option with a progress channel
if o != nil && o.updates != nil {
pw = &progressWriter{
w: w,
updates: o.updates,
size: size,
}
tw = pw
}
tf := tar.NewWriter(tw)
defer tf.Close()
imageToTags := dedupRefToImage(refToImage)
var m Manifest
seenLayerDigests := make(map[string]struct{})
for img, tags := range imageToTags {
for img := range imageToTags {
// Write the config.
cfgName, err := img.ConfigName()
if err != nil {
return err
return sendProgressWriterReturn(pw, err)
}
cfgBlob, err := img.RawConfigFile()
if err != nil {
return err
return sendProgressWriterReturn(pw, err)
}
if err := writeTarEntry(tf, cfgName.String(), bytes.NewReader(cfgBlob), int64(len(cfgBlob))); err != nil {
return err
return sendProgressWriterReturn(pw, err)
}
// Store foreign layer info.
layerSources := make(map[v1.Hash]v1.Descriptor)
// Write the layers.
layers, err := img.Layers()
if err != nil {
return err
return sendProgressWriterReturn(pw, err)
}
layerFiles := make([]string, len(layers))
for i, l := range layers {
d, err := l.Digest()
if err != nil {
return err
return sendProgressWriterReturn(pw, err)
}
// Munge the file name to appease ancient technology.
//
@@ -136,31 +200,80 @@ func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error {
}
seenLayerDigests[hex] = struct{}{}
r, err := l.Compressed()
if err != nil {
return sendProgressWriterReturn(pw, err)
}
blobSize, err := l.Size()
if err != nil {
return sendProgressWriterReturn(pw, err)
}
if err := writeTarEntry(tf, layerFiles[i], r, blobSize); err != nil {
return sendProgressWriterReturn(pw, err)
}
}
}
if err := writeTarEntry(tf, "manifest.json", bytes.NewReader(m), int64(len(m))); err != nil {
return sendProgressWriterReturn(pw, err)
}
// be sure to close the tar writer so everything is flushed out before we send our EOF
if err := tf.Close(); err != nil {
return sendProgressWriterReturn(pw, err)
}
// send an EOF to indicate finished on the channel, but nil as our return error
_ = sendProgressWriterReturn(pw, io.EOF)
return nil
}
// calculateManifest calculates the manifest and optionally the size of the tar file
func calculateManifest(refToImage map[name.Reference]v1.Image) (m Manifest, err error) {
imageToTags := dedupRefToImage(refToImage)
for img, tags := range imageToTags {
cfgName, err := img.ConfigName()
if err != nil {
return nil, err
}
// Store foreign layer info.
layerSources := make(map[v1.Hash]v1.Descriptor)
// Write the layers.
layers, err := img.Layers()
if err != nil {
return nil, err
}
layerFiles := make([]string, len(layers))
for i, l := range layers {
d, err := l.Digest()
if err != nil {
return nil, err
}
// Munge the file name to appease ancient technology.
//
// tar assumes anything with a colon is a remote tape drive:
// https://www.gnu.org/software/tar/manual/html_section/tar_45.html
// Drop the algorithm prefix, e.g. "sha256:"
hex := d.Hex
// gunzip expects certain file extensions:
// https://www.gnu.org/software/gzip/manual/html_node/Overview.html
layerFiles[i] = fmt.Sprintf("%s.tar.gz", hex)
// Add to LayerSources if it's a foreign layer.
desc, err := partial.BlobDescriptor(img, d)
if err != nil {
return err
return nil, err
}
if !desc.MediaType.IsDistributable() {
diffid, err := partial.BlobToDiffID(img, d)
if err != nil {
return err
return nil, err
}
layerSources[diffid] = *desc
}
r, err := l.Compressed()
if err != nil {
return err
}
blobSize, err := l.Size()
if err != nil {
return err
}
if err := writeTarEntry(tf, layerFiles[i], r, blobSize); err != nil {
return err
}
}
// Generate the tar descriptor and write it.
@@ -171,12 +284,39 @@ func MultiRefWrite(refToImage map[name.Reference]v1.Image, w io.Writer) error {
LayerSources: layerSources,
})
}
// sort by name of the repotags so it is consistent. Alternatively, we could sort by hash of the
// descriptor, but that would make it hard for humans to process
sort.Slice(m, func(i, j int) bool {
return strings.Join(m[i].RepoTags, ",") < strings.Join(m[j].RepoTags, ",")
})
return m, nil
}
// calculateTarballSize calculates the size of the tar file
func calculateTarballSize(refToImage map[name.Reference]v1.Image, m []byte) (size int64, err error) {
imageToTags := dedupRefToImage(refToImage)
for img, name := range imageToTags {
manifest, err := img.Manifest()
if err != nil {
return size, fmt.Errorf("unable to get manifest for img %s: %v", name, err)
}
size += CalculateTarFileSize(manifest.Config.Size)
for _, l := range manifest.Layers {
size += CalculateTarFileSize(l.Size)
}
}
// add the manifest
mBytes, err := json.Marshal(m)
if err != nil {
return err
return size, err
}
return writeTarEntry(tf, "manifest.json", bytes.NewReader(mBytes), int64(len(mBytes)))
size += CalculateTarFileSize(int64(len(mBytes)))
// add the two padding blocks that indicate end of a tar file
size += 1024
return size, nil
}
func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]string {
@@ -199,7 +339,7 @@ func dedupRefToImage(refToImage map[name.Reference]v1.Image) map[v1.Image][]stri
return imageToTags
}
// Writes a file to the provided writer with a corresponding tar header
// writeTarEntry writes a file to the provided writer with a corresponding tar header
func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error {
hdr := &tar.Header{
Mode: 0644,
@@ -213,3 +353,78 @@ func writeTarEntry(tf *tar.Writer, path string, r io.Reader, size int64) error {
_, err := io.Copy(tf, r)
return err
}
// ComputeManifest get the manifest.json that will be written to the tarball
// for multiple references
func ComputeManifest(refToImage map[name.Reference]v1.Image) (Manifest, error) {
return calculateManifest(refToImage)
}
// WriteOption a function option to pass to Write()
type WriteOption func(*writeOptions) error
type writeOptions struct {
updates chan<- v1.Update
}
// WithProgress create a WriteOption for passing to Write() that enables
// a channel to receive updates as they are downloaded and written to disk.
func WithProgress(updates chan<- v1.Update) WriteOption {
return func(o *writeOptions) error {
o.updates = updates
return nil
}
}
// progressWriter is a writer which will send the download progress
type progressWriter struct {
w io.Writer
updates chan<- v1.Update
size, complete int64
}
func (pw *progressWriter) Write(p []byte) (int, error) {
n, err := pw.w.Write(p)
if err != nil {
return n, err
}
pw.complete += int64(n)
pw.updates <- v1.Update{
Total: pw.size,
Complete: pw.complete,
}
return n, err
}
func (pw *progressWriter) Error(err error) error {
pw.updates <- v1.Update{
Total: pw.size,
Complete: pw.complete,
Error: err,
}
return err
}
func (pw *progressWriter) Close() error {
pw.updates <- v1.Update{
Total: pw.size,
Complete: pw.complete,
Error: io.EOF,
}
return io.EOF
}
// CalculateTarFileSize calculate the size a file will take up in a tar archive,
// given the input data. Provided by rounding up to nearest whole block (512)
// and adding header 512
func CalculateTarFileSize(in int64) (out int64) {
// doing this manually, because math.Round() works with float64
out += in
if remainder := out % 512; remainder != 0 {
out += (512 - remainder)
}
out += 512
return out
}