chore(deps): bump github.com/go-git/go-git/v5 from 5.10.0 to 5.10.1 (#2890)

Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.10.0 to 5.10.1. - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/compare/v5.10.0...v5.10.1) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-10-20 21:34:02 +02:00 · 2023-12-04 11:02:55 -08:00
parent 111205f010
commit 1903ab4df2
30 changed files with 697 additions and 549 deletions
@@ -109,10 +109,10 @@ compatibility status with go-git.

 ## Server admin

-| Feature              | Sub-feature | Status | Notes | Examples |
-| -------------------- | ----------- | ------ | ----- | -------- |
-| `daemon`             |             | ❌     |       |          |
-| `update-server-info` |             | ❌     |       |          |
+| Feature              | Sub-feature | Status | Notes | Examples                                  |
+| -------------------- | ----------- | ------ | ----- | ----------------------------------------- |
+| `daemon`             |             | ❌     |       |                                           |
+| `update-server-info` |             | ✅     |       | [cli](./cli/go-git/update_server_info.go) |

 ## Advanced

@@ -17,8 +17,11 @@ const (
 	s = 16

 	// https://github.com/git/git/blob/f7466e94375b3be27f229c78873f0acf8301c0a5/diff-delta.c#L428
-	// Max size of a copy operation (64KB)
+	// Max size of a copy operation (64KB).
 	maxCopySize = 64 * 1024
+
+	// Min size of a copy operation.
+	minCopySize = 4
 )

 // GetDelta returns an EncodedObject of type OFSDeltaObject. Base and Target object,
@@ -3,6 +3,7 @@ package packfile
 import (
 	"bytes"
 	"errors"
+	"fmt"
 	"io"

 	"github.com/go-git/go-git/v5/plumbing"
@@ -174,13 +175,25 @@ func (p *Parser) init() error {
 	return nil
 }

+type objectHeaderWriter func(typ plumbing.ObjectType, sz int64) error
+
+type lazyObjectWriter interface {
+	// LazyWriter enables an object to be lazily written.
+	// It returns:
+	// - w: a writer to receive the object's content.
+	// - lwh: a func to write the object header.
+	// - err: any error from the initial writer creation process.
+	//
+	// Note that if the object header is not written BEFORE the writer
+	// is used, this will result in an invalid object.
+	LazyWriter() (w io.WriteCloser, lwh objectHeaderWriter, err error)
+}
+
 func (p *Parser) indexObjects() error {
 	buf := sync.GetBytesBuffer()
 	defer sync.PutBytesBuffer(buf)

 	for i := uint32(0); i < p.count; i++ {
-		buf.Reset()
-
 		oh, err := p.scanner.NextObjectHeader()
 		if err != nil {
 			return err
@@ -220,21 +233,60 @@ func (p *Parser) indexObjects() error {
 			ota = newBaseObject(oh.Offset, oh.Length, t)
 		}

-		buf.Grow(int(oh.Length))
-		_, crc, err := p.scanner.NextObject(buf)
+		hasher := plumbing.NewHasher(oh.Type, oh.Length)
+		writers := []io.Writer{hasher}
+		var obj *plumbing.MemoryObject
+
+		// Lazy writing is only available for non-delta objects.
+		if p.storage != nil && !delta {
+			// When a storage is set and supports lazy writing,
+			// use that instead of creating a memory object.
+			if low, ok := p.storage.(lazyObjectWriter); ok {
+				ow, lwh, err := low.LazyWriter()
+				if err != nil {
+					return err
+				}
+
+				if err = lwh(oh.Type, oh.Length); err != nil {
+					return err
+				}
+
+				defer ow.Close()
+				writers = append(writers, ow)
+			} else {
+				obj = new(plumbing.MemoryObject)
+				obj.SetSize(oh.Length)
+				obj.SetType(oh.Type)
+
+				writers = append(writers, obj)
+			}
+		}
+		if delta && !p.scanner.IsSeekable {
+			buf.Reset()
+			buf.Grow(int(oh.Length))
+			writers = append(writers, buf)
+		}
+
+		mw := io.MultiWriter(writers...)
+
+		_, crc, err := p.scanner.NextObject(mw)
 		if err != nil {
 			return err
 		}

+		// Non delta objects needs to be added into the storage. This
+		// is only required when lazy writing is not supported.
+		if obj != nil {
+			if _, err := p.storage.SetEncodedObject(obj); err != nil {
+				return err
+			}
+		}
+
 		ota.Crc32 = crc
 		ota.Length = oh.Length

-		data := buf.Bytes()
 		if !delta {
-			sha1, err := getSHA1(ota.Type, data)
-			if err != nil {
-				return err
-			}
+			sha1 := hasher.Sum()

 			// Move children of placeholder parent into actual parent, in case this
 			// was a non-external delta reference.
@@ -249,20 +301,8 @@ func (p *Parser) indexObjects() error {
 			p.oiByHash[ota.SHA1] = ota
 		}

-		if p.storage != nil && !delta {
-			obj := new(plumbing.MemoryObject)
-			obj.SetSize(oh.Length)
-			obj.SetType(oh.Type)
-			if _, err := obj.Write(data); err != nil {
-				return err
-			}
-
-			if _, err := p.storage.SetEncodedObject(obj); err != nil {
-				return err
-			}
-		}
-
 		if delta && !p.scanner.IsSeekable {
+			data := buf.Bytes()
 			p.deltas[oh.Offset] = make([]byte, len(data))
 			copy(p.deltas[oh.Offset], data)
 		}
@@ -280,23 +320,29 @@ func (p *Parser) resolveDeltas() error {

 	for _, obj := range p.oi {
 		buf.Reset()
+		buf.Grow(int(obj.Length))
 		err := p.get(obj, buf)
 		if err != nil {
 			return err
 		}
-		content := buf.Bytes()

 		if err := p.onInflatedObjectHeader(obj.Type, obj.Length, obj.Offset); err != nil {
 			return err
 		}

-		if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, content); err != nil {
+		if err := p.onInflatedObjectContent(obj.SHA1, obj.Offset, obj.Crc32, nil); err != nil {
 			return err
 		}

 		if !obj.IsDelta() && len(obj.Children) > 0 {
+			// Dealing with an io.ReaderAt object, means we can
+			// create it once and reuse across all children.
+			r := bytes.NewReader(buf.Bytes())
 			for _, child := range obj.Children {
-				if err := p.resolveObject(io.Discard, child, content); err != nil {
+				// Even though we are discarding the output, we still need to read it to
+				// so that the scanner can advance to the next object, and the SHA1 can be
+				// calculated.
+				if err := p.resolveObject(io.Discard, child, r); err != nil {
 					return err
 				}
 				p.resolveExternalRef(child)
@@ -361,13 +407,13 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) {
 	if o.DiskType.IsDelta() {
 		b := sync.GetBytesBuffer()
 		defer sync.PutBytesBuffer(b)
+		buf.Grow(int(o.Length))
 		err := p.get(o.Parent, b)
 		if err != nil {
 			return err
 		}
-		base := b.Bytes()

-		err = p.resolveObject(buf, o, base)
+		err = p.resolveObject(buf, o, bytes.NewReader(b.Bytes()))
 		if err != nil {
 			return err
 		}
@@ -378,6 +424,13 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) {
 		}
 	}

+	// If the scanner is seekable, caching this data into
+	// memory by offset seems wasteful.
+	// There is a trade-off to be considered here in terms
+	// of execution time vs memory consumption.
+	//
+	// TODO: improve seekable execution time, so that we can
+	// skip this cache.
 	if len(o.Children) > 0 {
 		data := make([]byte, buf.Len())
 		copy(data, buf.Bytes())
@@ -386,10 +439,25 @@ func (p *Parser) get(o *objectInfo, buf *bytes.Buffer) (err error) {
 	return nil
 }

+// resolveObject resolves an object from base, using information
+// provided by o.
+//
+// This call has the side-effect of changing field values
+// from the object info o:
+//   - Type: OFSDeltaObject may become the target type (e.g. Blob).
+//   - Size: The size may be update with the target size.
+//   - Hash: Zero hashes will be calculated as part of the object
+//     resolution. Hence why this process can't be avoided even when w
+//     is an io.Discard.
+//
+// base must be an io.ReaderAt, which is a requirement from
+// patchDeltaStream. The main reason being that reversing an
+// delta object may lead to going backs and forths within base,
+// which is not supported by io.Reader.
 func (p *Parser) resolveObject(
 	w io.Writer,
 	o *objectInfo,
-	base []byte,
+	base io.ReaderAt,
 ) error {
 	if !o.DiskType.IsDelta() {
 		return nil
@@ -400,26 +468,46 @@ func (p *Parser) resolveObject(
 	if err != nil {
 		return err
 	}
-	data := buf.Bytes()

-	data, err = applyPatchBase(o, data, base)
+	writers := []io.Writer{w}
+	var obj *plumbing.MemoryObject
+	var lwh objectHeaderWriter
+
+	if p.storage != nil {
+		if low, ok := p.storage.(lazyObjectWriter); ok {
+			ow, wh, err := low.LazyWriter()
+			if err != nil {
+				return err
+			}
+			lwh = wh
+
+			defer ow.Close()
+			writers = append(writers, ow)
+		} else {
+			obj = new(plumbing.MemoryObject)
+			ow, err := obj.Writer()
+			if err != nil {
+				return err
+			}
+
+			writers = append(writers, ow)
+		}
+	}
+
+	mw := io.MultiWriter(writers...)
+
+	err = applyPatchBase(o, base, buf, mw, lwh)
 	if err != nil {
 		return err
 	}

-	if p.storage != nil {
-		obj := new(plumbing.MemoryObject)
-		obj.SetSize(o.Size())
+	if obj != nil {
 		obj.SetType(o.Type)
-		if _, err := obj.Write(data); err != nil {
-			return err
-		}
-
+		obj.SetSize(o.Size()) // Size here is correct as it was populated by applyPatchBase.
 		if _, err := p.storage.SetEncodedObject(obj); err != nil {
 			return err
 		}
 	}
-	_, err = w.Write(data)
 	return err
 }

@@ -443,24 +531,31 @@ func (p *Parser) readData(w io.Writer, o *objectInfo) error {
 	return nil
 }

-func applyPatchBase(ota *objectInfo, data, base []byte) ([]byte, error) {
-	patched, err := PatchDelta(base, data)
+// applyPatchBase applies the patch to target.
+//
+// Note that ota will be updated based on the description in resolveObject.
+func applyPatchBase(ota *objectInfo, base io.ReaderAt, delta io.Reader, target io.Writer, wh objectHeaderWriter) error {
+	if target == nil {
+		return fmt.Errorf("cannot apply patch against nil target")
+	}
+
+	typ := ota.Type
+	if ota.SHA1 == plumbing.ZeroHash {
+		typ = ota.Parent.Type
+	}
+
+	sz, h, err := patchDeltaWriter(target, base, delta, typ, wh)
 	if err != nil {
-		return nil, err
+		return err
 	}

 	if ota.SHA1 == plumbing.ZeroHash {
-		ota.Type = ota.Parent.Type
-		sha1, err := getSHA1(ota.Type, patched)
-		if err != nil {
-			return nil, err
-		}
-
-		ota.SHA1 = sha1
-		ota.Length = int64(len(patched))
+		ota.Type = typ
+		ota.Length = int64(sz)
+		ota.SHA1 = h
 	}

-	return patched, nil
+	return nil
 }

 func getSHA1(t plumbing.ObjectType, data []byte) (plumbing.Hash, error) {
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 	"math"

@@ -17,7 +18,33 @@ import (
 // and https://github.com/tarruda/node-git-core/blob/master/src/js/delta.js
 // for details about the delta format.

-const deltaSizeMin = 4
+var (
+	ErrInvalidDelta = errors.New("invalid delta")
+	ErrDeltaCmd     = errors.New("wrong delta command")
+)
+
+const (
+	payload      = 0x7f // 0111 1111
+	continuation = 0x80 // 1000 0000
+)
+
+type offset struct {
+	mask  byte
+	shift uint
+}
+
+var offsets = []offset{
+	{mask: 0x01, shift: 0},
+	{mask: 0x02, shift: 8},
+	{mask: 0x04, shift: 16},
+	{mask: 0x08, shift: 24},
+}
+
+var sizes = []offset{
+	{mask: 0x10, shift: 0},
+	{mask: 0x20, shift: 8},
+	{mask: 0x40, shift: 16},
+}

 // ApplyDelta writes to target the result of applying the modification deltas in delta to base.
 func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) {
@@ -58,11 +85,6 @@ func ApplyDelta(target, base plumbing.EncodedObject, delta []byte) (err error) {
 	return err
 }

-var (
-	ErrInvalidDelta = errors.New("invalid delta")
-	ErrDeltaCmd     = errors.New("wrong delta command")
-)
-
 // PatchDelta returns the result of applying the modification deltas in delta to src.
 // An error will be returned if delta is corrupted (ErrDeltaLen) or an action command
 // is not copy from source or copy from delta (ErrDeltaCmd).
@@ -120,7 +142,8 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo
 				return
 			}

-			if isCopyFromSrc(cmd) {
+			switch {
+			case isCopyFromSrc(cmd):
 				offset, err := decodeOffsetByteReader(cmd, deltaBuf)
 				if err != nil {
 					_ = dstWr.CloseWithError(err)
@@ -173,7 +196,8 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo
 				}
 				remainingTargetSz -= sz
 				basePos += sz
-			} else if isCopyFromDelta(cmd) {
+
+			case isCopyFromDelta(cmd):
 				sz := uint(cmd) // cmd is the size itself
 				if invalidSize(sz, targetSz) {
 					_ = dstWr.CloseWithError(ErrInvalidDelta)
@@ -185,10 +209,12 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo
 				}

 				remainingTargetSz -= sz
-			} else {
+
+			default:
 				_ = dstWr.CloseWithError(ErrDeltaCmd)
 				return
 			}
+
 			if remainingTargetSz <= 0 {
 				_ = dstWr.Close()
 				return
@@ -200,7 +226,7 @@ func ReaderFromDelta(base plumbing.EncodedObject, deltaRC io.Reader) (io.ReadClo
 }

 func patchDelta(dst *bytes.Buffer, src, delta []byte) error {
-	if len(delta) < deltaSizeMin {
+	if len(delta) < minCopySize {
 		return ErrInvalidDelta
 	}

@@ -221,7 +247,9 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error {

 		cmd = delta[0]
 		delta = delta[1:]
-		if isCopyFromSrc(cmd) {
+
+		switch {
+		case isCopyFromSrc(cmd):
 			var offset, sz uint
 			var err error
 			offset, delta, err = decodeOffset(cmd, delta)
@@ -240,7 +268,8 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error {
 			}
 			dst.Write(src[offset : offset+sz])
 			remainingTargetSz -= sz
-		} else if isCopyFromDelta(cmd) {
+
+		case isCopyFromDelta(cmd):
 			sz := uint(cmd) // cmd is the size itself
 			if invalidSize(sz, targetSz) {
 				return ErrInvalidDelta
@@ -253,7 +282,8 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error {
 			dst.Write(delta[0:sz])
 			remainingTargetSz -= sz
 			delta = delta[sz:]
-		} else {
+
+		default:
 			return ErrDeltaCmd
 		}

@@ -265,6 +295,107 @@ func patchDelta(dst *bytes.Buffer, src, delta []byte) error {
 	return nil
 }

+func patchDeltaWriter(dst io.Writer, base io.ReaderAt, delta io.Reader,
+	typ plumbing.ObjectType, writeHeader objectHeaderWriter) (uint, plumbing.Hash, error) {
+	deltaBuf := bufio.NewReaderSize(delta, 1024)
+	srcSz, err := decodeLEB128ByteReader(deltaBuf)
+	if err != nil {
+		if err == io.EOF {
+			return 0, plumbing.ZeroHash, ErrInvalidDelta
+		}
+		return 0, plumbing.ZeroHash, err
+	}
+
+	if r, ok := base.(*bytes.Reader); ok && srcSz != uint(r.Size()) {
+		return 0, plumbing.ZeroHash, ErrInvalidDelta
+	}
+
+	targetSz, err := decodeLEB128ByteReader(deltaBuf)
+	if err != nil {
+		if err == io.EOF {
+			return 0, plumbing.ZeroHash, ErrInvalidDelta
+		}
+		return 0, plumbing.ZeroHash, err
+	}
+
+	// If header still needs to be written, caller will provide
+	// a LazyObjectWriterHeader. This seems to be the case when
+	// dealing with thin-packs.
+	if writeHeader != nil {
+		err = writeHeader(typ, int64(targetSz))
+		if err != nil {
+			return 0, plumbing.ZeroHash, fmt.Errorf("could not lazy write header: %w", err)
+		}
+	}
+
+	remainingTargetSz := targetSz
+
+	hasher := plumbing.NewHasher(typ, int64(targetSz))
+	mw := io.MultiWriter(dst, hasher)
+
+	bufp := sync.GetByteSlice()
+	defer sync.PutByteSlice(bufp)
+
+	sr := io.NewSectionReader(base, int64(0), int64(srcSz))
+	// Keep both the io.LimitedReader types, so we can reset N.
+	baselr := io.LimitReader(sr, 0).(*io.LimitedReader)
+	deltalr := io.LimitReader(deltaBuf, 0).(*io.LimitedReader)
+
+	for {
+		buf := *bufp
+		cmd, err := deltaBuf.ReadByte()
+		if err == io.EOF {
+			return 0, plumbing.ZeroHash, ErrInvalidDelta
+		}
+		if err != nil {
+			return 0, plumbing.ZeroHash, err
+		}
+
+		if isCopyFromSrc(cmd) {
+			offset, err := decodeOffsetByteReader(cmd, deltaBuf)
+			if err != nil {
+				return 0, plumbing.ZeroHash, err
+			}
+			sz, err := decodeSizeByteReader(cmd, deltaBuf)
+			if err != nil {
+				return 0, plumbing.ZeroHash, err
+			}
+
+			if invalidSize(sz, targetSz) ||
+				invalidOffsetSize(offset, sz, srcSz) {
+				return 0, plumbing.ZeroHash, err
+			}
+
+			if _, err := sr.Seek(int64(offset), io.SeekStart); err != nil {
+				return 0, plumbing.ZeroHash, err
+			}
+			baselr.N = int64(sz)
+			if _, err := io.CopyBuffer(mw, baselr, buf); err != nil {
+				return 0, plumbing.ZeroHash, err
+			}
+			remainingTargetSz -= sz
+		} else if isCopyFromDelta(cmd) {
+			sz := uint(cmd) // cmd is the size itself
+			if invalidSize(sz, targetSz) {
+				return 0, plumbing.ZeroHash, ErrInvalidDelta
+			}
+			deltalr.N = int64(sz)
+			if _, err := io.CopyBuffer(mw, deltalr, buf); err != nil {
+				return 0, plumbing.ZeroHash, err
+			}
+
+			remainingTargetSz -= sz
+		} else {
+			return 0, plumbing.ZeroHash, err
+		}
+		if remainingTargetSz <= 0 {
+			break
+		}
+	}
+
+	return targetSz, hasher.Sum(), nil
+}
+
 // Decodes a number encoded as an unsigned LEB128 at the start of some
 // binary data and returns the decoded number and the rest of the
 // stream.
@@ -306,48 +437,24 @@ func decodeLEB128ByteReader(input io.ByteReader) (uint, error) {
 	return num, nil
 }

-const (
-	payload      = 0x7f // 0111 1111
-	continuation = 0x80 // 1000 0000
-)
-
 func isCopyFromSrc(cmd byte) bool {
-	return (cmd & 0x80) != 0
+	return (cmd & continuation) != 0
 }

 func isCopyFromDelta(cmd byte) bool {
-	return (cmd&0x80) == 0 && cmd != 0
+	return (cmd&continuation) == 0 && cmd != 0
 }

 func decodeOffsetByteReader(cmd byte, delta io.ByteReader) (uint, error) {
 	var offset uint
-	if (cmd & 0x01) != 0 {
-		next, err := delta.ReadByte()
-		if err != nil {
-			return 0, err
+	for _, o := range offsets {
+		if (cmd & o.mask) != 0 {
+			next, err := delta.ReadByte()
+			if err != nil {
+				return 0, err
+			}
+			offset |= uint(next) << o.shift
 		}
-		offset = uint(next)
-	}
-	if (cmd & 0x02) != 0 {
-		next, err := delta.ReadByte()
-		if err != nil {
-			return 0, err
-		}
-		offset |= uint(next) << 8
-	}
-	if (cmd & 0x04) != 0 {
-		next, err := delta.ReadByte()
-		if err != nil {
-			return 0, err
-		}
-		offset |= uint(next) << 16
-	}
-	if (cmd & 0x08) != 0 {
-		next, err := delta.ReadByte()
-		if err != nil {
-			return 0, err
-		}
-		offset |= uint(next) << 24
 	}

 	return offset, nil
@@ -355,33 +462,14 @@ func decodeOffsetByteReader(cmd byte, delta io.ByteReader) (uint, error) {

 func decodeOffset(cmd byte, delta []byte) (uint, []byte, error) {
 	var offset uint
-	if (cmd & 0x01) != 0 {
-		if len(delta) == 0 {
-			return 0, nil, ErrInvalidDelta
+	for _, o := range offsets {
+		if (cmd & o.mask) != 0 {
+			if len(delta) == 0 {
+				return 0, nil, ErrInvalidDelta
+			}
+			offset |= uint(delta[0]) << o.shift
+			delta = delta[1:]
 		}
-		offset = uint(delta[0])
-		delta = delta[1:]
-	}
-	if (cmd & 0x02) != 0 {
-		if len(delta) == 0 {
-			return 0, nil, ErrInvalidDelta
-		}
-		offset |= uint(delta[0]) << 8
-		delta = delta[1:]
-	}
-	if (cmd & 0x04) != 0 {
-		if len(delta) == 0 {
-			return 0, nil, ErrInvalidDelta
-		}
-		offset |= uint(delta[0]) << 16
-		delta = delta[1:]
-	}
-	if (cmd & 0x08) != 0 {
-		if len(delta) == 0 {
-			return 0, nil, ErrInvalidDelta
-		}
-		offset |= uint(delta[0]) << 24
-		delta = delta[1:]
 	}

 	return offset, delta, nil
@@ -389,29 +477,18 @@ func decodeOffset(cmd byte, delta []byte) (uint, []byte, error) {

 func decodeSizeByteReader(cmd byte, delta io.ByteReader) (uint, error) {
 	var sz uint
-	if (cmd & 0x10) != 0 {
-		next, err := delta.ReadByte()
-		if err != nil {
-			return 0, err
+	for _, s := range sizes {
+		if (cmd & s.mask) != 0 {
+			next, err := delta.ReadByte()
+			if err != nil {
+				return 0, err
+			}
+			sz |= uint(next) << s.shift
 		}
-		sz = uint(next)
-	}
-	if (cmd & 0x20) != 0 {
-		next, err := delta.ReadByte()
-		if err != nil {
-			return 0, err
-		}
-		sz |= uint(next) << 8
-	}
-	if (cmd & 0x40) != 0 {
-		next, err := delta.ReadByte()
-		if err != nil {
-			return 0, err
-		}
-		sz |= uint(next) << 16
 	}
+
 	if sz == 0 {
-		sz = 0x10000
+		sz = maxCopySize
 	}

 	return sz, nil
@@ -419,29 +496,17 @@ func decodeSizeByteReader(cmd byte, delta io.ByteReader) (uint, error) {

 func decodeSize(cmd byte, delta []byte) (uint, []byte, error) {
 	var sz uint
-	if (cmd & 0x10) != 0 {
-		if len(delta) == 0 {
-			return 0, nil, ErrInvalidDelta
+	for _, s := range sizes {
+		if (cmd & s.mask) != 0 {
+			if len(delta) == 0 {
+				return 0, nil, ErrInvalidDelta
+			}
+			sz |= uint(delta[0]) << s.shift
+			delta = delta[1:]
 		}
-		sz = uint(delta[0])
-		delta = delta[1:]
-	}
-	if (cmd & 0x20) != 0 {
-		if len(delta) == 0 {
-			return 0, nil, ErrInvalidDelta
-		}
-		sz |= uint(delta[0]) << 8
-		delta = delta[1:]
-	}
-	if (cmd & 0x40) != 0 {
-		if len(delta) == 0 {
-			return 0, nil, ErrInvalidDelta
-		}
-		sz |= uint(delta[0]) << 16
-		delta = delta[1:]
 	}
 	if sz == 0 {
-		sz = 0x10000
+		sz = maxCopySize
 	}

 	return sz, delta, nil
@@ -7,6 +7,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+
+	"github.com/go-git/go-git/v5/utils/trace"
 )

 // An Encoder writes pkt-lines to an output stream.
@@ -43,6 +45,7 @@ func NewEncoder(w io.Writer) *Encoder {

 // Flush encodes a flush-pkt to the output stream.
 func (e *Encoder) Flush() error {
+	defer trace.Packet.Print("packet: > 0000")
 	_, err := e.w.Write(FlushPkt)
 	return err
 }
@@ -70,6 +73,7 @@ func (e *Encoder) encodeLine(p []byte) error {
 	}

 	n := len(p) + 4
+	defer trace.Packet.Printf("packet: > %04x %s", n, p)
 	if _, err := e.w.Write(asciiHex16(n)); err != nil {
 		return err
 	}
@@ -0,0 +1,51 @@
+package pktline
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"strings"
+)
+
+var (
+	// ErrInvalidErrorLine is returned by Decode when the packet line is not an
+	// error line.
+	ErrInvalidErrorLine = errors.New("expected an error-line")
+
+	errPrefix = []byte("ERR ")
+)
+
+// ErrorLine is a packet line that contains an error message.
+// Once this packet is sent by client or server, the data transfer process is
+// terminated.
+// See https://git-scm.com/docs/pack-protocol#_pkt_line_format
+type ErrorLine struct {
+	Text string
+}
+
+// Error implements the error interface.
+func (e *ErrorLine) Error() string {
+	return e.Text
+}
+
+// Encode encodes the ErrorLine into a packet line.
+func (e *ErrorLine) Encode(w io.Writer) error {
+	p := NewEncoder(w)
+	return p.Encodef("%s%s\n", string(errPrefix), e.Text)
+}
+
+// Decode decodes a packet line into an ErrorLine.
+func (e *ErrorLine) Decode(r io.Reader) error {
+	s := NewScanner(r)
+	if !s.Scan() {
+		return s.Err()
+	}
+
+	line := s.Bytes()
+	if !bytes.HasPrefix(line, errPrefix) {
+		return ErrInvalidErrorLine
+	}
+
+	e.Text = strings.TrimSpace(string(line[4:]))
+	return nil
+}
@@ -1,8 +1,12 @@
 package pktline

 import (
+	"bytes"
 	"errors"
 	"io"
+	"strings"
+
+	"github.com/go-git/go-git/v5/utils/trace"
 )

 const (
@@ -65,6 +69,14 @@ func (s *Scanner) Scan() bool {
 		return false
 	}
 	s.payload = s.payload[:l]
+	trace.Packet.Printf("packet: < %04x %s", l, s.payload)
+
+	if bytes.HasPrefix(s.payload, errPrefix) {
+		s.err = &ErrorLine{
+			Text: strings.TrimSpace(string(s.payload[4:])),
+		}
+		return false
+	}

 	return true
 }
@@ -48,6 +48,11 @@ func isFlush(payload []byte) bool {
 	return len(payload) == 0
 }

+var (
+	// ErrNilWriter is returned when a nil writer is passed to the encoder.
+	ErrNilWriter = fmt.Errorf("nil writer")
+)
+
 // ErrUnexpectedData represents an unexpected data decoding a message
 type ErrUnexpectedData struct {
 	Msg  string
@@ -0,0 +1,120 @@
+package packp
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing/format/pktline"
+)
+
+var (
+	// ErrInvalidGitProtoRequest is returned by Decode if the input is not a
+	// valid git protocol request.
+	ErrInvalidGitProtoRequest = fmt.Errorf("invalid git protocol request")
+)
+
+// GitProtoRequest is a command request for the git protocol.
+// It is used to send the command, endpoint, and extra parameters to the
+// remote.
+// See https://git-scm.com/docs/pack-protocol#_git_transport
+type GitProtoRequest struct {
+	RequestCommand string
+	Pathname       string
+
+	// Optional
+	Host string
+
+	// Optional
+	ExtraParams []string
+}
+
+// validate validates the request.
+func (g *GitProtoRequest) validate() error {
+	if g.RequestCommand == "" {
+		return fmt.Errorf("%w: empty request command", ErrInvalidGitProtoRequest)
+	}
+
+	if g.Pathname == "" {
+		return fmt.Errorf("%w: empty pathname", ErrInvalidGitProtoRequest)
+	}
+
+	return nil
+}
+
+// Encode encodes the request into the writer.
+func (g *GitProtoRequest) Encode(w io.Writer) error {
+	if w == nil {
+		return ErrNilWriter
+	}
+
+	if err := g.validate(); err != nil {
+		return err
+	}
+
+	p := pktline.NewEncoder(w)
+	req := fmt.Sprintf("%s %s\x00", g.RequestCommand, g.Pathname)
+	if host := g.Host; host != "" {
+		req += fmt.Sprintf("host=%s\x00", host)
+	}
+
+	if len(g.ExtraParams) > 0 {
+		req += "\x00"
+		for _, param := range g.ExtraParams {
+			req += param + "\x00"
+		}
+	}
+
+	if err := p.Encode([]byte(req)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Decode decodes the request from the reader.
+func (g *GitProtoRequest) Decode(r io.Reader) error {
+	s := pktline.NewScanner(r)
+	if !s.Scan() {
+		err := s.Err()
+		if err == nil {
+			return ErrInvalidGitProtoRequest
+		}
+		return err
+	}
+
+	line := string(s.Bytes())
+	if len(line) == 0 {
+		return io.EOF
+	}
+
+	if line[len(line)-1] != 0 {
+		return fmt.Errorf("%w: missing null terminator", ErrInvalidGitProtoRequest)
+	}
+
+	parts := strings.SplitN(line, " ", 2)
+	if len(parts) != 2 {
+		return fmt.Errorf("%w: short request", ErrInvalidGitProtoRequest)
+	}
+
+	g.RequestCommand = parts[0]
+	params := strings.Split(parts[1], string(null))
+	if len(params) < 1 {
+		return fmt.Errorf("%w: missing pathname", ErrInvalidGitProtoRequest)
+	}
+
+	g.Pathname = params[0]
+	if len(params) > 1 {
+		g.Host = strings.TrimPrefix(params[1], "host=")
+	}
+
+	if len(params) > 2 {
+		for _, param := range params[2:] {
+			if param != "" {
+				g.ExtraParams = append(g.ExtraParams, param)
+			}
+		}
+	}
+
+	return nil
+}
@@ -11,7 +11,6 @@ import (

 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/internal/common"
-	"github.com/go-git/go-git/v5/utils/ioutil"
 	"golang.org/x/sys/execabs"
 )

@@ -112,7 +111,7 @@ func (c *command) Start() error {
 func (c *command) StderrPipe() (io.Reader, error) {
 	// Pipe returned by Command.StderrPipe has a race with Read + Command.Wait.
 	// We use an io.Pipe and close it after the command finishes.
-	r, w := ioutil.Pipe()
+	r, w := io.Pipe()
 	c.cmd.Stderr = w
 	c.stderrCloser = r
 	return r, nil
@@ -2,12 +2,11 @@
 package git

 import (
-	"fmt"
 	"io"
 	"net"
 	"strconv"

-	"github.com/go-git/go-git/v5/plumbing/format/pktline"
+	"github.com/go-git/go-git/v5/plumbing/protocol/packp"
 	"github.com/go-git/go-git/v5/plumbing/transport"
 	"github.com/go-git/go-git/v5/plumbing/transport/internal/common"
 	"github.com/go-git/go-git/v5/utils/ioutil"
@@ -42,10 +41,18 @@ type command struct {

 // Start executes the command sending the required message to the TCP connection
 func (c *command) Start() error {
-	cmd := endpointToCommand(c.command, c.endpoint)
+	req := packp.GitProtoRequest{
+		RequestCommand: c.command,
+		Pathname:       c.endpoint.Path,
+	}
+	host := c.endpoint.Host
+	if c.endpoint.Port != DefaultPort {
+		host = net.JoinHostPort(c.endpoint.Host, strconv.Itoa(c.endpoint.Port))
+	}

-	e := pktline.NewEncoder(c.conn)
-	return e.Encode([]byte(cmd))
+	req.Host = host
+
+	return req.Encode(c.conn)
 }

 func (c *command) connect() error {
@@ -90,15 +97,6 @@ func (c *command) StdoutPipe() (io.Reader, error) {
 	return c.conn, nil
 }

-func endpointToCommand(cmd string, ep *transport.Endpoint) string {
-	host := ep.Host
-	if ep.Port != DefaultPort {
-		host = net.JoinHostPort(ep.Host, strconv.Itoa(ep.Port))
-	}
-
-	return fmt.Sprintf("%s %s%chost=%s%c", cmd, ep.Path, 0, host, 0)
-}
-
 // Close closes the TCP connection and connection.
 func (c *command) Close() error {
 	if !c.connected {
@@ -203,9 +203,22 @@ func (s *session) AdvertisedReferencesContext(ctx context.Context) (*packp.AdvRe
 }

 func (s *session) handleAdvRefDecodeError(err error) error {
+	var errLine *pktline.ErrorLine
+	if errors.As(err, &errLine) {
+		if isRepoNotFoundError(errLine.Text) {
+			return transport.ErrRepositoryNotFound
+		}
+
+		return errLine
+	}
+
 	// If repository is not found, we get empty stdout and server writes an
 	// error to stderr.
-	if err == packp.ErrEmptyInput {
+	if errors.Is(err, packp.ErrEmptyInput) {
+		// TODO:(v6): handle this error in a better way.
+		// Instead of checking the stderr output for a specific error message,
+		// define an ExitError and embed the stderr output and exit (if one
+		// exists) in the error struct. Just like exec.ExitError.
 		s.finished = true
 		if err := s.checkNotFoundError(); err != nil {
 			return err
@@ -245,6 +258,12 @@ func (s *session) handleAdvRefDecodeError(err error) error {
 // returned with the packfile content. The reader must be closed after reading.
 func (s *session) UploadPack(ctx context.Context, req *packp.UploadPackRequest) (*packp.UploadPackResponse, error) {
 	if req.IsEmpty() {
+		// XXX: IsEmpty means haves are a subset of wants, in that case we have
+		// everything we asked for. Close the connection and return nil.
+		if err := s.finish(); err != nil {
+			return nil, err
+		}
+		// TODO:(v6) return nil here
 		return nil, transport.ErrEmptyUploadPackRequest
 	}

@@ -393,59 +412,43 @@ func (s *session) checkNotFoundError() error {
 			return transport.ErrRepositoryNotFound
 		}

+		// TODO:(v6): return server error just as it is without a prefix
 		return fmt.Errorf("unknown error: %s", line)
 	}
 }

-var (
-	githubRepoNotFoundErr      = "ERROR: Repository not found."
-	bitbucketRepoNotFoundErr   = "conq: repository does not exist."
+const (
+	githubRepoNotFoundErr      = "Repository not found."
+	bitbucketRepoNotFoundErr   = "repository does not exist."
 	localRepoNotFoundErr       = "does not appear to be a git repository"
-	gitProtocolNotFoundErr     = "ERR \n  Repository not found."
-	gitProtocolNoSuchErr       = "ERR no such repository"
-	gitProtocolAccessDeniedErr = "ERR access denied"
-	gogsAccessDeniedErr        = "Gogs: Repository does not exist or you do not have access"
-	gitlabRepoNotFoundErr      = "remote: ERROR: The project you were looking for could not be found"
+	gitProtocolNotFoundErr     = "Repository not found."
+	gitProtocolNoSuchErr       = "no such repository"
+	gitProtocolAccessDeniedErr = "access denied"
+	gogsAccessDeniedErr        = "Repository does not exist or you do not have access"
+	gitlabRepoNotFoundErr      = "The project you were looking for could not be found"
 )

 func isRepoNotFoundError(s string) bool {
-	if strings.HasPrefix(s, githubRepoNotFoundErr) {
-		return true
-	}
-
-	if strings.HasPrefix(s, bitbucketRepoNotFoundErr) {
-		return true
-	}
-
-	if strings.HasSuffix(s, localRepoNotFoundErr) {
-		return true
-	}
-
-	if strings.HasPrefix(s, gitProtocolNotFoundErr) {
-		return true
-	}
-
-	if strings.HasPrefix(s, gitProtocolNoSuchErr) {
-		return true
-	}
-
-	if strings.HasPrefix(s, gitProtocolAccessDeniedErr) {
-		return true
-	}
-
-	if strings.HasPrefix(s, gogsAccessDeniedErr) {
-		return true
-	}
-
-	if strings.HasPrefix(s, gitlabRepoNotFoundErr) {
-		return true
+	for _, err := range []string{
+		githubRepoNotFoundErr,
+		bitbucketRepoNotFoundErr,
+		localRepoNotFoundErr,
+		gitProtocolNotFoundErr,
+		gitProtocolNoSuchErr,
+		gitProtocolAccessDeniedErr,
+		gogsAccessDeniedErr,
+		gitlabRepoNotFoundErr,
+	} {
+		if strings.Contains(s, err) {
+			return true
+		}
 	}

 	return false
 }

 // uploadPack implements the git-upload-pack protocol.
-func uploadPack(w io.WriteCloser, r io.Reader, req *packp.UploadPackRequest) error {
+func uploadPack(w io.WriteCloser, _ io.Reader, req *packp.UploadPackRequest) error {
 	// TODO support multi_ack mode
 	// TODO support multi_ack_detailed mode
 	// TODO support acks for common objects
@@ -166,7 +166,7 @@ func (s *upSession) UploadPack(ctx context.Context, req *packp.UploadPackRequest
 		return nil, err
 	}

-	pr, pw := ioutil.Pipe()
+	pr, pw := io.Pipe()
 	e := packfile.NewEncoder(pw, s.storer, false)
 	go func() {
 		// TODO: plumb through a pack window.
@@ -168,7 +168,7 @@ func dial(network, addr string, proxyOpts transport.ProxyOptions, config *ssh.Cl
 	defer cancel()

 	var conn net.Conn
-	var err error
+	var dialErr error

 	if proxyOpts.URL != "" {
 		proxyUrl, err := proxyOpts.FullURL()
@@ -186,12 +186,12 @@ func dial(network, addr string, proxyOpts transport.ProxyOptions, config *ssh.Cl
 			return nil, fmt.Errorf("expected ssh proxy dialer to be of type %s; got %s",
 				reflect.TypeOf(ctxDialer), reflect.TypeOf(dialer))
 		}
-		conn, err = ctxDialer.DialContext(ctx, "tcp", addr)
+		conn, dialErr = ctxDialer.DialContext(ctx, "tcp", addr)
 	} else {
-		conn, err = proxy.Dial(ctx, network, addr)
+		conn, dialErr = proxy.Dial(ctx, network, addr)
 	}
-	if err != nil {
-		return nil, err
+	if dialErr != nil {
+		return nil, dialErr
 	}

 	c, chans, reqs, err := ssh.NewClientConn(conn, addr, config)
@@ -552,6 +552,10 @@ func (r *Remote) fetchPack(ctx context.Context, o *FetchOptions, s transport.Upl

 	reader, err := s.UploadPack(ctx, req)
 	if err != nil {
+		if errors.Is(err, transport.ErrEmptyUploadPackRequest) {
+			// XXX: no packfile provided, everything is up-to-date.
+			return nil
+		}
 		return err
 	}

@@ -1198,9 +1202,9 @@ func (r *Remote) updateLocalReferenceStorage(
 			old, _ := storer.ResolveReference(r.s, localName)
 			new := plumbing.NewHashReference(localName, ref.Hash())

-			// If the ref exists locally as a branch and force is not specified,
-			// only update if the new ref is an ancestor of the old
-			if old != nil && old.Name().IsBranch() && !force && !spec.IsForceUpdate() {
+			// If the ref exists locally as a non-tag and force is not
+			// specified, only update if the new ref is an ancestor of the old
+			if old != nil && !old.Name().IsTag() && !force && !spec.IsForceUpdate() {
 				ff, err := isFastForward(r.s, old.Hash(), new.Hash())
 				if err != nil {
 					return updated, err
@@ -1387,7 +1391,7 @@ func pushHashes(
 	allDelete bool,
 ) (*packp.ReportStatus, error) {

-	rd, wr := ioutil.Pipe()
+	rd, wr := io.Pipe()

 	config, err := s.Config()
 	if err != nil {
@@ -146,6 +146,19 @@ func (s *ObjectStorage) SetEncodedObject(o plumbing.EncodedObject) (h plumbing.H
 	return o.Hash(), err
 }

+// LazyWriter returns a lazy ObjectWriter that is bound to a DotGit file.
+// It first write the header passing on the object type and size, so
+// that the object contents can be written later, without the need to
+// create a MemoryObject and buffering its entire contents into memory.
+func (s *ObjectStorage) LazyWriter() (w io.WriteCloser, wh func(typ plumbing.ObjectType, sz int64) error, err error) {
+	ow, err := s.dir.NewObject()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return ow, ow.WriteHeader, nil
+}
+
 // HasEncodedObject returns nil if the object exists, without actually
 // reading the object data from storage.
 func (s *ObjectStorage) HasEncodedObject(h plumbing.Hash) (err error) {
@@ -195,7 +195,7 @@ func NewWriterOnError(w io.Writer, notify func(error)) io.Writer {
 }

 // NewWriteCloserOnError returns a io.WriteCloser that call the notify function
-//when an unexpected (!io.EOF) error happens, after call Write function.
+// when an unexpected (!io.EOF) error happens, after call Write function.
 func NewWriteCloserOnError(w io.WriteCloser, notify func(error)) io.WriteCloser {
 	return NewWriteCloser(NewWriterOnError(w, notify), w)
 }
@@ -208,13 +208,3 @@ func (r *writerOnError) Write(p []byte) (n int, err error) {

 	return
 }
-
-type PipeReader interface {
-	io.ReadCloser
-	CloseWithError(err error) error
-}
-
-type PipeWriter interface {
-	io.WriteCloser
-	CloseWithError(err error) error
-}
@@ -1,9 +0,0 @@
-// +build !js
-
-package ioutil
-
-import "io"
-
-func Pipe() (PipeReader, PipeWriter) {
-	return io.Pipe()
-}
@@ -1,9 +0,0 @@
-// +build js
-
-package ioutil
-
-import "github.com/acomagu/bufpipe"
-
-func Pipe() (PipeReader, PipeWriter) {
-	return bufpipe.New(nil)
-}
@@ -103,6 +103,10 @@ func (n *node) calculateChildren() error {
 			continue
 		}

+		if file.Mode()&os.ModeSocket != 0 {
+			continue
+		}
+
 		c, err := n.newChildNode(file)
 		if err != nil {
 			return err
@@ -0,0 +1,55 @@
+package trace
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"sync/atomic"
+)
+
+var (
+	// logger is the logger to use for tracing.
+	logger = newLogger()
+
+	// current is the targets that are enabled for tracing.
+	current atomic.Int32
+)
+
+func newLogger() *log.Logger {
+	return log.New(os.Stderr, "", log.Ltime|log.Lmicroseconds|log.Lshortfile)
+}
+
+// Target is a tracing target.
+type Target int32
+
+const (
+	// General traces general operations.
+	General Target = 1 << iota
+
+	// Packet traces git packets.
+	Packet
+)
+
+// SetTarget sets the tracing targets.
+func SetTarget(target Target) {
+	current.Store(int32(target))
+}
+
+// SetLogger sets the logger to use for tracing.
+func SetLogger(l *log.Logger) {
+	logger = l
+}
+
+// Print prints the given message only if the target is enabled.
+func (t Target) Print(args ...interface{}) {
+	if int32(t)&current.Load() != 0 {
+		logger.Output(2, fmt.Sprint(args...)) // nolint: errcheck
+	}
+}
+
+// Printf prints the given message only if the target is enabled.
+func (t Target) Printf(format string, args ...interface{}) {
+	if int32(t)&current.Load() != 0 {
+		logger.Output(2, fmt.Sprintf(format, args...)) // nolint: errcheck
+	}
+}