syncthing/lib/db/structs.go

// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at https://mozilla.org/MPL/2.0/.

package db

import (
	"bytes"
	"fmt"
	"strings"

	"google.golang.org/protobuf/proto"

	"github.com/syncthing/syncthing/internal/gen/dbproto"
	"github.com/syncthing/syncthing/lib/protocol"
)

type CountsSet struct {
	Counts  []Counts
	Created int64 // unix nanos
}

type Counts struct {
	Files       int
	Directories int
	Symlinks    int
	Deleted     int
	Bytes       int64
	Sequence    int64             // zero for the global state
	DeviceID    protocol.DeviceID // device ID for remote devices, or special values for local/global
	LocalFlags  uint32            // the local flag for this count bucket
}

func (c Counts) toWire() *dbproto.Counts {
	return &dbproto.Counts{
		Files:       int32(c.Files),
		Directories: int32(c.Directories),
		Symlinks:    int32(c.Symlinks),
		Deleted:     int32(c.Deleted),
		Bytes:       c.Bytes,
		Sequence:    c.Sequence,
		DeviceId:    c.DeviceID[:],
		LocalFlags:  c.LocalFlags,
	}
}

func countsFromWire(w *dbproto.Counts) Counts {
	return Counts{
		Files:       int(w.Files),
		Directories: int(w.Directories),
		Symlinks:    int(w.Symlinks),
		Deleted:     int(w.Deleted),
		Bytes:       w.Bytes,
		Sequence:    w.Sequence,
		DeviceID:    protocol.DeviceID(w.DeviceId),
		LocalFlags:  w.LocalFlags,
	}
}

func (c Counts) Add(other Counts) Counts {
	return Counts{
		Files:       c.Files + other.Files,
		Directories: c.Directories + other.Directories,
		Symlinks:    c.Symlinks + other.Symlinks,
		Deleted:     c.Deleted + other.Deleted,
		Bytes:       c.Bytes + other.Bytes,
		Sequence:    c.Sequence + other.Sequence,
		DeviceID:    protocol.EmptyDeviceID,
		LocalFlags:  c.LocalFlags | other.LocalFlags,
	}
}

func (c Counts) TotalItems() int {
	return c.Files + c.Directories + c.Symlinks + c.Deleted
}

func (c Counts) String() string {
	var flags strings.Builder
	if c.LocalFlags&needFlag != 0 {
		flags.WriteString("Need")
	}
	if c.LocalFlags&protocol.FlagLocalIgnored != 0 {
		flags.WriteString("Ignored")
	}
	if c.LocalFlags&protocol.FlagLocalMustRescan != 0 {
		flags.WriteString("Rescan")
	}
	if c.LocalFlags&protocol.FlagLocalReceiveOnly != 0 {
		flags.WriteString("Recvonly")
	}
	if c.LocalFlags&protocol.FlagLocalUnsupported != 0 {
		flags.WriteString("Unsupported")
	}
	if c.LocalFlags != 0 {
		flags.WriteString(fmt.Sprintf("(%x)", c.LocalFlags))
	}
	if flags.Len() == 0 {
		flags.WriteString("---")
	}
	return fmt.Sprintf("{Device:%v, Files:%d, Dirs:%d, Symlinks:%d, Del:%d, Bytes:%d, Seq:%d, Flags:%s}", c.DeviceID, c.Files, c.Directories, c.Symlinks, c.Deleted, c.Bytes, c.Sequence, flags.String())
}

// Equal compares the numbers only, not sequence/dev/flags.
func (c Counts) Equal(o Counts) bool {
	return c.Files == o.Files && c.Directories == o.Directories && c.Symlinks == o.Symlinks && c.Deleted == o.Deleted && c.Bytes == o.Bytes
}

// update brings the VersionList up to date with file. It returns the updated
// VersionList, a device that has the global/newest version, a device that previously
// had the global/newest version, a boolean indicating if the global version has
// changed and if any error occurred (only possible in db interaction).
func vlUpdate(vl *dbproto.VersionList, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (*dbproto.FileVersion, *dbproto.FileVersion, *dbproto.FileVersion, bool, bool, bool, error) {
	if len(vl.Versions) == 0 {
		nv := newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted())
		vl.Versions = append(vl.Versions, nv)
		return nv, nil, nil, false, false, true, nil
	}

	// Get the current global (before updating)
	oldFV, haveOldGlobal := vlGetGlobal(vl)
	oldFV = fvCopy(oldFV)

	// Remove ourselves first
	removedFV, haveRemoved, _ := vlPop(vl, device)
	// Find position and insert the file
	err := vlInsert(vl, folder, device, file, t)
	if err != nil {
		return nil, nil, nil, false, false, false, err
	}

	newFV, _ := vlGetGlobal(vl) // We just inserted something above, can't be empty

	if !haveOldGlobal {
		return newFV, nil, removedFV, false, haveRemoved, true, nil
	}

	globalChanged := true
	if fvIsInvalid(oldFV) == fvIsInvalid(newFV) && protocol.VectorFromWire(oldFV.Version).Equal(protocol.VectorFromWire(newFV.Version)) {
		globalChanged = false
	}

	return newFV, oldFV, removedFV, true, haveRemoved, globalChanged, nil
}

func vlInsert(vl *dbproto.VersionList, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) error {
	var added bool
	var err error
	i := 0
	for ; i < len(vl.Versions); i++ {
		// Insert our new version
		added, err = vlCheckInsertAt(vl, i, folder, device, file, t)
		if err != nil {
			return err
		}
		if added {
			break
		}
	}
	if i == len(vl.Versions) {
		// Append to the end
		vl.Versions = append(vl.Versions, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
	}
	return nil
}

func vlInsertAt(vl *dbproto.VersionList, i int, v *dbproto.FileVersion) {
	vl.Versions = append(vl.Versions, &dbproto.FileVersion{})
	copy(vl.Versions[i+1:], vl.Versions[i:])
	vl.Versions[i] = v
}

// pop removes the given device from the VersionList and returns the FileVersion
// before removing the device, whether it was found/removed at all and whether
// the global changed in the process.
func vlPop(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool, bool) {
	invDevice, i, j, ok := vlFindDevice(vl, device)
	if !ok {
		return nil, false, false
	}
	globalPos := vlFindGlobal(vl)

	fv := vl.Versions[i]
	if fvDeviceCount(fv) == 1 {
		vlPopVersionAt(vl, i)
		return fv, true, globalPos == i
	}

	oldFV := fvCopy(fv)
	if invDevice {
		vl.Versions[i].InvalidDevices = popDeviceAt(vl.Versions[i].InvalidDevices, j)
		return oldFV, true, false
	}
	vl.Versions[i].Devices = popDeviceAt(vl.Versions[i].Devices, j)
	// If the last valid device of the previous global was removed above,
	// the global changed.
	return oldFV, true, len(vl.Versions[i].Devices) == 0 && globalPos == i
}

// Get returns a FileVersion that contains the given device and whether it has
// been found at all.
func vlGet(vl *dbproto.VersionList, device []byte) (*dbproto.FileVersion, bool) {
	_, i, _, ok := vlFindDevice(vl, device)
	if !ok {
		return &dbproto.FileVersion{}, false
	}
	return vl.Versions[i], true
}

// GetGlobal returns the current global FileVersion. The returned FileVersion
// may be invalid, if all FileVersions are invalid. Returns false only if
// VersionList is empty.
func vlGetGlobal(vl *dbproto.VersionList) (*dbproto.FileVersion, bool) {
	i := vlFindGlobal(vl)
	if i == -1 {
		return nil, false
	}
	return vl.Versions[i], true
}

// findGlobal returns the first version that isn't invalid, or if all versions are
// invalid just the first version (i.e. 0) or -1, if there's no versions at all.
func vlFindGlobal(vl *dbproto.VersionList) int {
	for i := range vl.Versions {
		if !fvIsInvalid(vl.Versions[i]) {
			return i
		}
	}
	if len(vl.Versions) == 0 {
		return -1
	}
	return 0
}

// findDevice returns whether the device is in InvalidVersions or Versions and
// in InvalidDevices or Devices (true for invalid), the positions in the version
// and device slices and whether it has been found at all.
func vlFindDevice(vl *dbproto.VersionList, device []byte) (bool, int, int, bool) {
	for i, v := range vl.Versions {
		if j := deviceIndex(v.Devices, device); j != -1 {
			return false, i, j, true
		}
		if j := deviceIndex(v.InvalidDevices, device); j != -1 {
			return true, i, j, true
		}
	}
	return false, -1, -1, false
}

func vlPopVersionAt(vl *dbproto.VersionList, i int) {
	vl.Versions = append(vl.Versions[:i], vl.Versions[i+1:]...)
}

// checkInsertAt determines if the given device and associated file should be
// inserted into the FileVersion at position i or into a new FileVersion at
// position i.
func vlCheckInsertAt(vl *dbproto.VersionList, i int, folder, device []byte, file protocol.FileInfo, t readOnlyTransaction) (bool, error) {
	fv := vl.Versions[i]
	ordering := protocol.VectorFromWire(fv.Version).Compare(file.FileVersion())
	if ordering == protocol.Equal {
		if !file.IsInvalid() {
			fv.Devices = append(fv.Devices, device)
		} else {
			fv.InvalidDevices = append(fv.InvalidDevices, device)
		}
		return true, nil
	}
	existingDevice, _ := fvFirstDevice(fv)
	insert, err := shouldInsertBefore(ordering, folder, existingDevice, fvIsInvalid(fv), file, t)
	if err != nil {
		return false, err
	}
	if insert {
		vlInsertAt(vl, i, newFileVersion(device, file.FileVersion(), file.IsInvalid(), file.IsDeleted()))
		return true, nil
	}
	return false, nil
}

// shouldInsertBefore determines whether the file comes before an existing
// entry, given the version ordering (existing compared to new one), existing
// device and if the existing version is invalid.
func shouldInsertBefore(ordering protocol.Ordering, folder, existingDevice []byte, existingInvalid bool, file protocol.FileInfo, t readOnlyTransaction) (bool, error) {
	switch ordering {
	case protocol.Lesser:
		// The version at this point in the list is lesser
		// ("older") than us. We insert ourselves in front of it.
		return true, nil

	case protocol.ConcurrentLesser, protocol.ConcurrentGreater:
		// The version in conflict with us.
		// Check if we can shortcut due to one being invalid.
		if existingInvalid != file.IsInvalid() {
			return existingInvalid, nil
		}
		// We must pull the actual file metadata to determine who wins.
		// If we win, we insert ourselves in front of the loser here.
		// (The "Lesser" and "Greater" in the condition above is just
		// based on the device IDs in the version vector, which is not
		// the only thing we use to determine the winner.)
		of, ok, err := t.getFile(folder, existingDevice, []byte(file.FileName()))
		if err != nil {
			return false, err
		}
		// A surprise missing file entry here is counted as a win for us.
		if !ok {
			return true, nil
		}
		if file.WinsConflict(of) {
			return true, nil
		}
	}
	return false, nil
}

func deviceIndex(devices [][]byte, device []byte) int {
	for i, dev := range devices {
		if bytes.Equal(device, dev) {
			return i
		}
	}
	return -1
}

func popDeviceAt(devices [][]byte, i int) [][]byte {
	return append(devices[:i], devices[i+1:]...)
}

func newFileVersion(device []byte, version protocol.Vector, invalid, deleted bool) *dbproto.FileVersion {
	fv := &dbproto.FileVersion{
		Version: version.ToWire(),
		Deleted: deleted,
	}
	if invalid {
		fv.InvalidDevices = [][]byte{device}
	} else {
		fv.Devices = [][]byte{device}
	}
	return fv
}

func fvFirstDevice(fv *dbproto.FileVersion) ([]byte, bool) {
	if len(fv.Devices) != 0 {
		return fv.Devices[0], true
	}
	if len(fv.InvalidDevices) != 0 {
		return fv.InvalidDevices[0], true
	}
	return nil, false
}

func fvIsInvalid(fv *dbproto.FileVersion) bool {
	return fv == nil || len(fv.Devices) == 0
}

func fvDeviceCount(fv *dbproto.FileVersion) int {
	return len(fv.Devices) + len(fv.InvalidDevices)
}

func fvCopy(fv *dbproto.FileVersion) *dbproto.FileVersion {
	return proto.Clone(fv).(*dbproto.FileVersion)
}