files

Documentation

Overview

Package files handles the reading/writing of files (compression, encryption, etc.)

Index

• Constants
• type ByVolumeNumber
  • func (a ByVolumeNumber) Len() int
  • func (a ByVolumeNumber) Less(i, j int) bool
  • func (a ByVolumeNumber) Swap(i, j int)
• type JobInfo
  • func (j *JobInfo) BackupVolumeObjectName(volumeNumber int64) string
  • func (j *JobInfo) ManifestObjectName() string
  • func (j *JobInfo) String() string
  • func (j *JobInfo) TotalBytesStreamedAndVols() (total uint64, volnum int64)
  • func (j *JobInfo) TotalBytesWritten() uint64
  • func (j *JobInfo) ValidateSendFlags() error
• type SnapshotInfo
  • func (s *SnapshotInfo) Equal(t *SnapshotInfo) bool
• type VolumeInfo
  • func CreateBackupVolume(ctx context.Context, j *JobInfo, volnum int64) (*VolumeInfo, error)
  • func CreateManifestVolume(ctx context.Context, j *JobInfo) (*VolumeInfo, error)
  • func CreateSimpleVolume(ctx context.Context, pipe bool) (*VolumeInfo, error)
  • func ExtractLocal(ctx context.Context, j *JobInfo, path string, isManifest bool) (*VolumeInfo, error)
  • func (v *VolumeInfo) Close() error
  • func (v *VolumeInfo) CopyTo(dest string) (err error)
  • func (v *VolumeInfo) Counter() uint64
  • func (v *VolumeInfo) DeleteVolume() error
  • func (v *VolumeInfo) Extract(ctx context.Context, j *JobInfo, isManifest bool) error
  • func (v *VolumeInfo) IsUsingPipe() bool
  • func (v *VolumeInfo) OpenVolume() error
  • func (v *VolumeInfo) Read(p []byte) (int, error)
  • func (v *VolumeInfo) ReadAt(p []byte, off int64) (int, error)
  • func (v *VolumeInfo) Seek(offset int64, whence int) (int64, error)
  • func (v *VolumeInfo) Write(p []byte) (int, error)

Constants

const (
	// BufferSize is the size of various buffers and copy limits around the application
	BufferSize = 256 * humanize.KiByte // 256KiB
	// InternalCompressor is the key used to indicate we want to utilize the internal compressor
	InternalCompressor = "internal"
	ZfsCompressor      = "zfs"
)

Types

type ByVolumeNumber

type ByVolumeNumber []*VolumeInfo

ByVolumeNumber is used to sort a VolumeInfo slice by VolumeNumber.

func (ByVolumeNumber) Len

func (a ByVolumeNumber) Len() int

func (ByVolumeNumber) Less

func (a ByVolumeNumber) Less(i, j int) bool

func (ByVolumeNumber) Swap

func (a ByVolumeNumber) Swap(i, j int)

type JobInfo

type JobInfo struct {
	StartTime               time.Time
	EndTime                 time.Time
	VolumeName              string
	BaseSnapshot            SnapshotInfo
	IncrementalSnapshot     SnapshotInfo
	SnapshotPrefix          string
	Raw                     bool
	Compressor              string
	CompressionLevel        int
	Separator               string
	ZFSCommandLine          string
	ZFSStreamBytes          uint64
	Volumes                 []*VolumeInfo
	Version                 float64
	EncryptTo               string
	SignFrom                string
	Replication             bool
	SkipMissing             bool
	Deduplication           bool
	Properties              bool
	IntermediaryIncremental bool
	Resume                  bool `json:"-"`
	// "Smart" Options
	Full            bool          `json:"-"`
	Incremental     bool          `json:"-"`
	FullIfOlderThan time.Duration `json:"-"`

	// ZFS Receive options
	Force       bool   `json:"-"`
	FullPath    bool   `json:"-"`
	LastPath    bool   `json:"-"`
	NotMounted  bool   `json:"-"`
	Origin      string `json:"-"`
	LocalVolume string `json:"-"`
	AutoRestore bool   `json:"-"`

	Destinations       []string        `json:"-"`
	VolumeSize         uint64          `json:"-"`
	ManifestPrefix     string          `json:"-"`
	MaxBackoffTime     time.Duration   `json:"-"`
	MaxRetryTime       time.Duration   `json:"-"`
	MaxParallelUploads int             `json:"-"`
	MaxFileBuffer      int             `json:"-"`
	EncryptKey         *openpgp.Entity `json:"-"`
	SignKey            *openpgp.Entity `json:"-"`
	ParentSnap         *JobInfo        `json:"-"`
	UploadChunkSize    int             `json:"-"`
}

JobInfo represents the relevant information for a job that can be used to read in details of that job at a later time.

func (*JobInfo) BackupVolumeObjectName

func (j *JobInfo) BackupVolumeObjectName(volumeNumber int64) string

func (*JobInfo) ManifestObjectName

func (j *JobInfo) ManifestObjectName() string

func (*JobInfo) String

func (j *JobInfo) String() string

String will return a string representation of this JobInfo.

func (*JobInfo) TotalBytesStreamedAndVols

func (j *JobInfo) TotalBytesStreamedAndVols() (total uint64, volnum int64)

TotalBytesStreamedAndVols will sum up the streamed bytes of all underlying Volumes to give a total that represents how many bytes have been streamed. It will stop at any out of order volume number.

func (*JobInfo) TotalBytesWritten

func (j *JobInfo) TotalBytesWritten() uint64

TotalBytesWritten will sum up the size of all underlying Volumes to give a total that represents how many bytes have been written.

func (*JobInfo) ValidateSendFlags

func (j *JobInfo) ValidateSendFlags() error

ValidateSendFlags will check if the options assigned to this JobInfo object is properly within the bounds for a send backup operation. nolint:golint,stylecheck // Error strings used as log messages for user

type SnapshotInfo

type SnapshotInfo struct {
	CreationTime time.Time
	Name         string
	Bookmark     bool
}

SnapshotInfo represents a snapshot with relevant information.

func (*SnapshotInfo) Equal

func (s *SnapshotInfo) Equal(t *SnapshotInfo) bool

Equal will test two SnapshotInfo objects for equality. This is based on the snapshot name and the time of creation

type VolumeInfo

type VolumeInfo struct {
	ObjectName      string
	VolumeNumber    int64
	SHA256          hash.Hash   `json:"-"`
	MD5             hash.Hash   `json:"-"`
	CRC32C          hash.Hash32 `json:"-"`
	SHA1            hash.Hash   `json:"-"`
	SHA1Sum         string      `json:"-"`
	SHA256Sum       string
	MD5Sum          string
	CRC32CSum32     uint32
	Size            uint64
	ZFSStreamBytes  uint64
	CreateTime      time.Time
	CloseTime       time.Time
	IsManifest      bool
	IsFinalManifest bool
	// contains filtered or unexported fields
}

VolumeInfo holds all necessary information for a Volume as part of a backup

func CreateBackupVolume

func CreateBackupVolume(ctx context.Context, j *JobInfo, volnum int64) (*VolumeInfo, error)

CreateBackupVolume will call CreateSimpleVolume and add options to compress, encrypt, and/or sign the file as it is written depending on the provided options. It will also name the file accordingly as a volume as part of backup set.

func CreateManifestVolume

func CreateManifestVolume(ctx context.Context, j *JobInfo) (*VolumeInfo, error)

CreateManifestVolume will call CreateSimpleVolume and add options to compress, encrypt, and/or sign the file as it is written depending on the provided options. It will also name the file accordingly as a manifest file.

func CreateSimpleVolume

func CreateSimpleVolume(ctx context.Context, pipe bool) (*VolumeInfo, error)

CreateSimpleVolume will create a temporary file to write to. If MaxParallelUploads is set to 0, no temporary file will be used and an OS Pipe will be used instead.

func ExtractLocal

func ExtractLocal(ctx context.Context, j *JobInfo, path string, isManifest bool) (*VolumeInfo, error)

ExtractLocal will try and open a local file for extraction

func (*VolumeInfo) Close

func (v *VolumeInfo) Close() error

Close should be called after creating a new volume or after calling OpenVolume nolint:funlen,gocyclo // Difficult to break this apart

func (*VolumeInfo) CopyTo

func (v *VolumeInfo) CopyTo(dest string) (err error)

CopyTo will write out the volume to the path specified

func (*VolumeInfo) Counter

func (v *VolumeInfo) Counter() uint64

Counter will return how many bytes have been written to this volume.

func (*VolumeInfo) DeleteVolume

func (v *VolumeInfo) DeleteVolume() error

DeleteVolume will delete the volume from the temporary directory it was written to. Only valid to be called after creating a new Volume and closing it.

func (*VolumeInfo) Extract

func (v *VolumeInfo) Extract(ctx context.Context, j *JobInfo, isManifest bool) error

Extract will setup the volume for reading such that reading from it will handle any decryption, signature verification, and decompression that was used on it.

func (*VolumeInfo) IsUsingPipe

func (v *VolumeInfo) IsUsingPipe() bool

IsUsingPipe will return true when the volume is a glorified pipe

func (*VolumeInfo) OpenVolume

func (v *VolumeInfo) OpenVolume() error

OpenVolume will open this VolumeInfo in a read-only mode. It will automatically rate limit the amount of bytes that can be read at a time so no buffer should be used for reading from this Reader. Only valid to be called after creating a new Volume and closing it or when a MaxFileBuffer of 0 in which case this does nothing.

func (*VolumeInfo) Read

func (v *VolumeInfo) Read(p []byte) (int, error)

Read will passthru the command to the underlying io.Reader, which will be setup to ratelimit where applicable.

func (*VolumeInfo) ReadAt

func (v *VolumeInfo) ReadAt(p []byte, off int64) (int, error)

ReadAt will passthru the command to the underlying *os.File

func (*VolumeInfo) Seek

func (v *VolumeInfo) Seek(offset int64, whence int) (int64, error)

Seek will passthru the command to the underlying *os.File

func (*VolumeInfo) Write

func (v *VolumeInfo) Write(p []byte) (int, error)

Write writes through to the underlying writer, satisfying the io.Writer interface.