Documentation ¶
Overview ¶
Package format manages kopia.repository and other central format blobs.
Index ¶
- Constants
- Variables
- func BackupBlobID(l UpgradeLockIntent) blob.ID
- func Initialize(ctx context.Context, st blob.Storage, formatBlob *KopiaRepositoryJSON, ...) error
- func NewDiskCache(cacheDir string) blobCache
- func NewFormatBlobCache(cacheDir string, validDuration time.Duration, timeNow func() time.Time) blobCache
- func NewMemoryBlobCache(timeNow func() time.Time) blobCache
- func RecoverFormatBlob(ctx context.Context, st blob.Storage, blobID blob.ID, optionalLength int64) ([]byte, error)
- func WriteLegacyIndexPoisonBlob(ctx context.Context, st blob.Storage) error
- type BlobStorageConfiguration
- type ContentFormat
- func (f *ContentFormat) GetCachedMutableParameters() MutableParameters
- func (f *ContentFormat) GetECCAlgorithm() string
- func (f *ContentFormat) GetECCOverheadPercent() int
- func (f *ContentFormat) GetEncryptionAlgorithm() string
- func (f *ContentFormat) GetHashFunction() string
- func (f *ContentFormat) GetHmacSecret() []byte
- func (f *ContentFormat) GetMasterKey() []byte
- func (f *ContentFormat) GetMutableParameters(ctx context.Context) (MutableParameters, error)
- func (f *ContentFormat) ResolveFormatVersion() error
- func (f *ContentFormat) SupportsPasswordChange() bool
- type EncryptedRepositoryConfig
- type KopiaRepositoryJSON
- func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error)
- func (f *KopiaRepositoryJSON) EncryptRepositoryConfig(format *RepositoryConfig, masterKey []byte) error
- func (f *KopiaRepositoryJSON) WriteBlobCfgBlob(ctx context.Context, st blob.Storage, blobcfg BlobStorageConfiguration, ...) error
- func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlob(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration) error
- func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlobWithID(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration, ...) error
- type Manager
- func (m *Manager) BlobCfgBlob(ctx context.Context) (BlobStorageConfiguration, error)
- func (m *Manager) ChangePassword(ctx context.Context, newPassword string) error
- func (m *Manager) CommitUpgrade(ctx context.Context) error
- func (m *Manager) Encryptor() encryption.Encryptor
- func (m *Manager) FormatEncryptionKey() []byte
- func (m *Manager) GetCachedMutableParameters() MutableParameters
- func (m *Manager) GetECCAlgorithm() string
- func (m *Manager) GetECCOverheadPercent() int
- func (m *Manager) GetEncryptionAlgorithm() string
- func (m *Manager) GetHashFunction() string
- func (m *Manager) GetHmacSecret() []byte
- func (m *Manager) GetMasterKey() []byte
- func (m *Manager) GetMutableParameters(ctx context.Context) (MutableParameters, error)
- func (m *Manager) GetUpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error)
- func (m *Manager) HashFunc() hashing.HashFunc
- func (m *Manager) LoadedTime() time.Time
- func (m *Manager) ObjectFormat() ObjectFormat
- func (m *Manager) RefreshCount() int
- func (m *Manager) RepositoryFormatBytes(ctx context.Context) ([]byte, error)
- func (m *Manager) RequiredFeatures(ctx context.Context) ([]feature.Required, error)
- func (m *Manager) RollbackUpgrade(ctx context.Context) error
- func (m *Manager) ScrubbedContentFormat() ContentFormat
- func (m *Manager) SetParameters(ctx context.Context, mp MutableParameters, blobcfg BlobStorageConfiguration, ...) error
- func (m *Manager) SetUpgradeLockIntent(ctx context.Context, l UpgradeLockIntent) (*UpgradeLockIntent, error)
- func (m *Manager) SupportsPasswordChange() bool
- func (m *Manager) UniqueID() []byte
- func (m *Manager) UpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error)
- func (m *Manager) ValidCacheDuration() time.Duration
- type MutableParameters
- type ObjectFormat
- type Provider
- type RepositoryConfig
- type UpgradeLockIntent
- func (l *UpgradeLockIntent) Clone() *UpgradeLockIntent
- func (l *UpgradeLockIntent) IsLocked(now time.Time) (locked, writersDrained bool)
- func (l *UpgradeLockIntent) Update(other *UpgradeLockIntent) (*UpgradeLockIntent, error)
- func (l *UpgradeLockIntent) UpgradeTime() time.Time
- func (l *UpgradeLockIntent) Validate() error
- type Version
Constants ¶
const ( // CurrentWriteVersion is the version of the repository applied to new repositories. CurrentWriteVersion = FormatVersion3 // MinSupportedWriteVersion is the minimum version that this kopia client can write. MinSupportedWriteVersion = FormatVersion1 // MaxSupportedWriteVersion is the maximum version that this kopia client can write. MaxSupportedWriteVersion = FormatVersion3 // MinSupportedReadVersion is the minimum version that this kopia client can read. MinSupportedReadVersion = FormatVersion1 // MaxSupportedReadVersion is the maximum version that this kopia client can read. MaxSupportedReadVersion = FormatVersion3 )
const ( // BackupBlobIDPrefix is the prefix for all identifiers of the BLOBs that // keep a backup copy of the FormatBlobID BLOB for the purposes of rollback // during upgrade. BackupBlobIDPrefix = "kopia.repository.backup." // LegacyIndexPoisonBlobID used to pollute V0 indexes after upgrade to prevent legacy clients from corrupting V1 indexes. LegacyIndexPoisonBlobID = "n00000000000000000000000000000000-repository_unreadable_by_this_kopia_version_upgrade_required" )
const DefaultFormatEncryption = "AES256_GCM"
DefaultFormatEncryption is the identifier of the default format blob encryption algorithm.
const DefaultKeyDerivationAlgorithm = crypto.DefaultKeyDerivationAlgorithm
DefaultKeyDerivationAlgorithm is the key derivation algorithm for new configurations.
const DefaultRepositoryBlobCacheDuration = 15 * time.Minute
DefaultRepositoryBlobCacheDuration is the duration for which we treat cached kopia.repository as valid.
const KopiaBlobCfgBlobID = "kopia.blobcfg"
KopiaBlobCfgBlobID is the identifier of a BLOB that describes BLOB retention settings for the repository.
const KopiaRepositoryBlobID = "kopia.repository"
KopiaRepositoryBlobID is the identifier of a BLOB that describes repository format.
const UniqueIDLengthBytes = 32
UniqueIDLengthBytes is the length of random unique ID of each repository.
Variables ¶
var ErrAlreadyInitialized = errors.Errorf("repository already initialized")
ErrAlreadyInitialized indicates that repository has already been initialized.
var ErrFormatUptoDate = errors.New("repository format is up to date") // +checklocksignore
ErrFormatUptoDate is returned whenever a lock intent is attempted to be set on a repository that is already using the latest format version.
var ErrInvalidPassword = errors.Errorf("invalid repository password") // +checklocksignore
ErrInvalidPassword is returned when repository password is invalid.
Functions ¶
func BackupBlobID ¶
func BackupBlobID(l UpgradeLockIntent) blob.ID
BackupBlobID gets the upgrade backu pblob-id fro mthe lock.
func Initialize ¶
func Initialize(ctx context.Context, st blob.Storage, formatBlob *KopiaRepositoryJSON, repoConfig *RepositoryConfig, blobcfg BlobStorageConfiguration, password string) error
Initialize initializes the format blob in a given storage.
func NewDiskCache ¶
func NewDiskCache(cacheDir string) blobCache
NewDiskCache returns on-disk blob cache.
func NewFormatBlobCache ¶
func NewFormatBlobCache(cacheDir string, validDuration time.Duration, timeNow func() time.Time) blobCache
NewFormatBlobCache creates an implementationof blobCache for particular cache settings.
func NewMemoryBlobCache ¶
NewMemoryBlobCache returns in-memory blob cache.
func RecoverFormatBlob ¶
func RecoverFormatBlob(ctx context.Context, st blob.Storage, blobID blob.ID, optionalLength int64) ([]byte, error)
RecoverFormatBlob attempts to recover format blob replica from the specified file. The format blob can be either the prefix or a suffix of the given file. optionally the length can be provided (if known) to speed up recovery.
func WriteLegacyIndexPoisonBlob ¶ added in v0.13.0
WriteLegacyIndexPoisonBlob writes a "poison blob" that will prevent old kopia clients that have not been upgraded from being able to open the repository after its format has been upgraded.
Types ¶
type BlobStorageConfiguration ¶
type BlobStorageConfiguration struct { RetentionMode blob.RetentionMode `json:"retentionMode,omitempty"` RetentionPeriod time.Duration `json:"retentionPeriod,omitempty"` }
BlobStorageConfiguration is the content for `kopia.blobcfg` blob which contains the blob storage configuration options.
func (*BlobStorageConfiguration) IsRetentionEnabled ¶
func (r *BlobStorageConfiguration) IsRetentionEnabled() bool
IsRetentionEnabled returns true if retention is enabled on the blob-config object.
func (*BlobStorageConfiguration) Validate ¶
func (r *BlobStorageConfiguration) Validate() error
Validate validates the blob config parameters.
type ContentFormat ¶
type ContentFormat struct { Hash string `json:"hash,omitempty"` // identifier of the hash algorithm used Encryption string `json:"encryption,omitempty"` // identifier of the encryption algorithm used ECC string `json:"ecc,omitempty"` // identifier of the ecc algorithm used ECCOverheadPercent int `json:"eccOverheadPercent,omitempty"` // space overhead for ecc HMACSecret []byte `json:"secret,omitempty" kopia:"sensitive"` // HMAC secret used to generate encryption keys MasterKey []byte `json:"masterKey,omitempty" kopia:"sensitive"` // master encryption key (SIV-mode encryption only) MutableParameters EnablePasswordChange bool `json:"enablePasswordChange"` // disables replication of kopia.repository blob in packs }
ContentFormat describes the rules for formatting contents in repository.
func (*ContentFormat) GetCachedMutableParameters ¶ added in v0.16.0
func (f *ContentFormat) GetCachedMutableParameters() MutableParameters
GetCachedMutableParameters implements FormattingOptionsProvider.
func (*ContentFormat) GetECCAlgorithm ¶
func (f *ContentFormat) GetECCAlgorithm() string
GetECCAlgorithm implements ecc.Parameters.
func (*ContentFormat) GetECCOverheadPercent ¶
func (f *ContentFormat) GetECCOverheadPercent() int
GetECCOverheadPercent implements ecc.Parameters.
func (*ContentFormat) GetEncryptionAlgorithm ¶
func (f *ContentFormat) GetEncryptionAlgorithm() string
GetEncryptionAlgorithm implements encryption.Parameters.
func (*ContentFormat) GetHashFunction ¶
func (f *ContentFormat) GetHashFunction() string
GetHashFunction implements hashing.Parameters.
func (*ContentFormat) GetHmacSecret ¶
func (f *ContentFormat) GetHmacSecret() []byte
GetHmacSecret implements hashing.Parameters.
func (*ContentFormat) GetMasterKey ¶
func (f *ContentFormat) GetMasterKey() []byte
GetMasterKey implements encryption.Parameters.
func (*ContentFormat) GetMutableParameters ¶
func (f *ContentFormat) GetMutableParameters(ctx context.Context) (MutableParameters, error)
GetMutableParameters implements FormattingOptionsProvider.
func (*ContentFormat) ResolveFormatVersion ¶
func (f *ContentFormat) ResolveFormatVersion() error
ResolveFormatVersion applies format options parameters based on the format version.
func (*ContentFormat) SupportsPasswordChange ¶
func (f *ContentFormat) SupportsPasswordChange() bool
SupportsPasswordChange implements FormattingOptionsProvider.
type EncryptedRepositoryConfig ¶
type EncryptedRepositoryConfig struct {
Format RepositoryConfig `json:"format"`
}
EncryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format.
type KopiaRepositoryJSON ¶
type KopiaRepositoryJSON struct { Tool string `json:"tool"` BuildVersion string `json:"buildVersion"` BuildInfo string `json:"buildInfo"` UniqueID []byte `json:"uniqueID"` KeyDerivationAlgorithm string `json:"keyAlgo"` EncryptionAlgorithm string `json:"encryption"` // encrypted, serialized JSON encryptedRepositoryConfig{} EncryptedFormatBytes []byte `json:"encryptedBlockFormat,omitempty"` }
KopiaRepositoryJSON represents JSON contents of 'kopia.repository' blob.
func ParseKopiaRepositoryJSON ¶
func ParseKopiaRepositoryJSON(b []byte) (*KopiaRepositoryJSON, error)
ParseKopiaRepositoryJSON parses the provided byte slice into KopiaRepositoryJSON.
func (*KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword ¶
func (f *KopiaRepositoryJSON) DeriveFormatEncryptionKeyFromPassword(password string) ([]byte, error)
DeriveFormatEncryptionKeyFromPassword derives encryption key using the provided password and per-repository unique ID.
func (*KopiaRepositoryJSON) EncryptRepositoryConfig ¶
func (f *KopiaRepositoryJSON) EncryptRepositoryConfig(format *RepositoryConfig, masterKey []byte) error
EncryptRepositoryConfig encrypts the provided repository config and stores it in EncryptedFormatBytes.
func (*KopiaRepositoryJSON) WriteBlobCfgBlob ¶
func (f *KopiaRepositoryJSON) WriteBlobCfgBlob(ctx context.Context, st blob.Storage, blobcfg BlobStorageConfiguration, formatEncryptionKey []byte) error
WriteBlobCfgBlob writes `kopia.blobcfg` encrypted using the provided key.
func (*KopiaRepositoryJSON) WriteKopiaRepositoryBlob ¶
func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlob(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration) error
WriteKopiaRepositoryBlob writes `kopia.repository` blob to a given storage.
func (*KopiaRepositoryJSON) WriteKopiaRepositoryBlobWithID ¶
func (f *KopiaRepositoryJSON) WriteKopiaRepositoryBlobWithID(ctx context.Context, st blob.Storage, blobCfg BlobStorageConfiguration, id blob.ID) error
WriteKopiaRepositoryBlobWithID writes `kopia.repository` blob to a given storage under an alternate blobID.
type Manager ¶
type Manager struct {
// contains filtered or unexported fields
}
Manager manages the contents of `kopia.repository` and `kopia.blobcfg`.
func NewManager ¶
func NewManager( ctx context.Context, st blob.Storage, cacheDir string, validDuration time.Duration, password string, timeNow func() time.Time, ) (*Manager, error)
NewManager creates new format manager which automatically refreshes format blob on reads (in a blocking manner).
func NewManagerWithCache ¶
func NewManagerWithCache( ctx context.Context, st blob.Storage, validDuration time.Duration, password string, timeNow func() time.Time, cache blobCache, ) (*Manager, error)
NewManagerWithCache creates new format manager which automatically refreshes format blob on reads (in a blocking manner) and uses the provided cache.
func (*Manager) BlobCfgBlob ¶
func (m *Manager) BlobCfgBlob(ctx context.Context) (BlobStorageConfiguration, error)
BlobCfgBlob gets the BlobStorageConfiguration.
func (*Manager) ChangePassword ¶
ChangePassword changes the repository password and rewrites `kopia.repository` & `kopia.blobcfg`.
func (*Manager) CommitUpgrade ¶
CommitUpgrade removes the upgrade lock from the from the repository format blob. This in-effect commits the new repository format to the repository and resumes all access to the repository.
func (*Manager) Encryptor ¶
func (m *Manager) Encryptor() encryption.Encryptor
Encryptor returns the resolved encryptor.
func (*Manager) FormatEncryptionKey ¶
FormatEncryptionKey gets the format encryption key derived from the password.
func (*Manager) GetCachedMutableParameters ¶ added in v0.16.0
func (m *Manager) GetCachedMutableParameters() MutableParameters
GetCachedMutableParameters gets mutable paramers of the repository without blocking.
func (*Manager) GetECCAlgorithm ¶
GetECCAlgorithm returns the ECC algorithm.
func (*Manager) GetECCOverheadPercent ¶
GetECCOverheadPercent returns the ECC overhead percent.
func (*Manager) GetEncryptionAlgorithm ¶
GetEncryptionAlgorithm returns the encryption algorithm.
func (*Manager) GetHashFunction ¶
GetHashFunction returns the hash function.
func (*Manager) GetHmacSecret ¶
GetHmacSecret returns the HMAC function.
func (*Manager) GetMasterKey ¶
GetMasterKey gets the master key.
func (*Manager) GetMutableParameters ¶
func (m *Manager) GetMutableParameters(ctx context.Context) (MutableParameters, error)
GetMutableParameters gets mutable paramers of the repository. This function blocks to refresh the format blob if necessary.
func (*Manager) GetUpgradeLockIntent ¶
func (m *Manager) GetUpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error)
GetUpgradeLockIntent gets the current upgrade lock intent.
func (*Manager) LoadedTime ¶
LoadedTime gets the time when the config was last reloaded.
func (*Manager) ObjectFormat ¶
func (m *Manager) ObjectFormat() ObjectFormat
ObjectFormat gets the object format.
func (*Manager) RefreshCount ¶
RefreshCount returns the number of time the format has been refreshed.
func (*Manager) RepositoryFormatBytes ¶
RepositoryFormatBytes returns the bytes of `kopia.repository` blob. This function blocks to refresh the format blob if necessary.
func (*Manager) RequiredFeatures ¶
RequiredFeatures returns the list of features required to open the repository.
func (*Manager) RollbackUpgrade ¶
RollbackUpgrade removes the upgrade lock while also restoring the format-blob's original version. This method does not restore the original repository data format and neither does it validate against any repository changes. Rolling back the repository format is currently not supported and hence using this API could render the repository corrupted and unreadable by clients.
func (*Manager) ScrubbedContentFormat ¶
func (m *Manager) ScrubbedContentFormat() ContentFormat
ScrubbedContentFormat returns scrubbed content format with all sensitive data replaced.
func (*Manager) SetParameters ¶
func (m *Manager) SetParameters( ctx context.Context, mp MutableParameters, blobcfg BlobStorageConfiguration, requiredFeatures []feature.Required, ) error
SetParameters sets the mutable repository parameters.
func (*Manager) SetUpgradeLockIntent ¶
func (m *Manager) SetUpgradeLockIntent(ctx context.Context, l UpgradeLockIntent) (*UpgradeLockIntent, error)
SetUpgradeLockIntent sets the upgrade lock intent on the repository format blob for other clients to notice. If a lock intent was already placed then it updates the existing lock using the output of the UpgradeLock.Update().
This method also backs up the original format version on the upgrade lock intent and sets the latest format-version to the repository blob. This should cause the unsupporting clients (non-upgrade capable) to fail connecting to the repository.
func (*Manager) SupportsPasswordChange ¶
SupportsPasswordChange returns true if the repository supports password change.
func (*Manager) UpgradeLockIntent ¶
func (m *Manager) UpgradeLockIntent(ctx context.Context) (*UpgradeLockIntent, error)
UpgradeLockIntent returns the current lock intent.
func (*Manager) ValidCacheDuration ¶
ValidCacheDuration returns the duration for which each blob in the cache is valid.
type MutableParameters ¶
type MutableParameters struct { Version Version `json:"version,omitempty"` // version number, must be "1", "2" or "3" MaxPackSize int `json:"maxPackSize,omitempty"` // maximum size of a pack object IndexVersion int `json:"indexVersion,omitempty"` // force particular index format version (1,2,..) EpochParameters epoch.Parameters `json:"epochParameters,omitempty"` // epoch manager parameters }
MutableParameters represents parameters of the content manager that can be mutated after the repository is created.
func (*MutableParameters) Validate ¶
func (v *MutableParameters) Validate() error
Validate validates the parameters.
type ObjectFormat ¶
type ObjectFormat struct {
Splitter string `json:"splitter,omitempty"` // splitter used to break objects into pieces of content
}
ObjectFormat describes the format of objects in a repository.
type Provider ¶
type Provider interface { encryption.Parameters hashing.Parameters ecc.Parameters HashFunc() hashing.HashFunc Encryptor() encryption.Encryptor // this is typically cached, but sometimes refreshes MutableParameters from // the repository so the results should not be cached. GetMutableParameters(ctx context.Context) (MutableParameters, error) GetCachedMutableParameters() MutableParameters SupportsPasswordChange() bool GetMasterKey() []byte RepositoryFormatBytes(ctx context.Context) ([]byte, error) }
Provider provides current formatting options. The options returned should not be cached for more than a few seconds as they are subject to change.
func NewFormattingOptionsProvider ¶
func NewFormattingOptionsProvider(f0 *ContentFormat, formatBytes []byte) (Provider, error)
NewFormattingOptionsProvider validates the provided formatting options and returns static FormattingOptionsProvider based on them.
type RepositoryConfig ¶
type RepositoryConfig struct { ContentFormat ObjectFormat UpgradeLock *UpgradeLockIntent `json:"upgradeLock,omitempty"` RequiredFeatures []feature.Required `json:"requiredFeatures,omitempty"` }
RepositoryConfig describes the format of objects in a repository. The contents of this object are stored encrypted since they contain sensitive key material.
type UpgradeLockIntent ¶
type UpgradeLockIntent struct { OwnerID string `json:"ownerID,omitempty"` CreationTime time.Time `json:"creationTime,omitempty"` AdvanceNoticeDuration time.Duration `json:"advanceNoticeDuration,omitempty"` IODrainTimeout time.Duration `json:"ioDrainTimeout,omitempty"` StatusPollInterval time.Duration `json:"statusPollInterval,omitempty"` Message string `json:"message,omitempty"` MaxPermittedClockDrift time.Duration `json:"maxPermittedClockDrift,omitempty"` }
UpgradeLockIntent represents the intent to lock a kopia repository for upgrade related maintenance activity. This signals a request for exclusive access to the repository. The lock object is set on the Kopia repository format blob 'kopia.repository' and must be respected by all clients accessing the repository.
func (*UpgradeLockIntent) Clone ¶
func (l *UpgradeLockIntent) Clone() *UpgradeLockIntent
Clone creates a copy of the UpgradeLock instance.
func (*UpgradeLockIntent) IsLocked ¶
func (l *UpgradeLockIntent) IsLocked(now time.Time) (locked, writersDrained bool)
IsLocked indicates whether a lock intent has been placed and whether all other repository accessors have been drained.
func (*UpgradeLockIntent) Update ¶
func (l *UpgradeLockIntent) Update(other *UpgradeLockIntent) (*UpgradeLockIntent, error)
Update upgrades an existing lock intent. This method controls what mutations are allowed on an upgrade lock once it has been placed on the repository.
func (*UpgradeLockIntent) UpgradeTime ¶
func (l *UpgradeLockIntent) UpgradeTime() time.Time
UpgradeTime returns the absolute time in future by when the upgrade lock will be fully established, i.e. all non-upgrading-owner kopia accessors would be drained.
func (*UpgradeLockIntent) Validate ¶
func (l *UpgradeLockIntent) Validate() error
Validate verifies the parameters of an upgrade lock.