providers

Documentation

Index

• Constants
• func GetProviderNameForRemote(remoteName string) string
• func GetSyncProviders() map[string]SyncProvider
• func InitCoreProviders()
• func RegisterSyncProvider(p SyncProvider) error
• type FileSystemSyncProvider
  • func (self *FileSystemSyncProvider) Download(remoteName string, filenames []string, toDir string, force bool, ...) error
  • func (self *FileSystemSyncProvider) FileExists(remoteName, filename string) bool
  • func (self *FileSystemSyncProvider) FileExistsAndIsOfSize(remoteName, filename string, sz int64) bool
  • func (*FileSystemSyncProvider) HelpTextDetail() string
  • func (*FileSystemSyncProvider) HelpTextSummary() string
  • func (*FileSystemSyncProvider) Release()
  • func (*FileSystemSyncProvider) TypeID() string
  • func (self *FileSystemSyncProvider) Upload(remoteName string, filenames []string, fromDir string, force bool, ...) error
  • func (*FileSystemSyncProvider) ValidateConfig(remoteName string) error
• type S3SyncProvider
  • func (self *S3SyncProvider) Download(remoteName string, filenames []string, toDir string, force bool, ...) error
  • func (self *S3SyncProvider) FileExists(remoteName, filename string) bool
  • func (self *S3SyncProvider) FileExistsAndIsOfSize(remoteName, filename string, sz int64) bool
  • func (*S3SyncProvider) HelpTextDetail() string
  • func (*S3SyncProvider) HelpTextSummary() string
  • func (self *S3SyncProvider) Release()
  • func (*S3SyncProvider) TypeID() string
  • func (self *S3SyncProvider) Upload(remoteName string, filenames []string, fromDir string, force bool, ...) error
  • func (self *S3SyncProvider) ValidateConfig(remoteName string) error
• type SmartSyncProvider
  • func UpgradeToSmartSyncProvider(provider SyncProvider) SmartSyncProvider
• type SyncProgressCallback
• type SyncProgressReader
  • func NewSyncProgressReader(r io.Reader, filename string, totalBytes int64, callback SyncProgressCallback) *SyncProgressReader
  • func (self *SyncProgressReader) Read(p []byte) (n int, err error)
• type SyncProvider
  • func GetProviderForRemote(remoteName string) (SyncProvider, error)
  • func GetSyncProvider(typeID string) (SyncProvider, error)

Constants

const FileSystemBufferSize = 131072

const S3BufferSize = 131072

Functions

func GetProviderNameForRemote

func GetProviderNameForRemote(remoteName string) string

Get the provider name specified for the named remote in the current git repo May return "" if not specified

func GetSyncProviders

func GetSyncProviders() map[string]SyncProvider

Retrieve all sync providers, keyed on name

func InitCoreProviders

func InitCoreProviders()

Install the core providers

func RegisterSyncProvider

func RegisterSyncProvider(p SyncProvider) error

Registers an instance of a SyncProvider for later use Must only be called from the main thread, not thread safe Repeat calls for providers using the same TypeID will overrule previous

Types

type FileSystemSyncProvider

type FileSystemSyncProvider struct {
}

FileSystemSyncProvider implements the basic SyncProvider interface

func (*FileSystemSyncProvider) Download

func (self *FileSystemSyncProvider) Download(remoteName string, filenames []string, toDir string,
	force bool, callback SyncProgressCallback) error

func (*FileSystemSyncProvider) FileExists

func (self *FileSystemSyncProvider) FileExists(remoteName, filename string) bool

func (*FileSystemSyncProvider) FileExistsAndIsOfSize

func (self *FileSystemSyncProvider) FileExistsAndIsOfSize(remoteName, filename string, sz int64) bool

func (*FileSystemSyncProvider) HelpTextDetail

func (*FileSystemSyncProvider) HelpTextDetail() string

func (*FileSystemSyncProvider) HelpTextSummary

func (*FileSystemSyncProvider) HelpTextSummary() string

func (*FileSystemSyncProvider) Release

func (*FileSystemSyncProvider) Release()

func (*FileSystemSyncProvider) TypeID

func (*FileSystemSyncProvider) TypeID() string

func (*FileSystemSyncProvider) Upload

func (self *FileSystemSyncProvider) Upload(remoteName string, filenames []string, fromDir string,
	force bool, callback SyncProgressCallback) error

func (*FileSystemSyncProvider) ValidateConfig

func (*FileSystemSyncProvider) ValidateConfig(remoteName string) error

type S3SyncProvider

type S3SyncProvider struct {
	S3Connection *s3.S3
	Buckets      []string
}

S3SyncProvider implements the basic SyncProvider interface for S3

func (*S3SyncProvider) Download

func (self *S3SyncProvider) Download(remoteName string, filenames []string, toDir string, force bool, callback SyncProgressCallback) error

func (*S3SyncProvider) FileExists

func (self *S3SyncProvider) FileExists(remoteName, filename string) bool

func (*S3SyncProvider) FileExistsAndIsOfSize

func (self *S3SyncProvider) FileExistsAndIsOfSize(remoteName, filename string, sz int64) bool

func (*S3SyncProvider) HelpTextDetail

func (*S3SyncProvider) HelpTextDetail() string

func (*S3SyncProvider) HelpTextSummary

func (*S3SyncProvider) HelpTextSummary() string

func (*S3SyncProvider) Release

func (self *S3SyncProvider) Release()

func (*S3SyncProvider) TypeID

func (*S3SyncProvider) TypeID() string

func (*S3SyncProvider) Upload

func (self *S3SyncProvider) Upload(remoteName string, filenames []string, fromDir string,
	force bool, callback SyncProgressCallback) error

func (*S3SyncProvider) ValidateConfig

func (self *S3SyncProvider) ValidateConfig(remoteName string) error

type SmartSyncProvider

type SmartSyncProvider interface {
	// Everything from core SyncProvider
	SyncProvider

	// Whether a LOB exists in full on the remote, and gets its size
	LOBExists(remoteName, sha string) (ex bool, sz int64)
	// Prepare a delta from a list of candidate shas and report the size of it, the chosen base SHA. If this fails caller should use standard Download()
	PrepareDeltaForDownload(remoteName, sha string, candidateBaseSHAs []string) (sz int64, base string, e error)
	// Download delta of LOB content (must be applied later)
	DownloadDelta(remoteName, basesha, targetsha string, out io.Writer, callback SyncProgressCallback) error
	// Return the LOB which the server has a complete copy of, from a list of candidates
	// Server must test in the order provided & return the earliest one which is complete on the server
	// Server doesn't have to test full integrity of LOB, just completeness (check size against meta)
	// Return a blank string if none are available
	GetFirstCompleteLOBFromList(remoteName string, candidateSHAs []string) (string, error)
	// Upload delta of LOB content (must be calculated first)
	UploadDelta(remoteName, basesha, targetsha string, in io.Reader, size int64, callback SyncProgressCallback) error
}

Smart sync provider interface with more options

func UpgradeToSmartSyncProvider

func UpgradeToSmartSyncProvider(provider SyncProvider) SmartSyncProvider

'Upgrade' a pointer to a SyncProvider to a SmartSyncProvider, if possible (returns nil if not)

type SyncProgressCallback

type SyncProgressCallback func(fileInProgress string, progressType util.ProgressCallbackType, bytesDone, totalBytes int64) (abort bool)

Callback when progress is made uploading / downloading fileInProgress: relative path of file, isSkipped: whether file was up to date, bytesDone/totalBytes: progress for current file return true to abort the process for this and all other files in the batch

type SyncProgressReader

type SyncProgressReader struct {
	Aborted   bool
	BytesRead int64
	// contains filtered or unexported fields
}

This is a passthrough reader that reports progress, for cases where you need to give a Reader to a lower level process

func NewSyncProgressReader

func NewSyncProgressReader(r io.Reader, filename string, totalBytes int64, callback SyncProgressCallback) *SyncProgressReader

func (*SyncProgressReader) Read

func (self *SyncProgressReader) Read(p []byte) (n int, err error)

type SyncProvider

type SyncProvider interface {
	// Return the type identifier for this provider
	TypeID() string

	// Return this provider's help text summary for inclusion in the response to
	// 'git-lob providers' (where all are listed)
	HelpTextSummary() string

	// Return this provider's full help text, including all details of configuration
	// parameters in gitconfig, for use as a response to
	// 'git-lob provider <TypeID>'
	HelpTextDetail() string

	// Return whether the configuration for a given remote is valid
	ValidateConfig(remoteName string) error

	// Release any resources being used by the provider
	Release()

	// Upload a given list of files (binary storage). The paths are relative to fromDir and are provided
	// like that to make it easier for the provider since it will likely use the same
	// relative paths inside its own storage
	// For each file, if the remote already has this file and it's the same size, skip.
	// Must only return nil if remote is considered fully up to date with these files
	// If force = true, files should be uploaded even if they're already there & the correct size
	Upload(remoteName string, filenames []string, fromDir string, force bool, callback SyncProgressCallback) error
	// Download the list of files (binary storage). The paths are relative and files should
	// be placed relative to toDir. Ideally in-progress downloads should go to other locations
	// and be moved to the final location on success, although git-lob will detect files
	// of incorrect size.
	// For each file, if the local copy already has this file and it's the same size, skip.
	// Must only return nil if all files were successfully uploaded
	// If force = true, files should be downloaded even if they're already there & the correct size
	// Files not found should not cause an error, since we do not query presence beforehand. Instead,
	// the callback should be called with ProgressNotFound and proceed to the next one.
	Download(remoteName string, filenames []string, toDir string, force bool, callback SyncProgressCallback) error
	// Validate that the passed in file exists on the named remote
	// filename is relative to the root of the store
	FileExists(remoteName, filename string) bool
	// Validate that the passed in file exists on the named remote and is of the correct size
	// filename is relative to the root of the store
	FileExistsAndIsOfSize(remoteName, filename string, sz int64) bool
}

General interface that all sync providers must implement

func GetProviderForRemote

func GetProviderForRemote(remoteName string) (SyncProvider, error)

Get the provider for a given remote, and validate that it's configured correctly

func GetSyncProvider

func GetSyncProvider(typeID string) (SyncProvider, error)

Retrieve a SyncProvider with the associated typeID