file

package
v2.0.239 Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Aug 17, 2023 License: MIT Imports: 40 Imported by: 2

Documentation

Index

Constants

View Source
const (
	DownloadPartChunkSize = int64(1024 * 1024 * 5)
	DownloadPartLimit     = 15
)
View Source
const (
	RetryAll                    = RetryPolicyType("RetryAll")
	RetryUnfinished             = RetryPolicyType("RetryUnfinished")
	RetryErroredIfSomeCompleted = RetryPolicyType("RetryErroredIfSomeCompleted")
)

Variables

View Source
var UntrustedSizeRangeRequestSizeExpectedReceived = fmt.Errorf("received size did not match server expected size")
View Source
var UntrustedSizeRangeRequestSizeSentLessThanExpected = fmt.Errorf("server send less than expected")
View Source
var UntrustedSizeRangeRequestSizeSentMoreThanExpected = fmt.Errorf("server send more than expected")
View Source
var UntrustedSizeRangeRequestSizeSentReceived = fmt.Errorf("received size did not match server send size")

Functions

func BeginUpload

func BeginUpload(params files_sdk.FileBeginUploadParams, opts ...files_sdk.RequestResponseOption) (fileUploadPartCollection files_sdk.FileUploadPartCollection, err error)

func Copy

func Copy(params files_sdk.FileCopyParams, opts ...files_sdk.RequestResponseOption) (fileAction files_sdk.FileAction, err error)

func Create

func Create(params files_sdk.FileCreateParams, opts ...files_sdk.RequestResponseOption) (file files_sdk.File, err error)

func Delete

func Delete(params files_sdk.FileDeleteParams, opts ...files_sdk.RequestResponseOption) (err error)

func Find

func Get

func Move

func Move(params files_sdk.FileMoveParams, opts ...files_sdk.RequestResponseOption) (fileAction files_sdk.FileAction, err error)

func RetryByPolicy

func RetryByPolicy(ctx context.Context, job *status.Job, policy RetryPolicy, signalEvents bool)

func RetryByStatus

func RetryByStatus(ctx context.Context, job *status.Job, signalEvents bool, policy RetryPolicy, s ...status.GetStatus)

func SetJobParams

func SetJobParams(r *status.Job, d direction.Direction, params interface{}, logger retryablehttp.Logger, remoteFs fs.FS)

func Update

func Update(params files_sdk.FileUpdateParams, opts ...files_sdk.RequestResponseOption) (file files_sdk.File, err error)

func UpdateWithMap

func UpdateWithMap(params map[string]interface{}, opts ...files_sdk.RequestResponseOption) (file files_sdk.File, err error)

Types

type ByteOffset

type ByteOffset struct {
	PartSizes []int64
}

func (ByteOffset) BySize

func (b ByteOffset) BySize(size *int64) Iterator

func (ByteOffset) Resume

func (b ByteOffset) Resume(size *int64, off int64, index int) Iterator

func (ByteOffset) WithDefaultChunkSize

func (b ByteOffset) WithDefaultChunkSize(size *int64, off int64, index int, defaultChunkSize int64) Iterator

type Client

type Client struct {
	files_sdk.Config
}

func (*Client) BeginUpload

func (c *Client) BeginUpload(params files_sdk.FileBeginUploadParams, opts ...files_sdk.RequestResponseOption) (fileUploadPartCollection files_sdk.FileUploadPartCollection, err error)

func (*Client) Copy

func (c *Client) Copy(params files_sdk.FileCopyParams, opts ...files_sdk.RequestResponseOption) (fileAction files_sdk.FileAction, err error)

func (*Client) Create

func (c *Client) Create(params files_sdk.FileCreateParams, opts ...files_sdk.RequestResponseOption) (file files_sdk.File, err error)

func (*Client) Delete

func (c *Client) Delete(params files_sdk.FileDeleteParams, opts ...files_sdk.RequestResponseOption) (err error)

func (*Client) Download

func (*Client) DownloadRequestStatus

func (c *Client) DownloadRequestStatus(fileDownloadUrl string, downloadRequestId string, opts ...files_sdk.RequestResponseOption) (files_sdk.ResponseError, error)

func (*Client) DownloadRetry

func (c *Client) DownloadRetry(job status.Job, opts ...files_sdk.RequestResponseOption) *status.Job

func (*Client) DownloadToFile

func (c *Client) DownloadToFile(params files_sdk.FileDownloadParams, filePath string, opts ...files_sdk.RequestResponseOption) (files_sdk.File, error)

func (*Client) DownloadUri

func (*Client) Downloader

func (c *Client) Downloader(params DownloaderParams, opts ...files_sdk.RequestResponseOption) *status.Job

func (*Client) FileStats

func (c *Client) FileStats(file files_sdk.File, opts ...files_sdk.RequestResponseOption) (files_sdk.File, error)

File{}.Size and File{}.Mtime are not always up to date. This calls HEAD on File{}.DownloadUri to get the latest info. Some Download URLs won't support HEAD. In this case the size is reported as UntrustedSizeValue. The size can be known post download using Client{}.DownloadRequestStatus. This applies to the remote mount types FTP, SFTP, and WebDAV.

func (*Client) Find

func (c *Client) Find(params files_sdk.FileFindParams, opts ...files_sdk.RequestResponseOption) (file files_sdk.File, err error)

func (*Client) Get

func (c *Client) Get(Path string, opts ...files_sdk.RequestResponseOption) (files_sdk.File, error)

func (*Client) ListFor

func (*Client) Move

func (c *Client) Move(params files_sdk.FileMoveParams, opts ...files_sdk.RequestResponseOption) (fileAction files_sdk.FileAction, err error)

func (*Client) Update

func (c *Client) Update(params files_sdk.FileUpdateParams, opts ...files_sdk.RequestResponseOption) (file files_sdk.File, err error)

func (*Client) UpdateWithMap

func (c *Client) UpdateWithMap(params map[string]interface{}, opts ...files_sdk.RequestResponseOption) (file files_sdk.File, err error)

func (*Client) Upload

func (c *Client) Upload(opts ...UploadOption) error

func (*Client) UploadFile

func (c *Client) UploadFile(sourcePath string, destinationPath string, opts ...UploadOption) error

UploadFile Deprecated use c.Upload(UploadWithFile(sourcePath), UploadWithDestinationPath(destinationPath))

func (*Client) UploadRetry

func (c *Client) UploadRetry(job status.Job, opts ...files_sdk.RequestResponseOption) *status.Job

func (*Client) UploadWithResume

func (c *Client) UploadWithResume(opts ...UploadOption) (UploadResumable, error)

func (*Client) Uploader

func (c *Client) Uploader(params UploaderParams, opts ...files_sdk.RequestResponseOption) *status.Job

type CustomTransport

type CustomTransport struct {
	http.Transport
	Addr string
}

func (*CustomTransport) RoundTrip

func (t *CustomTransport) RoundTrip(req *http.Request) (*http.Response, error)

type DeleteSource

type DeleteSource struct {
	direction.Direction
	Config files_sdk.Config
}

DeleteSource files after a sync

job.RegisterFileEvent(func(file status.File) {
		log, err := file.DeleteSource{Direction: f.Direction, Config: config}.Call(ctx, f)
}, status.Complete, status.Skipped)

func (DeleteSource) Call

type DirEntryError

type DirEntryError struct {
	DirEntries []goFs.DirEntry
	// contains filtered or unexported fields
}

type DownloadParts

type DownloadParts struct {
	context.CancelFunc
	context.Context

	fs.File
	fs.FileInfo
	lib.WriterAndAt

	CloseError error
	files_sdk.Config

	*sync.RWMutex
	// contains filtered or unexported fields
}

func (*DownloadParts) FinalSize

func (d *DownloadParts) FinalSize() int64

func (*DownloadParts) Init

func (d *DownloadParts) Init(file fs.File, info fs.FileInfo, globalWait manager.ConcurrencyManager, writer lib.WriterAndAt, config files_sdk.Config) *DownloadParts

func (*DownloadParts) Run

func (d *DownloadParts) Run(ctx context.Context) error

type DownloadStatus

type DownloadStatus struct {
	fs.FS
	fs.FileInfo

	DownloadedBytes int64

	Sync bool

	Mutex         *sync.RWMutex
	PreserveTimes bool

	status.Changes
	// contains filtered or unexported fields
}

func (*DownloadStatus) EndedAt

func (d *DownloadStatus) EndedAt() time.Time

func (*DownloadStatus) Err

func (d *DownloadStatus) Err() error

func (*DownloadStatus) File

func (d *DownloadStatus) File() files_sdk.File

func (*DownloadStatus) Id

func (d *DownloadStatus) Id() string

func (*DownloadStatus) IncrementTransferBytes added in v2.0.219

func (d *DownloadStatus) IncrementTransferBytes(b int64)

func (*DownloadStatus) Job

func (d *DownloadStatus) Job() *status.Job

func (*DownloadStatus) LastByte

func (d *DownloadStatus) LastByte() time.Time

func (*DownloadStatus) LocalPath

func (d *DownloadStatus) LocalPath() string

func (*DownloadStatus) RecentError

func (d *DownloadStatus) RecentError() error

func (*DownloadStatus) RemotePath

func (d *DownloadStatus) RemotePath() string

func (*DownloadStatus) SetFinalSize

func (d *DownloadStatus) SetFinalSize(written int64)

func (*DownloadStatus) SetStatus

func (d *DownloadStatus) SetStatus(s status.Status, err error)

func (*DownloadStatus) Size

func (d *DownloadStatus) Size() (size int64)

func (*DownloadStatus) StartedAt added in v2.0.219

func (d *DownloadStatus) StartedAt() time.Time

func (*DownloadStatus) Status

func (d *DownloadStatus) Status() status.Status

func (*DownloadStatus) StatusChanges

func (d *DownloadStatus) StatusChanges() status.Changes

func (*DownloadStatus) TransferBytes

func (d *DownloadStatus) TransferBytes() int64

type DownloadUri

type DownloadUri string

func (DownloadUri) ToTime

func (d DownloadUri) ToTime() (t time.Time, err error)

func (DownloadUri) ToUrl

func (d DownloadUri) ToUrl() (u *url.URL, err error)

func (DownloadUri) Valid

func (d DownloadUri) Valid(within time.Duration) (s DownloadUriValid, valid bool, err error)

type DownloadUriValid

type DownloadUriValid struct {
	*url.URL
	time.Time
	Remaining time.Duration
}

type DownloaderParams

type DownloaderParams struct {
	RemotePath    string
	RemoteFile    files_sdk.File
	LocalPath     string
	Sync          bool
	PreserveTimes bool
	RetryPolicy
	*manager.Manager
	status.EventsReporter
	files_sdk.Config
	DryRun bool
}

type Entity

type Entity struct {
	fs.File
	fs.FS
	// contains filtered or unexported fields
}

type FS

type FS struct {
	files_sdk.Config
	context.Context
	Root string
	// contains filtered or unexported fields
}

func (*FS) ClearCache

func (f *FS) ClearCache()

func (*FS) Create

func (f *FS) Create(path string) (io.WriteCloser, error)

Create Not for performant use cases.

func (*FS) Init

func (f *FS) Init(config files_sdk.Config, cache bool) *FS

func (*FS) MkdirAll

func (f *FS) MkdirAll(dir string, _ goFs.FileMode) error

func (*FS) MkdirTemp

func (f *FS) MkdirTemp(dir, pattern string) (string, error)

func (*FS) Open

func (f *FS) Open(name string) (goFs.File, error)

func (*FS) PathJoin

func (f *FS) PathJoin(s ...string) string

func (*FS) PathSeparator

func (f *FS) PathSeparator() string

func (*FS) ReadDir

func (f *FS) ReadDir(name string) ([]goFs.DirEntry, error)

func (*FS) RelPath

func (f *FS) RelPath(parent, child string) (string, error)

func (*FS) Remove

func (f *FS) Remove(path string) error

func (*FS) RemoveAll

func (f *FS) RemoveAll(path string) error

func (*FS) SplitPath

func (f *FS) SplitPath(path string) (string, string)

func (*FS) TempDir

func (f *FS) TempDir() string

func (*FS) WithContext

func (f *FS) WithContext(ctx context.Context) interface{}

type FakeDownloadServer

type FakeDownloadServer struct {
	Addr string
	*http.Server
	Port int

	MockFiles map[string]mockFile
	*testing.T
	TrackRequest map[string][]string
	// contains filtered or unexported fields
}

func (*FakeDownloadServer) Client

func (f *FakeDownloadServer) Client() *Client

func (*FakeDownloadServer) Do

func (*FakeDownloadServer) GetFile

func (f *FakeDownloadServer) GetFile(file mockFile) (r io.Reader, contentLengthOk bool, contentLength int64, realSize int64, err error)

func (*FakeDownloadServer) GetRouter

func (f *FakeDownloadServer) GetRouter() *gin.Engine

func (*FakeDownloadServer) Routes

func (f *FakeDownloadServer) Routes()

func (*FakeDownloadServer) Shutdown

func (f *FakeDownloadServer) Shutdown() error

type File

type File struct {
	*files_sdk.File
	*FS
	io.ReadCloser

	MaxConnections int

	SizeTrust
	// contains filtered or unexported fields
}

func (*File) Close

func (f *File) Close() error

func (*File) Info

func (f *File) Info() (goFs.FileInfo, error)

func (*File) Init

func (f *File) Init() *File

func (*File) IsDir

func (f *File) IsDir() bool

func (*File) Name

func (f *File) Name() string

func (*File) Read

func (f *File) Read(b []byte) (n int, err error)

func (*File) ReadAt

func (f *File) ReadAt(p []byte, off int64) (n int, err error)

func (*File) ReaderRange

func (f *File) ReaderRange(off int64, end int64) (r io.ReadCloser, err error)

func (*File) Stat

func (f *File) Stat() (goFs.FileInfo, error)

func (*File) Type

func (f *File) Type() goFs.FileMode

func (*File) WithContext

func (f *File) WithContext(ctx context.Context) goFs.File

type Info

type Info struct {
	files_sdk.File
	// contains filtered or unexported fields
}

func (Info) IsDir

func (i Info) IsDir() bool

func (Info) ModTime

func (i Info) ModTime() time.Time

func (Info) Mode

func (i Info) Mode() goFs.FileMode

func (Info) Name

func (i Info) Name() string

func (Info) PossibleSize

func (i Info) PossibleSize() int64

func (Info) RemoteMount

func (i Info) RemoteMount() bool

func (Info) Size

func (i Info) Size() int64

func (Info) SizeTrust

func (i Info) SizeTrust() SizeTrust

func (Info) Sys

func (i Info) Sys() interface{}

func (Info) UntrustedSize

func (i Info) UntrustedSize() bool

type Iter

type Iter struct {
	*folder.Iter
}

func (Iter) Iterate

func (i Iter) Iterate(identifier interface{}, opts ...files_sdk.RequestResponseOption) (files_sdk.IterI, error)

func (Iter) LoadResource

func (i Iter) LoadResource(identifier interface{}, opts ...files_sdk.RequestResponseOption) (interface{}, error)

type Iterator

type Iterator func() (OffSet, Iterator, int)

type Len

type Len interface {
	Len() int
}

type MoveSource

type MoveSource struct {
	direction.Direction
	Path   string
	Config files_sdk.Config
}

MoveSource files after a sync

job.RegisterFileEvent(func(file status.File) {
		log, err := file.MoveSource{Direction: f.Direction, Config: config}.Call(ctx, f)
}, status.Complete, status.Skipped)

func (MoveSource) Call

type OffSet

type OffSet struct {
	// contains filtered or unexported fields
}

type Part

type Part struct {
	OffSet
	files_sdk.EtagsParam

	context.Context
	context.CancelFunc
	*sync.RWMutex

	ProxyReader
	// contains filtered or unexported fields
}

func (*Part) Clear

func (p *Part) Clear()

func (*Part) Done

func (p *Part) Done() *Part

func (*Part) Err

func (p *Part) Err() error

func (*Part) SetError

func (p *Part) SetError(err error)

func (*Part) Start

func (p *Part) Start(ctx ...context.Context) *Part

func (*Part) Successful

func (p *Part) Successful() bool

func (*Part) Touch

func (p *Part) Touch()

func (*Part) WithContext

func (p *Part) WithContext(ctx context.Context) *Part

type PartRunnerReturn

type PartRunnerReturn int

type Parts

type Parts []*Part

func (Parts) SuccessfulBytes

func (p Parts) SuccessfulBytes() (b int64)

type PossibleSize

type PossibleSize interface {
	PossibleSize() int64
}

type Progress

type Progress func(int64)

type ProxyRead

type ProxyRead struct {
	io.Reader
	// contains filtered or unexported fields
}

func (*ProxyRead) BytesRead

func (x *ProxyRead) BytesRead() int

func (*ProxyRead) Close

func (x *ProxyRead) Close() error

func (*ProxyRead) Len

func (x *ProxyRead) Len() int

func (*ProxyRead) Read

func (x *ProxyRead) Read(p []byte) (int, error)

type ProxyReader

type ProxyReader interface {
	io.ReadCloser
	Len() int
	BytesRead() int
}

type ProxyReaderAt

type ProxyReaderAt struct {
	io.ReaderAt
	// contains filtered or unexported fields
}

func (*ProxyReaderAt) BytesRead

func (x *ProxyReaderAt) BytesRead() int

func (*ProxyReaderAt) Close

func (x *ProxyReaderAt) Close() error

func (*ProxyReaderAt) Len

func (x *ProxyReaderAt) Len() int

func (*ProxyReaderAt) Read

func (x *ProxyReaderAt) Read(p []byte) (int, error)

func (*ProxyReaderAt) Seek

func (x *ProxyReaderAt) Seek(offset int64, whence int) (int64, error)

type ReadAtLeastWrapper

type ReadAtLeastWrapper struct {
	io.ReadCloser
	io.Reader
}

func (ReadAtLeastWrapper) Close

func (r ReadAtLeastWrapper) Close() error

func (ReadAtLeastWrapper) Read

func (f ReadAtLeastWrapper) Read(b []byte) (n int, err error)

type ReadDirFile

type ReadDirFile struct {
	*File
	// contains filtered or unexported fields
}

func (ReadDirFile) ReadDir

func (f ReadDirFile) ReadDir(n int) ([]goFs.DirEntry, error)

type ReadWrapper

type ReadWrapper struct {
	io.ReadCloser
	// contains filtered or unexported fields
}

func (*ReadWrapper) Read

func (r *ReadWrapper) Read(p []byte) (n int, err error)

type ReaderCloserDownloadStatus

type ReaderCloserDownloadStatus struct {
	*ReadWrapper

	UntrustedSizeRangeRequestSize
	// contains filtered or unexported fields
}

func (ReaderCloserDownloadStatus) Close

type ReaderRange

type ReaderRange interface {
	ReaderRange(off int64, end int64) (io.ReadCloser, error)
	goFs.File
}

type RecursiveItem

type RecursiveItem struct {
	files_sdk.File
	// contains filtered or unexported fields
}

func (RecursiveItem) Err

func (r RecursiveItem) Err() error

type RetryPolicy

type RetryPolicy struct {
	Type       RetryPolicyType
	RetryCount int
	Backoff    int
}

func (RetryPolicy) WaitSec

func (p RetryPolicy) WaitSec(retry int) time.Duration

type RetryPolicyType

type RetryPolicyType string

type SizeTrust

type SizeTrust int
const (
	NullSizeTrust SizeTrust = iota
	UntrustedSizeValue
	TrustedSizeValue
)

type TestLogger

type TestLogger struct {
	*testing.T
}

func (TestLogger) Printf

func (t TestLogger) Printf(format string, args ...any)

func (TestLogger) Write

func (t TestLogger) Write(p []byte) (n int, err error)

type UntrustedSize

type UntrustedSize interface {
	UntrustedSize() bool
	SizeTrust() SizeTrust
	goFs.FileInfo
}

type UntrustedSizeRangeRequestSize

type UntrustedSizeRangeRequestSize struct {
	ExpectedSize int64
	SentSize     int64
	ReceivedSize int64
	Status       string
}

func (UntrustedSizeRangeRequestSize) Log

func (u UntrustedSizeRangeRequestSize) Log() map[string]interface{}

func (UntrustedSizeRangeRequestSize) Mismatch

func (u UntrustedSizeRangeRequestSize) Mismatch() error

func (UntrustedSizeRangeRequestSize) VerifyReceived

func (u UntrustedSizeRangeRequestSize) VerifyReceived() error

type UploadOption

type UploadOption func(uploadIO) (uploadIO, error)

func UploadRewindAllProgressOnFailure

func UploadRewindAllProgressOnFailure() UploadOption

UploadRewindAllProgressOnFailure on upload failure rewind all successfully parts

func UploadWithContext

func UploadWithContext(ctx context.Context) UploadOption

func UploadWithDestinationPath

func UploadWithDestinationPath(destinationPath string) UploadOption

func UploadWithFile

func UploadWithFile(sourcePath string) UploadOption

func UploadWithProgress

func UploadWithProgress(progress Progress) UploadOption

func UploadWithProvidedMtime

func UploadWithProvidedMtime(providedMtime time.Time) UploadOption

func UploadWithReader

func UploadWithReader(reader io.Reader) UploadOption

func UploadWithReaderAt

func UploadWithReaderAt(readerAt io.ReaderAt) UploadOption

func UploadWithResume

func UploadWithResume(resume UploadResumable) UploadOption

func UploadWithSize

func UploadWithSize(size int64) UploadOption

type UploadResumable

type UploadResumable struct {
	files_sdk.FileUploadPart
	Parts
	files_sdk.File
}

type UploadStatus

type UploadStatus struct {
	Sync bool

	Uploader
	UploadResumable
	Mutex *sync.RWMutex

	status.Changes
	// contains filtered or unexported fields
}

func (*UploadStatus) EndedAt

func (u *UploadStatus) EndedAt() time.Time

func (*UploadStatus) Err

func (u *UploadStatus) Err() error

func (*UploadStatus) File

func (u *UploadStatus) File() files_sdk.File

func (*UploadStatus) Id

func (u *UploadStatus) Id() string

func (*UploadStatus) IncrementTransferBytes added in v2.0.219

func (u *UploadStatus) IncrementTransferBytes(b int64)

func (*UploadStatus) Job

func (u *UploadStatus) Job() *status.Job

func (*UploadStatus) LastByte

func (u *UploadStatus) LastByte() time.Time

func (*UploadStatus) LocalPath

func (u *UploadStatus) LocalPath() string

func (*UploadStatus) RecentError

func (u *UploadStatus) RecentError() error

func (*UploadStatus) RemotePath

func (u *UploadStatus) RemotePath() string

func (*UploadStatus) SetStatus

func (u *UploadStatus) SetStatus(s status.Status, err error)

func (*UploadStatus) SetUploadedBytes

func (u *UploadStatus) SetUploadedBytes(b int64)

func (*UploadStatus) Size

func (u *UploadStatus) Size() int64

func (*UploadStatus) StartedAt added in v2.0.219

func (u *UploadStatus) StartedAt() time.Time

func (*UploadStatus) Status

func (u *UploadStatus) Status() status.Status

func (*UploadStatus) StatusChanges

func (u *UploadStatus) StatusChanges() status.Changes

func (*UploadStatus) TransferBytes

func (u *UploadStatus) TransferBytes() int64

type Uploader

type Uploader interface {
	UploadWithResume(...UploadOption) (UploadResumable, error)
	Find(files_sdk.FileFindParams, ...files_sdk.RequestResponseOption) (files_sdk.File, error)
}

type UploaderParams

type UploaderParams struct {
	Ignore []string
	*status.Job
	Sync       bool
	LocalPath  string
	RemotePath string
	DryRun     bool
	RetryPolicy
	status.EventsReporter
	*manager.Manager
	files_sdk.Config
}

type WritableFile

type WritableFile struct {
	*Client
	*FS

	*bytes.Buffer
	// contains filtered or unexported fields
}

func (WritableFile) Close

func (w WritableFile) Close() (err error)

func (WritableFile) Write

func (w WritableFile) Write(p []byte) (int, error)

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL