fileservice

package

Go to main page

Versions in this module

v1

v1.2.4

Sep 20, 2024

v1.2.3

Sep 10, 2024

Changes in this version

+ var GlobalDiskCacheSizeHint atomic.Int64

+ var GlobalMemoryCacheSizeHint atomic.Int64

+ func EvictDiskCaches() map[string]int64

+ func EvictMemoryCaches() map[string]int64

type CacheDataAllocator

+ AllocateCacheData func(size int) fscache.Data

type DiskCache

+ func (d *DiskCache) Close()

+ func (d *DiskCache) Evict(done chan int64)

type IOVectorCache

+ Evict func(done chan int64)

type LocalFS

+ func (l *LocalFS) AllocateCacheData(size int) fscache.Data

type MemCache

+ func (m *MemCache) Close()

+ func (m *MemCache) Evict(done chan int64)

type RemoteCache

+ func (r *RemoteCache) Evict(done chan int64)

type S3FS

+ func (s *S3FS) AllocateCacheData(size int) fscache.Data

v1.2.3-hotfix-20241016

Oct 16, 2024

v1.2.3-hotfix-20241010

Oct 11, 2024

v1.2.3-hotfix-20240916

Sep 14, 2024

v1.2.2

Jul 11, 2024

Changes in this version

type ObjectStorageArguments

+ Concurrency int64

v1.2.2-20240715

Jul 15, 2024

v1.2.1

Jun 28, 2024

Changes in this version

+ var EventLoggerKey eventLoggerKey

+ func LogEvent(ctx context.Context, ev string)

+ func LogSlowEvent(ctx context.Context, threshold time.Duration)

+ func WithEventLogger(ctx context.Context) context.Context

type CacheConfig

+ CheckOverlaps bool

type CacheDataAllocator

+ func GetDefaultCacheDataAllocator() CacheDataAllocator

+ type CostAttr struct

+ List CostItem

+ type CostItem uint8

+ const CostHigh

+ const CostLow

type FileService

+ Cost func() *CostAttr

type FileServices

+ func (f *FileServices) Cost() *CostAttr

type LocalETLFS

+ func (l *LocalETLFS) Cost() *CostAttr

type LocalFS

+ func (l *LocalFS) Cost() *CostAttr

type MemoryFS

+ func (m *MemoryFS) Cost() *CostAttr

type S3FS

+ func (s *S3FS) Cost() *CostAttr

v1.2.1-20240709

Jul 9, 2024

v1.2.0

May 17, 2024

Changes in this version

+ func DoWithRetry[T any](what string, fn func() (T, error), maxAttemps int, ...) (res T, err error)

+ func IsRetryableError(err error) bool

+ func NewMemoryCache(capacity int64, checkOverlaps bool, callbacks *CacheCallbacks) *memorycache.Cache

type CacheConfig

+ QueryClient client.QueryClient

type FileService

+ Close func()

type FileServices

+ func (f *FileServices) Close()

+ type IOMergeKey struct

+ End int64

+ Offset int64

+ Path string

+ Policy Policy

+ type IOMerger struct

+ func NewIOMerger() *IOMerger

+ func (i *IOMerger) Merge(key IOMergeKey) (done func(), wait func())

type LocalETLFS

+ func (l *LocalETLFS) Close()

type LocalFS

+ func (l *LocalFS) Close()

type MemCache

+ func (m *MemCache) Alloc(n int) memorycache.CacheData

type MemoryFS

+ func (m *MemoryFS) Close()

type S3FS

+ func (s *S3FS) Close()

v1.1.3

Apr 15, 2024

v1.1.2

Apr 2, 2024

Changes in this version

+ var NoDefaultCredentialsForETL = os.Getenv("MO_NO_DEFAULT_CREDENTIALS") != ""

type ObjectStorageArguments

+ NoBucketValidation bool

+ NoDefaultCredentials bool

v1.1.1

Feb 2, 2024

Changes in this version

type ObjectStorageArguments

+ RoleARN string

v1.1.0

Dec 29, 2023

Changes in this version

type DataCache

+ DeletePaths func(ctx context.Context, paths []string)

type DiskCache

+ func (d *DiskCache) DeletePaths(ctx context.Context, paths []string) error

type IOEntry

+ func (i IOEntry) String() string

type IOVectorCache

+ DeletePaths func(ctx context.Context, paths []string) error

type MemCache

+ func (m *MemCache) DeletePaths(ctx context.Context, paths []string) error

type RemoteCache

+ func (r *RemoteCache) DeletePaths(ctx context.Context, paths []string) error

v1.0.2

Dec 21, 2023

v1.0.1

Dec 15, 2023

Changes in this version

type FileService

+ PrefetchFile func(ctx context.Context, filePath string) error

type FileServices

+ func (f *FileServices) PrefetchFile(ctx context.Context, filePath string) error

type IOVector

+ Caches []IOVectorCache

type LocalETLFS

+ func (l *LocalETLFS) PrefetchFile(ctx context.Context, filePath string) error

type LocalFS

+ func (l *LocalFS) PrefetchFile(ctx context.Context, filePath string) error

type MemoryFS

+ func (m *MemoryFS) PrefetchFile(ctx context.Context, filePath string) error

type ObjectStorageArguments

+ CertFiles []string

+ func (o ObjectStorageArguments) String() string

type S3FS

+ func (s *S3FS) PrefetchFile(ctx context.Context, filePath string) error

v1.0.0

Nov 13, 2023

Changes in this version

+ const SkipAllCache

+ const SkipCacheReads

+ const SkipCacheWrites

+ const SkipDiskCache

+ const SkipDiskCacheReads

+ const SkipDiskCacheWrites

+ const SkipFullFilePreloads

+ const SkipMemoryCache

+ const SkipMemoryCacheReads

+ const SkipMemoryCacheWrites

+ var CtxKeyDiskCacheCallbacks ctxKeyDiskCacheCallbacks

+ var CtxKeyStatementProfiler ctxKeyStatementProfiler

+ var DefaultCacheDataAllocator = new(bytesAllocator)

+ var ErrNotCacheFile = errorStr("not a cache file")

+ var PerStatementProfileDir = os.Getenv("PER_STMT_PROFILE_DIR")

+ var PerStatementProfileThreshold = func() time.Duration { ... }()

+ var RCBytesPool = func() *rcBytesPool { ... }()

+ func EnsureStatementProfiler(ctx context.Context, from context.Context) context.Context

+ func HandleRemoteRead(ctx context.Context, fs FileService, req *pb.Request, resp *pb.Response) error

+ func NewLRUCache(capacity int64, checkOverlaps bool, callbacks *CacheCallbacks) *lrucache.LRU[CacheKey, CacheData]

+ func NewStatementProfiler(ctx context.Context) (newCtx context.Context, end func(fileSuffixFunc func() string))

+ func OnDiskCacheEvict(ctx context.Context, fn OnDiskCacheEvictFunc) (ret context.Context)

+ func OnDiskCacheWritten(ctx context.Context, fn OnDiskCacheWrittenFunc) (ret context.Context)

+ func StatementProfileNewSpan(ctx context.Context) (_ context.Context, end func())

+ type AliyunSDK struct

+ func NewAliyunSDK(ctx context.Context, args ObjectStorageArguments, ...) (*AliyunSDK, error)

+ func (a *AliyunSDK) Delete(ctx context.Context, keys ...string) (err error)

+ func (a *AliyunSDK) Exists(ctx context.Context, key string) (bool, error)

+ func (a *AliyunSDK) List(ctx context.Context, prefix string, fn func(bool, string, int64) (bool, error)) error

+ func (a *AliyunSDK) Read(ctx context.Context, key string, min *int64, max *int64) (r io.ReadCloser, err error)

+ func (a *AliyunSDK) Stat(ctx context.Context, key string) (size int64, err error)

+ func (a *AliyunSDK) Write(ctx context.Context, key string, r io.Reader, size int64, expire *time.Time) (err error)

+ type AwsSDKv1 struct

+ func NewAwsSDKv1(ctx context.Context, args ObjectStorageArguments, ...) (*AwsSDKv1, error)

+ func (a *AwsSDKv1) Delete(ctx context.Context, keys ...string) (err error)

+ func (a *AwsSDKv1) Exists(ctx context.Context, key string) (bool, error)

+ func (a *AwsSDKv1) List(ctx context.Context, prefix string, fn func(bool, string, int64) (bool, error)) error

+ func (a *AwsSDKv1) Read(ctx context.Context, key string, min *int64, max *int64) (r io.ReadCloser, err error)

+ func (a *AwsSDKv1) Stat(ctx context.Context, key string) (size int64, err error)

+ func (a *AwsSDKv1) Write(ctx context.Context, key string, r io.Reader, size int64, expire *time.Time) (err error)

+ type AwsSDKv2 struct

+ func NewAwsSDKv2(ctx context.Context, args ObjectStorageArguments, ...) (*AwsSDKv2, error)

+ func (a *AwsSDKv2) Delete(ctx context.Context, keys ...string) (err error)

+ func (a *AwsSDKv2) Exists(ctx context.Context, key string) (bool, error)

+ func (a *AwsSDKv2) List(ctx context.Context, prefix string, fn func(bool, string, int64) (bool, error)) error

+ func (a *AwsSDKv2) Read(ctx context.Context, key string, min *int64, max *int64) (r io.ReadCloser, err error)

+ func (a *AwsSDKv2) Stat(ctx context.Context, key string) (size int64, err error)

+ func (a *AwsSDKv2) Write(ctx context.Context, key string, r io.Reader, size int64, expire *time.Time) (err error)

+ type Bytes []byte

+ func (b Bytes) Bytes() []byte

+ func (b Bytes) Release()

+ func (b Bytes) Retain()

+ func (b Bytes) Size() int64

+ func (b Bytes) Slice(length int) CacheData

+ type CacheCallbackFunc = func(CacheKey, CacheData)

+ type CacheCallbacks struct

+ PostEvict []CacheCallbackFunc

+ PostGet []CacheCallbackFunc

+ PostSet []CacheCallbackFunc

type CacheConfig

+ CacheClient client.CacheClient

+ InitKeyRouter *sync.Once

+ KeyRouter KeyRouter

+ KeyRouterFactory KeyRouterFactory

+ RPC morpc.Config

+ RemoteCacheEnabled bool

+ func (c *CacheConfig) SetRemoteCacheCallback()

+ type CacheData interface

+ Bytes func() []byte

+ Release func()

+ Retain func()

+ Slice func(length int) CacheData

+ func CacheOriginalData(r io.Reader, data []byte, allocator CacheDataAllocator) (cacheData CacheData, err error)

+ type CacheDataAllocator interface

+ Alloc func(size int) CacheData

type Config

+ FixMissing bool

+ type DataCache interface

+ Available func() int64

+ Capacity func() int64

+ Flush func()

+ Get func(ctx context.Context, key CacheKey) (value CacheData, ok bool)

+ Set func(ctx context.Context, key CacheKey, value CacheData)

+ Used func() int64

type DiskCache

+ func (d *DiskCache) SetFile(ctx context.Context, path string, content []byte) error

+ type DiskCacheCallbacks struct

+ OnEvict []OnDiskCacheEvictFunc

+ OnWritten []OnDiskCacheWrittenFunc

+ type FileCache interface

+ SetFile func(ctx context.Context, path string, content []byte) error

type FileService

+ func GetForBackup(ctx context.Context, spec string) (res FileService, err error)

+ ReadCache func(ctx context.Context, vector *IOVector) error

type FileServices

+ func (f *FileServices) ReadCache(ctx context.Context, vector *IOVector) error

type IOEntry

+ CachedData CacheData

+ ToCacheData func(reader io.Reader, data []byte, allocator CacheDataAllocator) (cacheData CacheData, err error)

+ type IOLockKey struct

+ File string

+ type IOLocks struct

+ func (i *IOLocks) Lock(key IOLockKey) (unlock func(), wait func())

type IOVector

+ Policy Policy

+ func (i *IOVector) Release()

+ type KeyRouter interface

+ AddItem func(key CacheKey, operation gpb.Operation)

+ Target func(k CacheKey) string

+ type KeyRouterFactory func() KeyRouter

type LocalETLFS

+ func (l *LocalETLFS) ReadCache(ctx context.Context, vector *IOVector) error

type LocalFS

+ func (l *LocalFS) ReadCache(ctx context.Context, vector *IOVector) (err error)

type MemoryFS

+ func (m *MemoryFS) ReadCache(ctx context.Context, vector *IOVector) (err error)

+ type MinioSDK struct

+ func NewMinioSDK(ctx context.Context, args ObjectStorageArguments, ...) (*MinioSDK, error)

+ func (a *MinioSDK) Delete(ctx context.Context, keys ...string) (err error)

+ func (a *MinioSDK) Exists(ctx context.Context, key string) (bool, error)

+ func (a *MinioSDK) List(ctx context.Context, prefix string, fn func(bool, string, int64) (bool, error)) error

+ func (a *MinioSDK) Read(ctx context.Context, key string, min *int64, max *int64) (r io.ReadCloser, err error)

+ func (a *MinioSDK) Stat(ctx context.Context, key string) (size int64, err error)

+ func (a *MinioSDK) Write(ctx context.Context, key string, r io.Reader, size int64, expire *time.Time) (err error)

+ type ObjectStorage interface

+ Delete func(ctx context.Context, keys ...string) (err error)

+ Exists func(ctx context.Context, key string) (bool, error)

+ List func(ctx context.Context, prefix string, ...) (err error)

+ Read func(ctx context.Context, key string, min *int64, max *int64) (r io.ReadCloser, err error)

+ Stat func(ctx context.Context, key string) (size int64, err error)

+ Write func(ctx context.Context, key string, r io.Reader, size int64, expire *time.Time) (err error)

+ type ObjectStorageArguments struct

+ AssumeRoleARN string

+ BearerToken string

+ Bucket string

+ Endpoint string

+ ExternalID string

+ IsMinio bool

+ KeyID string

+ KeyPrefix string

+ KeySecret string

+ Name string

+ OIDCProviderARN string

+ OIDCRoleARN string

+ OIDCTokenFilePath string

+ RAMRole string

+ Region string

+ RoleSessionName string

+ SecurityToken string

+ SessionToken string

+ SharedConfigProfile string

+ func (o *ObjectStorageArguments) SetFromString(arguments []string) error

+ type OnDiskCacheEvictFunc = func(diskFilePath string)

+ type OnDiskCacheWrittenFunc = func(filePath string, entry IOEntry)

+ type Policy uint64

+ func (c Policy) Any(policies ...Policy) bool

+ func (c Policy) CacheFullFile() bool

+ func (c Policy) CacheIOEntry() bool

+ type ProfileInfo struct

+ func NewProfileInfo() *ProfileInfo

+ type ProfileSpan struct

+ BeginTime time.Time

+ Locations []*profile.Location

+ SubSpanDuration time.Duration

+ type RCBytes struct

+ func (r RCBytes) Bytes() []byte

+ func (r RCBytes) Copy() []byte

+ func (r RCBytes) Release()

+ func (r RCBytes) Slice(length int) CacheData

+ type RCPool struct

+ func NewRCPool[T any](newFunc func() T) *RCPool[T]

+ func (r *RCPool[T]) Get() *RCPoolItem[T]

+ type RCPoolItem struct

+ Value T

+ func (r *RCPoolItem[T]) Release()

+ func (r *RCPoolItem[T]) Retain()

+ type RemoteCache struct

+ func NewRemoteCache(client client.CacheClient, factory KeyRouterFactory) *RemoteCache

+ func (r *RemoteCache) Flush()

+ func (r *RemoteCache) Read(ctx context.Context, vector *IOVector) error

+ func (r *RemoteCache) Update(ctx context.Context, vector *IOVector, async bool) error

type S3FS

+ func (s *S3FS) ReadCache(ctx context.Context, vector *IOVector) (err error)

+ type SpanProfiler struct

+ func NewSpanProfiler() *SpanProfiler

+ func (s *SpanProfiler) Begin(skip int) (profiler *SpanProfiler, end func())

+ func (s *SpanProfiler) Write(w io.Writer) error

+ type TargetCacheKeys map[string][]*pb.RequestCacheKey

v1.0.0-rc2

Oct 24, 2023

v1.0.0-rc1

Aug 24, 2023

v0

v0.8.1

Oct 9, 2023

v0.8.0

Jun 30, 2023

Changes in this version

+ const DisableCacheCapacity

+ var DisabledCacheConfig = CacheConfig

+ var FSProfileHandler = NewProfileHandler()

+ func NewFileWithChecksumOSFile(ctx context.Context, underlying *os.File, blockContentSize int, ...) (*FileWithChecksum[*os.File], PutBack[*FileWithChecksum[*os.File]])

type CacheConfig

+ DiskEvictTarget float64

+ DiskMinEvictInterval toml.Duration

+ func (c *CacheConfig) SetDefaults()

type CachingFileService

+ SetAsyncUpdate func(bool)

type DiskCache

+ func (d *DiskCache) GetFileContent(ctx context.Context, filePath string, offset int64) (r io.ReadCloser, err error)

+ func (d *DiskCache) SetFileContent(ctx context.Context, filePath string, ...) (err error)

+ type FileContentCache interface

+ GetFileContent func(ctx context.Context, path string, offset int64) (r io.ReadCloser, err error)

+ SetFileContent func(ctx context.Context, path string, ...) (err error)

type FileService

+ func SubPath(upstream FileService, path string) FileService

+ Preload func(ctx context.Context, filePath string) error

type FileServices

+ func (f *FileServices) Preload(ctx context.Context, dirPath string) error

type IOEntry

+ ObjectBytes []byte

+ ToObjectBytes func(reader io.Reader, data []byte) (object []byte, objectSize int64, err error)

+ func (e *IOEntry) ReadFromOSFile(file *os.File) error

type IOVector

+ NoCache bool

+ Preloading bool

+ type IOVectorCache interface

+ Flush func()

+ Read func(ctx context.Context, vector *IOVector) error

+ Update func(ctx context.Context, vector *IOVector, async bool) error

+ type IOVectorCacheKey struct

+ Offset int64

+ Path string

+ Size int64

type LocalETLFS

+ func (l *LocalETLFS) Preload(ctx context.Context, filePath string) error

type LocalFS

+ func (l *LocalFS) Preload(ctx context.Context, filePath string) error

+ func (l *LocalFS) SetAsyncUpdate(b bool)

+ type MemCacheOptionFunc func(*memCacheOptions)

+ func WithClock(capacity int64) MemCacheOptionFunc

+ func WithLRU(capacity int64) MemCacheOptionFunc

+ func WithPerfCounterSets(counterSets []*perfcounter.CounterSet) MemCacheOptionFunc

type MemoryFS

+ func (m *MemoryFS) Preload(ctx context.Context, filePath string) error

+ type ObjectCache interface

+ Available func() int64

+ Capacity func() int64

+ Flush func()

+ Get func(key any, preloading bool) (value []byte, size int64, ok bool)

+ Set func(key any, value []byte, size int64, preloading bool)

+ Used func() int64

type Path

+ func (p Path) ServiceString() string

+ func (p Path) String() string

+ type Pool struct

+ func NewPool(capacity uint32, newFunc func() T, resetFunc func(T), finallyFunc func(T)) *Pool[T]

+ func (p *Pool[T]) Get(ptr *T) PutBack[T]

+ func (p *Pool[T]) Put(idx int, ptr *T)

+ type ProfileHandler struct

+ func NewProfileHandler() *ProfileHandler

+ func (p *ProfileHandler) AddSample()

+ func (p *ProfileHandler) ServeHTTP(w http.ResponseWriter, req *http.Request)

+ func (p *ProfileHandler) StartProfile(w io.Writer) (stop func())

+ type PutBack struct

+ func (pb PutBack[T]) Put()

type S3FS

+ func (s *S3FS) Preload(ctx context.Context, filePath string) error

+ func (s *S3FS) SetAsyncUpdate(b bool)

v0.7.0

Feb 18, 2023

Changes in this version

+ type Cache interface

+ CacheStats func() *CacheStats

+ Flush func()

+ Read func(ctx context.Context, vector *IOVector) error

+ Update func(ctx context.Context, vector *IOVector) error

type CacheConfig

+ DiskPath string

+ type DiskCache struct

+ func NewDiskCache(path string, capacity int64) (*DiskCache, error)

+ func (d *DiskCache) CacheStats() *CacheStats

+ func (d *DiskCache) Flush()

+ func (d *DiskCache) Read(ctx context.Context, vector *IOVector) (err error)

+ func (d *DiskCache) Update(ctx context.Context, vector *IOVector) (err error)

type FileService

+ StatFile func(ctx context.Context, filePath string) (*DirEntry, error)

type FileServices

+ func (f *FileServices) StatFile(ctx context.Context, filePath string) (*DirEntry, error)

type IOVector

+ ExpireAt time.Time

type LocalETLFS

+ func (l *LocalETLFS) StatFile(ctx context.Context, filePath string) (*DirEntry, error)

type LocalFS

+ func (l *LocalFS) StatFile(ctx context.Context, filePath string) (*DirEntry, error)

type MemCache

+ func (m *MemCache) Update(ctx context.Context, vector *IOVector) error

type MemoryFS

+ func (m *MemoryFS) StatFile(ctx context.Context, filePath string) (*DirEntry, error)

type S3FS

+ func (s *S3FS) StatFile(ctx context.Context, filePath string) (*DirEntry, error)

v0.6.0

Nov 29, 2022

Changes in this version

+ const ServiceNameSeparator

+ var ErrChecksumNotMatch = moerr.NewInternalError("checksum not match")

+ func Get(fs FileService, name string) (res T, err error)

+ func JoinPath(serviceName string, path string) string

+ type CacheConfig struct

+ DiskCapacity toml.ByteSize

+ MemoryCapacity toml.ByteSize

+ type CacheKey struct

+ Offset int64

+ Path string

+ Size int64

+ type CacheStats struct

+ NumHit int64

+ NumRead int64

+ type CachingFileService interface

+ CacheStats func() *CacheStats

+ FlushCache func()

+ type Config struct

+ Backend string

+ Cache CacheConfig

+ DataDir string

+ Name string

+ S3 S3Config

+ type ETLFileService interface

+ ETLCompatible func()

+ func GetForETL(fs FileService, path string) (res ETLFileService, readPath string, err error)

+ type FileLike interface

type FileService

+ func NewFileService(cfg Config) (FileService, error)

+ Name func() string

+ type FileServices struct

+ func NewFileServices(defaultName string, fss ...FileService) (*FileServices, error)

+ func (f *FileServices) Delete(ctx context.Context, filePaths ...string) error

+ func (f *FileServices) List(ctx context.Context, dirPath string) ([]DirEntry, error)

+ func (f *FileServices) Name() string

+ func (f *FileServices) Read(ctx context.Context, vector *IOVector) error

+ func (f *FileServices) Write(ctx context.Context, vector IOVector) error

+ type FileWithChecksum struct

+ func NewFileWithChecksum(underlying T, blockContentSize int) *FileWithChecksum[T]

+ func (f *FileWithChecksum[T]) Read(buf []byte) (n int, err error)

+ func (f *FileWithChecksum[T]) ReadAt(buf []byte, offset int64) (n int, err error)

+ func (f *FileWithChecksum[T]) Seek(offset int64, whence int) (int64, error)

+ func (f *FileWithChecksum[T]) Write(buf []byte) (n int, err error)

+ func (f *FileWithChecksum[T]) WriteAt(buf []byte, offset int64) (n int, err error)

type IOEntry

+ Object any

+ ObjectSize int64

+ ToObject func(reader io.Reader, data []byte) (object any, objectSize int64, err error)

+ type LRU struct

+ func NewLRU(capacity int64) *LRU

+ func (l *LRU) Flush()

+ func (l *LRU) Get(key any) (value any, size int64, ok bool)

+ func (l *LRU) Set(key any, value any, size int64)

+ type LocalETLFS struct

+ func NewLocalETLFS(name string, rootPath string) (*LocalETLFS, error)

+ func (l *LocalETLFS) Delete(ctx context.Context, filePaths ...string) error

+ func (l *LocalETLFS) ETLCompatible()

+ func (l *LocalETLFS) List(ctx context.Context, dirPath string) (ret []DirEntry, err error)

+ func (l *LocalETLFS) Name() string

+ func (l *LocalETLFS) NewMutator(filePath string) (Mutator, error)

+ func (l *LocalETLFS) Read(ctx context.Context, vector *IOVector) error

+ func (l *LocalETLFS) Write(ctx context.Context, vector IOVector) error

+ type LocalETLFSMutator struct

+ func (l *LocalETLFSMutator) Append(ctx context.Context, entries ...IOEntry) error

+ func (l *LocalETLFSMutator) Close() error

+ func (l *LocalETLFSMutator) Mutate(ctx context.Context, entries ...IOEntry) error

type LocalFS

+ func (l *LocalFS) CacheStats() *CacheStats

+ func (l *LocalFS) FlushCache()

+ func (l *LocalFS) Name() string

+ func (l *LocalFS) NewMutator(filePath string) (Mutator, error)

+ func (l *LocalFS) Replace(ctx context.Context, vector IOVector) error

+ type LocalFSMutator struct

+ func (l *LocalFSMutator) Append(ctx context.Context, entries ...IOEntry) error

+ func (l *LocalFSMutator) Close() error

+ func (l *LocalFSMutator) Mutate(ctx context.Context, entries ...IOEntry) error

+ type MemCache struct

+ func NewMemCache(capacity int64) *MemCache

+ func (m *MemCache) CacheStats() *CacheStats

+ func (m *MemCache) Flush()

+ func (m *MemCache) Read(ctx context.Context, vector *IOVector, ...) (err error)

type MemoryFS

+ func (m *MemoryFS) ETLCompatible()

+ func (m *MemoryFS) Name() string

+ func (m *MemoryFS) Replace(ctx context.Context, vector IOVector) error

type MutableFileService

+ NewMutator func(filePath string) (Mutator, error)

+ type Mutator interface

+ Append func(ctx context.Context, entries ...IOEntry) error

+ Close func() error

+ Mutate func(ctx context.Context, entries ...IOEntry) error

+ type NewFileServicesFunc = func(defaultName string) (*FileServices, error)

+ type Path struct

+ File string

+ Service string

+ ServiceArguments []string

+ func ParsePath(s string) (path Path, err error)

+ func ParsePathAtService(s string, serviceName string) (path Path, err error)

+ type RC struct

+ Value T

+ func NewRC(value T) *RC[T]

+ func (r *RC[T]) DecRef()

+ func (r *RC[T]) IncRef()

+ func (r *RC[T]) RefCount() int64

+ type ReplaceableFileService interface

+ Replace func(ctx context.Context, vector IOVector) error

type S3Config

+ SharedConfigProfile string

type S3FS

+ func NewS3FSOnMinio(sharedConfigProfile string, name string, endpoint string, bucket string, ...) (*S3FS, error)

+ func (*S3FS) ETLCompatible()

+ func (s *S3FS) CacheStats() *CacheStats

+ func (s *S3FS) FlushCache()

+ func (s *S3FS) Name() string

v0.5.1

Aug 17, 2022

v0.5.0

Jul 14, 2022

Changes in this version

+ var ErrEmptyRange = errors.New("empty range")

+ var ErrFileExisted = errors.New("file existed")

+ var ErrFileNotFound = errors.New("file not found")

+ var ErrSizeNotMatch = errors.New("size not match")

+ var ErrUnexpectedEOF = io.ErrUnexpectedEOF

+ type DirEntry struct

+ IsDir bool

+ Name string

+ Size int

+ type FileService interface

+ Delete func(ctx context.Context, filePath string) error

+ List func(ctx context.Context, dirPath string) ([]DirEntry, error)

+ Read func(ctx context.Context, vector *IOVector) error

+ Write func(ctx context.Context, vector IOVector) error

+ type IOEntry struct

+ Data []byte

+ Offset int

+ ReadCloserForRead *io.ReadCloser

+ ReaderForWrite io.Reader

+ Size int

+ WriterForRead io.Writer

+ type IOVector struct

+ Entries []IOEntry

+ FilePath string

+ type LocalFS struct

+ func NewLocalFS(rootPath string) (*LocalFS, error)

+ func (l *LocalFS) Delete(ctx context.Context, filePath string) error

+ func (l *LocalFS) List(ctx context.Context, dirPath string) (ret []DirEntry, err error)

+ func (l *LocalFS) Mutate(ctx context.Context, vector IOVector) error

+ func (l *LocalFS) Read(ctx context.Context, vector *IOVector) error

+ func (l *LocalFS) Write(ctx context.Context, vector IOVector) error

+ type MemoryFS struct

+ func NewMemoryFS() (*MemoryFS, error)

+ func (m *MemoryFS) Delete(ctx context.Context, filePath string) error

+ func (m *MemoryFS) List(ctx context.Context, dirPath string) (entries []DirEntry, err error)

+ func (m *MemoryFS) Read(ctx context.Context, vector *IOVector) error

+ func (m *MemoryFS) Write(ctx context.Context, vector IOVector) error

+ type MutableFileService interface

+ Mutate func(ctx context.Context, vector IOVector) error

+ type S3Config struct

+ APIKey string

+ APISecret string

+ Bucket string

+ Endpoint string

+ KeyPrefix string

+ Region string

+ type S3FS struct

+ func NewS3FS(endpoint string, bucket string, keyPrefix string) (*S3FS, error)

+ func (m *S3FS) Delete(ctx context.Context, filePath string) error

+ func (m *S3FS) List(ctx context.Context, dirPath string) (entries []DirEntry, err error)

+ func (m *S3FS) Read(ctx context.Context, vector *IOVector) error

+ func (m *S3FS) Write(ctx context.Context, vector IOVector) error

+ type S3FSMinio struct

+ func NewS3FSMinio(config S3Config) (*S3FSMinio, error)

+ func (m *S3FSMinio) Delete(ctx context.Context, filePath string) error

+ func (m *S3FSMinio) List(ctx context.Context, dirPath string) (entries []DirEntry, err error)

+ func (m *S3FSMinio) Read(ctx context.Context, vector *IOVector) error

+ func (m *S3FSMinio) Write(ctx context.Context, vector IOVector) error

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL