store

Documentation

Index

• Variables
• type FileMeta
  • func (f *FileMeta) Path() string
  • func (f *FileMeta) Size() uint64
  • func (f *FileMeta) TimeModified() time.Time
• type Local
  • func NewStoreLocal(cfg *LocalConfig) *Local
  • func (l *Local) ListFrom(path string) (iter.Iter[*FileMeta], error)
  • func (l *Local) Read(path string) (iter.Iter[string], error)
  • func (l *Local) Root() string
• type LocalConfig
• type S3
  • func NewStoreS3(config *S3Config) (*S3, error)
  • func (s S3) ListFrom(path string) (iter.Iter[*FileMeta], error)
  • func (s S3) Read(path string) (iter.Iter[string], error)
  • func (s S3) Root() string
• type S3Config
• type Store
  • func New(config StoreConfig) (Store, error)
• type StoreConfig

Variables

var (
	ErrFileNotFound = xerrors.New("file not found")
)

Types

type FileMeta

type FileMeta struct {
	// contains filtered or unexported fields
}

func (*FileMeta) Path

func (f *FileMeta) Path() string

func (*FileMeta) Size

func (f *FileMeta) Size() uint64

func (*FileMeta) TimeModified

func (f *FileMeta) TimeModified() time.Time

type Local

type Local struct {
	Path string
}

func NewStoreLocal

func NewStoreLocal(cfg *LocalConfig) *Local

func (*Local) ListFrom

func (l *Local) ListFrom(path string) (iter.Iter[*FileMeta], error)

func (*Local) Read

func (l *Local) Read(path string) (iter.Iter[string], error)

func (*Local) Root

func (l *Local) Root() string

type LocalConfig

type LocalConfig struct {
	Path string
}

type S3

type S3 struct {
	// contains filtered or unexported fields
}

func NewStoreS3

func NewStoreS3(config *S3Config) (*S3, error)

func (S3) ListFrom

func (s S3) ListFrom(path string) (iter.Iter[*FileMeta], error)

func (S3) Read

func (s S3) Read(path string) (iter.Iter[string], error)

func (S3) Root

func (s S3) Root() string

type S3Config

type S3Config struct {
	Endpoint         string
	TablePath        string
	Region           string
	AccessKey        string
	S3ForcePathStyle bool
	Secret           string
	Bucket           string
	UseSSL           bool
	VerifySSL        bool
}

type Store

type Store interface {
	// Root return root path for delta-table store
	Root() string

	// Read the given file and return an `Iterator` of lines, with line breaks removed from each line.
	// Callers of this function are responsible to close the iterator if they are done with it.
	Read(path string) (iter.Iter[string], error)

	// ListFrom resolve the paths in the same directory that are lexicographically greater or equal to (UTF-8 sorting) the given `path`.
	// The result should also be sorted by the file name.
	ListFrom(path string) (iter.Iter[*FileMeta], error)
}

Store is general interface for all critical file system operations required to read and write the Delta logs. The correctness is predicated on the atomicity and durability guarantees of the implementation of this interface. Specifically, Consistent listing: Once a file has been written in a directory, all future listings for that directory must return that file. All subclasses of this interface is required to have a constructor that takes StoreConfig as a single parameter. This constructor is used to dynamically create the Store. Store and its implementations are not meant for direct access but for configuration based on storage system. See [[https://docs.delta.io/latest/delta-storage.html]] for details.

func New

func New(config StoreConfig) (Store, error)

type StoreConfig

type StoreConfig interface {
	// contains filtered or unexported methods
}