protocol

package
v0.0.0-rc2 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 19, 2024 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const LastCheckpointPath string = "_last_checkpoint"

Variables

View Source 
var (
	LogRetentionProp = &TableConfig[time.Duration]{
		Key:          "logRetentionDuration",
		DefaultValue: "interval 30 days",
		FromString:   parseDuration,
	}
	TombstoneRetentionProp = &TableConfig[time.Duration]{
		Key:          "deletedFileRetentionDuration",
		DefaultValue: "interval 1 week",
		FromString:   parseDuration,
	}
	DeltaConfigCheckpointInterval = &TableConfig[int]{
		Key:          "checkpointInterval",
		DefaultValue: "10",
		FromString: func(s string) (int, error) {
			return strconv.Atoi(s)
		},
	}
	EnableExpiredLogCleanupProp = &TableConfig[bool]{
		Key:          "enableExpiredLogCleanup",
		DefaultValue: "true",
		FromString: func(s string) (bool, error) {
			return strings.ToLower(s) == "true", nil
		},
	}
	IsAppendOnlyProp = &TableConfig[bool]{
		Key:          "appendOnly",
		DefaultValue: "false",
		FromString: func(s string) (bool, error) {
			return strings.ToLower(s) == "true", nil
		},
	}
)
View Source 
var MaxInstance = CheckpointInstance{Version: -1, NumParts: 0}

Functions

func CheckpointFileSingular 

func CheckpointFileSingular(dir string, version int64) string

func CheckpointFileWithParts 

func CheckpointFileWithParts(dir string, version int64, numParts int) []string

func CheckpointPrefix 

func CheckpointPrefix(path string, version int64) string

func CheckpointVersion 

func CheckpointVersion(path string) (int64, error)

func DeltaFile 

func DeltaFile(path string, version int64) string

func GetFileVersion 

func GetFileVersion(path string) (int64, error)

func IsCheckpointFile 

func IsCheckpointFile(path string) bool

func IsDeltaFile 

func IsDeltaFile(path string) bool

func LogVersion 

func LogVersion(path string) (int64, error)

func NumCheckpointParts 

func NumCheckpointParts(path string) (int, error)

Types

type CheckpointInstance 

type CheckpointInstance struct {
	Version  int64
	NumParts int64
}

func FindLastCompleteCheckpoint 

func FindLastCompleteCheckpoint(s store.Store, cv CheckpointInstance) (*CheckpointInstance, error)

func FromMetadata 

func FromMetadata(metadata CheckpointMetaData) *CheckpointInstance

func FromPath 

func FromPath(path string) (*CheckpointInstance, error)

func LatestCompleteCheckpoint 

func LatestCompleteCheckpoint(instances []*CheckpointInstance, notLaterThan CheckpointInstance) *CheckpointInstance

func (*CheckpointInstance) Compare 

func (t *CheckpointInstance) Compare(other CheckpointInstance) int

func (*CheckpointInstance) GetCorrespondingFiles 

func (t *CheckpointInstance) GetCorrespondingFiles(dir string) (res []string)

func (*CheckpointInstance) IsEarlierThan 

func (t *CheckpointInstance) IsEarlierThan(other CheckpointInstance) bool

IsEarlierThan compare based just on version and amount of parts in checkpoint

func (*CheckpointInstance) IsNotLaterThan 

func (t *CheckpointInstance) IsNotLaterThan(other CheckpointInstance) bool

IsNotLaterThan compare based just on version

type CheckpointMetaData 

type CheckpointMetaData struct {
	Version int64  `json:"version,omitempty"`
	Size    int64  `json:"size,omitempty"`
	Parts   *int64 `json:"parts,omitempty"`
}

func LastCheckpoint 

func LastCheckpoint(s store.Store) (*CheckpointMetaData, error)

func LoadMetadataFromFile 

func LoadMetadataFromFile(s store.Store) (*CheckpointMetaData, error)

type CheckpointReader 

type CheckpointReader interface {
	Read(path string) (iter.Iter[action.Container], error)
}

type LogSegment 

type LogSegment struct {
	LogPath           string
	Version           int64
	Deltas            []*store.FileMeta
	Checkpoints       []*store.FileMeta
	CheckpointVersion int64
	LastCommitTS      time.Time
}

type MemoryOptimizedLogReplay 

type MemoryOptimizedLogReplay struct {
	// contains filtered or unexported fields
}

func (*MemoryOptimizedLogReplay) GetReverseIterator 

func (m *MemoryOptimizedLogReplay) GetReverseIterator() iter.Iter[*replayTuple]

type Replayer 

type Replayer struct {
	MinTS int64
	// contains filtered or unexported fields
}

func NewReplayer 

func NewReplayer(minTS int64) *Replayer

func (*Replayer) Append 

func (r *Replayer) Append(version int64, iter iter.Iter[action.Container]) error

func (*Replayer) GetActiveFiles 

func (r *Replayer) GetActiveFiles() iter.Iter[*action.AddFile]

func (*Replayer) GetSetTransactions 

func (r *Replayer) GetSetTransactions() []*action.SetTransaction

func (*Replayer) GetTombstones 

func (r *Replayer) GetTombstones() iter.Iter[*action.RemoveFile]

type Snapshot 

type Snapshot struct {
	// contains filtered or unexported fields
}

Snapshot provides APIs to access the Delta table state (such as table metadata, active files) at some version. See Delta Transaction Log Protocol for more details about the transaction logs.

func NewInitialSnapshot 

func NewInitialSnapshot(path string, store store2.Store, cpReader CheckpointReader) (*Snapshot, error)

func NewSnapshot 

func NewSnapshot(
	path string,
	version int64,
	logsegment *LogSegment,
	minTS int64,
	commitTS int64,
	store store2.Store,
	checkpointReader CheckpointReader,
) (*Snapshot, error)

func (*Snapshot) AllFiles 

func (s *Snapshot) AllFiles() ([]*action.AddFile, error)

func (*Snapshot) CommitTS 

func (s *Snapshot) CommitTS() time.Time

CommitTS returns the time of commit for this Snapshot

func (*Snapshot) Metadata 

func (s *Snapshot) Metadata() (*action.Metadata, error)

func (*Snapshot) Version 

func (s *Snapshot) Version() int64

Version returns the version of this Snapshot

type SnapshotReader 

type SnapshotReader struct {
	// contains filtered or unexported fields
}

func NewSnapshotReader 

func NewSnapshotReader(cpReader CheckpointReader, logStore store.Store, history *history) (*SnapshotReader, error)

type StoreCheckpointReader 

type StoreCheckpointReader struct {
	// contains filtered or unexported fields
}

func NewCheckpointReader 

func NewCheckpointReader(store store.Store) (*StoreCheckpointReader, error)

func (*StoreCheckpointReader) Read 

func (l *StoreCheckpointReader) Read(path string) (iter.Iter[action.Container], error)

type TableConfig 

type TableConfig[T any] struct {
	Key          string
	DefaultValue string
	FromString   func(s string) (T, error)
}

TableConfig generic config structure from any string-val to typed-val.

type TableLog 

type TableLog struct {
	// contains filtered or unexported fields
}

func NewTableLog 

func NewTableLog(dataPath string, logStore store2.Store) (*TableLog, error)

NewTableLog Create a DeltaLog instance representing the table located at the provided path.

func (*TableLog) CommitInfoAt 

func (l *TableLog) CommitInfoAt(version int64) (*action.CommitInfo, error)

func (*TableLog) Path 

func (l *TableLog) Path() string

func (*TableLog) Snapshot 

func (l *TableLog) Snapshot() (*Snapshot, error)

Snapshot the current Snapshot of the Delta table. You may need to call update() to access the latest Snapshot if the current Snapshot is stale.

func (*TableLog) SnapshotForTimestamp 

func (l *TableLog) SnapshotForTimestamp(timestamp int64) (*Snapshot, error)

func (*TableLog) SnapshotForVersion 

func (l *TableLog) SnapshotForVersion(version int64) (*Snapshot, error)

func (*TableLog) TableExists 

func (l *TableLog) TableExists() bool

func (*TableLog) Update 

func (l *TableLog) Update() (*Snapshot, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.