diskstore

Documentation

Index

• func GetPathForRedologFile(prefix, table string, shardID int, creationTime int64) string
• func GetPathForTableArchiveBatchColumnFile(prefix, table string, shardID int, batchID string, batchVersion uint32, ...) string
• func GetPathForTableArchiveBatchDir(prefix, table string, shardID int, batchID string, batchVersion uint32, ...) string
• func GetPathForTableArchiveBatchRootDir(prefix, table string, shardID int) string
• func GetPathForTableRedologs(prefix, table string, shardID int) string
• func GetPathForTableSnapshotBatchDir(prefix, table string, shardID int, redoLogFile int64, offset uint32, ...) string
• func GetPathForTableSnapshotColumnFilePath(prefix, table string, shardID int, redoLogFile int64, offset uint32, ...) string
• func GetPathForTableSnapshotDir(prefix, table string, shardID int) string
• func GetPathForTableSnapshotDirPath(prefix, table string, shardID int, redoLogFile int64, offset uint32) string
• func ParseBatchIDAndVersionName(batchIDAndVersion string) (string, uint32, uint32, error)
• type DiskStore
  • func NewLocalDiskStore(rootPath string) DiskStore
• type LocalDiskStore
  • func (l LocalDiskStore) DeleteBatchVersions(table string, shard, batchID int, batchVersion uint32, seqNum uint32) error
  • func (l LocalDiskStore) DeleteBatches(table string, shard, batchIDStart, batchIDEnd int) (int, error)
  • func (l LocalDiskStore) DeleteColumn(table string, columnID int, shard int) error
  • func (l LocalDiskStore) DeleteLogFile(table string, shard int, creationTime int64) error
  • func (l LocalDiskStore) DeleteSnapshot(table string, shard int, latestRedoLogFile int64, latestOffset uint32) error
  • func (l LocalDiskStore) DeleteTableShard(table string, shard int) error
  • func (l LocalDiskStore) ListLogFiles(table string, shard int) (creationUnixTime []int64, err error)
  • func (l LocalDiskStore) ListSnapshotBatches(table string, shard int, redoLogFile int64, offset uint32) (batches []int, err error)
  • func (l LocalDiskStore) ListSnapshotVectorPartyFiles(table string, shard int, redoLogFile int64, offset uint32, batchID int) (columnIDs []int, err error)
  • func (l LocalDiskStore) OpenLogFileForAppend(table string, shard int, creationTime int64) (io.WriteCloser, error)
  • func (l LocalDiskStore) OpenLogFileForReplay(table string, shard int, creationTime int64) (utils.ReaderSeekerCloser, error)
  • func (l LocalDiskStore) OpenSnapshotVectorPartyFileForRead(table string, shard int, redoLogFile int64, offset uint32, batchID int, ...) (io.ReadCloser, error)
  • func (l LocalDiskStore) OpenSnapshotVectorPartyFileForWrite(table string, shard int, redoLogFile int64, offset uint32, batchID int, ...) (io.WriteCloser, error)
  • func (l LocalDiskStore) OpenVectorPartyFileForRead(table string, columnID int, shard, batchID int, batchVersion uint32, ...) (io.ReadCloser, error)
  • func (l LocalDiskStore) OpenVectorPartyFileForWrite(table string, columnID int, shard, batchID int, batchVersion uint32, ...) (io.WriteCloser, error)
  • func (l LocalDiskStore) TruncateLogFile(table string, shard int, creationTime int64, offset int64) error

Functions

func GetPathForRedologFile

func GetPathForRedologFile(prefix, table string, shardID int, creationTime int64) string

GetPathForRedologFile is used to get on disk file path given path prefix, table name, shard id and creationTime.

func GetPathForTableArchiveBatchColumnFile

func GetPathForTableArchiveBatchColumnFile(prefix, table string, shardID int, batchID string, batchVersion uint32, seqNum uint32, columnID int) string

GetPathForTableArchiveBatchColumnFile is used to get the file path of a column inside an archive batch version given path prefix, table name, shard id, batch id, batch version and column id.

func GetPathForTableArchiveBatchDir

func GetPathForTableArchiveBatchDir(prefix, table string, shardID int, batchID string, batchVersion uint32, seqNum uint32) string

GetPathForTableArchiveBatchDir is used to get the dir path of an archive batch version given path prefix, table name, shard id, batch id and batch version.

func GetPathForTableArchiveBatchRootDir

func GetPathForTableArchiveBatchRootDir(prefix, table string, shardID int) string

GetPathForTableArchiveBatchRootDir is used to get root directory path for archive batch given path prefix, table name and shard id.

func GetPathForTableRedologs

func GetPathForTableRedologs(prefix, table string, shardID int) string

GetPathForTableRedologs is used to get the directory to store a table redolog given path prefix, table name and shard id.

func GetPathForTableSnapshotBatchDir

func GetPathForTableSnapshotBatchDir(prefix, table string, shardID int, redoLogFile int64, offset uint32,
	batchID int) string

GetPathForTableSnapshotBatchDir is used to get the dir path of a snapshot batch given path prefix, table name, shard id, redo log file, offset and batchID.

func GetPathForTableSnapshotColumnFilePath

func GetPathForTableSnapshotColumnFilePath(prefix, table string, shardID int, redoLogFile int64, offset uint32,
	batchID, columnID int) string

GetPathForTableSnapshotColumnFilePath is used to get the file path of a snapshot column given path prefix, table name, shard id, redo log file, offset, batchID and columnID

func GetPathForTableSnapshotDir

func GetPathForTableSnapshotDir(prefix, table string, shardID int) string

GetPathForTableSnapshotDir is used to get the dir path of a snapshot given path prefix, table name and shard id.

func GetPathForTableSnapshotDirPath

func GetPathForTableSnapshotDirPath(prefix, table string, shardID int, redoLogFile int64, offset uint32) string

GetPathForTableSnapshotDirPath is used to get the dir path of a snapshot given path prefix, table name, shard id redo log file and offset.

func ParseBatchIDAndVersionName

func ParseBatchIDAndVersionName(batchIDAndVersion string) (string, uint32, uint32, error)

ParseBatchIDAndVersionName will parse a batchIDAndVersion into batchID and batchVersion+seqNum.

Types

type DiskStore

type DiskStore interface {

	// Completely wipe out a table shard.
	DeleteTableShard(table string, shard int) error

	// Returns the file creation unix time in second for each log file as a sorted slice.
	ListLogFiles(table string, shard int) ([]int64, error)
	// Opens the specified log file for replay.
	OpenLogFileForReplay(table string, shard int, creationTime int64) (utils.ReaderSeekerCloser, error)
	// Opens/creates the specified log file for append.
	OpenLogFileForAppend(table string, shard int, creationTime int64) (io.WriteCloser, error)
	// Deletes the specified log file.
	DeleteLogFile(table string, shard int, creationTime int64) error
	// Truncate Redolog to drop the last incomplete/corrupted upsert batch.
	TruncateLogFile(table string, shard int, creationTime int64, offset int64) error

	// Returns the batch directories under a specific snapshot directory.
	ListSnapshotBatches(table string, shard int,
		redoLogFile int64, offset uint32) ([]int, error)

	// Returns the vector party files under a specific snapshot batch directory.
	// the return value is sorted
	ListSnapshotVectorPartyFiles(table string, shard int,
		redoLogFile int64, offset uint32, batchID int) ([]int, error)

	// Opens the snapshot vector party file for read.
	OpenSnapshotVectorPartyFileForRead(table string, shard int,
		redoLogFile int64, offset uint32, batchID int, columnID int) (io.ReadCloser, error)

	OpenSnapshotVectorPartyFileForWrite(table string, shard int,
		redoLogFile int64, offset uint32, batchID int, columnID int) (io.WriteCloser, error)
	// Deletes snapshot files **older than** the specified version.
	DeleteSnapshot(table string, shard int, redoLogFile int64, offset uint32) error

	// Opens the vector party file at the specified batchVersion for read.
	OpenVectorPartyFileForRead(table string, column, shard, batchID int, batchVersion uint32,
		seqNum uint32) (io.ReadCloser, error)
	// Creates/truncates the vector party file at the specified batchVersion for write.
	OpenVectorPartyFileForWrite(table string, column, shard, batchID int, batchVersion uint32,
		seqNum uint32) (io.WriteCloser, error)
	// Deletes all old batches with the specified batchID that have version lower than or equal to the specified batch
	// version. All columns of those batches will be deleted.
	DeleteBatchVersions(table string, shard, batchID int, batchVersion uint32, seqNum uint32) error
	// Deletes all batches within range [batchIDStart, batchIDEnd)
	DeleteBatches(table string, shard, batchIDStart, batchIDEnd int) (int, error)
	// Deletes all batches of the specified column.
	DeleteColumn(table string, column, shard int) error
}

DiskStore defines the interface for reading/writing redo logs, snapshot files, and archived vector party files.

func NewLocalDiskStore

func NewLocalDiskStore(rootPath string) DiskStore

NewLocalDiskStore is used to init a LocalDiskStore with rootPath.

type LocalDiskStore

type LocalDiskStore struct {
	// contains filtered or unexported fields
}

LocalDiskStore is the implementation of Diskstore for local disk.

func (LocalDiskStore) DeleteBatchVersions

func (l LocalDiskStore) DeleteBatchVersions(table string, shard, batchID int, batchVersion uint32, seqNum uint32) error

DeleteBatchVersions deletes all old batches with the specified batchID that have version lower than or equal to the specified batch version. All columns of those batches will be deleted.

func (LocalDiskStore) DeleteBatches

func (l LocalDiskStore) DeleteBatches(table string, shard, batchIDStart, batchIDEnd int) (int, error)

DeleteBatches : Deletes all batches within [batchIDStart, batchIDEnd)

func (LocalDiskStore) DeleteColumn

func (l LocalDiskStore) DeleteColumn(table string, columnID int, shard int) error

DeleteColumn : Deletes all batches of the specified column.

func (LocalDiskStore) DeleteLogFile

func (l LocalDiskStore) DeleteLogFile(table string, shard int, creationTime int64) error

DeleteLogFile is used to delete a specified redolog.

func (LocalDiskStore) DeleteSnapshot

func (l LocalDiskStore) DeleteSnapshot(table string, shard int, latestRedoLogFile int64, latestOffset uint32) error

DeleteSnapshot : Deletes snapshot directories **older than** the specified version (redolog file and offset).

func (LocalDiskStore) DeleteTableShard

func (l LocalDiskStore) DeleteTableShard(table string, shard int) error

DeleteTableShard : Completely wipe out a table shard.

func (LocalDiskStore) ListLogFiles

func (l LocalDiskStore) ListLogFiles(table string, shard int) (creationUnixTime []int64, err error)

ListLogFiles : Returns the file creation unix time in second for each log file as a sorted slice.

func (LocalDiskStore) ListSnapshotBatches

func (l LocalDiskStore) ListSnapshotBatches(table string, shard int,
	redoLogFile int64, offset uint32) (batches []int, err error)

ListSnapshotBatches : Returns the batch directories at the specified version.

func (LocalDiskStore) ListSnapshotVectorPartyFiles

func (l LocalDiskStore) ListSnapshotVectorPartyFiles(table string, shard int,
	redoLogFile int64, offset uint32, batchID int) (columnIDs []int, err error)

ListSnapshotVectorPartyFiles : Returns the vector party files under specific batch directory.

func (LocalDiskStore) OpenLogFileForAppend

func (l LocalDiskStore) OpenLogFileForAppend(table string, shard int, creationTime int64) (io.WriteCloser, error)

OpenLogFileForAppend : Opens/creates the specified log file for append.

func (LocalDiskStore) OpenLogFileForReplay

func (l LocalDiskStore) OpenLogFileForReplay(table string, shard int,
	creationTime int64) (utils.ReaderSeekerCloser, error)

OpenLogFileForReplay : Opens the specified log file for replay.

func (LocalDiskStore) OpenSnapshotVectorPartyFileForRead

func (l LocalDiskStore) OpenSnapshotVectorPartyFileForRead(table string, shard int,
	redoLogFile int64, offset uint32, batchID int, columnID int) (io.ReadCloser, error)

OpenSnapshotVectorPartyFileForRead : Opens the snapshot file for read at the specified version.

func (LocalDiskStore) OpenSnapshotVectorPartyFileForWrite

func (l LocalDiskStore) OpenSnapshotVectorPartyFileForWrite(table string, shard int,
	redoLogFile int64, offset uint32, batchID int, columnID int) (io.WriteCloser, error)

OpenSnapshotVectorPartyFileForWrite : Creates/truncates the snapshot file for write at the specified version.

func (LocalDiskStore) OpenVectorPartyFileForRead

func (l LocalDiskStore) OpenVectorPartyFileForRead(table string, columnID int, shard, batchID int, batchVersion uint32,
	seqNum uint32) (io.ReadCloser, error)

OpenVectorPartyFileForRead : Opens the vector party file at the specified batchVersion for read.

func (LocalDiskStore) OpenVectorPartyFileForWrite

func (l LocalDiskStore) OpenVectorPartyFileForWrite(table string, columnID int, shard, batchID int, batchVersion uint32,
	seqNum uint32) (io.WriteCloser, error)

OpenVectorPartyFileForWrite : Creates/truncates the vector party file at the specified batchVersion for write.

func (LocalDiskStore) TruncateLogFile

func (l LocalDiskStore) TruncateLogFile(table string, shard int, creationTime int64, offset int64) error

TruncateLogFile is used to truncate redolog to drop the last incomplete/corrupted upsert batch.