fs

Documentation

Overview

Package fs implements write-ahead-log on FileSystem.

Index

• func NewCompactor(ctx context.Context, pipelineName string, vertexName string, ...) (unaligned.Compactor, error)
• func NewFSManager(ctx context.Context, segmentWALPath string, compactWALPath string, ...) wal.Manager
• func NewGCEventsWAL(ctx context.Context, pipelineName string, vertexName string, ...) (unaligned.GCEventsWAL, error)
• func NewUnalignedReadWriteWAL(ctx context.Context, pipelineName string, vertexName string, ...) (wal.WAL, error)
• func NewUnalignedWriteOnlyWAL(ctx context.Context, pipelineName string, vertexName string, ...) (wal.WAL, error)
• type CompactorOption
  • func WithCompactionDuration(maxDuration time.Duration) CompactorOption
  • func WithCompactorMaxFileSize(maxFileSize int64) CompactorOption
  • func WithCompactorSyncDuration(maxDuration time.Duration) CompactorOption
• type GCEventsWALOption
  • func WithEventsPath(path string) GCEventsWALOption
  • func WithGCTrackerRotationDuration(rotationDuration time.Duration) GCEventsWALOption
  • func WithGCTrackerRotationEventsCount(count int) GCEventsWALOption
  • func WithGCTrackerSyncDuration(maxDuration time.Duration) GCEventsWALOption
• type WALOption
  • func WithCompactWALPath(path string) WALOption
  • func WithMaxBatchSize(size int64) WALOption
  • func WithSegmentRotationDuration(maxDuration time.Duration) WALOption
  • func WithSegmentSize(size int64) WALOption
  • func WithSegmentWALPath(path string) WALOption
  • func WithSyncDuration(maxDuration time.Duration) WALOption

Functions

func NewCompactor

func NewCompactor(ctx context.Context, pipelineName string, vertexName string, vertexReplica int32, partitionId *partition.ID, gcEventsPath string, dataSegmentWALPath string, compactedSegWALPath string, opts ...CompactorOption) (unaligned.Compactor, error)

NewCompactor returns a new WAL compactor instance

func NewFSManager

func NewFSManager(ctx context.Context, segmentWALPath string, compactWALPath string, vertexInstance *dfv1.VertexInstance, opts ...WALOption) wal.Manager

NewFSManager is a FileSystem Stores Manager.

func NewGCEventsWAL

func NewGCEventsWAL(ctx context.Context, pipelineName string, vertexName string, replicaIndex int32, opts ...GCEventsWALOption) (unaligned.GCEventsWAL, error)

NewGCEventsWAL returns a new GCEventsWAL

func NewUnalignedReadWriteWAL

func NewUnalignedReadWriteWAL(ctx context.Context, pipelineName string, vertexName string, vertexReplica int32, filesToReplay []string, opts ...WALOption) (wal.WAL, error)

NewUnalignedReadWriteWAL returns a new WAL instance for reading and writing

func NewUnalignedWriteOnlyWAL

func NewUnalignedWriteOnlyWAL(ctx context.Context, pipelineName string, vertexName string, vertexReplica int32, partitionId *partition.ID, opts ...WALOption) (wal.WAL, error)

NewUnalignedWriteOnlyWAL returns a new store writer instance

Types

type CompactorOption

type CompactorOption func(c *compactor)

func WithCompactionDuration

func WithCompactionDuration(maxDuration time.Duration) CompactorOption

WithCompactionDuration sets the compaction duration for the compactor

func WithCompactorMaxFileSize

func WithCompactorMaxFileSize(maxFileSize int64) CompactorOption

WithCompactorMaxFileSize sets the max file size for the compactor

func WithCompactorSyncDuration

func WithCompactorSyncDuration(maxDuration time.Duration) CompactorOption

WithCompactorSyncDuration sets the sync duration for the compactor

type GCEventsWALOption

type GCEventsWALOption func(tracker *gcEventsWAL)

func WithEventsPath

func WithEventsPath(path string) GCEventsWALOption

WithEventsPath sets the path for the GC events WAL

func WithGCTrackerRotationDuration

func WithGCTrackerRotationDuration(rotationDuration time.Duration) GCEventsWALOption

WithGCTrackerRotationDuration sets the rotation duration for the GC events WAL

func WithGCTrackerRotationEventsCount

func WithGCTrackerRotationEventsCount(count int) GCEventsWALOption

WithGCTrackerRotationEventsCount sets the rotation events count for the GC events WAL

func WithGCTrackerSyncDuration

func WithGCTrackerSyncDuration(maxDuration time.Duration) GCEventsWALOption

WithGCTrackerSyncDuration sets the sync duration for the GC events WAL

type WALOption

type WALOption func(stores *unalignedWAL)

func WithCompactWALPath

func WithCompactWALPath(path string) WALOption

WithCompactWALPath sets the compact WAL path

func WithMaxBatchSize

func WithMaxBatchSize(size int64) WALOption

WithMaxBatchSize sets the max batch size

func WithSegmentRotationDuration

func WithSegmentRotationDuration(maxDuration time.Duration) WALOption

WithSegmentRotationDuration sets the segment rotation duration

func WithSegmentSize

func WithSegmentSize(size int64) WALOption

WithSegmentSize sets the segment size

func WithSegmentWALPath

func WithSegmentWALPath(path string) WALOption

WithSegmentWALPath sets the segment WAL path

func WithSyncDuration

func WithSyncDuration(maxDuration time.Duration) WALOption

WithSyncDuration sets the sync duration