filereader

Documentation

Index

• func CreateWorkers(ctx context.Context, n int, opt WorkersOpt) ([]chan<- tasks.ReadFile, *errgroup.Group)
• func RoundRobin[T any](ts []T) func() T
• func TrySync(ctx context.Context, workers []chan<- tasks.ReadFile) error
• func WarmUpFooters(glob string, sendToWorker sendFileHook) error
• func WarmUpWholeFile(glob string, sendToWorker sendFileHook) error
• type Config
• type ExecContext
  • func New(config Config) *ExecContext
  • func (execCtx *ExecContext) RunAndClose(ctx context.Context) error
• type StatedFile
  • func StatFilesByGlob(glob string) ([]StatedFile, error)
• type WorkersOpt

Functions

func CreateWorkers

func CreateWorkers(ctx context.Context, n int, opt WorkersOpt) ([]chan<- tasks.ReadFile, *errgroup.Group)

func RoundRobin

func RoundRobin[T any](ts []T) func() T

func TrySync

func TrySync(ctx context.Context, workers []chan<- tasks.ReadFile) error

func WarmUpFooters

func WarmUpFooters(glob string, sendToWorker sendFileHook) error

func WarmUpWholeFile

func WarmUpWholeFile(glob string, sendToWorker sendFileHook) error

Types

type Config

type Config struct {
	Files           string
	Type            string
	RateLimit       float64
	NWorkers        int
	Direct          bool
	CheckpointEvery uint64
	CheckpointFile  string

	OnStep        worker.OnStepHook
	OnFireRequest func(*tasks.ReadFile)
}

type ExecContext

type ExecContext struct {
	// contains filtered or unexported fields
}

func New

func New(config Config) *ExecContext

func (*ExecContext) RunAndClose

func (execCtx *ExecContext) RunAndClose(ctx context.Context) error

type StatedFile

type StatedFile struct {
	Info os.FileInfo
	Path string
}

func StatFilesByGlob

func StatFilesByGlob(glob string) ([]StatedFile, error)

type WorkersOpt

type WorkersOpt struct {
	ObserveTotalSize *uint64
	RateLimitInMiB   float64
	OnStep           worker.OnStepHook
}