datasetworker

Documentation

Index

• func EnsureParentDirectories(db *gorm.DB, item *model.Item, rootDirID uint64, ...) error
• func ExtractFromFsObject(ctx context.Context, info fs.ObjectInfo) (size int64, hashValue string, lastModified time.Time, ...)
• func MaxSizeToSplitSize(m int64) int64
• func PushItem(ctx context.Context, db *gorm.DB, obj fs.ObjectInfo, source model.Source, ...) (*model.Item, []model.ItemPart, error)
• type DatasetWorker
  • func NewDatasetWorker(db *gorm.DB, config DatasetWorkerConfig) *DatasetWorker
  • func (w DatasetWorker) Run(parent context.Context) error
• type DatasetWorkerConfig
• type DatasetWorkerThread

Functions

func EnsureParentDirectories

func EnsureParentDirectories(db *gorm.DB,
	item *model.Item, rootDirID uint64,
	directoryCache map[string]uint64) error

func ExtractFromFsObject

func ExtractFromFsObject(ctx context.Context, info fs.ObjectInfo) (size int64, hashValue string, lastModified time.Time, lastModifiedReliable bool)

func MaxSizeToSplitSize

func MaxSizeToSplitSize(m int64) int64

func PushItem

func PushItem(ctx context.Context, db *gorm.DB, obj fs.ObjectInfo,
	source model.Source, dataset model.Dataset,
	directoryCache map[string]uint64) (*model.Item, []model.ItemPart, error)

Types

type DatasetWorker

type DatasetWorker struct {
	// contains filtered or unexported fields
}

func NewDatasetWorker

func NewDatasetWorker(db *gorm.DB, config DatasetWorkerConfig) *DatasetWorker

func (DatasetWorker) Run

func (w DatasetWorker) Run(parent context.Context) error

type DatasetWorkerConfig

type DatasetWorkerConfig struct {
	Concurrency    int
	ExitOnComplete bool
	EnableScan     bool
	EnablePack     bool
	EnableDag      bool
	ExitOnError    bool
}

type DatasetWorkerThread

type DatasetWorkerThread struct {
	// contains filtered or unexported fields
}