data

package
v0.26.28 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 22, 2022 License: MIT Imports: 24 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type Batch 

type Batch struct {
	X            []*tf.Tensor
	Y            *tf.Tensor
	ClassWeights *tf.Tensor
}

type Dataset 

type Dataset interface {
	Len() int
	NumCategoricalClasses() int
	SetMode(mode GeneratorMode) Dataset
	Shuffle(seed int64)
	Unshuffle() error
	GetColumnNames() []string
	GeneratorChan(batchSize int, preFetch int) chan Batch
	Generate(batchSize int) ([]*tf.Tensor, *tf.Tensor, *tf.Tensor, error)
	Reset() error
	SaveProcessors(saveDir string) error
}

type GeneratorMode 

type GeneratorMode string
var (
	ErrGeneratorEnd                  = errors.New("end of data")
	GeneratorModeTrain GeneratorMode = "train"
	GeneratorModeVal   GeneratorMode = "val"
	GeneratorModeTest  GeneratorMode = "test"
)

type ImgFolderDataset added in v0.26.28 

type ImgFolderDataset struct {
	ClassCounts  map[int]int
	ClassWeights map[int]float32
	Count        int
	// contains filtered or unexported fields
}

func NewImgFolderDataset added in v0.26.28 

func NewImgFolderDataset(
	logger *cblog.Logger,
	errorHandler *cberrors.ErrorsContainer,
	config ImgFolderDatasetConfig,
	processor *preprocessor.Processor,
) (*ImgFolderDataset, error)

func (*ImgFolderDataset) Generate added in v0.26.28 

func (d *ImgFolderDataset) Generate(batchSize int) ([]*tf.Tensor, *tf.Tensor, *tf.Tensor, error)

func (*ImgFolderDataset) GeneratorChan added in v0.26.28 

func (d *ImgFolderDataset) GeneratorChan(batchSize int, preFetch int) chan Batch

func (*ImgFolderDataset) GetColumnNames added in v0.26.28 

func (d *ImgFolderDataset) GetColumnNames() []string

func (*ImgFolderDataset) Len added in v0.26.28 

func (d *ImgFolderDataset) Len() int

func (*ImgFolderDataset) NumCategoricalClasses added in v0.26.28 

func (d *ImgFolderDataset) NumCategoricalClasses() int

func (*ImgFolderDataset) Reset added in v0.26.28 

func (d *ImgFolderDataset) Reset() error

func (*ImgFolderDataset) SaveProcessors added in v0.26.28 

func (d *ImgFolderDataset) SaveProcessors(saveDir string) error

func (*ImgFolderDataset) SetMode added in v0.26.28 

func (d *ImgFolderDataset) SetMode(mode GeneratorMode) Dataset

func (*ImgFolderDataset) Shuffle added in v0.26.28 

func (d *ImgFolderDataset) Shuffle(seed int64)

func (*ImgFolderDataset) Unshuffle added in v0.26.28 

func (d *ImgFolderDataset) Unshuffle() error

type ImgFolderDatasetConfig added in v0.26.28 

type ImgFolderDatasetConfig struct {
	ParentDir           string
	CacheDir            string
	TrainPercent        float32
	ValPercent          float32
	TestPercent         float32
	IgnoreParseErrors   bool
	ConcurrentFileLimit int32
	ClassWeights        map[int]float32
}

type Inference added in v0.26.28 

type Inference struct {
	// contains filtered or unexported fields
}

func NewInference added in v0.26.28 

func NewInference(
	logger *cblog.Logger,
	errorHandler *cberrors.ErrorsContainer,
	processorsSaveDir string,
	columnProcessors ...*preprocessor.Processor,
) (*Inference, error)

func (*Inference) GenerateInputs added in v0.26.28 

func (d *Inference) GenerateInputs(rawInputValues ...interface{}) ([]*tf.Tensor, error)

func (*Inference) GetColumnNames added in v0.26.28 

func (d *Inference) GetColumnNames() []string

type SingleFileDataset 

type SingleFileDataset struct {
	ClassCounts  map[int]int
	ClassWeights map[int]float32
	Count        int
	// contains filtered or unexported fields
}

func NewSingleFileDataset 

func NewSingleFileDataset(
	logger *cblog.Logger,
	errorHandler *cberrors.ErrorsContainer,
	config SingleFileDatasetConfig,
	yProcessor *preprocessor.Processor,
	columnProcessors ...*preprocessor.Processor,
) (*SingleFileDataset, error)

func (*SingleFileDataset) Generate 

func (d *SingleFileDataset) Generate(batchSize int) ([]*tf.Tensor, *tf.Tensor, *tf.Tensor, error)

func (*SingleFileDataset) GeneratorChan 

func (d *SingleFileDataset) GeneratorChan(batchSize int, preFetch int) chan Batch

func (*SingleFileDataset) GetColumnNames 

func (d *SingleFileDataset) GetColumnNames() []string

func (*SingleFileDataset) Len 

func (d *SingleFileDataset) Len() int

func (*SingleFileDataset) NumCategoricalClasses added in v0.26.28 

func (d *SingleFileDataset) NumCategoricalClasses() int

func (*SingleFileDataset) Reset 

func (d *SingleFileDataset) Reset() error

func (*SingleFileDataset) SaveProcessors added in v0.26.28 

func (d *SingleFileDataset) SaveProcessors(saveDir string) error

func (*SingleFileDataset) SetMode 

func (d *SingleFileDataset) SetMode(mode GeneratorMode) Dataset

func (*SingleFileDataset) Shuffle 

func (d *SingleFileDataset) Shuffle(seed int64)

func (*SingleFileDataset) Unshuffle 

func (d *SingleFileDataset) Unshuffle() error

type SingleFileDatasetConfig added in v0.26.28 

type SingleFileDatasetConfig struct {
	FilePath               string
	CacheDir               string
	TrainPercent           float32
	ValPercent             float32
	TestPercent            float32
	IgnoreParseErrors      bool
	SkipHeaders            bool
	RowFilter              func(line []string) bool
	ConcurrentFileLimit    int32
	MaxRowsForProcessorFit int
	ClassWeights           map[int]float32
}

type ValuesDataset added in v0.26.28 

type ValuesDataset struct {
	ClassCounts  map[int]int
	ClassWeights map[int]float32
	Count        int
	// contains filtered or unexported fields
}

func NewValuesDataset added in v0.26.28 

func NewValuesDataset(
	logger *cblog.Logger,
	errorHandler *cberrors.ErrorsContainer,
	config ValuesDatasetConfig,
	yProcessor *preprocessor.Processor,
	columnProcessors ...*preprocessor.Processor,
) (*ValuesDataset, error)

func (*ValuesDataset) Generate added in v0.26.28 

func (d *ValuesDataset) Generate(batchSize int) ([]*tf.Tensor, *tf.Tensor, *tf.Tensor, error)

func (*ValuesDataset) GeneratorChan added in v0.26.28 

func (d *ValuesDataset) GeneratorChan(batchSize int, preFetch int) chan Batch

func (*ValuesDataset) GetColumnNames added in v0.26.28 

func (d *ValuesDataset) GetColumnNames() []string

func (*ValuesDataset) Len added in v0.26.28 

func (d *ValuesDataset) Len() int

func (*ValuesDataset) NumCategoricalClasses added in v0.26.28 

func (d *ValuesDataset) NumCategoricalClasses() int

func (*ValuesDataset) Reset added in v0.26.28 

func (d *ValuesDataset) Reset() error

func (*ValuesDataset) SaveProcessors added in v0.26.28 

func (d *ValuesDataset) SaveProcessors(saveDir string) error

func (*ValuesDataset) SetMode added in v0.26.28 

func (d *ValuesDataset) SetMode(mode GeneratorMode) Dataset

func (*ValuesDataset) SetValues added in v0.26.28 

func (d *ValuesDataset) SetValues(yValues []interface{}, xValues ...[]interface{}) error

func (*ValuesDataset) Shuffle added in v0.26.28 

func (d *ValuesDataset) Shuffle(seed int64)

func (*ValuesDataset) Unshuffle added in v0.26.28 

func (d *ValuesDataset) Unshuffle() error

type ValuesDatasetConfig added in v0.26.28 

type ValuesDatasetConfig struct {
	CacheDir     string
	TrainPercent float32
	ValPercent   float32
	TestPercent  float32
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.