data

package
v0.0.0-...-e537141 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 9, 2023 License: GPL-3.0 Imports: 15 Imported by: 1

Documentation

Index

Constants

View Source 
const (
	KindCsvRead  ReadKind  = "data.io.read.csv"
	KindCsvWrite WriteKind = "data.io.write.csv"
)
View Source 
const NoConcurrency int = -1

Variables

View Source 
var ErrNoDetectAndColumnTypes = fmt.Errorf("field 'DetectTypes' cannot be true when ColumnTypes are also defined")

Functions

func Float32Slice 

func Float32Slice(r *rand.Rand, maxGoroutines int, n uint64) []float32

func Float64Slice 

func Float64Slice(r *rand.Rand, maxGoroutines int, n uint64) []float64

func Int64Slice 

func Int64Slice(r *rand.Rand, maxGoroutines int, n uint64) []int64

func IntSlice 

func IntSlice(r *rand.Rand, maxGoroutines int, n uint64) []int

func SerializeCsv 

func SerializeCsv(w *csv.Writer, d StringData) error

func StringSlice 

func StringSlice(r *rand.Rand, sliceLen, stringLen uint64) []string

func StringSliceRandLen 

func StringSliceRandLen(r *rand.Rand, sliceLen uint64) []string

func TransformIntoNew 

func TransformIntoNew(tr RowTransformer, in []string) (out []string, err error)

Types

type BinaryReadConfig 

type BinaryReadConfig struct {
	Endianness Endianness `json:"endianness"`
	Structure  Struct     `json:"structure"`
	Fields     Fields     `json:"fields"`
}

func (BinaryReadConfig) ReadKind 

func (BinaryReadConfig) ReadKind() ReadKind

type BinaryReadRequest 

type BinaryReadRequest struct {
	Config BinaryReadConfig
	Reader io.Reader
	Into   any
}

type Config 

type Config struct {
	TypeConfig TypeConfig `json:"type_config"`
}

type CsvConfig 

type CsvConfig struct {
	Comma            rune
	Comment          rune
	LazyQuotes       bool
	TrimLeadingSpace bool
}

type CsvRead 

type CsvRead struct {
	Out    fs.ResourceID `json:"out"`
	Config CsvConfig     `json:"config"`
	Fields Fields        `json:"fields"`
}

func (CsvRead) ReadKind 

func (CsvRead) ReadKind() ReadKind

type CsvWrite 

type CsvWrite struct {
	Out    fs.ResourceID `json:"out"`
	Config CsvConfig     `json:"config"`
	Fields Fields        `json:"fields"`
}

func (CsvWrite) WriteKind 

func (CsvWrite) WriteKind() WriteKind

type DataTransaction 

type DataTransaction struct {
	Transform RowTransformer
	IndexIn   int
}

type Deps 

type Deps struct {
	Log *slog.Logger
}

type Endianness 

type Endianness int

type ErrMultipleFieldsLen 

type ErrMultipleFieldsLen struct {
	Lens        map[string]int
	ExcludedVal int
}

func (*ErrMultipleFieldsLen) Error 

func (err *ErrMultipleFieldsLen) Error() string

type Field 

type Field struct {
	Name FieldName `json:"name"`
	Meta TypeMeta  `json:"meta"`
}

type FieldMeta 

type FieldMeta struct {
	TypeID TypeID    `json:"type_id"`
	Name   FieldName `json:"field_name"`
}

type FieldName 

type FieldName string

type FieldTransaction 

type FieldTransaction struct {
	Transform RowTransformer
	FieldIn   string
	FieldOut  string
}

type FieldTransactions 

type FieldTransactions struct {
	Fields InOutFields
	Trans  RowTransformer
}

type Fields 

type Fields struct {
	Names []FieldName `json:"names"`
}

type Generator 

type Generator struct {
	StringGen     func(*rand.Rand) string
	MaxGoroutines int
	// contains filtered or unexported fields
}

func (Generator) Float32Slice 

func (g Generator) Float32Slice(n uint64) []float32

func (Generator) Float64Slice 

func (g Generator) Float64Slice(n uint64) []float64

func (Generator) Int64Slice 

func (g Generator) Int64Slice(n uint64) []int64

func (Generator) IntSlice 

func (g Generator) IntSlice(n uint64) []int

func (Generator) Rng 

func (g Generator) Rng() *rand.Rand

type IO 

type IO struct {
	Deps    *Deps
	Methods IOMethods
}

func (IO) ReadBinaryRecord 

func (io IO) ReadBinaryRecord(br *BinaryReadRequest) error

type IOMethods 

type IOMethods struct {
	ReadBinaryRecord func(d *Deps, br *BinaryReadRequest) error
}

func (IOMethods) WithDeps 

func (m IOMethods) WithDeps(d *Deps) IO

type InOutFields 

type InOutFields map[string]string

type LoadOptions 

type LoadOptions struct {
	ColumnTypes map[string]Type
	NanValues   []string
	Names       []string
	DefaultType Type
	DetectTypes bool
	HasHeader   bool
}

func (LoadOptions) Sanitize 

func (l LoadOptions) Sanitize() error

type Meta 

type Meta struct {
	TypeID     TypeID        `json:"type_id"`
	Attributes attribute.Map `json:"attributes"`
}

type Methods 

type Methods struct {
	Mapping   *mapping.Methods
	KeyConfig field.KeyConfig
	NamedData NamedMethods
	Transform TransformMethods
	Field     field.Methods
}

func (Methods) JoinKeyParts 

func (m Methods) JoinKeyParts(parts ...string) (key string, err error)

func (Methods) RefNamedData 

func (m Methods) RefNamedData(nf *NamedFields) (ref NamedFieldsRef)

func (Methods) TransformMap 

func (m Methods) TransformMap(trans FieldTransactions, nf *NamedFields) (err error)

func (Methods) TransformNamed 

func (m Methods) TransformNamed(trans FieldTransaction, nf *NamedFields) (err error)

type Named 

type Named map[string][]string

type NamedFields 

type NamedFields struct {
	FieldIndices field.Indices
	// contains filtered or unexported fields
}

func (NamedFields) Fields 

func (nf NamedFields) Fields() (s []string)

func (NamedFields) GetData 

func (n NamedFields) GetData() (d StringData)

func (NamedFields) HasField 

func (nf NamedFields) HasField(field string) (has bool)

func (NamedFields) MarshalCsv 

func (n NamedFields) MarshalCsv(w *csv.Writer) (err error)

func (NamedFields) Rows 

func (nf NamedFields) Rows() (n int)

type NamedFieldsRef 

type NamedFieldsRef struct {
	// contains filtered or unexported fields
}

func (NamedFieldsRef) Data 

func (nf NamedFieldsRef) Data() (d StringData)

func (NamedFieldsRef) GetAt 

func (nf NamedFieldsRef) GetAt(field string, irow int) (elem string, err error)

func (NamedFieldsRef) GetRow 

func (nf NamedFieldsRef) GetRow(field string) (s []string, err error)

func (NamedFieldsRef) NamedFields 

func (nr NamedFieldsRef) NamedFields() (nf *NamedFields)

func (*NamedFieldsRef) UnmarshalCsv 

func (n *NamedFieldsRef) UnmarshalCsv(r *csv.Reader) (err error)

type NamedMethods 

type NamedMethods struct {
	FieldMethods field.Methods
	// contains filtered or unexported fields
}

func NewNamedMethods 

func NewNamedMethods(log zerolog.Logger, fm field.Methods) (nm NamedMethods)

func (NamedMethods) FromData 

func (nm NamedMethods) FromData(fi field.Indices, data StringData) (nf NamedFields)

func (NamedMethods) FromMap 

func (man NamedMethods) FromMap(data Named) (nf NamedFields, err error)

func (NamedMethods) FromNamed 

func (nm NamedMethods) FromNamed(fi field.Indices, nd Named) (n NamedFields, err error)

func (NamedMethods) GetCol 

func (nm NamedMethods) GetCol(nf NamedFields, field string) (row []string, err error)

func (NamedMethods) NamesAndData 

func (nm NamedMethods) NamesAndData(names []string, data StringData) (nf NamedFields, err error)

func (*NamedMethods) Ref 

func (man *NamedMethods) Ref(nf *NamedFields) (ref NamedFieldsRef)

type Number 

type Number interface {
	constraints.Float | constraints.Integer
}

type PathEncoder 

type PathEncoder struct {
	// contains filtered or unexported fields
}

func NewPathEncoder 

func NewPathEncoder(fsys fs.FS, encoder fs.Encoding) (pe PathEncoder)

func (PathEncoder) Encode 

func (pe PathEncoder) Encode(in string) (out string, err error)

func (PathEncoder) TransformRow 

func (pe PathEncoder) TransformRow(in, out []string) (err error)

type RawStringData 

type RawStringData [][]string

type ReadKind 

type ReadKind string
const (
	KindBinaryRead  ReadKind = "data.io.read.binary"
	KindBinaryWrite ReadKind = "data.io.write.binary"
)

type ReadOp 

type ReadOp interface {
	ReadKind() ReadKind
}

type RowTransformer 

type RowTransformer interface {
	TransformRow(in, out []string) (err error)
}

type StringData 

type StringData struct {
	// contains filtered or unexported fields
}

func DeserializeCsv 

func DeserializeCsv(r *csv.Reader, ncols int) (d StringData, err error)

func NewStringData 

func NewStringData(raw RawStringData) (d StringData, err error)

func (*StringData) AddColumn 

func (d *StringData) AddColumn(col []string) (newIndex int, err error)

func (StringData) At 

func (d StringData) At(irow, icol int) (s string)

func (StringData) GetRecords 

func (d StringData) GetRecords(index int) (field []string, err error)

func (StringData) MarshalCsv 

func (d StringData) MarshalCsv(w *csv.Writer) (err error)

func (StringData) NumCols 

func (d StringData) NumCols() (rec int)

func (StringData) NumRows 

func (d StringData) NumRows() (rec int)

func (StringData) RawData 

func (d StringData) RawData() (raw RawStringData)

func (StringData) TotalRecords 

func (d StringData) TotalRecords() (n int)

type Struct 

type Struct struct {
	Fields  []FieldMeta          `json:"fields"`
	DataNum map[FieldName]uint64 `json:"data_num"`
}

type TransformFunc 

type TransformFunc func(in, out []string) (err error)

func (TransformFunc) TransformRow 

func (tf TransformFunc) TransformRow(in, out []string) (err error)

type TransformMethods 

type TransformMethods struct {
	// contains filtered or unexported fields
}

func NewTransformMethods 

func NewTransformMethods(log zerolog.Logger, methods field.Methods) (tm TransformMethods)

func (TransformMethods) DataTransform 

func (t TransformMethods) DataTransform(trans DataTransaction, data *StringData) (newIndex int, err error)

func (TransformMethods) NamedFieldsTransform 

func (t TransformMethods) NamedFieldsTransform(trans FieldTransaction, nf *NamedFields, nm NamedMethods) (err error)

type Transformer 

type Transformer interface {
	DataTransform(trans DataTransaction, data *StringData) (newIndex int, err error)
	NamedFieldsTransform(trans FieldTransaction, nf *NamedFields) (err error)
}

type Type 

type Type int

ENUM( 

Int8
Int16
Int32
Int64

Uint8
Uint16
Uint32
Uint64

Float32
Float64

Complex64
Complex128

Pointer
Struct

) 

const (
	// TypeInt8 is a Type of type Int8.
	TypeInt8 Type = iota
	// TypeInt16 is a Type of type Int16.
	TypeInt16
	// TypeInt32 is a Type of type Int32.
	TypeInt32
	// TypeInt64 is a Type of type Int64.
	TypeInt64
	// TypeUint8 is a Type of type Uint8.
	TypeUint8
	// TypeUint16 is a Type of type Uint16.
	TypeUint16
	// TypeUint32 is a Type of type Uint32.
	TypeUint32
	// TypeUint64 is a Type of type Uint64.
	TypeUint64
	// TypeFloat32 is a Type of type Float32.
	TypeFloat32
	// TypeFloat64 is a Type of type Float64.
	TypeFloat64
	// TypeComplex64 is a Type of type Complex64.
	TypeComplex64
	// TypeComplex128 is a Type of type Complex128.
	TypeComplex128
	// TypePointer is a Type of type Pointer.
	TypePointer
	// TypeStruct is a Type of type Struct.
	TypeStruct
)

func ParseType 

func ParseType(name string) (Type, error)

ParseType attempts to convert a string to a Type.

func (Type) MarshalText 

func (x Type) MarshalText() ([]byte, error)

MarshalText implements the text marshaller method.

func (Type) String 

func (x Type) String() string

String implements the Stringer interface.

func (*Type) UnmarshalText 

func (x *Type) UnmarshalText(text []byte) error

UnmarshalText implements the text unmarshaller method.

type TypeConfig 

type TypeConfig struct {
	Types map[TypeID]TypeMeta `json:"types"`
}

type TypeID 

type TypeID uint64

type TypeMeta 

type TypeMeta struct {
	Size uint64   `json:"size"`
	Type Type     `json:"type"`
	Name TypeName `json:"name"`
}

type TypeName 

type TypeName string

type UnaryOp 

type UnaryOp int
const (
	UnaryNegate UnaryOp = iota
	UnaryAbs
)

type WriteKind 

type WriteKind string

type WriteOp 

type WriteOp interface {
	WriteKind() WriteKind
}

Source Files

View all Source files

Directories

Path Synopsis
impl

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.