datagen

package
v1.2.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Oct 24, 2023 License: GPL-2.0 Imports: 24 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

func AddFloat 

func AddFloat(seqNum int, args ...string) (interface{}, error)

AddFloat calculates the sum of float values retrieved from the fake data cache.

func AddInt 

func AddInt(seqNum int, args ...string) (interface{}, error)

AddInt calculates the sum of integer values retrieved from the fake data cache.

func AddRandomNumber 

func AddRandomNumber(seqNum int, args ...string) (interface{}, error)

AddRandomNumber adds a random number to a given value retrieved from the fake data cache.

func AddRandomTimeMs 

func AddRandomTimeMs(seqNum int, args ...string) (interface{}, error)

AddRandomTimeMs adds a random number of milliseconds to a given value retrieved from the fake data cache.

func AddString 

func AddString(seqNum int, args ...string) (interface{}, error)

AddString concatenates string values retrieved from the fake data cache.

func And 

func And(seqNum int, args ...string) (interface{}, error)

And calculates the logical AND operation on boolean values retrieved from the fake data cache.

func ApplyOperations 

func ApplyOperations(outputBuilder *OutputBuilder, seqNum int) error

ApplyOperations applies the operations defined in the output builder to generate the final output.

func BinaryAtCache2ZipAtBytes 

func BinaryAtCache2ZipAtBytes(schemaNames []string, numberOfFilesPerCompressedFileMap map[string]map[string]int, seqNum int, numRecordsMap map[string]int) ([]byte, error)

BinaryAtCache2ZipAtBytes converts binary data stored in cache to a zip file and returns it as bytes.

func BinaryAtCache2ZipAtFile 

func BinaryAtCache2ZipAtFile(outputBuilder *OutputBuilder, seqNum int) error

BinaryAtCache2ZipAtFile converts binary data stored in cache to a zip file.

func BinaryAtCache2ZipAtFileBySchema 

func BinaryAtCache2ZipAtFileBySchema(schemaName string, numberOfFilesPerCompressedFile map[string]int, zipFilePath string, seqNum int, numRecords int, zipWriter *zip.Writer, compressPerFile bool) error

BinaryAtCache2ZipAtFileBySchema converts binary data stored in cache to a zip file for a specific schema.

func CSVAtCache2ZipAtBytes 

func CSVAtCache2ZipAtBytes(schemaNames []string, numberOfFilesPerCompressedFile map[string]map[string]int, seqNum int, numRecordsMap map[string]int) ([]byte, error)

CSVAtCache2ZipAtBytes converts CSV data stored in cache to a zip file and returns it as bytes.

func CSVAtCache2ZipAtFile 

func CSVAtCache2ZipAtFile(outputBuilder *OutputBuilder, seqNum int) error

CSVAtCache2ZipAtFile converts CSV data stored in cache to a zip file.

func CSVAtCache2ZipAtFileBySchema 

func CSVAtCache2ZipAtFileBySchema(schemaName string, numberOfFilesPerCompressedFile map[string]int, zipFilePath string, seqNum int, numRecords int, zipWriter *zip.Writer, compressPerFile bool) error

CSVAtCache2ZipAtFileBySchema converts CSV data stored in cache to a zip file for a specific schema.

func Copy 

func Copy(seqNum int, args ...string) (interface{}, error)

Copy retrieves a value from the fake data cache.

func CreateJobByDataSet 

func CreateJobByDataSet(jobName string, pvcName string, dataGenerator *windtunnelv1alpha1.DataSet, schemaMap map[string]*windtunnelv1alpha1.Schema) (*kbatch.Job, error)

CreateJobByDataSet creates a Kubernetes Job based on the DataSet configuration.

func CreatePVC 

func CreatePVC(name types.NamespacedName) *corev1.PersistentVolumeClaim

CreatePVC creates a PersistentVolumeClaim for the data generator job.

func CurrentTimeMs 

func CurrentTimeMs(seqNum int, args ...string) (interface{}, error)

CurrentTimeMs returns the current time in milliseconds.

func GetColumnNames 

func GetColumnNames(schemaName string) []string

GetColumnNames retrieves column names from the column names cache for a specific schema.

func GetFakeData 

func GetFakeData(key string, recordID int) (interface{}, error)

GetFakeData retrieves fake data from the fake data cache for a specific key and record ID.

func GetKey 

func GetKey(schBldr *SchemaBuilder, colBldr *ColumnBuilder) string

GetKey generates a unique key for a schema builder and column builder combination.

func MakeOutputDir 

func MakeOutputDir(dataGeneratorConfig *windtunnelv1alpha1.DataSet, seqNum int) error

MakeOutputDir creates the output directory for a schema in the dataset.

func NewFakeDataCache 

func NewFakeDataCache(outputBuilder *OutputBuilder)

NewFakeDataCache creates a new fake data cache based on the provided output builder.

func Or 

func Or(seqNum int, args ...string) (interface{}, error)

Or calculates the logical OR operation on boolean values retrieved from the fake data cache.

func PutColumnNames 

func PutColumnNames(schemaName string, columnNames []string)

PutColumnNames adds column names to the column names cache for a specific schema.

func PutFakeData 

func PutFakeData(key string, recordID int, v interface{})

PutFakeData stores fake data in the fake data cache for a specific key and record ID.

func PutFormulaLookup 

func PutFormulaLookup(formulaName string, formula Formula)

PutFormulaLookup adds a formula function to the formula lookups map.

func PutOpLookups 

func PutOpLookups(name string, op Operation)

PutOpLookups registers an operation function with a name in the opLookups map.

func PutParams 

func PutParams(in windtunnelv1alpha1.Column, params []gofakeit.Param) *gofakeit.MapParams

PutParams creates a gofakeit.MapParams instance based on the provided column and parameters.

func PutSchemaBuilder 

func PutSchemaBuilder(name string, schBldr *SchemaBuilder)

PutSchemaBuilder adds a schema builder to the schema builder cache.

func Raw2BinaryAtBytesBySchema 

func Raw2BinaryAtBytesBySchema(numRecords int, schemaName string) ([]byte, error)

Raw2CSVAtBytesBySchema converts raw data to CSV format for a specific schema and returns it as bytes.

func Raw2BinaryAtCache 

func Raw2BinaryAtCache(outputBuilder *OutputBuilder, seqNum int) error

Raw2BinaryAtCache converts raw data to binary format and stores it in cache.

func Raw2BinaryAtCacheBySchema 

func Raw2BinaryAtCacheBySchema(numRecords int, schemaName string) error

Raw2BinaryAtCacheBySchema converts raw data to binary format for a specific schema and stores it in cache.

func Raw2BinaryAtFile 

func Raw2BinaryAtFile(outputBuilder *OutputBuilder, seqNum int) error

Raw2BinaryAtFile converts raw data to binary format and writes it to a file.

func Raw2BinaryAtFileBySchema 

func Raw2BinaryAtFileBySchema(numRecords int, schemaName, filePath string) error

Raw2BinaryAtFileBySchema converts raw data to binary format for a specific schema and writes it to a file.

func Raw2CSVAtBytesBySchema 

func Raw2CSVAtBytesBySchema(numRecords int, schemaName string) ([]byte, error)

Raw2CSVAtBytesBySchema converts raw data to CSV format for a specific schema and returns it as bytes.

func Raw2CSVAtCache 

func Raw2CSVAtCache(outputBuilder *OutputBuilder, seqNum int) error

Raw2CSVAtCache converts raw data to CSV format and stores it in cache.

func Raw2CSVAtCacheBySchema 

func Raw2CSVAtCacheBySchema(numRecords int, schemaName string) error

Raw2CSVAtCacheBySchema converts raw data to CSV format for a specific schema and stores it in cache.

func Raw2CSVAtFile 

func Raw2CSVAtFile(outputBuilder *OutputBuilder, seqNum int) error

Raw2CSVAtFile converts raw data to CSV format and writes it to a file.

func Raw2CSVAtFileBySchema 

func Raw2CSVAtFileBySchema(numRecords int, schemaName, filePath string) error

Raw2CSVAtFileBySchema converts raw data to CSV format for a specific schema and writes it to a file.

func ToUnixMilli 

func ToUnixMilli(seqNum int, args ...string) (interface{}, error)

ToUnixMilli converts a string representation of a date to Unix milliseconds.

func XOrInt 

func XOrInt(seqNum int, args ...string) (interface{}, error)

XOrInt calculates the XOR operation on integer values retrieved from the fake data cache.

Types

type BuildBasedDataGeneratorJob 

type BuildBasedDataGeneratorJob struct {
	RepeatStart int
	RepeatEnd   int
	Namespace   string
	DGName      string
	Dataset     *windtunnelv1alpha1.DataSet
	SchemaMap   map[string]*windtunnelv1alpha1.Schema
}

BuildBasedDataGeneratorJob is a data generator job based on the Build strategy.

func (*BuildBasedDataGeneratorJob) GenerateData 

func (dg *BuildBasedDataGeneratorJob) GenerateData() error

GenerateData generates the data using the Build strategy.

type ColumnBuilder 

type ColumnBuilder struct {
	Name          string
	Info          *gofakeit.Info
	InfoMapParams *gofakeit.MapParams
	Formula       Formula
	FormulaArgs   []string
}

type ColumnNamesCache 

type ColumnNamesCache map[string][]string

type DataGeneratorJob 

type DataGeneratorJob interface {
	GenerateData() error
}

DataGeneratorJob is an interface for generating data.

func NewBuildBasedDataGeneratorJob 

func NewBuildBasedDataGeneratorJob(start int, end int, dgNamespace string, dgName string, dataset *windtunnelv1alpha1.DataSet, schemaMap map[string]*windtunnelv1alpha1.Schema) DataGeneratorJob

NewBuildBasedDataGeneratorJob creates a new BuildBasedDataGeneratorJob instance.

type FakeDataCache 

type FakeDataCache map[string][]interface{} // key is schema name + column name

type Formula 

type Formula func(seqNum int, args ...string) (interface{}, error)

Formula represents a function that generates data based on a sequence number and arguments.

func GetFormulaLookup 

func GetFormulaLookup(formulaName string) Formula

GetFormulaLookup retrieves a formula function from the formula lookups map by name.

type JobConfig 

type JobConfig struct {
	RepeatStart int
	RepeatEnd   int
}

JobConfig holds the configuration for a data generator job.

type Operation 

type Operation func(outputBuilder *OutputBuilder, seqNum int) error

Operation represents a function that performs a data generation operation.

func GetOpLookups 

func GetOpLookups(name string) Operation

GetOpLookups retrieves an operation function by its name from the opLookups map.

type OutputBuilder 

type OutputBuilder struct {
	CompressPerSchema bool
	SchBuilders       []*SchemaBuilder
	Operations        []Operation
	Path              string
	Name              string
}

func NewOutputBuilder 

func NewOutputBuilder(output *windtunnelv1alpha1.DataSet) (*OutputBuilder, error)

NewOutputBuilder creates a new OutputBuilder based on the provided output configuration.

type SchemaBuilder 

type SchemaBuilder struct {
	ColBulders                     []*ColumnBuilder
	ParentPath                     string
	SchemaName                     string
	NumRecords                     int
	NumberOfFilesPerCompressedFile map[string]int
}

func GetSchemaBuilder 

func GetSchemaBuilder(name string) *SchemaBuilder

GetSchemaBuilder retrieves a schema builder from the schema builder cache by name.

func NewSchemaBuilder 

func NewSchemaBuilder(schema *windtunnelv1alpha1.Schema) (*SchemaBuilder, error)

NewSchemaBuilder creates a new SchemaBuilder based on the provided schema.

func (*SchemaBuilder) Build 

func (schBldr *SchemaBuilder) Build(r *rand.Rand) error

Build generates fake data based on the provided SchemaBuilder.

type SchemaBuilderCache 

type SchemaBuilderCache map[string]*SchemaBuilder

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.