loader

Documentation

Index

• func CSVtoNumpyMulti(csvReader *csv.Reader, tbk io.TimeBucketKey, cvm *CSVMetadata, chunkSize int, ...) (npm *io.NumpyMultiDataset, endReached bool, err error)
• type CSVConfig
• type CSVMetadata
  • func ReadMetadata(dataFD, controlFD *os.File, dbDataShapes []io.DataShape) (csvReader *csv.Reader, cvm *CSVMetadata, err error)

Functions

func CSVtoNumpyMulti

func CSVtoNumpyMulti(csvReader *csv.Reader, tbk io.TimeBucketKey, cvm *CSVMetadata, chunkSize int,
	isVariable bool) (npm *io.NumpyMultiDataset, endReached bool, err error)

Types

type CSVConfig

type CSVConfig struct {
	FirstRowHasColumnNames bool     `yaml:"firstRowHasColumnNames"`
	TimeFormat             string   `yaml:"timeFormat"`
	Timezone               string   `yaml:"timeZone"`
	ColumnNameMap          []string `yaml:"columnNameMap"`
}

CSVConfig is constructed from the control file that specifies the formatting of the csv data.

type CSVMetadata

type CSVMetadata struct {
	Config      *CSVConfig     // Configuration of the CSV file, including the names of the columns
	DSV         []io.DataShape // Datashapes inside this CSV file
	ColumnIndex []int          // Maps the index of the columns in the CSV file to each time bucket in the DB
}

func ReadMetadata

func ReadMetadata(dataFD, controlFD *os.File, dbDataShapes []io.DataShape) (csvReader *csv.Reader, cvm *CSVMetadata, err error)

ReadMetadata returns formatting info about the csv file containing the data to be loaded into the database.