userconfig

package
v0.4.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 14, 2019 License: Apache-2.0 Imports: 22 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// Shared
	UnknownKey    = "unknown"
	NameKey       = "name"
	KindKey       = "kind"
	InputKey      = "input"
	ComputeKey    = "compute"
	TypeKey       = "type"
	PathKey       = "path"
	OutputTypeKey = "output_type"
	TagsKey       = "tags"

	// input schema options
	OptionalOptKey = "_optional"
	DefaultOptKey  = "_default"
	MinCountOptKey = "_min_count"
	MaxCountOptKey = "_max_count"

	// environment
	DataKey           = "data"
	SchemaKey         = "schema"
	LogLevelKey       = "log_level"
	LimitKey          = "limit"
	NumRowsKey        = "num_rows"
	FractionOfRowsKey = "fraction_of_rows"
	RandomizeKey      = "randomize"
	RandomSeedKey     = "random_seed"

	// templates / embeds
	TemplateKey = "template"
	YAMLKey     = "yaml"
	ArgsKey     = "args"

	// constants
	ValueKey = "value"

	// raw columns
	RequiredKey = "required"
	MinKey      = "min"
	MaxKey      = "max"
	ValuesKey   = "values"

	// aggregator / aggregate
	AggregatorKey     = "aggregator"
	AggregatorPathKey = "aggregator_path"

	// transformer / transformed_column
	TransformerKey     = "transformer"
	TransformerPathKey = "transformer_path"

	// estimator / model
	EstimatorKey                 = "estimator"
	EstimatorPathKey             = "estimator_path"
	TrainingInputKey             = "training_input"
	HparamsKey                   = "hparams"
	TargetColumnKey              = "target_column"
	PredictionKeyKey             = "prediction_key"
	DataPartitionRatioKey        = "data_partition_ratio"
	TrainingKey                  = "training"
	EvaluationKey                = "evaluation"
	BatchSizeKey                 = "batch_size"
	NumStepsKey                  = "num_steps"
	NumEpochsKey                 = "num_epochs"
	ShuffleKey                   = "shuffle"
	TfRandomSeedKey              = "tf_random_seed"
	TfRandomizeSeedKey           = "tf_randomize_seed"
	SaveSummaryStepsKey          = "save_summary_steps"
	SaveCheckpointsSecsKey       = "save_checkpoints_secs"
	SaveCheckpointsStepsKey      = "save_checkpoints_steps"
	LogStepCountStepsKey         = "log_step_count_steps"
	KeepCheckpointMaxKey         = "keep_checkpoint_max"
	KeepCheckpointEveryNHoursKey = "keep_checkpoint_every_n_hours"
	StartDelaySecsKey            = "start_delay_secs"
	ThrottleSecsKey              = "throttle_secs"
	DatasetComputeKey            = "dataset_compute"

	// API
	ModelKey         = "model"
	ModelNameKey     = "model_name"
	ExternalModelKey = "external_model"
)

Variables

This section is empty.

Functions

func CastInputValue 

func CastInputValue(value interface{}, inputSchema *InputSchema) (interface{}, error)

func CastOutputValue 

func CastOutputValue(value interface{}, outputSchema OutputSchema) (interface{}, error)

func DataTypeStr 

func DataTypeStr(dataType interface{}) string

func DataTypeStrsOr 

func DataTypeStrsOr(dataTypes []interface{}) string

func DataTypeUserStr 

func DataTypeUserStr(dataType interface{}) string

func EnvironmentDataTypeStrings 

func EnvironmentDataTypeStrings() []string

func ErrorCannotBeNull 

func ErrorCannotBeNull() error

func ErrorCannotMixValueAndColumnTypes 

func ErrorCannotMixValueAndColumnTypes(provided interface{}) error

func ErrorColumnTypeLiteral 

func ErrorColumnTypeLiteral(provided interface{}) error

func ErrorColumnTypeNotAllowed 

func ErrorColumnTypeNotAllowed(provided interface{}) error

func ErrorCompoundTypeInOutputType 

func ErrorCompoundTypeInOutputType(provided interface{}) error

func ErrorDuplicateConfig 

func ErrorDuplicateConfig(resourceType resource.Type) error

func ErrorDuplicateResourceName 

func ErrorDuplicateResourceName(resources ...Resource) error

func ErrorDuplicateResourceValue 

func ErrorDuplicateResourceValue(value string, keys ...string) error

func ErrorDuplicateTypeInTypeString 

func ErrorDuplicateTypeInTypeString(duplicated string, provided string) error

func ErrorEnvSchemaMismatch 

func ErrorEnvSchemaMismatch(env1, env2 *Environment) error

func ErrorExternalModelNotFound 

func ErrorExternalModelNotFound(path string) error

func ErrorExtraResourcesWithExternalAPIs 

func ErrorExtraResourcesWithExternalAPIs(res Resource) error

func ErrorGenericTypeMapLength 

func ErrorGenericTypeMapLength(provided interface{}) error

func ErrorImplDoesNotExist 

func ErrorImplDoesNotExist(path string) error

func ErrorInvalidCompoundType 

func ErrorInvalidCompoundType(provided interface{}) error

func ErrorInvalidInputType 

func ErrorInvalidInputType(provided interface{}) error

func ErrorInvalidOutputType 

func ErrorInvalidOutputType(provided interface{}) error

func ErrorK8sQuantityMustBeInt 

func ErrorK8sQuantityMustBeInt(quantityStr string) error

func ErrorMalformedConfig 

func ErrorMalformedConfig() error

func ErrorMinCountGreaterThanMaxCount 

func ErrorMinCountGreaterThanMaxCount() error

func ErrorMissingAppDefinition 

func ErrorMissingAppDefinition() error

func ErrorMixedInputArgOptionsAndUserKeys 

func ErrorMixedInputArgOptionsAndUserKeys() error

func ErrorMustBeDefined 

func ErrorMustBeDefined(allowedType interface{}) error

func ErrorOneOfPrerequisitesNotDefined 

func ErrorOneOfPrerequisitesNotDefined(argName string, prerequisites ...string) error

func ErrorOptionOnNonIterable 

func ErrorOptionOnNonIterable(key string) error

func ErrorParseConfig 

func ErrorParseConfig() error

func ErrorPredictionKeyOnModelWithEstimator 

func ErrorPredictionKeyOnModelWithEstimator() error

func ErrorRawColumnNotInEnv 

func ErrorRawColumnNotInEnv(envName string) error

func ErrorReadConfig 

func ErrorReadConfig() error

func ErrorResourceWrongType 

func ErrorResourceWrongType(resources []Resource, validResourceTypes ...resource.Type) error

func ErrorSpecifyAllOrNone 

func ErrorSpecifyAllOrNone(vals ...string) error

func ErrorSpecifyOnlyOne 

func ErrorSpecifyOnlyOne(vals ...string) error

func ErrorSpecifyOnlyOneMissing 

func ErrorSpecifyOnlyOneMissing(vals ...string) error

func ErrorTemplateExtraArg 

func ErrorTemplateExtraArg(template *Template, argName string) error

func ErrorTemplateMissingArg 

func ErrorTemplateMissingArg(template *Template, argName string) error

func ErrorTooFewElements 

func ErrorTooFewElements(t configreader.PrimitiveType, minCount int64) error

func ErrorTooManyElements 

func ErrorTooManyElements(t configreader.PrimitiveType, maxCount int64) error

func ErrorTypeListLength 

func ErrorTypeListLength(provided interface{}) error

func ErrorTypeMapZeroLength 

func ErrorTypeMapZeroLength(provided interface{}) error

func ErrorUndefinedConfig 

func ErrorUndefinedConfig(resourceType resource.Type) error

func ErrorUndefinedResource 

func ErrorUndefinedResource(resourceName string, resourceTypes ...resource.Type) error

func ErrorUnsupportedConfigKey 

func ErrorUnsupportedConfigKey() error

func ErrorUnsupportedLiteralMapKey 

func ErrorUnsupportedLiteralMapKey(key interface{}, allowedType interface{}) error

func ErrorUnsupportedLiteralType 

func ErrorUnsupportedLiteralType(provided interface{}, allowedType interface{}) error

func ErrorUnsupportedOutputType 

func ErrorUnsupportedOutputType(provided interface{}, allowedType interface{}) error

func ErrorUserKeysCannotStartWithUnderscore 

func ErrorUserKeysCannotStartWithUnderscore(key string) error

func Identify 

func Identify(r Resource) string

func QuantityParser 

func QuantityParser(v *QuantityValidation) func(string) (interface{}, error)

func QuantityPtrID 

func QuantityPtrID(quantity *Quantity) string

func QuantityPtrsEqual 

func QuantityPtrsEqual(quantity *Quantity, quantity2 *Quantity) bool

func ReadAppName 

func ReadAppName(filePath string, relativePath string) (string, error)

func ValidColumnTypeStrings 

func ValidColumnTypeStrings() []string

func ValueTypeStrings 

func ValueTypeStrings() []string

Types

type API 

type API struct {
	ResourceFields
	Model         *string        `json:"model" yaml:"model"`
	ExternalModel *ExternalModel `json:"external_model" yaml:"external_model"`
	Compute       *APICompute    `json:"compute" yaml:"compute"`
	Tags          Tags           `json:"tags" yaml:"tags"`
}

func (*API) GetResourceType 

func (api *API) GetResourceType() resource.Type

func (*API) Validate 

func (api *API) Validate() error

type APICompute 

type APICompute struct {
	Replicas int32     `json:"replicas" yaml:"replicas"`
	CPU      *Quantity `json:"cpu" yaml:"cpu"`
	Mem      *Quantity `json:"mem" yaml:"mem"`
	GPU      int64     `json:"gpu" yaml:"gpu"`
}

func (*APICompute) Equal 

func (apiCompute *APICompute) Equal(apiCompute2 APICompute) bool

func (*APICompute) ID 

func (apiCompute *APICompute) ID() string

func (*APICompute) IDWithoutReplicas 

func (apiCompute *APICompute) IDWithoutReplicas() string

type APIs 

type APIs []*API

func (APIs) Names 

func (apis APIs) Names() []string

func (APIs) Validate 

func (apis APIs) Validate() error

type Aggregate 

type Aggregate struct {
	ResourceFields
	Aggregator     string        `json:"aggregator" yaml:"aggregator"`
	AggregatorPath *string       `json:"aggregator_path" yaml:"aggregator_path"`
	Input          interface{}   `json:"input" yaml:"input"`
	Compute        *SparkCompute `json:"compute" yaml:"compute"`
	Tags           Tags          `json:"tags" yaml:"tags"`
}

func (*Aggregate) GetResourceType 

func (aggregate *Aggregate) GetResourceType() resource.Type

func (*Aggregate) Validate 

func (aggregate *Aggregate) Validate() error

type Aggregates 

type Aggregates []*Aggregate

func (Aggregates) Get 

func (aggregates Aggregates) Get(name string) *Aggregate

func (Aggregates) Names 

func (aggregates Aggregates) Names() []string

func (Aggregates) Validate 

func (aggregates Aggregates) Validate() error

type Aggregator 

type Aggregator struct {
	ResourceFields
	Input      *InputSchema `json:"input" yaml:"input"`
	OutputType OutputSchema `json:"output_type"  yaml:"output_type"`
	Path       string       `json:"path"  yaml:"path"`
}

func (*Aggregator) GetResourceType 

func (aggregator *Aggregator) GetResourceType() resource.Type

type Aggregators 

type Aggregators []*Aggregator

func (Aggregators) Get 

func (aggregators Aggregators) Get(name string) *Aggregator

func (Aggregators) Names 

func (aggregators Aggregators) Names() []string

func (Aggregators) Validate 

func (aggregators Aggregators) Validate() error

type App 

type App struct {
	Name string `json:"name" yaml:"name"`
}

func (*App) Validate 

func (app *App) Validate() error

type CSVConfig 

type CSVConfig struct {
	Sep                       *string `json:"sep" yaml:"sep"`
	Encoding                  *string `json:"encoding" yaml:"encoding"`
	Quote                     *string `json:"quote" yaml:"quote"`
	Escape                    *string `json:"escape" yaml:"escape"`
	Comment                   *string `json:"comment" yaml:"comment"`
	Header                    *bool   `json:"header" yaml:"header"`
	IgnoreLeadingWhiteSpace   *bool   `json:"ignore_leading_white_space" yaml:"ignore_leading_white_space"`
	IgnoreTrailingWhiteSpace  *bool   `json:"ignore_trailing_white_space" yaml:"ignore_trailing_white_space"`
	NullValue                 *string `json:"null_value" yaml:"null_value"`
	NanValue                  *string `json:"nan_value" yaml:"nan_value"`
	PositiveInf               *string `json:"positive_inf" yaml:"positive_inf"`
	NegativeInf               *string `json:"negative_inf" yaml:"negative_inf"`
	MaxColumns                *int32  `json:"max_columns" yaml:"max_columns"`
	MaxCharsPerColumn         *int32  `json:"max_chars_per_column" yaml:"max_chars_per_column"`
	Multiline                 *bool   `json:"multiline" yaml:"multiline"`
	CharToEscapeQuoteEscaping *string `json:"char_to_escape_quote_escaping" yaml:"char_to_escape_quote_escaping"`
	EmptyValue                *string `json:"empty_value" yaml:"empty_value"`
}

CSVConfig is SPARK_VERSION dependent

type CSVData 

type CSVData struct {
	Type      EnvironmentDataType `json:"type" yaml:"type"`
	Schema    []string            `json:"schema" yaml:"schema"`
	DropNull  bool                `json:"drop_null" yaml:"drop_null"`
	CSVConfig *CSVConfig          `json:"csv_config" yaml:"csv_config"`
	ExternalData
}

func (*CSVData) GetExternalData 

func (csvData *CSVData) GetExternalData() ExternalData

func (*CSVData) GetIngestedColumnNames 

func (csvData *CSVData) GetIngestedColumnNames() []string

func (*CSVData) Validate 

func (csvData *CSVData) Validate() error

type Column 

type Column interface {
	Resource
	IsRaw() bool
}

type ColumnType 

type ColumnType int
const (
	UnknownColumnType ColumnType = iota
	InferredColumnType
	IntegerColumnType
	FloatColumnType
	StringColumnType
	IntegerListColumnType
	FloatListColumnType
	StringListColumnType
)

func ColumnTypeFromString 

func ColumnTypeFromString(s string) ColumnType

func (ColumnType) JSONPlaceholder 

func (t ColumnType) JSONPlaceholder() string

func (ColumnType) MarshalBinary 

func (t ColumnType) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (ColumnType) MarshalText 

func (t ColumnType) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (ColumnType) String 

func (t ColumnType) String() string

func (*ColumnType) UnmarshalBinary 

func (t *ColumnType) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*ColumnType) UnmarshalText 

func (t *ColumnType) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type ColumnTypes 

type ColumnTypes []ColumnType

func (ColumnTypes) String 

func (ts ColumnTypes) String() string

func (ColumnTypes) StringList 

func (ts ColumnTypes) StringList() []string

type CompoundType 

type CompoundType string

func CompoundTypeFromString 

func CompoundTypeFromString(val interface{}) (CompoundType, error)

func (*CompoundType) CastValue 

func (compoundType *CompoundType) CastValue(value interface{}) (interface{}, error)

func (*CompoundType) IsColumns 

func (compoundType *CompoundType) IsColumns() bool

func (*CompoundType) IsValues 

func (compoundType *CompoundType) IsValues() bool

func (*CompoundType) String 

func (compoundType *CompoundType) String() string

func (*CompoundType) SupportsType 

func (compoundType *CompoundType) SupportsType(t interface{}) bool

type Config 

type Config struct {
	App                *App               `json:"app" yaml:"app"`
	Environments       Environments       `json:"environments" yaml:"environments"`
	Environment        *Environment       `json:"environment" yaml:"environment"`
	RawColumns         RawColumns         `json:"raw_columns" yaml:"raw_columns"`
	Aggregates         Aggregates         `json:"aggregates" yaml:"aggregates"`
	TransformedColumns TransformedColumns `json:"transformed_columns" yaml:"transformed_columns"`
	Models             Models             `json:"models" yaml:"models"`
	APIs               APIs               `json:"apis" yaml:"apis"`
	Aggregators        Aggregators        `json:"aggregators" yaml:"aggregators"`
	Transformers       Transformers       `json:"transformers" yaml:"transformers"`
	Estimators         Estimators         `json:"estimators" yaml:"estimators"`
	Constants          Constants          `json:"constants" yaml:"constants"`
	Templates          Templates          `json:"templates" yaml:"templates"`
	Embeds             Embeds             `json:"embeds" yaml:"embeds"`
	Resources          map[string][]Resource
}

func New 

func New(configs map[string][]byte, envName string) (*Config, error)

func NewPartialPath 

func NewPartialPath(filePath string) (*Config, error)

func (*Config) ColumnNames 

func (config *Config) ColumnNames() []string

func (*Config) IsRawColumn 

func (config *Config) IsRawColumn(name string) bool

func (*Config) IsTransformedColumn 

func (config *Config) IsTransformedColumn(name string) bool

func (*Config) MergeBytes 

func (config *Config) MergeBytes(configBytes []byte, filePath string, emb *Embed, template *Template) (*Config, error)

func (*Config) Validate 

func (config *Config) Validate(envName string) error

func (*Config) ValidatePartial 

func (config *Config) ValidatePartial() error

type Constant 

type Constant struct {
	ResourceFields
	Type  OutputSchema `json:"type" yaml:"type"`
	Value interface{}  `json:"value" yaml:"value"`
	Tags  Tags         `json:"tags" yaml:"tags"`
}

func (*Constant) GetResourceType 

func (constant *Constant) GetResourceType() resource.Type

func (*Constant) Validate 

func (constant *Constant) Validate() error

type Constants 

type Constants []*Constant

func (Constants) Names 

func (constants Constants) Names() []string

func (Constants) Validate 

func (constants Constants) Validate() error

type Data 

type Data interface {
	GetIngestedColumnNames() []string
	GetExternalData() ExternalData
	Validate() error
}

type Embed 

type Embed struct {
	ResourceFields
	Template string                 `json:"template" yaml:"template"`
	Args     map[string]interface{} `json:"args" yaml:"args"`
}

func (*Embed) GetResourceType 

func (embed *Embed) GetResourceType() resource.Type

type Embeds 

type Embeds []*Embed

type Environment 

type Environment struct {
	ResourceFields
	LogLevel *LogLevel `json:"log_level" yaml:"log_level"`
	Limit    *Limit    `json:"limit" yaml:"limit"`
	Data     Data      `json:"-" yaml:"-"`
}

func (*Environment) GetResourceType 

func (env *Environment) GetResourceType() resource.Type

func (*Environment) Validate 

func (env *Environment) Validate() error

type EnvironmentDataType 

type EnvironmentDataType int
const (
	UnknownEnvironmentDataType EnvironmentDataType = iota
	CSVEnvironmentDataType
	ParquetEnvironmentDataType
)

func EnvironmentDataTypeFromString 

func EnvironmentDataTypeFromString(s string) EnvironmentDataType

func (EnvironmentDataType) MarshalBinary 

func (t EnvironmentDataType) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (EnvironmentDataType) MarshalText 

func (t EnvironmentDataType) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (EnvironmentDataType) String 

func (t EnvironmentDataType) String() string

func (*EnvironmentDataType) UnmarshalBinary 

func (t *EnvironmentDataType) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*EnvironmentDataType) UnmarshalText 

func (t *EnvironmentDataType) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type Environments 

type Environments []*Environment

func (Environments) Names 

func (environments Environments) Names() []string

func (Environments) Validate 

func (environments Environments) Validate() error

type Error 

type Error struct {
	Kind ErrorKind
	// contains filtered or unexported fields
}

func (Error) Error 

func (e Error) Error() string

type ErrorKind 

type ErrorKind int
const (
	ErrUnknown ErrorKind = iota
	ErrDuplicateResourceName
	ErrDuplicateResourceValue
	ErrDuplicateConfig
	ErrMalformedConfig
	ErrParseConfig
	ErrReadConfig
	ErrMissingAppDefinition
	ErrUndefinedConfig
	ErrRawColumnNotInEnv
	ErrUndefinedResource
	ErrResourceWrongType
	ErrSpecifyAllOrNone
	ErrSpecifyOnlyOne
	ErrOneOfPrerequisitesNotDefined
	ErrTemplateExtraArg
	ErrTemplateMissingArg
	ErrInvalidCompoundType
	ErrDuplicateTypeInTypeString
	ErrCannotMixValueAndColumnTypes
	ErrColumnTypeLiteral
	ErrColumnTypeNotAllowed
	ErrCompoundTypeInOutputType
	ErrUserKeysCannotStartWithUnderscore
	ErrMixedInputArgOptionsAndUserKeys
	ErrOptionOnNonIterable
	ErrMinCountGreaterThanMaxCount
	ErrTooManyElements
	ErrTooFewElements
	ErrInvalidInputType
	ErrInvalidOutputType
	ErrUnsupportedLiteralType
	ErrUnsupportedLiteralMapKey
	ErrUnsupportedOutputType
	ErrMustBeDefined
	ErrCannotBeNull
	ErrUnsupportedConfigKey
	ErrTypeListLength
	ErrTypeMapZeroLength
	ErrGenericTypeMapLength
	ErrK8sQuantityMustBeInt
	ErrPredictionKeyOnModelWithEstimator
	ErrSpecifyOnlyOneMissing
	ErrEnvSchemaMismatch
	ErrExtraResourcesWithExternalAPIs
	ErrImplDoesNotExist
	ErrExternalModelNotFound
)

func (ErrorKind) MarshalBinary 

func (t ErrorKind) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (ErrorKind) MarshalText 

func (t ErrorKind) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (ErrorKind) String 

func (t ErrorKind) String() string

func (*ErrorKind) UnmarshalBinary 

func (t *ErrorKind) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*ErrorKind) UnmarshalText 

func (t *ErrorKind) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type Estimator 

type Estimator struct {
	ResourceFields
	TargetColumn  *CompoundType `json:"target_column"  yaml:"target_column"`
	Input         *InputSchema  `json:"input" yaml:"input"`
	TrainingInput *InputSchema  `json:"training_input" yaml:"training_input"`
	Hparams       *InputSchema  `json:"hparams"  yaml:"hparams"`
	PredictionKey string        `json:"prediction_key" yaml:"prediction_key"`
	Path          string        `json:"path"  yaml:"path"`
}

func (*Estimator) GetResourceType 

func (estimator *Estimator) GetResourceType() resource.Type

type Estimators 

type Estimators []*Estimator

func (Estimators) Get 

func (estimators Estimators) Get(name string) *Estimator

func (Estimators) Names 

func (estimators Estimators) Names() []string

func (Estimators) Validate 

func (estimators Estimators) Validate() error

type ExternalData 

type ExternalData struct {
	Path   string `json:"path" yaml:"path"`
	Region string `json:"region" yaml:"region"`
}

type ExternalModel 

type ExternalModel struct {
	Path   string `json:"path" yaml:"path"`
	Region string `json:"region" yaml:"region"`
}

type InputSchema 

type InputSchema struct {
	Type      InputTypeSchema `json:"_type" yaml:"_type"`
	Optional  bool            `json:"_optional" yaml:"_optional"`
	Default   interface{}     `json:"_default" yaml:"_default"`
	AllowNull bool            `json:"_allow_null" yaml:"_allow_null"`
	MinCount  *int64          `json:"_min_count" yaml:"_min_count"`
	MaxCount  *int64          `json:"_max_count" yaml:"_max_count"`
}

func ValidateInputSchema 

func ValidateInputSchema(in interface{}, disallowColumnTypes bool, isAlreadyParsed bool) (*InputSchema, error)

type InputTypeSchema 

type InputTypeSchema interface{} // CompundType, length-one array of *InputSchema, or map of {scalar|CompoundType -> *InputSchema}

func ValidateInputTypeSchema 

func ValidateInputTypeSchema(in interface{}, disallowColumnTypes bool, isAlreadyParsed bool) (InputTypeSchema, error)

type Limit 

type Limit struct {
	NumRows        *int64   `json:"num_rows" yaml:"num_rows"`
	FractionOfRows *float32 `json:"fraction_of_rows" yaml:"fraction_of_rows"`
	Randomize      *bool    `json:"randomize" yaml:"randomize"`
	RandomSeed     *int64   `json:"random_seed" yaml:"random_seed"`
}

type LogLevel 

type LogLevel struct {
	Tensorflow string `json:"tensorflow" yaml:"tensorflow"`
	Spark      string `json:"spark" yaml:"spark"`
}

type Model 

type Model struct {
	ResourceFields
	Estimator          string                   `json:"estimator" yaml:"estimator"`
	EstimatorPath      *string                  `json:"estimator_path" yaml:"estimator_path"`
	TargetColumn       string                   `json:"target_column" yaml:"target_column"`
	Input              interface{}              `json:"input" yaml:"input"`
	TrainingInput      interface{}              `json:"training_input" yaml:"training_input"`
	Hparams            interface{}              `json:"hparams"  yaml:"hparams"`
	PredictionKey      string                   `json:"prediction_key" yaml:"prediction_key"`
	DataPartitionRatio *ModelDataPartitionRatio `json:"data_partition_ratio" yaml:"data_partition_ratio"`
	Training           *ModelTraining           `json:"training" yaml:"training"`
	Evaluation         *ModelEvaluation         `json:"evaluation" yaml:"evaluation"`
	Compute            *TFCompute               `json:"compute" yaml:"compute"`
	DatasetCompute     *SparkCompute            `json:"dataset_compute" yaml:"dataset_compute"`
	Tags               Tags                     `json:"tags" yaml:"tags"`
}

func (*Model) GetResourceType 

func (model *Model) GetResourceType() resource.Type

func (*Model) Validate 

func (model *Model) Validate() error

type ModelDataPartitionRatio 

type ModelDataPartitionRatio struct {
	Training   *float64 `json:"training"`
	Evaluation *float64 `json:"evaluation"`
}

type ModelEvaluation 

type ModelEvaluation struct {
	BatchSize      int64  `json:"batch_size" yaml:"batch_size"`
	NumSteps       *int64 `json:"num_steps" yaml:"num_steps"`
	NumEpochs      *int64 `json:"num_epochs" yaml:"num_epochs"`
	Shuffle        bool   `json:"shuffle" yaml:"shuffle"`
	StartDelaySecs int64  `json:"start_delay_secs" yaml:"start_delay_secs"`
	ThrottleSecs   int64  `json:"throttle_secs" yaml:"throttle_secs"`
}

type ModelTraining 

type ModelTraining struct {
	BatchSize                 int64  `json:"batch_size" yaml:"batch_size"`
	NumSteps                  *int64 `json:"num_steps" yaml:"num_steps"`
	NumEpochs                 *int64 `json:"num_epochs" yaml:"num_epochs"`
	Shuffle                   bool   `json:"shuffle" yaml:"shuffle"`
	TfRandomSeed              int64  `json:"tf_random_seed" yaml:"tf_random_seed"`
	TfRandomizeSeed           bool   `json:"tf_randomize_seed" yaml:"tf_randomize_seed"`
	SaveSummarySteps          int64  `json:"save_summary_steps" yaml:"save_summary_steps"`
	SaveCheckpointsSecs       *int64 `json:"save_checkpoints_secs" yaml:"save_checkpoints_secs"`
	SaveCheckpointsSteps      *int64 `json:"save_checkpoints_steps" yaml:"save_checkpoints_steps"`
	LogStepCountSteps         int64  `json:"log_step_count_steps" yaml:"log_step_count_steps"`
	KeepCheckpointMax         int64  `json:"keep_checkpoint_max" yaml:"keep_checkpoint_max"`
	KeepCheckpointEveryNHours int64  `json:"keep_checkpoint_every_n_hours" yaml:"keep_checkpoint_every_n_hours"`
}

type Models 

type Models []*Model

func (Models) Names 

func (models Models) Names() []string

func (Models) Validate 

func (models Models) Validate() error

type OutputSchema 

type OutputSchema interface{} // ValueType, length-one array of OutputSchema, or map of {scalar|ValueType -> OutputSchema} (no *_COLUMN types, compound types, or input options like _default)

func ValidateOutputSchema 

func ValidateOutputSchema(in interface{}) (OutputSchema, error)

type ParquetColumn 

type ParquetColumn struct {
	ParquetColumnName string `json:"parquet_column_name" yaml:"parquet_column_name"`
	RawColumn         string `json:"raw_column" yaml:"raw_column"`
}

type ParquetData 

type ParquetData struct {
	Type     EnvironmentDataType `json:"type" yaml:"type"`
	Schema   []*ParquetColumn    `json:"schema" yaml:"schema"`
	DropNull bool                `json:"drop_null" yaml:"drop_null"`
	ExternalData
}

func (*ParquetData) GetExternalData 

func (parqData *ParquetData) GetExternalData() ExternalData

func (*ParquetData) GetIngestedColumnNames 

func (parqData *ParquetData) GetIngestedColumnNames() []string

func (*ParquetData) Validate 

func (parqData *ParquetData) Validate() error

type Quantity 

type Quantity struct {
	k8sresource.Quantity
	UserString string
}

func (*Quantity) Equal 

func (quantity *Quantity) Equal(quantity2 Quantity) bool

func (*Quantity) ID 

func (quantity *Quantity) ID() string

func (Quantity) MarshalBinary 

func (quantity Quantity) MarshalBinary() ([]byte, error)

func (Quantity) MarshalJSON 

func (quantity Quantity) MarshalJSON() ([]byte, error)

func (Quantity) MarshalText 

func (quantity Quantity) MarshalText() ([]byte, error)

func (*Quantity) SplitInTwo 

func (quantity *Quantity) SplitInTwo() (*k8sresource.Quantity, *k8sresource.Quantity)

SplitInTwo divides the quantity in two and return both halves (ensuring they add up to the original value)

func (*Quantity) String 

func (quantity *Quantity) String() string

func (*Quantity) ToFloat32 

func (quantity *Quantity) ToFloat32() float32

func (*Quantity) ToKi 

func (quantity *Quantity) ToKi() int64

func (*Quantity) UnmarshalBinary 

func (quantity *Quantity) UnmarshalBinary(data []byte) error

func (*Quantity) UnmarshalJSON 

func (quantity *Quantity) UnmarshalJSON(data []byte) error

func (*Quantity) UnmarshalText 

func (quantity *Quantity) UnmarshalText(data []byte) error

type QuantityValidation 

type QuantityValidation struct {
	Min k8sresource.Quantity
	Int bool
}

type RawColumn 

type RawColumn interface {
	Column
	GetColumnType() ColumnType
	GetCompute() *SparkCompute
}

type RawColumns 

type RawColumns []RawColumn

func (RawColumns) Get 

func (rawColumns RawColumns) Get(name string) RawColumn

func (RawColumns) Names 

func (rawColumns RawColumns) Names() []string

func (RawColumns) Validate 

func (rawColumns RawColumns) Validate() error

type RawFloatColumn 

type RawFloatColumn struct {
	ResourceFields
	Type     ColumnType    `json:"type" yaml:"type"`
	Required bool          `json:"required" yaml:"required"`
	Min      *float32      `json:"min" yaml:"min"`
	Max      *float32      `json:"max" yaml:"max"`
	Values   []float32     `json:"values" yaml:"values"`
	Compute  *SparkCompute `json:"compute" yaml:"compute"`
	Tags     Tags          `json:"tags" yaml:"tags"`
}

func (*RawFloatColumn) GetColumnType 

func (column *RawFloatColumn) GetColumnType() ColumnType

func (*RawFloatColumn) GetCompute 

func (column *RawFloatColumn) GetCompute() *SparkCompute

func (*RawFloatColumn) GetResourceType 

func (column *RawFloatColumn) GetResourceType() resource.Type

func (*RawFloatColumn) IsRaw 

func (column *RawFloatColumn) IsRaw() bool

type RawInferredColumn 

type RawInferredColumn struct {
	ResourceFields
	Type    ColumnType    `json:"type" yaml:"type"`
	Compute *SparkCompute `json:"compute" yaml:"compute"`
}

func (*RawInferredColumn) GetColumnType 

func (column *RawInferredColumn) GetColumnType() ColumnType

func (*RawInferredColumn) GetCompute 

func (column *RawInferredColumn) GetCompute() *SparkCompute

func (*RawInferredColumn) GetResourceType 

func (column *RawInferredColumn) GetResourceType() resource.Type

func (*RawInferredColumn) IsRaw 

func (column *RawInferredColumn) IsRaw() bool

type RawIntColumn 

type RawIntColumn struct {
	ResourceFields
	Type     ColumnType    `json:"type" yaml:"type"`
	Required bool          `json:"required" yaml:"required"`
	Min      *int64        `json:"min" yaml:"min"`
	Max      *int64        `json:"max" yaml:"max"`
	Values   []int64       `json:"values" yaml:"values"`
	Compute  *SparkCompute `json:"compute" yaml:"compute"`
	Tags     Tags          `json:"tags" yaml:"tags"`
}

func (*RawIntColumn) GetColumnType 

func (column *RawIntColumn) GetColumnType() ColumnType

func (*RawIntColumn) GetCompute 

func (column *RawIntColumn) GetCompute() *SparkCompute

func (*RawIntColumn) GetResourceType 

func (column *RawIntColumn) GetResourceType() resource.Type

func (*RawIntColumn) IsRaw 

func (column *RawIntColumn) IsRaw() bool

type RawStringColumn 

type RawStringColumn struct {
	ResourceFields
	Type     ColumnType    `json:"type" yaml:"type"`
	Required bool          `json:"required" yaml:"required"`
	Values   []string      `json:"values" yaml:"values"`
	Compute  *SparkCompute `json:"compute" yaml:"compute"`
	Tags     Tags          `json:"tags" yaml:"tags"`
}

func (*RawStringColumn) GetColumnType 

func (column *RawStringColumn) GetColumnType() ColumnType

func (*RawStringColumn) GetCompute 

func (column *RawStringColumn) GetCompute() *SparkCompute

func (*RawStringColumn) GetResourceType 

func (column *RawStringColumn) GetResourceType() resource.Type

func (*RawStringColumn) IsRaw 

func (column *RawStringColumn) IsRaw() bool

type Resource 

type Resource interface {
	GetName() string
	GetResourceType() resource.Type
	GetIndex() int
	SetIndex(int)
	GetFilePath() string
	SetFilePath(string)
	GetEmbed() *Embed
	SetEmbed(*Embed)
}

func FindDuplicateResourceName 

func FindDuplicateResourceName(resources ...Resource) []Resource

type ResourceFields 

type ResourceFields struct {
	Name     string `json:"name" yaml:"name"`
	Index    int    `json:"index" yaml:"-"`
	FilePath string `json:"file_path" yaml:"-"`
	Embed    *Embed `json:"embed" yaml:"-"`
}

func (*ResourceFields) GetEmbed 

func (resourceFields *ResourceFields) GetEmbed() *Embed

func (*ResourceFields) GetFilePath 

func (resourceFields *ResourceFields) GetFilePath() string

func (*ResourceFields) GetIndex 

func (resourceFields *ResourceFields) GetIndex() int

func (*ResourceFields) GetName 

func (resourceFields *ResourceFields) GetName() string

func (*ResourceFields) SetEmbed 

func (resourceFields *ResourceFields) SetEmbed(embed *Embed)

func (*ResourceFields) SetFilePath 

func (resourceFields *ResourceFields) SetFilePath(filePath string)

func (*ResourceFields) SetIndex 

func (resourceFields *ResourceFields) SetIndex(index int)

type SparkCompute 

type SparkCompute struct {
	Executors           int32     `json:"executors" yaml:"executors"`
	DriverCPU           Quantity  `json:"driver_cpu" yaml:"driver_cpu"`
	DriverMem           Quantity  `json:"driver_mem" yaml:"driver_mem"`
	DriverMemOverhead   *Quantity `json:"driver_mem_overhead" yaml:"driver_mem_overhead"`
	ExecutorCPU         Quantity  `json:"executor_cpu" yaml:"executor_cpu"`
	ExecutorMem         Quantity  `json:"executor_mem" yaml:"executor_mem"`
	ExecutorMemOverhead *Quantity `json:"executor_mem_overhead" yaml:"executor_mem_overhead"`
	MemOverheadFactor   *float64  `json:"mem_overhead_factor" yaml:"mem_overhead_factor"`
}

func MaxSparkCompute 

func MaxSparkCompute(sparkComputes ...*SparkCompute) *SparkCompute

func (*SparkCompute) ID 

func (sparkCompute *SparkCompute) ID() string

type TFCompute 

type TFCompute struct {
	CPU *Quantity `json:"cpu" yaml:"cpu"`
	Mem *Quantity `json:"mem" yaml:"mem"`
	GPU *int64    `json:"gpu" yaml:"gpu"`
}

func MaxTFCompute 

func MaxTFCompute(tfComputes ...*TFCompute) *TFCompute

func (*TFCompute) ID 

func (tfCompute *TFCompute) ID() string

type Tags 

type Tags map[string]interface{}

func (Tags) ID 

func (tags Tags) ID() string

type Template 

type Template struct {
	ResourceFields
	YAML string `json:"yaml" yaml:"yaml"`
}

func (*Template) GetResourceType 

func (template *Template) GetResourceType() resource.Type

func (*Template) Populate 

func (template *Template) Populate(emb *Embed) (string, error)

func (*Template) VariableNames 

func (template *Template) VariableNames() []string

type Templates 

type Templates []*Template

func (Templates) Map 

func (templates Templates) Map() map[string]*Template

func (Templates) Names 

func (templates Templates) Names() []string

func (Templates) Validate 

func (templates Templates) Validate() error

type TransformedColumn 

type TransformedColumn struct {
	ResourceFields
	Transformer     string        `json:"transformer" yaml:"transformer"`
	TransformerPath *string       `json:"transformer_path" yaml:"transformer_path"`
	Input           interface{}   `json:"input" yaml:"input"`
	Compute         *SparkCompute `json:"compute" yaml:"compute"`
	Tags            Tags          `json:"tags" yaml:"tags"`
}

func (*TransformedColumn) GetResourceType 

func (column *TransformedColumn) GetResourceType() resource.Type

func (*TransformedColumn) IsRaw 

func (column *TransformedColumn) IsRaw() bool

func (*TransformedColumn) Validate 

func (column *TransformedColumn) Validate() error

type TransformedColumns 

type TransformedColumns []*TransformedColumn

func (TransformedColumns) Get 

func (columns TransformedColumns) Get(name string) *TransformedColumn

func (TransformedColumns) Names 

func (columns TransformedColumns) Names() []string

func (TransformedColumns) Validate 

func (columns TransformedColumns) Validate() error

type Transformer 

type Transformer struct {
	ResourceFields
	Input      *InputSchema `json:"input" yaml:"input"`
	OutputType ColumnType   `json:"output_type"  yaml:"output_type"`
	Path       string       `json:"path"  yaml:"path"`
}

func (*Transformer) GetResourceType 

func (transformer *Transformer) GetResourceType() resource.Type

type Transformers 

type Transformers []*Transformer

func (Transformers) Get 

func (transformers Transformers) Get(name string) *Transformer

func (Transformers) Names 

func (transformers Transformers) Names() []string

func (Transformers) Validate 

func (transformers Transformers) Validate() error

type ValueType 

type ValueType int
const (
	UnknownValueType ValueType = iota
	IntegerValueType
	FloatValueType
	StringValueType
	BoolValueType
)

func ValueTypeFromString 

func ValueTypeFromString(s string) ValueType

func (*ValueType) CastValue 

func (t *ValueType) CastValue(value interface{}) (interface{}, error)

func (ValueType) MarshalBinary 

func (t ValueType) MarshalBinary() ([]byte, error)

MarshalBinary satisfies BinaryMarshaler

func (ValueType) MarshalText 

func (t ValueType) MarshalText() ([]byte, error)

MarshalText satisfies TextMarshaler

func (ValueType) String 

func (t ValueType) String() string

func (*ValueType) UnmarshalBinary 

func (t *ValueType) UnmarshalBinary(data []byte) error

UnmarshalBinary satisfies BinaryUnmarshaler Needed for msgpack

func (*ValueType) UnmarshalText 

func (t *ValueType) UnmarshalText(text []byte) error

UnmarshalText satisfies TextUnmarshaler

type ValueTypes 

type ValueTypes []ValueType

func (ValueTypes) String 

func (ts ValueTypes) String() string

func (ValueTypes) StringList 

func (ts ValueTypes) StringList() []string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.