model

package
v0.0.0-...-0d40728 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 12, 2021 License: Apache-2.0 Imports: 17 Imported by: 1

Documentation

Index

Constants

View Source 
const (
	// MaxNumBuckets is the maximum number of buckets to use for histograms
	MaxNumBuckets = 50

	// HourInterval represents an hour time bucketing interval
	HourInterval = 60 * 60
	// DayInterval represents an day time bucketing interval
	DayInterval = HourInterval * 24
	// WeekInterval represents a week time bucketing interval
	WeekInterval = DayInterval * 7
	// MonthInterval represents a month time bucketing interval
	MonthInterval = WeekInterval * 4
)
View Source 
const (
	// DefaultDataMode use the id field for filtering, ex. clustering not applied
	DefaultDataMode = iota + 1
	// ClusterDataMode use computed cluster information for filtering if availble, ex. timeseries clusters
	ClusterDataMode
)
View Source 
const (
	// SolutionResultTypeInference is the solution result type for inferences.
	SolutionResultTypeInference = "inference"
	// SolutionResultTypeTest is the solution result type for tests.
	SolutionResultTypeTest = "test"
)
View Source 
const (
	// TimeseriesAddOp indicates that bucket values should be added
	TimeseriesAddOp = "add"
	// TimeseriesMinOp indicates that the min of bucket values should be taken
	TimeseriesMinOp = "min"
	// TimeseriesMaxOp indicates that the max of bucket values should be taken
	TimeseriesMaxOp = "max"
	// TimeseriesMeanOp indicates that the mean of bucket values should be taken
	TimeseriesMeanOp = "mean"
	// TimeseriesDefaultOp is the operation to use when none is specified
	TimeseriesDefaultOp = TimeseriesAddOp
)
View Source 
const (
	// MinAggPrefix is the prefix used for min aggregations.
	MinAggPrefix = "min_"
	// MaxAggPrefix is the prefix used for max aggregations.
	MaxAggPrefix = "max_"
	// TermsAggPrefix is the prefix used for terms aggregations.
	TermsAggPrefix = "terms_"
	// HistogramAggPrefix is the prefix used for histogram aggregations.
	HistogramAggPrefix = "histogram_"
	// VariableValueField is the field which stores the variable value.
	VariableValueField = "value"
	// VariableTypeField is the field which stores the variable's schema type value.
	VariableTypeField = "schemaType"
)
View Source 
const (
	// DefaultMode use the default facet for a variable summary given its type, ex. a horizontal histogram for numeric values.
	DefaultMode = iota + 1
	// ClusterMode use computed cluster information for a variable summary if availble, ex. timeseries clusters
	ClusterMode
	// TimeseriesMode use the timeseries grouping to return timeseries counts rather than observation counts.
	TimeseriesMode
	// MultiBandImageMode use the multi-band image grouping to return tile counts rather than image counts.
	MultiBandImageMode
)

Variables

View Source 
var (
	// FilterModeInverse provides a quick way to inverse filter modes
	FilterModeInverse = map[string]string{model.ExcludeFilter: model.IncludeFilter, model.IncludeFilter: model.ExcludeFilter}
)

Functions

func FetchDatasetVariables 

func FetchDatasetVariables(dataset string, metaStore MetadataStorage) ([]*model.Variable, error)

FetchDatasetVariables fetches the variable list for a dataset, and removes any variables that don't have corresponding dataset values (grouped variables).

func FetchSummaryVariables 

func FetchSummaryVariables(dataset string, metaStore MetadataStorage) ([]*model.Variable, error)

FetchSummaryVariables fetches the variable list from the data store and filters/expands it to contain only those variables that should be displayed to a client.

func GetClusterColFromGrouping 

func GetClusterColFromGrouping(group model.BaseGrouping) (string, bool)

GetClusterColFromGrouping return the cluster column if it exists in the group.

func GetConfidenceKey 

func GetConfidenceKey(solutionID string) string

GetConfidenceKey returns a solutions error col key.

func GetErrorKey 

func GetErrorKey(solutionID string) string

GetErrorKey returns a solutions error col key.

func GetFilterVariables 

func GetFilterVariables(filterVariables []string, variables []*model.Variable) []*model.Variable

GetFilterVariables builds the filtered list of fields based on the filtering parameters.

func GetPredictedKey 

func GetPredictedKey(solutionID string) string

GetPredictedKey returns a solutions predicted col key.

func GetRankKey 

func GetRankKey(solutionID string) string

GetRankKey returns a solutions rank col key.

func HasClusterData 

func HasClusterData(datasetName string, variableName string, metaStore MetadataStorage) bool

HasClusterData checks to see if a grouped variable has associated cluster data available. If the cluster info has not yet been computed (it can be a long running task) then this willl return false.

func IsConfidenceKey 

func IsConfidenceKey(key string) bool

IsConfidenceKey returns true if the key matches a confidence key.

func IsErrorKey 

func IsErrorKey(key string) bool

IsErrorKey returns true if the key matches an error key.

func IsPredictedKey 

func IsPredictedKey(key string) bool

IsPredictedKey returns true if the key matches a predicted key.

func IsRankKey 

func IsRankKey(key string) bool

IsRankKey returns true if the key matches a rank key

func IsResultKey 

func IsResultKey(key string) bool

IsResultKey returns true if the key matches an predicted or error key.

func MapVariables 

func MapVariables(variables []*model.Variable, mapper func(variable *model.Variable) string) map[string]*model.Variable

MapVariables creates a variable map using the mapper function to create the key.

func ParseDatasetOriginsFromJSON 

func ParseDatasetOriginsFromJSON(originsJSON []map[string]interface{}) []*model.DatasetOrigin

ParseDatasetOriginsFromJSON parses dataset origins from string maps.

func StripKeySuffix 

func StripKeySuffix(key string) string

StripKeySuffix removes any result key suffix.

func UpdateDiskDataset 

func UpdateDiskDataset(ds *Dataset, data [][]string) error

UpdateDiskDataset updates a disk dataset to have the new and updated data.

func UpdateExtremas 

func UpdateExtremas(dataset string, varName string, storageMeta MetadataStorage, storageData DataStorage) error

UpdateExtremas updates the variable extremas based on the data stored.

func UpdateFilterKey 

func UpdateFilterKey(metaStore MetadataStorage, dataset string, dataMode DataMode, filter *model.Filter, variable *model.Variable)

UpdateFilterKey updates the supplied filter key to point to a group-specific column, rather than relying on the group variable name.

Types

type BinningArgs 

type BinningArgs struct {
	Rounded  *Extrema
	Count    int
	Interval float64
}

BinningArgs represent timeseries binning args.

type Bucket 

type Bucket struct {
	Key     string    `json:"key"`
	Count   int64     `json:"count"`
	Buckets []*Bucket `json:"buckets,omitempty"`
}

Bucket represents a single histogram bucket.

type Column 

type Column struct {
	Label  string  `json:"label"`
	Key    string  `json:"key"`
	Type   string  `json:"type"`
	Weight float64 `json:"weight"`
	Index  int     `json:"index"`
}

Column represents a column for filtered data.

type DataMode 

type DataMode int

DataMode defines the data filter modes.

func DataModeFromString 

func DataModeFromString(s string) (DataMode, error)

DataModeFromString creates a DataMode from the supplied string

type DataStorage 

type DataStorage interface {
	FetchNumRows(storageName string, variables []*model.Variable) (int, error)
	FetchData(dataset string, storageName string, filterParams *FilterParams, includeGroupingCol bool, orderByVar *model.Variable) (*FilteredData, error)
	FetchDataset(dataset string, storageName string, includeMetadata bool, limitSelectedFields bool, filterParams *FilterParams) ([][]string, error)
	FetchResultDataset(dataset string, storageName string, predictionName string, features []string, resultURI string, includeExplain bool) ([][]string, error)
	FetchSummary(dataset string, storageName string, varName string, filterParams *FilterParams, mode SummaryMode) (*VariableSummary, error)
	FetchSummaryByResult(dataset string, storageName string, varName string, resultURI string, filterParams *FilterParams, extrema *Extrema, mode SummaryMode) (*VariableSummary, error)
	PersistResult(dataset string, storageName string, resultURI string, target string) error
	PersistExplainedResult(dataset string, storageName string, resultURI string, explainResult *SolutionExplainResult) error
	PersistSolutionFeatureWeight(dataset string, storageName string, solutionID string, weights [][]string) error
	FetchResults(dataset string, storageName string, resultURI string, solutionID string, filterParams *FilterParams, removeTargetColumn bool) (*FilteredData, error)
	FetchPredictedSummary(dataset string, storageName string, resultURI string, filterParams *FilterParams, extrema *Extrema, mode SummaryMode) (*VariableSummary, error)
	FetchResultsExtremaByURI(dataset string, storageName string, resultURI string) (*Extrema, error)
	FetchCorrectnessSummary(dataset string, storageName string, resultURI string, filterParams *FilterParams, mode SummaryMode) (*VariableSummary, error)
	FetchConfidenceSummary(dataset string, storageName string, resultURI string, filterParams *FilterParams, mode SummaryMode) (map[string]*VariableSummary, error)
	FetchResidualsSummary(dataset string, storageName string, resultURI string, filterParams *FilterParams, extrema *Extrema, mode SummaryMode) (*VariableSummary, error)
	FetchResidualsExtremaByURI(dataset string, storageName string, resultURI string) (*Extrema, error)
	FetchExtrema(dataset string, storageName string, variable *model.Variable) (*Extrema, error)
	FetchExtremaByURI(dataset string, storageName string, resultURI string, variable string) (*Extrema, error)
	FetchTimeseries(dataset string, storageName string, variableKey string, seriesIDColName string, xColName string, yColName string,
		seriesIDs []string, operation TimeseriesOp, filterParams *FilterParams) ([]*TimeseriesData, error)
	FetchTimeseriesForecast(dataset string, storageName string, variableKey string, seriesIDColName string, xColName string, yColName string,
		seriesIDs []string, operation TimeseriesOp, resultUUID string, filterParams *FilterParams) ([]*TimeseriesData, error)
	FetchCategoryCounts(storageName string, variable *model.Variable) (map[string]int, error)
	FetchSolutionFeatureWeights(dataset string, storageName string, resultURI string, d3mIndex int64) (*SolutionFeatureWeight, error)
	// Dataset manipulation
	IsValidDataType(dataset string, storageName string, varName string, varType string) (bool, error)
	SetDataType(dataset string, storageName string, varName string, varType string) error
	AddVariable(dataset string, storageName string, varName string, varType string, defaultVal string) error
	AddField(dataset string, storageName string, varName string, varType string, defaultVal string) error
	DeleteVariable(dataset string, storageName string, varName string) error
	SetVariableValue(dataset string, storageName string, varName string, value string, filterParams *FilterParams) error
	UpdateVariableBatch(storageName string, varName string, updates map[string]string) error
	UpdateData(dataset string, storageName string, varName string, updates map[string]string, filterParams *FilterParams) error
	DoesVariableExist(dataset string, storageName string, varName string) (bool, error)
	VerifyData(datasetID string, tableName string) error
	// Raw data queries
	FetchRawDistinctValues(dataset string, storageName string, varNames []string) ([][]string, error)

	// Property queries
	GetStorageName(dataset string) (string, error)
	// SaveDataset is used to drop all the undesired values (only call for save dataset route)
	SaveDataset(dataset string, storageName string, filterParams *FilterParams) error
	// CloneDataset creates a copy of an existing dataset
	CloneDataset(dataset string, storageName string, datasetNew string, storageNameNew string) error
	// DeleteDataset drops all tables associated to storageName
	DeleteDataset(storageName string) error
	CreateIndices(dataset string, indexFields []string) error
	// IsKey verifies the unique property of the listed variables
	IsKey(dataset string, storageName string, variables []*model.Variable) (bool, error)
}

DataStorage defines the functions available to query the underlying data storage.

type DataStorageCtor 

type DataStorageCtor func() (DataStorage, error)

DataStorageCtor represents a client constructor to instantiate a data storage client.

type Dataset 

type Dataset struct {
	ID              string                 `json:"id"`
	Name            string                 `json:"name"`
	StorageName     string                 `json:"storageName"`
	Folder          string                 `json:"datasetFolder"`
	Description     string                 `json:"description"`
	Summary         string                 `json:"summary"`
	SummaryML       string                 `json:"summaryML"`
	Variables       []*model.Variable      `json:"variables"`
	NumRows         int64                  `json:"numRows"`
	NumBytes        int64                  `json:"numBytes"`
	Provenance      string                 `json:"provenance"`
	Source          metadata.DatasetSource `json:"source"`
	JoinSuggestions []*JoinSuggestion      `json:"joinSuggestion"`
	JoinScore       float64                `json:"joinScore"`
	Type            DatasetType            `json:"type"`
	LearningDataset string                 `json:"learningDataset"`
	Clone           bool                   `json:"clone"`
	Immutable       bool                   `json:"immutable"`
	ParentDataset   string                 `json:"parentDataset"`
	Deleted         bool                   `json:"deleted"`
}

Dataset represents a decsription of a dataset.

func (*Dataset) GetD3MIndexVariable 

func (d *Dataset) GetD3MIndexVariable() *model.Variable

GetD3MIndexVariable returns the D3M index variable.

func (*Dataset) GetLearningFolder 

func (d *Dataset) GetLearningFolder() string

GetLearningFolder returns the folder on disk that has the data for learning.

func (*Dataset) ToMetadata 

func (d *Dataset) ToMetadata() *model.Metadata

ToMetadata capture the dataset metadata in a d3m metadata struct.

type DatasetType 

type DatasetType string

DatasetType is used to identify the type of dataset ingested. 

const (
	// DatasetTypeModelling is a dataset used to build models.
	DatasetTypeModelling DatasetType = "modelling"
	// DatasetTypeInference is a dataset consumed by a model to infer predictions.
	DatasetTypeInference DatasetType = "inference"
)

type DiskDataset 

type DiskDataset struct {
	Dataset           *serialization.RawDataset
	FeaturizedDataset *DiskDataset
	// contains filtered or unexported fields
}

DiskDataset represents a dataset stored on disk.

func LoadDiskDataset 

func LoadDiskDataset(ds *Dataset) (*DiskDataset, error)

LoadDiskDataset loads a dataset from disk. It will load the learning dataset as the featurized dataset.

func LoadDiskDatasetFromFolder 

func LoadDiskDatasetFromFolder(folder string) (*DiskDataset, error)

LoadDiskDatasetFromFolder loads a dataset from disk at the specified location.

func (*DiskDataset) AddField 

func (d *DiskDataset) AddField(variable *model.Variable) error

AddField adds a field to the dataset.

func (*DiskDataset) Clone 

func (d *DiskDataset) Clone(targetFolder string, cloneDatasetID string, cloneStorageName string) (*DiskDataset, error)

Clone clones a dataset on disk.

func (*DiskDataset) FieldExists 

func (d *DiskDataset) FieldExists(variable *model.Variable) bool

FieldExists returns true if a field exists in the dataset.

func (*DiskDataset) GetLearningFolder 

func (d *DiskDataset) GetLearningFolder() string

GetLearningFolder returns the folder containing the learning dataset.

func (*DiskDataset) GetPath 

func (d *DiskDataset) GetPath() string

GetPath returns the schema path of the dataset on disk.

func (*DiskDataset) HasFields 

func (d *DiskDataset) HasFields(fields []*model.Variable, exactMatch bool) bool

HasFields returns true if the dataset has the specified fields, and if specified, only those fields.

func (*DiskDataset) ReorderFields 

func (d *DiskDataset) ReorderFields(fields []*model.Variable) error

ReorderFields will reorder the dataset fields to match the passed in order.

func (*DiskDataset) SaveDataset 

func (d *DiskDataset) SaveDataset() error

SaveDataset saves a dataset to disk.

func (*DiskDataset) SaveMetadata 

func (d *DiskDataset) SaveMetadata() error

SaveMetadata saves a dataset metadata to disk.

func (*DiskDataset) Update 

func (d *DiskDataset) Update(updates map[string]map[string]string, filterNotFound bool) error

Update updates a dataset, optionally filtering rows not being updated.

func (*DiskDataset) UpdateOnDisk 

func (d *DiskDataset) UpdateOnDisk(ds *Dataset, data [][]string, updateImmutable bool, filterNotFound bool) error

UpdateOnDisk updates a disk dataset to have the new and updated data.

func (*DiskDataset) UpdatePath 

func (d *DiskDataset) UpdatePath(datasetFolder string)

UpdatePath updates the path of the disk dataset to point to a new location.

func (*DiskDataset) UpdateRawData 

func (d *DiskDataset) UpdateRawData(varMap map[string]*model.Variable, data [][]string, updateImmutable bool, filterNotFound bool) error

UpdateRawData updates the data in a disk dataset but does not save it.

type ExportedModel 

type ExportedModel struct {
	ModelName        string              `json:"modelName"`
	ModelDescription string              `json:"modelDescription"`
	FilePath         string              `json:"filePath"`
	FittedSolutionID string              `json:"fittedSolutionId"`
	DatasetID        string              `json:"datasetId"`
	DatasetName      string              `json:"datasetName"`
	Target           *SolutionVariable   `json:"target"`
	Variables        []string            `json:"variables"`
	VariableDetails  []*SolutionVariable `json:"variableDetails"`
	Deleted          bool                `json:"deleted"`
}

ExportedModel represents a description of an exported model.

type ExportedModelStorage 

type ExportedModelStorage interface {
	PersistExportedModel(exportedModel *ExportedModel) error
	FetchModel(model string) (*ExportedModel, error)
	FetchModelByID(fittedSolutionID string) (*ExportedModel, error)
	FetchModels(includeDeleted bool) ([]*ExportedModel, error)
	SearchModels(terms string, includeDeleted bool) ([]*ExportedModel, error)
	DeleteModel(fittedSolutionID string) error
}

ExportedModelStorage defines the functions available to query the underlying model storage.

type ExportedModelStorageCtor 

type ExportedModelStorageCtor func() (ExportedModelStorage, error)

ExportedModelStorageCtor represents a client constructor to instantiate a model storage client.

type Extrema 

type Extrema struct {
	Key    string   `json:"-"`
	Type   string   `json:"-"`
	Min    float64  `json:"min"`
	Max    float64  `json:"max"`
	Values []string `json:"values"`
}

Extrema represents the extrema for a single variable.

func NewExtrema 

func NewExtrema(min float64, max float64) (*Extrema, error)

NewExtrema instantiates a new extrema struct.

func (*Extrema) GetBucketCount 

func (e *Extrema) GetBucketCount(numBuckets int) int

GetBucketCount calculates the number of buckets for the extrema given a target number of buckets.

func (*Extrema) GetBucketInterval 

func (e *Extrema) GetBucketInterval(numBuckets int) float64

GetBucketInterval calculates the size of the buckets given the extrema and a bucket count.

func (*Extrema) GetBucketMinMax 

func (e *Extrema) GetBucketMinMax(numBuckets int) *Extrema

GetBucketMinMax calculates the bucket min and max for the extrema.

func (*Extrema) GetTimeBucketCount 

func (e *Extrema) GetTimeBucketCount(interval int) int

GetTimeBucketCount calculates the number of buckets for the extrema.

func (*Extrema) GetTimeBucketMinMax 

func (e *Extrema) GetTimeBucketMinMax(interval int) *Extrema

GetTimeBucketMinMax calculates the bucket min and max for the extrema.

func (*Extrema) GetTimeseriesBinningArgs 

func (e *Extrema) GetTimeseriesBinningArgs(interval int) BinningArgs

GetTimeseriesBinningArgs returns the histogram binning args

type Feature 

type Feature struct {
	RequestID   string `json:"requestId"`
	FeatureName string `json:"featureName"`
	FeatureType string `json:"featureType"`
}

Feature represents a request feature metadata.

type FilterParams 

type FilterParams struct {
	Size       int                `json:"size"`
	Filters    []*model.FilterSet `json:"filters"`
	Highlights []*model.FilterSet `json:"highlights"`
	Variables  []string           `json:"variables"`
	DataMode   DataMode           `json:"dataMode"`
	Invert     bool               `json:"invert"`
}

FilterParams defines the set of filters to use. Note that this is to be used by the server only, and not the client. Filters are gathered by mode (include/exclude), with each mode being a list of features that are used as filters.

func ExpandFilterParams 

func ExpandFilterParams(dataset string, filterParams *FilterParams, includeHidden bool, metaStore MetadataStorage) (*FilterParams, error)

ExpandFilterParams examines filter parameters for grouped variables, and replaces them with their constituent components as necessary.

func GetBaselineFilter 

func GetBaselineFilter(filterParam *FilterParams) *FilterParams

GetBaselineFilter returns a filter params that only has the baseline filters.

func NewFilterParamsFromFilters 

func NewFilterParamsFromFilters(filters []*model.Filter) *FilterParams

NewFilterParamsFromFilters creates a wrapping container for all filters.

func ParseFilterParamsFromJSON 

func ParseFilterParamsFromJSON(params map[string]interface{}) (*FilterParams, error)

ParseFilterParamsFromJSON parses filter parameters out of a map[string]interface{}

func ParseFilterParamsFromJSONRaw 

func ParseFilterParamsFromJSONRaw(raw encoding.RawMessage) (*FilterParams, error)

ParseFilterParamsFromJSONRaw parses filter parameters out of a json.RawMessage

func (*FilterParams) AddFilter 

func (f *FilterParams) AddFilter(filter *model.Filter)

AddFilter adds a filter to the filter params, inserting it in the proper collection.

func (*FilterParams) AddVariable 

func (f *FilterParams) AddVariable(nv string)

AddVariable adds a variable, preventing duplicates

func (*FilterParams) Clone 

func (f *FilterParams) Clone() *FilterParams

Clone returns a deep copy of the filter params.

func (*FilterParams) InvertFilters 

func (f *FilterParams) InvertFilters()

InvertFilters inverts filters and highlights.

func (*FilterParams) IsEmpty 

func (f *FilterParams) IsEmpty(ignoreBaselineFilters bool) bool

IsEmpty returns true if the filter set is empty.

func (*FilterParams) MergeFilterObjects 

func (f *FilterParams) MergeFilterObjects(filters []model.FilterObject)

MergeFilterObjects merges a slice of filter objects with the existing filter params.

func (*FilterParams) ToJSON 

func (f *FilterParams) ToJSON() string

ToJSON creates a string representation of the json representation of the filter params.

type FilteredData 

type FilteredData struct {
	NumRows         int                    `json:"numRows"`
	NumRowsFiltered int                    `json:"numRowsFiltered"`
	Columns         map[string]*Column     `json:"columns"`
	Values          [][]*FilteredDataValue `json:"values"`
}

FilteredData provides the metadata and raw data values that match a supplied input filter.

func CreateFilteredData 

func CreateFilteredData(inputData [][]string, variables []*model.Variable, returnRaw bool, lineCount int) (*FilteredData, error)

CreateFilteredData creates a filtered data structure from raw string data.

func EmptyFilterData 

func EmptyFilterData() *FilteredData

EmptyFilterData returns an empty FilteredData object

func ReplaceNaNs 

func ReplaceNaNs(data *FilteredData, replacementType NaNReplacement) *FilteredData

ReplaceNaNs replaces NaN values found in numerical columns with empty values. This allows for downstream JSON encoding, as the Go JSON encoder doesn't properly handle NaN values.

type FilteredDataValue 

type FilteredDataValue struct {
	Value      interface{}     `json:"value"`
	Weight     float64         `json:"weight,omitempty"`
	Confidence NullableFloat64 `json:"confidence,omitempty"`
	Rank       NullableFloat64 `json:"rank,omitempty"`
}

FilteredDataValue represents a data value combined with an optional weight.

type Histogram 

type Histogram struct {
	Extrema         *Extrema             `json:"extrema,omitempty"`
	DefaultBucket   *Bucket              `json:"defaultBucket"`
	Buckets         []*Bucket            `json:"buckets"`
	CategoryBuckets map[string][]*Bucket `json:"categoryBuckets"`
	Exemplars       []string             `json:"exemplars"`
	StdDev          float64              `json:"stddev"`
	Mean            float64              `json:"mean"`
}

Histogram represents a single variable histogram.

func (*Histogram) IsEmpty 

func (h *Histogram) IsEmpty() bool

IsEmpty returns true if no data exists in the histogram (ie sum(buckets) == 0)

type JoinSuggestion 

type JoinSuggestion struct {
	BaseDataset   string               `json:"baseDataset"`
	BaseColumns   []string             `json:"baseColumns"`
	JoinDataset   string               `json:"joinDataset"`
	JoinColumns   []string             `json:"joinColumns"`
	JoinScore     float64              `json:"joinScore"`
	DatasetOrigin *model.DatasetOrigin `json:"datasetOrigin"`
	Index         int                  `json:"index"`
}

JoinSuggestion specifies potential joins between datasets.

type MetadataStorage 

type MetadataStorage interface {
	FetchVariables(dataset string, includeIndex bool, includeMeta bool, includeSystemData bool) ([]*model.Variable, error)
	FetchVariablesByName(dataset string, varNames []string, includeIndex bool, includeMeta bool, includeSystemData bool) ([]*model.Variable, error)
	FetchVariablesDisplay(dataset string) ([]*model.Variable, error)
	DoesVariableExist(dataset string, varName string) (bool, error)
	FetchVariable(dataset string, varName string) (*model.Variable, error)
	FetchVariableDisplay(dataset string, varName string) (*model.Variable, error)
	FetchDataset(dataset string, includeIndex bool, includeMeta bool, includeSystemData bool) (*Dataset, error)
	FetchDatasets(includeIndex bool, includeMeta bool, includeSystemData bool) ([]*Dataset, error)
	SearchDatasets(terms string, baseDataset *Dataset, includeIndex bool, includeMeta bool, includeSystemData bool) ([]*Dataset, error)
	ImportDataset(id string, uri string) (string, error)

	// Dataset manipulation
	SetDataType(dataset string, varName string, varType string) error
	SetExtrema(dataset string, varName string, extrema *Extrema) error
	AddVariable(dataset string, varName string, varDisplayName string, varType string, varDistilRole []string) error
	UpdateVariable(dataset string, varName string, variableValue *model.Variable) error
	DeleteVariable(dataset string, varName string) error
	AddGroupedVariable(dataset string, varName string, varDisplayName string, varType string, varRole []string, grouping model.BaseGrouping) error
	RemoveGroupedVariable(datasetName string, grouping model.BaseGrouping) error
	DeleteDataset(dataset string, softDelete bool) error
	IngestDataset(datasetSource metadata.DatasetSource, meta *model.Metadata) error
	UpdateDataset(dataset *Dataset) error
	DatasetExists(dataset string) (bool, error)

	// CloneDataset creates a copy of an existing dataset
	CloneDataset(dataset string, datasetNew string, storageNameNew string, folderNew string) error
}

MetadataStorage defines the functions available to query the underlying metadata storage.

type MetadataStorageCtor 

type MetadataStorageCtor func() (MetadataStorage, error)

MetadataStorageCtor represents a client constructor to instantiate a metadata storage client.

type NaNReplacement 

type NaNReplacement int

NaNReplacement defines the type of replacement value to use for NaNs 

const (
	// Null replaces NaN values with Nil, which will result in 'null' being encoded into the JSON structure
	Null NaNReplacement = iota + 1
	// EmptyString replaces NaN values with an empty string, which will result in "" being encoded into the JSON structure
	EmptyString
)

type NullableFloat64 

type NullableFloat64 float64

NullableFloat64 is float64 with custom JSON marshalling to allow for NaN values to be handled gracefully.

func (NullableFloat64) MarshalJSON 

func (f NullableFloat64) MarshalJSON() ([]byte, error)

MarshalJSON provides a custom float JSON marshaller that will handle a NaN float64 value by replacing it with empty data.

type Prediction 

type Prediction struct {
	RequestID        string    `json:"requestId"`
	Dataset          string    `json:"dataset"`
	Target           string    `json:"target"`
	FittedSolutionID string    `json:"fittedSolutionId"`
	Progress         string    `json:"progress"`
	CreatedTime      time.Time `json:"timestamp"`
	LastUpdatedTime  time.Time `json:"lastUpdatedTime"`
}

Prediction represents the prediction metadata.

type Request 

type Request struct {
	RequestID       string        `json:"requestId"`
	Dataset         string        `json:"dataset"`
	Progress        string        `json:"progress"`
	CreatedTime     time.Time     `json:"timestamp"`
	LastUpdatedTime time.Time     `json:"lastUpdatedTime"`
	Features        []*Feature    `json:"features"`
	Filters         *FilterParams `json:"filters"`
}

Request represents the request metadata.

func (*Request) TargetFeature 

func (r *Request) TargetFeature() string

TargetFeature returns the target feature out of the feature set.

type Solution 

type Solution struct {
	SolutionID          string            `json:"solutionId"`
	ExplainedSolutionID string            `json:"explainedSolutionId"`
	RequestID           string            `json:"requestId"`
	CreatedTime         time.Time         `json:"timestamp"`
	State               *SolutionState    `json:"state"`
	Results             []*SolutionResult `json:"results"`
	Scores              []*SolutionScore  `json:"scores"`
	IsBad               bool              `json:"isBad"`
}

Solution is a container for a TA2 solution.

type SolutionExplainResult 

type SolutionExplainResult struct {
	ResultURI       string
	Values          [][]string
	D3MIndexIndex   int
	ParsingFunction func([]string) (*SolutionExplainValues, error)
}

SolutionExplainResult captures the explainable output by row.

type SolutionExplainValues 

type SolutionExplainValues struct {
	LowConfidence  float64     `json:"lowConfidence,omitempty"`
	HighConfidence float64     `json:"highConfidence,omitempty"`
	GradCAM        [][]float64 `json:"gradCAM,omitempty"`
	Confidence     float64     `json:"confidence,omitempty"`
	Rank           float64     `json:"rank,omitempty"`
}

SolutionExplainValues represent use case specific explain output by row.

type SolutionFeatureWeight 

type SolutionFeatureWeight struct {
	ResultURI string
	D3MIndex  int64
	Weights   map[string]float64
}

SolutionFeatureWeight captures the weights for a given d3m index and result.

type SolutionResult 

type SolutionResult struct {
	FittedSolutionID string                         `json:"fittedSolutionId"`
	ProduceRequestID string                         `json:"produceRequestId"`
	SolutionID       string                         `json:"solutionId"`
	Dataset          string                         `json:"dataset"`
	ResultType       string                         `json:"result_type"`
	ResultURI        string                         `json:"requestUri"`
	ResultUUID       string                         `json:"resultId"`
	Progress         string                         `json:"progress"`
	OutputType       string                         `json:"outputType"`
	CreatedTime      time.Time                      `json:"timestamp"`
	ExplainOutput    []*SolutionResultExplainOutput `json:"-"`
}

SolutionResult represents the solution result metadata.

type SolutionResultExplainOutput 

type SolutionResultExplainOutput struct {
	ResultID string
	URI      string
	Type     string
}

SolutionResultExplainOutput captures the explainable output from a produce call.

type SolutionScore 

type SolutionScore struct {
	SolutionID     string  `json:"solutionId"`
	Metric         string  `json:"metric"`
	Label          string  `json:"label"`
	Score          float64 `json:"value"`
	SortMultiplier float64 `json:"sortMultiplier"`
}

SolutionScore represents the result score data.

type SolutionState 

type SolutionState struct {
	SolutionID  string    `json:"solutionId"`
	Progress    string    `json:"progress"`
	CreatedTime time.Time `json:"timestamp"`
}

SolutionState represents the state updates for a solution.

type SolutionStorage 

type SolutionStorage interface {
	PersistPrediction(requestID string, dataset string, target string, fittedSolutionID string, progress string, createdTime time.Time) error
	PersistRequest(requestID string, dataset string, progress string, createdTime time.Time) error
	PersistRequestFeature(requestID string, featureName string, featureType string) error
	PersistRequestFilters(requestID string, filters *FilterParams) error
	PersistSolution(requestID string, solutionID string, explainedSolutionID string, createdTime time.Time) error
	PersistSolutionWeight(solutionID string, featureName string, featureIndex int64, weight float64) error
	PersistSolutionState(solutionID string, progress string, createdTime time.Time) error
	PersistSolutionResult(solutionID string, fittedSolutionID string, produceRequestID string, resultType string, resultUUID string, resultURI string, progress string, createdTime time.Time) error
	PersistSolutionExplainedOutput(resultUUID string, explainOutput map[string]*SolutionExplainResult) error
	PersistSolutionScore(solutionID string, metric string, score float64) error
	UpdateRequest(requestID string, progress string, updatedTime time.Time) error
	UpdateSolution(solutionID string, explainedSolutionID string) error
	FetchRequest(requestID string) (*Request, error)
	FetchRequestBySolutionID(solutionID string) (*Request, error)
	FetchRequestByFittedSolutionID(fittedSolutionID string) (*Request, error)
	FetchRequestByDatasetTarget(dataset string, target string) ([]*Request, error)
	FetchRequestFeatures(requestID string) ([]*Feature, error)
	FetchRequestFilters(requestID string, features []*Feature) (*FilterParams, error)
	FetchSolution(solutionID string) (*Solution, error)
	FetchExplainValues(dataset string, storageName string, d3mIndex []int, resultUUID string) ([]SolutionExplainValues, error)
	FetchSolutionsByDatasetTarget(dataset string, target string) ([]*Solution, error)
	FetchSolutionsByRequestID(requestID string) ([]*Solution, error)
	FetchSolutionWeights(solutionID string) ([]*SolutionWeight, error)
	FetchSolutionResultByUUID(resultUUID string) (*SolutionResult, error)
	FetchSolutionResults(solutionID string) ([]*SolutionResult, error)
	FetchSolutionResultsByFittedSolutionID(fittedSolutionID string) ([]*SolutionResult, error)
	FetchSolutionResultByProduceRequestID(produceRequestID string) (*SolutionResult, error)
	FetchPredictionResultByProduceRequestID(produceRequestID string) (*SolutionResult, error)
	FetchPredictionResultByUUID(reusultUUID string) (*SolutionResult, error)
	FetchSolutionScores(solutionID string) ([]*SolutionScore, error)
	FetchPrediction(requestID string) (*Prediction, error)
	FetchPredictionsByFittedSolutionID(fittedSolutionID string) ([]*Prediction, error)
}

SolutionStorage defines the functions available to query the underlying solution storage.

type SolutionStorageCtor 

type SolutionStorageCtor func() (SolutionStorage, error)

SolutionStorageCtor represents a client constructor to instantiate a solution storage client.

type SolutionVariable 

type SolutionVariable struct {
	Key         string  `json:"key"`
	DisplayName string  `json:"displayName"`
	HeaderName  string  `json:"headerName"`
	Rank        float64 `json:"rank"`
	Type        string  `json:"varType"`
}

SolutionVariable represents the basic variable data for a solution

func SolutionVariableFromModelVariable 

func SolutionVariableFromModelVariable(variable *model.Variable, rank float64) *SolutionVariable

SolutionVariableFromModelVariable creates a solution variable from a model variable.

type SolutionWeight 

type SolutionWeight struct {
	SolutionID   string
	FeatureIndex int64
	FeatureName  string
	Weight       float64
}

SolutionWeight captures the weights for a given d3m index and result.

type SummaryMode 

type SummaryMode int

SummaryMode defines the summary display modes.

func SummaryModeFromString 

func SummaryModeFromString(s string) (SummaryMode, error)

SummaryModeFromString creates a SummaryMode from the supplied string

type TimeseriesData 

type TimeseriesData struct {
	VarKey     string                   `json:"variableKey"`
	SeriesID   string                   `json:"seriesID"`
	Timeseries []*TimeseriesObservation `json:"timeseries"`
	IsDateTime bool                     `json:"isDateTime"`
	Min        float64                  `json:"min"`
	Max        float64                  `json:"max"`
	Mean       float64                  `json:"mean"`
}

TimeseriesData represents the result of a timeseries request.

type TimeseriesObservation 

type TimeseriesObservation struct {
	Value          NullableFloat64 `json:"value"`
	Time           float64         `json:"time"`
	ConfidenceLow  NullableFloat64 `json:"confidenceLow"`
	ConfidenceHigh NullableFloat64 `json:"confidenceHigh"`
}

TimeseriesObservation represents a timeseries value along with confidences.

type TimeseriesOp 

type TimeseriesOp string

TimeseriesOp defines the operation to aggregate timeseries values that fall into the same bucket.

type VariableSummary 

type VariableSummary struct {
	Label            string     `json:"label"`
	Key              string     `json:"key"`
	Description      string     `json:"description"`
	DistilRole       []string   `json:"distilRole"`
	Type             string     `json:"type"`
	VarType          string     `json:"varType"`
	Dataset          string     `json:"dataset"`
	Baseline         *Histogram `json:"baseline"`
	Filtered         *Histogram `json:"filtered"`
	Timeline         *Histogram `json:"timeline"`
	TimelineBaseline *Histogram `json:"timelineBaseline"`
	TimelineType     string     `json:"timelineType"`
	Weighted         bool       `json:"weighted"`
}

VariableSummary represents a summary of the variable values.

func (*VariableSummary) EmptyFilteredHistogram 

func (s *VariableSummary) EmptyFilteredHistogram()

EmptyFilteredHistogram fills the filtered portion of the summary with empty bucket counts

type VariableUpdate 

type VariableUpdate struct {
	Index string `json:"index"`
	Name  string `json:"name"`
	Value string `json:"value"`
}

VariableUpdate captures the information to update the dataset data.

func ParseVariableUpdateList 

func ParseVariableUpdateList(data map[string]interface{}) ([]*VariableUpdate, error)

ParseVariableUpdateList returns a list of parsed variable updates.

Source Files

View all Source files

Directories

Path Synopsis
storage
datamart
elastic
file
postgres

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.