abstract

package
v0.0.0-...-2c27dae Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jul 23, 2022 License: Apache-2.0 Imports: 9 Imported by: 20

Documentation

Index

Constants

View Source 
const DefaultManifestFilename string = "MANIFEST.yaml"
View Source 
const (
	L_SCHEMA = "schema"
)

Variables

This section is empty.

Functions

func NewDatabaseBuilder 

func NewDatabaseBuilder() *databaseBuilder

NewDatabaseBuilder ... builder for a database asset type

func NewStreamBuilder 

func NewStreamBuilder() *streamBuilder

func NewTableBuilder 

func NewTableBuilder() *tableBuilder

NewTableBuilder ... table builder

Types

type Asset 

type Asset struct {
	// asset last found by crawler at - only added by service (not crawler/manifest itself, i.e. no yaml)
	LastDiscoveredAt time.Time `json:"last-discovered-at" yaml:"last-discovered-at,omitempty"`
	// asset publication datetime
	PublishedOn time.Time `yaml:"published-on" json:"published-on"`
	// name of the asset
	Name string `yaml:"name" json:"name"`
	// description of the asset
	Description string `yaml:"description" json:"description"`
	// the list of assets this depends on
	DependsOn []string `yaml:"depends-on" json:"depends-on"`
	// asset type
	Type AssetType `yaml:"type" json:"type"`
	// labels for the specific asset
	Labels map[string]interface{} `yaml:"labels" json:"labels"`
	// tags are flags used to simplify asset search
	Tags []string `yaml:"tags" json:"tags"`
	// versions specify available variants of the same asset
	Versions map[string]interface{} `yaml:"versions" json:"versions"`
}

Asset ... managed resource

func NewDatasetAsset 

func NewDatasetAsset() *Asset

func ParseAsset 

func ParseAsset(data []byte) (*Asset, error)

ParseAsset ... Parse an asset specification file

func (*Asset) Validate 

func (asset *Asset) Validate() error

Validate ... Validate asset specification file

type AssetDAOProvider 

type AssetDAOProvider interface {
	Init(*conf.DataSourceDefinition)
	Upsert(asset *Asset) error
	GetById(id string) (*Asset, error)
	GetByName(id string) (*Asset, error)
	SearchAssetsByTags(tags []string, limit int, page int) (*Paginated[Asset], error)
	ListAllAssets(limit int, page int) (*Paginated[Asset], error)
	Search(query string, limit int, page int) (*Paginated[Asset], error)
	CloseConnection()
}

AssetDAOProvider ... The interface each dao must implement

type AssetType 

type AssetType string

AssetType ... Asset type information

func (*AssetType) Validate 

func (assetType *AssetType) Validate() error

type CatalogueService 

type CatalogueService interface {
	Init(cfg *conf.Config) *resterrors.RestErr
	UpsertAssets(assets *[]Asset) (*[]Asset, *resterrors.RestErr)
	GetAssetByID(assetID string) (*Asset, *resterrors.RestErr)
	GetAssetByName(name string) (*Asset, *resterrors.RestErr)
	SearchAssetsByTags(tags []string, limit int, page int) (*Paginated[Asset], *resterrors.RestErr)
	Search(query string, limit int, page int) (*Paginated[Asset], *resterrors.RestErr)
	ListAllAssets(limit int, page int) (*Paginated[Asset], *resterrors.RestErr)
}

CatalogueService ... CatalogueService Interface listing service methods

type ColumnInfo 

type ColumnInfo struct {
	Type    string
	Comment string
}

ColumnInfo ... Type and description for a table column

type ConfigurableConnector 

type ConfigurableConnector struct {
	RequiredFields map[string]string
	OptionalFields map[string]string
}

func (*ConfigurableConnector) ValidateDataSourceDefinition 

func (c *ConfigurableConnector) ValidateDataSourceDefinition(def *conf.DataSourceDefinition) error

ValidateDataSourceDefinition ... validates the provided data source definition

type ConnectorProvider 

type ConnectorProvider interface {
	ValidateDataSourceDefinition(*conf.DataSourceDefinition) error
	InitConnection(*conf.DataSourceDefinition)
	CloseConnection()
}

ConnectorProvider ... The interface each connector must implement

type Crawler 

type Crawler interface {
	InitConnection(cfg *conf.Config) (Crawler, error)
	WalkWithFilter(root string, filenameFilter string) ([]Asset, error)
}

type DBInfo 

type DBInfo struct {
	Name    string
	Comment string
}

DBInfo ... Name and description for a database

func GetDBInfoByName 

func GetDBInfoByName(dbName string) (DBInfo, error)

Partial constructor for DBInfo

func (*DBInfo) BuildAsset 

func (db *DBInfo) BuildAsset() (*Asset, error)

type DeequAnalyzer 

type DeequAnalyzer struct {
	AnalyzerName string `json:"analyzerName"`
	Column       string `json:"column"`
}

type DeequAnalyzerContext 

type DeequAnalyzerContext struct {
	MetricMap []DeequMetricInstance `json:"metricMap"`
}

type DeequMetric 

type DeequMetric struct {
	ResultKey       *DeequResultKey       `json:"resultKey,omitempty"`
	AnalyzerContext *DeequAnalyzerContext `json:"analyzerContext,omitempty"`
}

DeequMetric ...

type DeequMetricInstance 

type DeequMetricInstance struct {
	Analyzer DeequAnalyzer    `json:"analyzer"`
	Metric   DeequMetricValue `json:"metric"`
}

type DeequMetricValue 

type DeequMetricValue struct {
	MetricName string  `json:"metricName"`
	Entity     string  `json:"entity"`
	Instance   string  `json:"instance"`
	Name       string  `json:"name"`
	Value      float64 `json:"value"`
}

type DeequResultKey 

type DeequResultKey struct {
	DataSetDate int64             `json:"dataSetDate"`
	Tags        map[string]string `json:"tags"`
}

type Embedding 

type Embedding struct {
	Id         string    `json:"id,omitempty"`
	Name       string    `json:"name,omitempty"`
	InsertedAt time.Time `json:"inserted_at,omitempty"`
	Vector     []float32 `json:"vector,omitempty"`
}

Embedding ... a named feature vector to be used for similarity search

func (*Embedding) Validate 

func (em *Embedding) Validate() error

Validate ... validate an embedding

type EmbeddingDAOProvider 

type EmbeddingDAOProvider interface {
	Init(*conf.DataSourceDefinition)
	Upsert(e []Embedding) error
	GetById(id string) (*Embedding, error)
	GetByName(name string) ([]Embedding, error)
	SimilarToThis(vector []float32, k int) ([]Embedding, error)
	DeleteByName(name string) error
	DeleteByIds(ids ...string) error
	CloseConnection()
}

EmbeddingDAOProvider ... The interface each dao must implement

type EmbeddingStoreService 

type EmbeddingStoreService interface {
	Init(cfg *conf.Config) *resterrors.RestErr
	UpsertEmbeddings(embeddings []Embedding) *resterrors.RestErr
	GetEmbeddingByID(id string) (*Embedding, *resterrors.RestErr)
	GetEmbeddingByName(name string) ([]Embedding, *resterrors.RestErr)
	SimilarToThis(vector []float32, k int) ([]Embedding, *resterrors.RestErr)
	DeleteEmbeddingByName(name string) *resterrors.RestErr
	DeleteEmbeddingByIds(ids ...string) *resterrors.RestErr
}

EmbeddingStoreService ... EmbeddingStoreService Interface listing service methods

type Feature 

type Feature struct {
	Name     string      `json:"name,omitempty"`
	Value    interface{} `json:"value,omitempty"`
	DataType string      `json:"data_type,omitempty"`
}

Feature ... a named variable with a data type

func (*Feature) Validate 

func (f *Feature) Validate() error

Validate ... validate a feature

type FeatureSet 

type FeatureSet struct {
	Name        string            `json:"name,omitempty"`
	InsertedAt  time.Time         `json:"inserted_at,omitempty"`
	Version     string            `json:"version,omitempty"`
	Features    []Feature         `json:"features,omitempty"`
	Description string            `json:"description,omitempty"`
	Labels      map[string]string `json:"labels,omitempty"`
}

FeatureSet ... a versioned set of features

func (*FeatureSet) Validate 

func (fs *FeatureSet) Validate() error

Validate ... validate a featureSet

type FeatureSetDAOProvider 

type FeatureSetDAOProvider interface {
	Init(*conf.DataSourceDefinition)
	Create(fs *FeatureSet) error
	GetById(id string) (*FeatureSet, error)
	GetByName(name string, limit int, page int) (*Paginated[FeatureSet], error)
	SearchFeatureSetsByLabels(labels map[string]string, limit int, page int) (*Paginated[FeatureSet], error)
	Search(query string, limit int, page int) (*Paginated[FeatureSet], error)
	ListAllFeatureSets(limit int, page int) (*Paginated[FeatureSet], error)
	CloseConnection()
}

FeatureSetDAOProvider ... The interface each dao must implement

type FeatureStoreService 

type FeatureStoreService interface {
	Init(cfg *conf.Config) *resterrors.RestErr
	CreateFeatureSet(fs FeatureSet) (*FeatureSet, *resterrors.RestErr)
	GetFeatureSetByID(fsID string) (*FeatureSet, *resterrors.RestErr)
	GetFeatureSetByName(fsName string, limit int, page int) (*Paginated[FeatureSet], *resterrors.RestErr)
	SearchFeatureSetsByLabels(labels map[string]string, limit int, page int) (*Paginated[FeatureSet], *resterrors.RestErr)
	Search(query string, limit int, page int) (*Paginated[FeatureSet], *resterrors.RestErr)
	ListAllFeatureSets(limit int, page int) (*Paginated[FeatureSet], *resterrors.RestErr)
}

Service ... FeatureStoreService Interface listing implemented methods

type Metric 

type Metric struct {
	*DeequMetric
}

func (*Metric) Validate 

func (m *Metric) Validate() error

Validate ... validate a metric

type MetricSet 

type MetricSet struct {
	// unique metric set name
	Name string `json:"name,omitempty"`
	// insertion time
	InsertedAt time.Time `json:"inserted_at,omitempty"`
	// version relates to the environment and the pipeline version
	Version string `json:"version,omitempty"`
	// description is related to the metrics and their extraction process and not the datasource they were calculated on
	Description string `json:"description,omitempty"`
	// labels used for query purposes
	Labels map[string]string `json:"labels,omitempty"`

	// actual metrics
	Metrics []Metric `json:"metrics,omitempty"`
}

MetricSet ... a timestamped set of Metrics

func (*MetricSet) Validate 

func (ms *MetricSet) Validate() error

Validate ... validate a metricSet

type MetricSetDAOProvider 

type MetricSetDAOProvider interface {
	Init(*conf.DataSourceDefinition)
	Create(m *MetricSet) error
	GetById(id string) (*MetricSet, error)
	GetByName(name string, limit int, page int) (*Paginated[MetricSet], error)
	SearchMetricSetsByLabels(labels map[string]string, limit int, page int) (*Paginated[MetricSet], error)
	ListAllMetricSets(limit int, page int) (*Paginated[MetricSet], error)
	Search(query string, limit int, page int) (*Paginated[MetricSet], error)
	CloseConnection()
}

MetricSetDAOProvider ... The interface each dao must implement

type MetricStoreService 

type MetricStoreService interface {
	Init(cfg *conf.Config) *resterrors.RestErr
	CreateMetricSet(ms MetricSet) (*MetricSet, *resterrors.RestErr)
	GetMetricSetByID(msID string) (*MetricSet, *resterrors.RestErr)
	GetMetricSetByName(msName string, limit int, page int) (*Paginated[MetricSet], *resterrors.RestErr)
	SearchMetricSetsByLabels(labels map[string]string, limit int, page int) (*Paginated[MetricSet], *resterrors.RestErr)
	Search(query string, limit int, page int) (*Paginated[MetricSet], *resterrors.RestErr)
	ListAllMetricSets(limit int, page int) (*Paginated[MetricSet], *resterrors.RestErr)
}

MetricStoreService ... MetricStoreService Interface listing service methods

type Paginable 

type Paginable interface {
	Asset | FeatureSet | MetricSet | Embedding
}

type Paginated 

type Paginated[T Paginable] struct {
	Data       *[]T           `json:"data"`
	Pagination PaginationData `json:"pagination"`
}

type PaginationData 

type PaginationData struct {
	Total     int64 `json:"total"`
	Page      int64 `json:"page"`
	PerPage   int64 `json:"perPage"`
	Prev      int64 `json:"prev"`
	Next      int64 `json:"next"`
	TotalPage int64 `json:"totalPage"`
}

func FromMongoPaginationData 

func FromMongoPaginationData(pagination paginate.PaginationData) PaginationData

type StreamInfo 

type StreamInfo struct {
	Name    string
	Comment string
	Schema  map[int]string
}

func GetStreamInfoByName 

func GetStreamInfoByName(streamName string) (StreamInfo, error)

Partial constructor for StreamInfo

func (*StreamInfo) BuildAsset 

func (si *StreamInfo) BuildAsset() (*Asset, error)

type TableInfo 

type TableInfo struct {
	Name    string
	Schema  map[string]ColumnInfo
	Comment string
}

TableInfo ... Name, schema and description for a table

func GetTableInfoByName 

func GetTableInfoByName(tableName string) (TableInfo, error)

Partial constructor for TableInfo

func (*TableInfo) BuildAsset 

func (tb *TableInfo) BuildAsset() (*Asset, error)

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.