datamart

package
v0.0.0-...-0d40728 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 12, 2021 License: Apache-2.0 Imports: 16 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	// ProvenanceNYU for NYU datamart
	ProvenanceNYU = "NYU"
	// ProvenanceISI for ISI datamart
	ProvenanceISI = "ISI"
)

Variables

This section is empty.

Functions

func NewISIMetadataStorage 

func NewISIMetadataStorage(outputPath string, config *env.Config, ingestConfig *task.IngestTaskConfig, clientCtor rest.ClientCtor) model.MetadataStorageCtor

NewISIMetadataStorage returns a constructor for an ISI datamart.

func NewNYUMetadataStorage 

func NewNYUMetadataStorage(outputPath string, config *env.Config, ingestConfig *task.IngestTaskConfig, clientCtor rest.ClientCtor) model.MetadataStorageCtor

NewNYUMetadataStorage returns a constructor for an NYU datamart.

Types

type ISIMaterializedDataset 

type ISIMaterializedDataset struct {
	Code    string `json:"code"`
	Message string `json:"message"`
	Data    string `json:"data"`
}

ISIMaterializedDataset container for the raw response from a materialize call to the ISI datamart.

type ISISearchResult 

type ISISearchResult struct {
	Summary         *ISISearchResultSummary     `json:"summary"`
	Score           float64                     `json:"score"`
	Metadata        []*ISISearchResultMetadata  `json:"metadata"`
	MaterializeInfo string                      `json:"materialize_info"`
	ID              string                      `json:"id"`
	Augmentation    *SearchResultAugmentation   `json:"augmentation,omitempty"`
	ColumnNames     *ISISearchResultColumnNames `json:"all_column_names,omitempty"`
	Sample          string                      `json:"sample"`
}

ISISearchResult contains a single result from a query to the ISI datamart.

type ISISearchResultColumnNames 

type ISISearchResultColumnNames struct {
	LeftNames  []string `json:"left_names"`
	RightNames []string `json:"right_names"`
}

ISISearchResultColumnNames is the name of struct that Phil did not comment and it caused linting warnings that were bothering me so now I've fixed it.

type ISISearchResultMaterialization 

type ISISearchResultMaterialization struct {
	ID           string                                      `json:"id"`
	Score        float64                                     `json:"score"`
	Metadata     *ISISearchResultMaterializationMetadata     `json:"metadata"`
	Augmentation *ISISearchResultMaterializationAugmentation `json:"augmentation"`
	DatamartType string                                      `json:"datamart_type"`
}

ISISearchResultMaterialization specifies how to materialize the dataset.

type ISISearchResultMaterializationAugmentation 

type ISISearchResultMaterializationAugmentation struct {
	Properties   string      `json:"properties"`
	LeftColumns  [][]float64 `json:"left_columns"`
	RightColumns [][]float64 `json:"right_columns"`
}

ISISearchResultMaterializationAugmentation specifies the materialization augmentation.

type ISISearchResultMaterializationMetadata 

type ISISearchResultMaterializationMetadata struct {
	ConnectionURL string                                              `json:"connection_url"`
	SearchResult  *ISISearchResultMaterializationMetadataSearchResult `json:"search_result"`
	QueryJSON     interface{}                                         `json:"query_json"`
	SearchType    string                                              `json:"search_type"`
}

ISISearchResultMaterializationMetadata specifies the materialization metadata.

type ISISearchResultMaterializationMetadataSearchResult 

type ISISearchResultMaterializationMetadataSearchResult struct {
	PNodesNeeded          []string `json:"p_nodes_needed"`
	TargetQNodeColumnName string   `json:"target_q_node_column_name"`
	NumberOfVectors       string   `json:"number_of_vectors"`
	QNodesList            []string `json:"q_nodes_list"`
}

ISISearchResultMaterializationMetadataSearchResult specifies the materialization search results.

type ISISearchResultMetadata 

type ISISearchResultMetadata struct {
	Selector []interface{}                    `json:"selector"`
	Metadata *ISISearchResultMetadataMetadata `json:"metadata"`
}

ISISearchResultMetadata specifies the metadata of the datamart dataset.

type ISISearchResultMetadataMetadata 

type ISISearchResultMetadataMetadata struct {
	StructuralType string                                    `json:"structural_type"`
	SemanticTypes  []interface{}                             `json:"semantic_types"`
	Dimension      *ISISearchResultMetadataMetadataDimension `json:"dimension"`
	Schema         string                                    `json:"schema"`
	Name           string                                    `json:"name"`
}

ISISearchResultMetadataMetadata specifies the structure of the datamart dataset.

type ISISearchResultMetadataMetadataDimension 

type ISISearchResultMetadataMetadataDimension struct {
	Name          string   `json:"name"`
	SemanticTypes []string `json:"semantic_types"`
	Length        float64  `json:"length"`
}

ISISearchResultMetadataMetadataDimension has the specification for a dimension in a dataset.

type ISISearchResultProvenance 

type ISISearchResultProvenance struct {
	Source string `json:"source"`
}

ISISearchResultProvenance defines the source of the data.

type ISISearchResultSummary 

type ISISearchResultSummary struct {
	Title                string   `json:"title"`
	DatamartID           string   `json:"Datamart ID"`
	Score                string   `json:"Score"`
	URL                  string   `json:"URL"`
	Columns              []string `json:"Columns"`
	RecommendJoinColumns string   `json:"Recommend Join Columns"`
}

ISISearchResultSummary has a summary of the search result.

type ISISearchResults 

type ISISearchResults struct {
	Results []*ISISearchResult `json:"results"`
}

ISISearchResults is the basic search result container for ISI searches.

type SearchQuery 

type SearchQuery struct {
	Dataset *SearchQueryDatasetProperties `json:"dataset,omitempty"`
}

SearchQuery contains the basic properties to query.

type SearchQueryDatasetProperties 

type SearchQueryDatasetProperties struct {
	About       string   `json:"about,omitempty"`
	Description []string `json:"description,omitempty"`
	Name        []string `json:"name,omitempty"`
	Keywords    []string `json:"keywords,omitempty"`
}

SearchQueryDatasetProperties represents queryin on metadata.

type SearchResult 

type SearchResult struct {
	ID                 string                    `json:"id"`
	Score              float64                   `json:"score"`
	Discoverer         string                    `json:"discoverer"`
	Metadata           *SearchResultMetadata     `json:"metadata"`
	Augmentation       *SearchResultAugmentation `json:"augmentation,omitempty"`
	SuppliedID         string                    `json:"supplied_id"`
	SuppliedResourceID string                    `json:"supplied_resource_id"`
}

SearchResult contains the basic dataset info.

type SearchResultAugmentation 

type SearchResultAugmentation struct {
	Type             string     `json:"type"`
	LeftColumns      [][]int    `json:"left_columns"`
	RightColumns     [][]int    `json:"right_columns"`
	LeftColumnsNames [][]string `json:"left_columns_names"`
}

SearchResultAugmentation contains data augmentation info.

type SearchResultColumn 

type SearchResultColumn struct {
	Name           string   `json:"name"`
	StructuralType string   `json:"structural_type"`
	SemanticTypes  []string `json:"semantic_types"`
}

SearchResultColumn has information on a dataset column.

type SearchResultMaterialize 

type SearchResultMaterialize struct {
	DirectURL string `json:"direct_url"`
	ID        string `json:"identifier"`
}

SearchResultMaterialize contains the materialization info.

type SearchResultMetadata 

type SearchResultMetadata struct {
	Name        string                   `json:"name"`
	Description string                   `json:"description"`
	Size        float64                  `json:"size"`
	NumRows     float64                  `json:"nb_rows"`
	Columns     []*SearchResultColumn    `json:"columns"`
	Materialize *SearchResultMaterialize `json:"materialize"`
	Date        string                   `json:"date"`
}

SearchResultMetadata represents the dataset metadata.

type SearchResults 

type SearchResults struct {
	Results []*SearchResult `json:"results"`
}

SearchResults is the basic search result container.

type Storage 

type Storage struct {
	// contains filtered or unexported fields
}

Storage accesses the underlying datamart instance.

func (*Storage) AddGroupedVariable 

func (s *Storage) AddGroupedVariable(dataset string, varName string, varDisplayName string, varType string, varRole []string, grouping model.BaseGrouping) error

AddGroupedVariable adds a variable grouping.

func (*Storage) AddVariable 

func (s *Storage) AddVariable(dataset string, varName string, varDisplayName string, varType string, varRole []string) error

AddVariable is not supported by the datamart.

func (*Storage) CloneDataset 

func (s *Storage) CloneDataset(dataset string, datasetNew string, storageNameNew string, folderNew string) error

CloneDataset is not supported (ES datasets are already ingested).

func (*Storage) DatasetExists 

func (s *Storage) DatasetExists(dataset string) (bool, error)

DatasetExists returns true if a dataset exists.

func (*Storage) DeleteDataset 

func (s *Storage) DeleteDataset(dataset string, softDelete bool) error

DeleteDataset deletes a dataset from the datamart.

func (*Storage) DeleteVariable 

func (s *Storage) DeleteVariable(dataset string, varName string) error

DeleteVariable is not supported by the datamart.

func (*Storage) DoesVariableExist 

func (s *Storage) DoesVariableExist(dataset string, varName string) (bool, error)

DoesVariableExist returns whether or not a variable exists.

func (*Storage) FetchDataset 

func (s *Storage) FetchDataset(datasetName string, includeIndex bool, includeMeta bool, includeSystemData bool) (*api.Dataset, error)

FetchDataset returns a dataset in the provided index.

func (*Storage) FetchDatasets 

func (s *Storage) FetchDatasets(includeIndex bool, includeMeta bool, includeSystemData bool) ([]*api.Dataset, error)

FetchDatasets returns all datasets in the provided index.

func (*Storage) FetchVariable 

func (s *Storage) FetchVariable(dataset string, varName string) (*model.Variable, error)

FetchVariable returns the variable for the provided dataset, and variable.

func (*Storage) FetchVariableDisplay 

func (s *Storage) FetchVariableDisplay(dataset string, varName string) (*model.Variable, error)

FetchVariableDisplay returns the display variable for the provided dataset, and variable.

func (*Storage) FetchVariables 

func (s *Storage) FetchVariables(dataset string, includeIndex bool, includeMeta bool, includeSystemData bool) ([]*model.Variable, error)

FetchVariables returns all the variables for the provided dataset.

func (*Storage) FetchVariablesByName 

func (s *Storage) FetchVariablesByName(dataset string, variables []string, includeIndex bool, includeMeta bool, includeSystemData bool) ([]*model.Variable, error)

FetchVariablesByName returns all the variables for the provided dataset and names.

func (*Storage) FetchVariablesDisplay 

func (s *Storage) FetchVariablesDisplay(dataset string) ([]*model.Variable, error)

FetchVariablesDisplay returns all the display variables for the provided dataset.

func (*Storage) ImportDataset 

func (s *Storage) ImportDataset(id string, uri string) (string, error)

ImportDataset makes the dataset available for ingest and returns the URI to use for ingest.

func (*Storage) IngestDataset 

func (s *Storage) IngestDataset(datasetSource metadata.DatasetSource, meta *model.Metadata) error

IngestDataset adds a document consisting of the metadata to the datamart.

func (*Storage) RemoveGroupedVariable 

func (s *Storage) RemoveGroupedVariable(datasetName string, grouping model.BaseGrouping) error

RemoveGroupedVariable removes a variable grouping.

func (*Storage) SearchDatasets 

func (s *Storage) SearchDatasets(terms string, baseDataset *api.Dataset, includeIndex bool, includeMeta bool, includeSystemData bool) ([]*api.Dataset, error)

SearchDatasets returns the datasets that match the search criteria in the provided index.

func (*Storage) SetDataType 

func (s *Storage) SetDataType(dataset string, varName string, varType string) error

SetDataType is not supported by the datamart.

func (*Storage) SetExtrema 

func (s *Storage) SetExtrema(dataset string, varName string, extrema *api.Extrema) error

SetExtrema is not supported by the datamart.

func (*Storage) UpdateDataset 

func (s *Storage) UpdateDataset(dataset *api.Dataset) error

UpdateDataset updates a document consisting of the metadata to the datamart.

func (*Storage) UpdateVariable 

func (s *Storage) UpdateVariable(dataset string, varName string, variableValue *model.Variable) error

UpdateVariable updates the values of a variable

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.