catalog

Documentation

Index

• Constants
• func GetDefaultGrpcOptions() []grpc_retry.CallOption
• func TransformFromParameters(m []*datacatalog.Parameter) (*core.LiteralMap, error)
• func TransformToInputParameters(ctx context.Context, m *core.LiteralMap) ([]*datacatalog.Parameter, error)
• func TransformToOutputParameters(ctx context.Context, m *core.LiteralMap) ([]*datacatalog.Parameter, error)
• type Client
  • func NewCatalogClient(ctx context.Context, store storage.ProtobufStore) (Client, error)
• type Config
  • func GetConfig() *Config
  • func (Config) GetPFlagSet(prefix string) *pflag.FlagSet
• type DiscoveryType
• type LegacyDiscovery
  • func NewLegacyDiscovery(discoveryEndpoint string, store storage.ProtobufStore) *LegacyDiscovery
  • func (d *LegacyDiscovery) Get(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)
  • func (d *LegacyDiscovery) Put(ctx context.Context, task *core.TaskTemplate, ...) error
• type MockCatalogClient
  • func (m *MockCatalogClient) Get(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)
  • func (m *MockCatalogClient) Put(ctx context.Context, task *core.TaskTemplate, ...) error
• type NoOpDiscovery
  • func NewNoOpDiscovery() *NoOpDiscovery
  • func (d *NoOpDiscovery) Get(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)
  • func (d *NoOpDiscovery) Put(ctx context.Context, task *core.TaskTemplate, ...) error

Constants

const ConfigSectionKey = "catalog-cache"

Functions

func GetDefaultGrpcOptions

func GetDefaultGrpcOptions() []grpc_retry.CallOption

func TransformFromParameters

func TransformFromParameters(m []*datacatalog.Parameter) (*core.LiteralMap, error)

func TransformToInputParameters

func TransformToInputParameters(ctx context.Context, m *core.LiteralMap) ([]*datacatalog.Parameter, error)

Hash each value in the map and return it as the parameter value to be used to generate the Provenance.

func TransformToOutputParameters

func TransformToOutputParameters(ctx context.Context, m *core.LiteralMap) ([]*datacatalog.Parameter, error)

Types

type Client

type Client interface {
	Get(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)
	Put(ctx context.Context, task *core.TaskTemplate, execID *core.TaskExecutionIdentifier, inputPath storage.DataReference, outputPath storage.DataReference) error
}

func NewCatalogClient

func NewCatalogClient(ctx context.Context, store storage.ProtobufStore) (Client, error)

type Config

type Config struct {
	Type     DiscoveryType `json:"type" pflag:"\"noop\", Catalog Implementation to use"`
	Endpoint string        `json:"endpoint" pflag:"\"\", Endpoint for catalog service"`
	Insecure bool          `json:"insecure" pflag:"false, Use insecure grpc connection"`
}

func GetConfig

func GetConfig() *Config

Gets loaded config for Discovery

func (Config) GetPFlagSet

func (Config) GetPFlagSet(prefix string) *pflag.FlagSet

GetPFlagSet will return strongly types pflags for all fields in Config and its nested types. The format of the flags is json-name.json-sub-name... etc.

type DiscoveryType

type DiscoveryType = string

const (
	NoOpDiscoveryType   DiscoveryType = "noop"
	LegacyDiscoveryType DiscoveryType = "legacy"
	DataCatalogType     DiscoveryType = "datacatalog"
)

type LegacyDiscovery

type LegacyDiscovery struct {
	// contains filtered or unexported fields
}

LegacyDiscovery encapsulates interactions with the Discovery service using a protobuf provided gRPC client.

func NewLegacyDiscovery

func NewLegacyDiscovery(discoveryEndpoint string, store storage.ProtobufStore) *LegacyDiscovery

func (*LegacyDiscovery) Get

func (d *LegacyDiscovery) Get(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)

func (*LegacyDiscovery) Put

func (d *LegacyDiscovery) Put(ctx context.Context, task *core.TaskTemplate, execID *core.TaskExecutionIdentifier, inputPath storage.DataReference, outputPath storage.DataReference) error

type MockCatalogClient

type MockCatalogClient struct {
	GetFunc func(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)
	PutFunc func(ctx context.Context, task *core.TaskTemplate, execID *core.TaskExecutionIdentifier, inputPath storage.DataReference, outputPath storage.DataReference) error
}

func (*MockCatalogClient) Get

func (m *MockCatalogClient) Get(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)

func (*MockCatalogClient) Put

func (m *MockCatalogClient) Put(ctx context.Context, task *core.TaskTemplate, execID *core.TaskExecutionIdentifier, inputPath storage.DataReference, outputPath storage.DataReference) error

type NoOpDiscovery

type NoOpDiscovery struct{}

NoOpDiscovery

func NewNoOpDiscovery

func NewNoOpDiscovery() *NoOpDiscovery

func (*NoOpDiscovery) Get

func (d *NoOpDiscovery) Get(ctx context.Context, task *core.TaskTemplate, inputPath storage.DataReference) (*core.LiteralMap, error)

func (*NoOpDiscovery) Put

func (d *NoOpDiscovery) Put(ctx context.Context, task *core.TaskTemplate, execID *core.TaskExecutionIdentifier, inputPath storage.DataReference, outputPath storage.DataReference) error