base

package
v0.28.0-beta Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 24, 2024 License: MIT Imports: 24 Imported by: 4

Documentation

Index

Constants

View Source 
const OpenAPITemplate = `` /* 1538-byte string literal not displayed */
View Source 
const SecretKeyword = "__INSTILL_SECRET"

SecretKeyword is a keyword to reference a secret in a component configuration. When a component detects this value in a configuration parameter, it will used the pre-configured value, injected at initialization.

Variables

View Source 
var InstillAcceptFormatsMeta = jsonschema.MustCompileString("instillAcceptFormats.json", `{
	"properties" : {
		"instillAcceptFormats": {
			"type": "array",
			"items": {
				"type": "string"
			}
		}
	}
}`)
View Source 
var InstillFormatMeta = jsonschema.MustCompileString("instillFormat.json", `{
	"properties" : {
		"instillFormat": {
			"type": "string"
		}
	}
}`)

Functions

func CompileInstillAcceptFormats 

func CompileInstillAcceptFormats(sch *structpb.Struct) error

func CompileInstillFormat 

func CompileInstillFormat(sch *structpb.Struct) error

func ConcurrentExecutor 

func ConcurrentExecutor(ctx context.Context, jobs []*Job, execute func(*structpb.Struct, *Job, context.Context) (*structpb.Struct, error)) error

func ConvertFromStructpb 

func ConvertFromStructpb(from *structpb.Struct, to any) error

ConvertFromStructpb converts from structpb.Struct to a struct

func ConvertToStructpb 

func ConvertToStructpb(from any) (*structpb.Struct, error)

ConvertToStructpb converts from a struct to structpb.Struct

func FormatErrors 

func FormatErrors(inputPath string, e jsonschema.Detailed, errors *[]string)

func GetBase64FileExtension 

func GetBase64FileExtension(b64 string) string

return the extension of the file from the base64 string, in the "jpeg" , "png" format, check with provided header

func NewInputReader 

func NewInputReader(input *structpb.Struct, schema string) *inputReader

func NewOutputWriter 

func NewOutputWriter(ow OutputWriter, schema string) *outputWriter

func NewUnresolvedCredential 

func NewUnresolvedCredential(key string) error

NewUnresolvedCredential returns an end-user error signaling that the component setup contains credentials that reference a global secret that wasn't injected into the component.

func ReadFromGlobalConfig 

func ReadFromGlobalConfig(key string, secrets map[string]any) string

ReadFromGlobalConfig looks up a component credential field from a secret map that comes from the environment variable configuration.

Config parameters are defined with snake_case, but the environment variable configuration loader replaces underscores by dots, so we can't use the parameter key directly. TODO using camelCase in configuration fields would fix this issue.

func RenderJSON 

func RenderJSON(tasksJSONBytes []byte, additionalJSONBytes map[string][]byte) ([]byte, error)

func SequentialExecutor 

func SequentialExecutor(ctx context.Context, jobs []*Job, execute func(*structpb.Struct) (*structpb.Struct, error)) error

func TaskIDToTitle 

func TaskIDToTitle(id string) string

TaskIDToTitle builds a Task title from its ID. This is used when the `title` key in the task definition isn't present.

func TrimBase64Mime 

func TrimBase64Mime(b64 string) string

func Validate 

func Validate(data *structpb.Struct, jsonSchema string, target string) error

Validate the input and output format

Types

type Component 

type Component struct {
	Logger          *zap.Logger
	NewUsageHandler UsageHandlerCreator
	// contains filtered or unexported fields
}

Component implements the common component methods.

func (*Component) GetDefinition 

func (c *Component) GetDefinition(sysVars map[string]any, compConfig *ComponentConfig) (*pb.ComponentDefinition, error)

func (*Component) GetDefinitionID 

func (c *Component) GetDefinitionID() string

GetDefinitionID returns the component definition ID.

func (*Component) GetDefinitionUID 

func (c *Component) GetDefinitionUID() uuid.UUID

GetDefinitionUID returns the component definition UID.

func (*Component) GetLogger 

func (c *Component) GetLogger() *zap.Logger

GetLogger returns the component's logger. If it hasn't been initialized, a no-op logger is returned.

func (*Component) GetTaskInputSchemas 

func (c *Component) GetTaskInputSchemas() map[string]string

func (*Component) GetTaskOutputSchemas 

func (c *Component) GetTaskOutputSchemas() map[string]string

func (*Component) HandleVerificationEvent 

func (c *Component) HandleVerificationEvent(header map[string][]string, req *structpb.Struct, setup map[string]any) (isVerification bool, resp *structpb.Struct, err error)

func (*Component) IsSecretField 

func (c *Component) IsSecretField(target string) bool

IsSecretField checks if the target field is secret field

func (*Component) ListInputAcceptFormatsFields 

func (c *Component) ListInputAcceptFormatsFields() (map[string]map[string][]string, error)

func (*Component) ListOutputFormatsFields 

func (c *Component) ListOutputFormatsFields() (map[string]map[string]string, error)

func (*Component) ListSecretFields 

func (c *Component) ListSecretFields() ([]string, error)

ListSecretFields lists the secret fields by definition id

func (*Component) LoadDefinition 

func (c *Component) LoadDefinition(definitionJSONBytes, setupJSONBytes, tasksJSONBytes []byte, additionalJSONBytes map[string][]byte) error

LoadDefinition loads the component definitions from json files

func (*Component) ParseEvent 

func (c *Component) ParseEvent(ctx context.Context, req *structpb.Struct, setup map[string]any) (parsed *structpb.Struct, err error)

func (*Component) Test 

func (c *Component) Test(sysVars map[string]any, setup *structpb.Struct) error

func (*Component) UsageHandlerCreator 

func (c *Component) UsageHandlerCreator() UsageHandlerCreator

UsageHandlerCreator returns a function to initialize a UsageHandler. If the component doesn't have such function initialized, a no-op usage handler creator is returned.

type ComponentConfig 

type ComponentConfig struct {
	Task  string
	Input map[string]any
	Setup map[string]any
}

type ComponentExecution 

type ComponentExecution struct {
	Component IComponent

	// Component ID is the ID of the component *as defined in the recipe*. This
	// identifies an instance of a component, which holds a given configuration
	// (task, setup, input parameters, etc.).
	//
	// NOTE: this is a property of the component not of the execution. However,
	// right now components are being created on startup and only executions
	// are created every time a pipeline is triggered. Therefore, at the moment
	// there's no intermediate entity reflecting "a component within a
	// pipeline". Since we need to access the component ID for e.g. logging /
	// metric collection purposes, for now this information will live in the
	// execution, but note that several executions might have the same
	// component ID.
	ComponentID     string
	SystemVariables map[string]any
	Setup           *structpb.Struct
	Task            string
}

ComponentExecution implements the common methods for component execution.

func (*ComponentExecution) FillInDefaultValues 

func (e *ComponentExecution) FillInDefaultValues(input *structpb.Struct) (*structpb.Struct, error)

func (*ComponentExecution) GetComponent 

func (e *ComponentExecution) GetComponent() IComponent

GetComponent returns the component interface that is triggering the execution.

func (*ComponentExecution) GetComponentID 

func (e *ComponentExecution) GetComponentID() string

GetComponentID returns the ID of the component that's being executed.

func (*ComponentExecution) GetLogger 

func (e *ComponentExecution) GetLogger() *zap.Logger

func (*ComponentExecution) GetSetup 

func (e *ComponentExecution) GetSetup() *structpb.Struct

func (*ComponentExecution) GetSystemVariables 

func (e *ComponentExecution) GetSystemVariables() map[string]any

func (*ComponentExecution) GetTask 

func (e *ComponentExecution) GetTask() string

func (*ComponentExecution) GetTaskInputSchema 

func (e *ComponentExecution) GetTaskInputSchema() string

func (*ComponentExecution) GetTaskOutputSchema 

func (e *ComponentExecution) GetTaskOutputSchema() string

func (*ComponentExecution) UsesInstillCredentials 

func (e *ComponentExecution) UsesInstillCredentials() bool

UsesInstillCredentials indicates wether the component setup includes the use of global secrets (as opposed to a bring-your-own-key configuration) to connect to external services. Components should override this method when they have the ability to read global secrets and be executed without explicit credentials.

type ErrorHandler 

type ErrorHandler interface {
	Error(ctx context.Context, err error)
}

type ExecutionWrapper 

type ExecutionWrapper struct {
	IExecution
}

ExecutionWrapper performs validation and usage collection around the execution of a component.

func (*ExecutionWrapper) Execute 

func (e *ExecutionWrapper) Execute(ctx context.Context, jobs []*Job) (err error)

Execute wraps the execution method with validation and usage collection.

type IComponent 

type IComponent interface {
	GetDefinitionID() string
	GetDefinitionUID() uuid.UUID
	GetLogger() *zap.Logger
	GetTaskInputSchemas() map[string]string
	GetTaskOutputSchemas() map[string]string

	LoadDefinition(definitionJSON, setupJSON, tasksJSON []byte, additionalJSONBytes map[string][]byte) error

	// Note: Some content in the definition JSON schema needs to be generated
	// by sysVars or component setting.
	GetDefinition(sysVars map[string]any, compConfig *ComponentConfig) (*pb.ComponentDefinition, error)

	// CreateExecution takes a ComponentExecution that can be used to compose
	// the core component behaviour with the particular business logic in the
	// implmentation.
	CreateExecution(base ComponentExecution) (IExecution, error)
	Test(sysVars map[string]any, config *structpb.Struct) error

	IsSecretField(target string) bool

	// Note: These two functions are for the pipeline run-on-event feature,
	// which is still experimental and may change at any time.
	HandleVerificationEvent(header map[string][]string, req *structpb.Struct, setup map[string]any) (isVerification bool, resp *structpb.Struct, err error)
	ParseEvent(ctx context.Context, req *structpb.Struct, setup map[string]any) (parsed *structpb.Struct, err error)

	UsageHandlerCreator() UsageHandlerCreator
}

IComponent is the interface that wraps the basic component methods. All component need to implement this interface.

type IExecution 

type IExecution interface {
	GetTask() string
	GetLogger() *zap.Logger
	GetTaskInputSchema() string
	GetTaskOutputSchema() string
	GetSystemVariables() map[string]any
	GetComponent() IComponent
	GetComponentID() string
	UsesInstillCredentials() bool

	Execute(context.Context, []*Job) error
}

IExecution allows components to be executed.

type InputReader 

type InputReader interface {
	Read(ctx context.Context) (input *structpb.Struct, err error)
}

type InstillAcceptFormatsCompiler 

type InstillAcceptFormatsCompiler struct{}

func (InstillAcceptFormatsCompiler) Compile 

func (InstillAcceptFormatsCompiler) Compile(ctx jsonschema.CompilerContext, m map[string]interface{}) (jsonschema.ExtSchema, error)

type InstillAcceptFormatsSchema 

type InstillAcceptFormatsSchema []string

func (InstillAcceptFormatsSchema) Validate 

func (s InstillAcceptFormatsSchema) Validate(ctx jsonschema.ValidationContext, v interface{}) error

type InstillExtension 

type InstillExtension struct {
	jsoniter.DummyExtension
}

func (*InstillExtension) UpdateStructDescriptor 

func (e *InstillExtension) UpdateStructDescriptor(structDescriptor *jsoniter.StructDescriptor)

type InstillFormatCompiler 

type InstillFormatCompiler struct{}

func (InstillFormatCompiler) Compile 

func (InstillFormatCompiler) Compile(ctx jsonschema.CompilerContext, m map[string]interface{}) (jsonschema.ExtSchema, error)

type InstillFormatSchema 

type InstillFormatSchema string

func (InstillFormatSchema) Validate 

func (s InstillFormatSchema) Validate(ctx jsonschema.ValidationContext, v interface{}) error

type Job 

type Job struct {
	Input  InputReader
	Output OutputWriter
	Error  ErrorHandler
}

func GenerateMockJob 

func GenerateMockJob(c minimock.Tester) (ir *mock.InputReaderMock, ow *mock.OutputWriterMock, eh *mock.ErrorHandlerMock, job *Job)

type OutputWriter 

type OutputWriter interface {
	Write(ctx context.Context, output *structpb.Struct) (err error)
}

type UsageHandler 

type UsageHandler interface {
	Check(ctx context.Context, inputs []*structpb.Struct) error
	Collect(ctx context.Context, inputs, outputs []*structpb.Struct) error
}

UsageHandler allows the component execution wrapper to add checks and collect usage metrics around a component execution.

func NewNoopUsageHandler 

func NewNoopUsageHandler(IExecution) (UsageHandler, error)

NewNoopUsageHandler is a no-op usage handler initializer.

type UsageHandlerCreator 

type UsageHandlerCreator func(IExecution) (UsageHandler, error)

UsageHandlerCreator returns a function to initialize a UsageHandler.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.