base

package
v0.28.0-beta Latest Latest
Warning

This package is not in the latest version of its module.

Go to latest
Published: Sep 24, 2024 License: MIT Imports: 24 Imported by: 1

Documentation

Index

Constants

View Source
const OpenAPITemplate = `` /* 1538-byte string literal not displayed */
View Source
const SecretKeyword = "__INSTILL_SECRET"

SecretKeyword is a keyword to reference a secret in a component configuration. When a component detects this value in a configuration parameter, it will used the pre-configured value, injected at initialization.

Variables

View Source
var InstillAcceptFormatsMeta = jsonschema.MustCompileString("instillAcceptFormats.json", `{
	"properties" : {
		"instillAcceptFormats": {
			"type": "array",
			"items": {
				"type": "string"
			}
		}
	}
}`)
View Source
var InstillFormatMeta = jsonschema.MustCompileString("instillFormat.json", `{
	"properties" : {
		"instillFormat": {
			"type": "string"
		}
	}
}`)

Functions

func CompileInstillAcceptFormats

func CompileInstillAcceptFormats(sch *structpb.Struct) error

func CompileInstillFormat

func CompileInstillFormat(sch *structpb.Struct) error

func ConcurrentExecutor

func ConcurrentExecutor(ctx context.Context, jobs []*Job, execute func(*structpb.Struct, *Job, context.Context) (*structpb.Struct, error)) error

func ConvertFromStructpb

func ConvertFromStructpb(from *structpb.Struct, to any) error

ConvertFromStructpb converts from structpb.Struct to a struct

func ConvertToStructpb

func ConvertToStructpb(from any) (*structpb.Struct, error)

ConvertToStructpb converts from a struct to structpb.Struct

func FormatErrors

func FormatErrors(inputPath string, e jsonschema.Detailed, errors *[]string)

func GetBase64FileExtension

func GetBase64FileExtension(b64 string) string

return the extension of the file from the base64 string, in the "jpeg" , "png" format, check with provided header

func NewInputReader

func NewInputReader(input *structpb.Struct, schema string) *inputReader

func NewOutputWriter

func NewOutputWriter(ow OutputWriter, schema string) *outputWriter

func NewUnresolvedCredential

func NewUnresolvedCredential(key string) error

NewUnresolvedCredential returns an end-user error signaling that the component setup contains credentials that reference a global secret that wasn't injected into the component.

func ReadFromGlobalConfig

func ReadFromGlobalConfig(key string, secrets map[string]any) string

ReadFromGlobalConfig looks up a component credential field from a secret map that comes from the environment variable configuration.

Config parameters are defined with snake_case, but the environment variable configuration loader replaces underscores by dots, so we can't use the parameter key directly. TODO using camelCase in configuration fields would fix this issue.

func RenderJSON

func RenderJSON(tasksJSONBytes []byte, additionalJSONBytes map[string][]byte) ([]byte, error)

func SequentialExecutor

func SequentialExecutor(ctx context.Context, jobs []*Job, execute func(*structpb.Struct) (*structpb.Struct, error)) error

func TaskIDToTitle

func TaskIDToTitle(id string) string

TaskIDToTitle builds a Task title from its ID. This is used when the `title` key in the task definition isn't present.

func TrimBase64Mime

func TrimBase64Mime(b64 string) string

func Validate

func Validate(data *structpb.Struct, jsonSchema string, target string) error

Validate the input and output format

Types

type Component

type Component struct {
	Logger          *zap.Logger
	NewUsageHandler UsageHandlerCreator
	// contains filtered or unexported fields
}

Component implements the common component methods.

func (*Component) GetDefinition

func (c *Component) GetDefinition(sysVars map[string]any, compConfig *ComponentConfig) (*pb.ComponentDefinition, error)

func (*Component) GetDefinitionID

func (c *Component) GetDefinitionID() string

GetDefinitionID returns the component definition ID.

func (*Component) GetDefinitionUID

func (c *Component) GetDefinitionUID() uuid.UUID

GetDefinitionUID returns the component definition UID.

func (*Component) GetLogger

func (c *Component) GetLogger() *zap.Logger

GetLogger returns the component's logger. If it hasn't been initialized, a no-op logger is returned.

func (*Component) GetTaskInputSchemas

func (c *Component) GetTaskInputSchemas() map[string]string

func (*Component) GetTaskOutputSchemas

func (c *Component) GetTaskOutputSchemas() map[string]string

func (*Component) HandleVerificationEvent

func (c *Component) HandleVerificationEvent(header map[string][]string, req *structpb.Struct, setup map[string]any) (isVerification bool, resp *structpb.Struct, err error)

func (*Component) IsSecretField

func (c *Component) IsSecretField(target string) bool

IsSecretField checks if the target field is secret field

func (*Component) ListInputAcceptFormatsFields

func (c *Component) ListInputAcceptFormatsFields() (map[string]map[string][]string, error)

func (*Component) ListOutputFormatsFields

func (c *Component) ListOutputFormatsFields() (map[string]map[string]string, error)

func (*Component) ListSecretFields

func (c *Component) ListSecretFields() ([]string, error)

ListSecretFields lists the secret fields by definition id

func (*Component) LoadDefinition

func (c *Component) LoadDefinition(definitionJSONBytes, setupJSONBytes, tasksJSONBytes []byte, additionalJSONBytes map[string][]byte) error

LoadDefinition loads the component definitions from json files

func (*Component) ParseEvent

func (c *Component) ParseEvent(ctx context.Context, req *structpb.Struct, setup map[string]any) (parsed *structpb.Struct, err error)

func (*Component) Test

func (c *Component) Test(sysVars map[string]any, setup *structpb.Struct) error

func (*Component) UsageHandlerCreator

func (c *Component) UsageHandlerCreator() UsageHandlerCreator

UsageHandlerCreator returns a function to initialize a UsageHandler. If the component doesn't have such function initialized, a no-op usage handler creator is returned.

type ComponentConfig

type ComponentConfig struct {
	Task  string
	Input map[string]any
	Setup map[string]any
}

type ComponentExecution

type ComponentExecution struct {
	Component IComponent

	// Component ID is the ID of the component *as defined in the recipe*. This
	// identifies an instance of a component, which holds a given configuration
	// (task, setup, input parameters, etc.).
	//
	// NOTE: this is a property of the component not of the execution. However,
	// right now components are being created on startup and only executions
	// are created every time a pipeline is triggered. Therefore, at the moment
	// there's no intermediate entity reflecting "a component within a
	// pipeline". Since we need to access the component ID for e.g. logging /
	// metric collection purposes, for now this information will live in the
	// execution, but note that several executions might have the same
	// component ID.
	ComponentID     string
	SystemVariables map[string]any
	Setup           *structpb.Struct
	Task            string
}

ComponentExecution implements the common methods for component execution.

func (*ComponentExecution) FillInDefaultValues

func (e *ComponentExecution) FillInDefaultValues(input *structpb.Struct) (*structpb.Struct, error)

func (*ComponentExecution) GetComponent

func (e *ComponentExecution) GetComponent() IComponent

GetComponent returns the component interface that is triggering the execution.

func (*ComponentExecution) GetComponentID

func (e *ComponentExecution) GetComponentID() string

GetComponentID returns the ID of the component that's being executed.

func (*ComponentExecution) GetLogger

func (e *ComponentExecution) GetLogger() *zap.Logger

func (*ComponentExecution) GetSetup

func (e *ComponentExecution) GetSetup() *structpb.Struct

func (*ComponentExecution) GetSystemVariables

func (e *ComponentExecution) GetSystemVariables() map[string]any

func (*ComponentExecution) GetTask

func (e *ComponentExecution) GetTask() string

func (*ComponentExecution) GetTaskInputSchema

func (e *ComponentExecution) GetTaskInputSchema() string

func (*ComponentExecution) GetTaskOutputSchema

func (e *ComponentExecution) GetTaskOutputSchema() string

func (*ComponentExecution) UsesInstillCredentials

func (e *ComponentExecution) UsesInstillCredentials() bool

UsesInstillCredentials indicates wether the component setup includes the use of global secrets (as opposed to a bring-your-own-key configuration) to connect to external services. Components should override this method when they have the ability to read global secrets and be executed without explicit credentials.

type ErrorHandler

type ErrorHandler interface {
	Error(ctx context.Context, err error)
}

type ExecutionWrapper

type ExecutionWrapper struct {
	IExecution
}

ExecutionWrapper performs validation and usage collection around the execution of a component.

func (*ExecutionWrapper) Execute

func (e *ExecutionWrapper) Execute(ctx context.Context, jobs []*Job) (err error)

Execute wraps the execution method with validation and usage collection.

type IComponent

type IComponent interface {
	GetDefinitionID() string
	GetDefinitionUID() uuid.UUID
	GetLogger() *zap.Logger
	GetTaskInputSchemas() map[string]string
	GetTaskOutputSchemas() map[string]string

	LoadDefinition(definitionJSON, setupJSON, tasksJSON []byte, additionalJSONBytes map[string][]byte) error

	// Note: Some content in the definition JSON schema needs to be generated
	// by sysVars or component setting.
	GetDefinition(sysVars map[string]any, compConfig *ComponentConfig) (*pb.ComponentDefinition, error)

	// CreateExecution takes a ComponentExecution that can be used to compose
	// the core component behaviour with the particular business logic in the
	// implmentation.
	CreateExecution(base ComponentExecution) (IExecution, error)
	Test(sysVars map[string]any, config *structpb.Struct) error

	IsSecretField(target string) bool

	// Note: These two functions are for the pipeline run-on-event feature,
	// which is still experimental and may change at any time.
	HandleVerificationEvent(header map[string][]string, req *structpb.Struct, setup map[string]any) (isVerification bool, resp *structpb.Struct, err error)
	ParseEvent(ctx context.Context, req *structpb.Struct, setup map[string]any) (parsed *structpb.Struct, err error)

	UsageHandlerCreator() UsageHandlerCreator
}

IComponent is the interface that wraps the basic component methods. All component need to implement this interface.

type IExecution

type IExecution interface {
	GetTask() string
	GetLogger() *zap.Logger
	GetTaskInputSchema() string
	GetTaskOutputSchema() string
	GetSystemVariables() map[string]any
	GetComponent() IComponent
	GetComponentID() string
	UsesInstillCredentials() bool

	Execute(context.Context, []*Job) error
}

IExecution allows components to be executed.

type InputReader

type InputReader interface {
	Read(ctx context.Context) (input *structpb.Struct, err error)
}

type InstillAcceptFormatsCompiler

type InstillAcceptFormatsCompiler struct{}

func (InstillAcceptFormatsCompiler) Compile

func (InstillAcceptFormatsCompiler) Compile(ctx jsonschema.CompilerContext, m map[string]interface{}) (jsonschema.ExtSchema, error)

type InstillAcceptFormatsSchema

type InstillAcceptFormatsSchema []string

func (InstillAcceptFormatsSchema) Validate

func (s InstillAcceptFormatsSchema) Validate(ctx jsonschema.ValidationContext, v interface{}) error

type InstillExtension

type InstillExtension struct {
	jsoniter.DummyExtension
}

func (*InstillExtension) UpdateStructDescriptor

func (e *InstillExtension) UpdateStructDescriptor(structDescriptor *jsoniter.StructDescriptor)

type InstillFormatCompiler

type InstillFormatCompiler struct{}

func (InstillFormatCompiler) Compile

func (InstillFormatCompiler) Compile(ctx jsonschema.CompilerContext, m map[string]interface{}) (jsonschema.ExtSchema, error)

type InstillFormatSchema

type InstillFormatSchema string

func (InstillFormatSchema) Validate

func (s InstillFormatSchema) Validate(ctx jsonschema.ValidationContext, v interface{}) error

type Job

type Job struct {
	Input  InputReader
	Output OutputWriter
	Error  ErrorHandler
}

func GenerateMockJob

func GenerateMockJob(c minimock.Tester) (ir *mock.InputReaderMock, ow *mock.OutputWriterMock, eh *mock.ErrorHandlerMock, job *Job)

type OutputWriter

type OutputWriter interface {
	Write(ctx context.Context, output *structpb.Struct) (err error)
}

type UsageHandler

type UsageHandler interface {
	Check(ctx context.Context, inputs []*structpb.Struct) error
	Collect(ctx context.Context, inputs, outputs []*structpb.Struct) error
}

UsageHandler allows the component execution wrapper to add checks and collect usage metrics around a component execution.

func NewNoopUsageHandler

func NewNoopUsageHandler(IExecution) (UsageHandler, error)

NewNoopUsageHandler is a no-op usage handler initializer.

type UsageHandlerCreator

type UsageHandlerCreator func(IExecution) (UsageHandler, error)

UsageHandlerCreator returns a function to initialize a UsageHandler.

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL